From af7c0ffc2f3a86e172a29545a85bd97b087255f7 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:46:33 +0000 Subject: [PATCH 01/14] [InfoExtractor] Use kwarg maxsplit for re.split * May become kw-only in future Pythons --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 9b0016d07ec..78704b55718 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -3170,7 +3170,7 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None, # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as # of jwplayer.flash.swf rtmp_url_parts = re.split( - r'((?:mp4|mp3|flv):)', source_url, 1) + r'((?:mp4|mp3|flv):)', source_url, maxsplit=1) if len(rtmp_url_parts) == 3: rtmp_url, prefix, play_path = rtmp_url_parts a_format.update({ From 7e93935ddaf1e52fc519b9d4f495db203b296f51 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 7 Dec 2024 03:37:39 +0000 Subject: [PATCH 02/14] [jsinterp] Implement `typeof` operator --- test/test_jsinterp.py | 15 +++- youtube_dl/jsinterp.py | 152 ++++++++++++++++++++++++++++------------- 2 files changed, 117 insertions(+), 50 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index c7a4f2cbf23..d063bbd36a5 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -266,7 +266,20 @@ def test_comma(self): self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5) def test_void(self): - self._test('function f() { return void 42; }', None) + self._test('function f() { return void 42; }', JS_Undefined) + + def test_typeof(self): + self._test('function f() { return typeof undefined; }', 'undefined') + self._test('function f() { return typeof NaN; }', 'number') + self._test('function f() { return typeof Infinity; }', 'number') + self._test('function f() { return typeof true; }', 'boolean') + self._test('function f() { return typeof null; }', 'object') + self._test('function f() { return typeof "a string"; }', 'string') + self._test('function f() { return typeof 42; }', 'number') + self._test('function f() { return typeof 42.42; }', 'number') + self._test('function f() { var g = function(){}; return typeof g; }', 'function') + self._test('function f() { return typeof {key: "value"}; }', 'object') + # not yet implemented: Symbol, BigInt def test_return_function(self): jsi = JSInterpreter(''' diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index a616ad070b2..acc68c5336c 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -5,7 +5,7 @@ import operator import re -from functools import update_wrapper +from functools import update_wrapper, wraps from .utils import ( error_to_compat_str, @@ -20,9 +20,11 @@ compat_basestring, compat_chr, compat_collections_chain_map as ChainMap, + compat_contextlib_suppress, compat_filter as filter, compat_itertools_zip_longest as zip_longest, compat_map as map, + compat_numeric_types, compat_str, ) @@ -138,6 +140,38 @@ def _js_ternary(cndn, if_true=True, if_false=False): return if_true +def _js_unary_op(op): + + @wraps_op(op) + def wrapped(_, a): + return op(a) + + return wrapped + + +# https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/typeof +def _js_typeof(expr): + with compat_contextlib_suppress(TypeError, KeyError): + return { + JS_Undefined: 'undefined', + _NaN: 'number', + _Infinity: 'number', + True: 'boolean', + False: 'boolean', + None: 'object', + }[expr] + for t, n in ( + (compat_basestring, 'string'), + (compat_numeric_types, 'number'), + ): + if isinstance(expr, t): + return n + if callable(expr): + return 'function' + # TODO: Symbol, BigInt + return 'object' + + # (op, definition) in order of binding priority, tightest first # avoid dict to maintain order # definition None => Defined in JSInterpreter._operator @@ -176,6 +210,11 @@ def _js_ternary(cndn, if_true=True, if_false=False): ('&&', None), ) +_UNARY_OPERATORS_X = ( + ('void', _js_unary_op(lambda _: JS_Undefined)), + ('typeof', _js_unary_op(_js_typeof)), +) + _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS)) _NAME_RE = r'[a-zA-Z_$][\w$]*' @@ -242,6 +281,7 @@ def truncate_string(s, left, right=0): @classmethod def wrap_interpreter(cls, f): + @wraps(f) def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs): if cls.ENABLED and stmt.strip(): cls.write(stmt, level=allow_recursion) @@ -347,6 +387,8 @@ def regex_flags(cls, expr): def __op_chars(cls): op_chars = set(';,[') for op in cls._all_operators(): + if op[0].isalpha(): + continue op_chars.update(op[0]) return op_chars @@ -425,7 +467,7 @@ def _all_operators(_cached=[]): if not _cached: _cached.extend(itertools.chain( # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence - _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)) + _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS, _UNARY_OPERATORS_X)) return _cached def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion): @@ -479,6 +521,52 @@ def _dump(self, obj, namespace): _FINALLY_RE = re.compile(r'finally\s*\{') _SWITCH_RE = re.compile(r'switch\s*\(') + def handle_operators(self, expr, local_vars, allow_recursion): + + for op, _ in self._all_operators(): + # hackety: have higher priority than <>, but don't confuse them + skip_delim = (op + op) if op in '<>*?' else None + if op == '?': + skip_delim = (skip_delim, '?.') + separated = list(self._separate(expr, op, skip_delims=skip_delim)) + if len(separated) < 2: + continue + + right_expr = separated.pop() + # handle operators that are both unary and binary, minimal BODMAS + if op in ('+', '-'): + # simplify/adjust consecutive instances of these operators + undone = 0 + separated = [s.strip() for s in separated] + while len(separated) > 1 and not separated[-1]: + undone += 1 + separated.pop() + if op == '-' and undone % 2 != 0: + right_expr = op + right_expr + elif op == '+': + while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS: + right_expr = separated.pop() + right_expr + if separated[-1][-1:] in self.OP_CHARS: + right_expr = separated.pop() + right_expr + # hanging op at end of left => unary + (strip) or - (push right) + left_val = separated[-1] if separated else '' + for dm_op in ('*', '%', '/', '**'): + bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim)) + if len(bodmas) > 1 and not bodmas[-1].strip(): + expr = op.join(separated) + op + right_expr + if len(separated) > 1: + separated.pop() + right_expr = op.join((left_val, right_expr)) + else: + separated = [op.join((left_val, right_expr))] + right_expr = None + break + if right_expr is None: + continue + + left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion) + return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), True + @Debugger.wrap_interpreter def interpret_statement(self, stmt, local_vars, allow_recursion=100): if allow_recursion < 0: @@ -533,9 +621,15 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): else: raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr) - if expr.startswith('void '): - left = self.interpret_expression(expr[5:], local_vars, allow_recursion) - return None, should_return + for op, _ in _UNARY_OPERATORS_X: + if not expr.startswith(op): + continue + operand = expr[len(op):] + if not operand or operand[0] != ' ': + continue + op_result = self.handle_operators(expr, local_vars, allow_recursion) + if op_result: + return op_result[0], should_return if expr.startswith('{'): inner, outer = self._separate_at_paren(expr) @@ -582,7 +676,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): if_expr, expr = self._separate_at_paren(expr) else: # may lose ... else ... because of ll.368-374 - if_expr, expr = self._separate_at_paren(expr, delim=';') + if_expr, expr = self._separate_at_paren(' %s;' % (expr,), delim=';') else_expr = None m = re.match(r'else\s*(?P\{)?', expr) if m: @@ -790,49 +884,9 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion) return self._index(val, idx), should_return - for op, _ in self._all_operators(): - # hackety: have higher priority than <>, but don't confuse them - skip_delim = (op + op) if op in '<>*?' else None - if op == '?': - skip_delim = (skip_delim, '?.') - separated = list(self._separate(expr, op, skip_delims=skip_delim)) - if len(separated) < 2: - continue - - right_expr = separated.pop() - # handle operators that are both unary and binary, minimal BODMAS - if op in ('+', '-'): - # simplify/adjust consecutive instances of these operators - undone = 0 - separated = [s.strip() for s in separated] - while len(separated) > 1 and not separated[-1]: - undone += 1 - separated.pop() - if op == '-' and undone % 2 != 0: - right_expr = op + right_expr - elif op == '+': - while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS: - right_expr = separated.pop() + right_expr - if separated[-1][-1:] in self.OP_CHARS: - right_expr = separated.pop() + right_expr - # hanging op at end of left => unary + (strip) or - (push right) - left_val = separated[-1] if separated else '' - for dm_op in ('*', '%', '/', '**'): - bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim)) - if len(bodmas) > 1 and not bodmas[-1].strip(): - expr = op.join(separated) + op + right_expr - if len(separated) > 1: - separated.pop() - right_expr = op.join((left_val, right_expr)) - else: - separated = [op.join((left_val, right_expr))] - right_expr = None - break - if right_expr is None: - continue - - left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion) - return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return + op_result = self.handle_operators(expr, local_vars, allow_recursion) + if op_result: + return op_result[0], should_return if md.get('attribute'): variable, member, nullish = m.group('var', 'member', 'nullish') From 527c137e4e87e211cb58aeed764077d3d59702f8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 00:00:32 +0000 Subject: [PATCH 03/14] [jsinterp] Fix and improve loose and strict equality operations * reimplement loose equality according to MDN (eg, 1 == "1") * improve strict equality (eg, "abc" === "abc" but 'abc' is not 'abc') * add tests for above --- test/test_jsinterp.py | 35 ++++++++++++++++-- youtube_dl/jsinterp.py | 80 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 101 insertions(+), 14 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index d063bbd36a5..86137d8e56a 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# coding: utf-8 from __future__ import unicode_literals @@ -11,7 +12,7 @@ import math import re -from youtube_dl.compat import compat_str +from youtube_dl.compat import compat_str as str from youtube_dl.jsinterp import JS_Undefined, JSInterpreter NaN = object() @@ -19,7 +20,7 @@ class TestJSInterpreter(unittest.TestCase): def _test(self, jsi_or_code, expected, func='f', args=()): - if isinstance(jsi_or_code, compat_str): + if isinstance(jsi_or_code, str): jsi_or_code = JSInterpreter(jsi_or_code) got = jsi_or_code.call_function(func, *args) if expected is NaN: @@ -89,7 +90,35 @@ def test_operators(self): self._test('function f(){return 19 & 21;}', 17) self._test('function f(){return 11 >> 2;}', 2) self._test('function f(){return []? 2+3: 4;}', 5) + # equality + self._test('function f(){return 1 == 1}', True) + self._test('function f(){return 1 == 1.0}', True) + self._test('function f(){return 1 == "1"}', True) self._test('function f(){return 1 == 2}', False) + self._test('function f(){return 1 != "1"}', False) + self._test('function f(){return 1 != 2}', True) + self._test('function f(){var x = {a: 1}; var y = x; return x == y}', True) + self._test('function f(){var x = {a: 1}; return x == {a: 1}}', False) + self._test('function f(){return NaN == NaN}', False) + self._test('function f(){return null == undefined}', True) + self._test('function f(){return "spam, eggs" == "spam, eggs"}', True) + # strict equality + self._test('function f(){return 1 === 1}', True) + self._test('function f(){return 1 === 1.0}', True) + self._test('function f(){return 1 === "1"}', False) + self._test('function f(){return 1 === 2}', False) + self._test('function f(){var x = {a: 1}; var y = x; return x === y}', True) + self._test('function f(){var x = {a: 1}; return x === {a: 1}}', False) + self._test('function f(){return NaN === NaN}', False) + self._test('function f(){return null === undefined}', False) + self._test('function f(){return null === null}', True) + self._test('function f(){return undefined === undefined}', True) + self._test('function f(){return "uninterned" === "uninterned"}', True) + self._test('function f(){return 1 === 1}', True) + self._test('function f(){return 1 === "1"}', False) + self._test('function f(){return 1 !== 1}', False) + self._test('function f(){return 1 !== "1"}', True) + # expressions self._test('function f(){return 0 && 1 || 2;}', 2) self._test('function f(){return 0 ?? 42;}', 0) self._test('function f(){return "life, the universe and everything" < 42;}', False) @@ -296,7 +325,7 @@ def test_null(self): def test_undefined(self): self._test('function f() { return undefined === undefined; }', True) self._test('function f() { return undefined; }', JS_Undefined) - self._test('function f() {return undefined ?? 42; }', 42) + self._test('function f() { return undefined ?? 42; }', 42) self._test('function f() { let v; return v; }', JS_Undefined) self._test('function f() { let v; return v**0; }', 1) self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }', diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index acc68c5336c..b7a3ef69ab9 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals import itertools @@ -64,6 +65,10 @@ def update_and_rename_wrapper(w): _Infinity = float('inf') +class JS_Undefined(object): + pass + + def _js_bit_op(op): def zeroise(x): @@ -107,12 +112,69 @@ def _js_exp(a, b): return (a or 0) ** b -def _js_eq_op(op): +def _js_to_primitive(v): + return ( + ','.join(map(_js_toString, v)) if isinstance(v, list) + else '[object Object]' if isinstance(v, dict) + else compat_str(v) if not isinstance(v, ( + compat_numeric_types, compat_basestring, bool)) + else v + ) + + +def _js_toString(v): + return ( + 'undefined' if v is JS_Undefined + else 'Infinity' if v == _Infinity + else 'NaN' if v is _NaN + else 'null' if v is None + else compat_str(v) if isinstance(v, compat_numeric_types) + else _js_to_primitive(v)) + + +_nullish = frozenset((None, JS_Undefined)) + + +def _js_eq(a, b): + # NaN != any + if _NaN in (a, b): + return False + # Object is Object + if isinstance(a, type(b)) and isinstance(b, (dict, list)): + return operator.is_(a, b) + # general case + if a == b: + return True + # null == undefined + a_b = set((a, b)) + if a_b & _nullish: + return a_b <= _nullish + a, b = _js_to_primitive(a), _js_to_primitive(b) + if not isinstance(a, compat_basestring): + a, b = b, a + # Number to String: convert the string to a number + # Conversion failure results in ... false + if isinstance(a, compat_basestring): + return float_or_none(a) == b + return a == b + + +def _js_neq(a, b): + return not _js_eq(a, b) + + +def _js_id_op(op): @wraps_op(op) def wrapped(a, b): - if set((a, b)) <= set((None, JS_Undefined)): - return op(a, a) + if _NaN in (a, b): + return op(_NaN, None) + if not isinstance(a, (compat_basestring, compat_numeric_types)): + a, b = b, a + # strings are === if == + # why 'a' is not 'a': https://stackoverflow.com/a/1504848 + if isinstance(a, (compat_basestring, compat_numeric_types)): + return a == b if op(0, 0) else a != b return op(a, b) return wrapped @@ -187,10 +249,10 @@ def _js_typeof(expr): ) _COMP_OPERATORS = ( - ('===', operator.is_), - ('!==', operator.is_not), - ('==', _js_eq_op(operator.eq)), - ('!=', _js_eq_op(operator.ne)), + ('===', _js_id_op(operator.is_)), + ('!==', _js_id_op(operator.is_not)), + ('==', _js_eq), + ('!=', _js_neq), ('<=', _js_comp_op(operator.le)), ('>=', _js_comp_op(operator.ge)), ('<', _js_comp_op(operator.lt)), @@ -222,10 +284,6 @@ def _js_typeof(expr): _QUOTES = '\'"/' -class JS_Undefined(object): - pass - - class JS_Break(ExtractorError): def __init__(self): ExtractorError.__init__(self, 'Invalid break') From 7d3045ce12310c9f6819947aa2170d1e9e911614 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 00:15:31 +0000 Subject: [PATCH 04/14] [jsinterp] Support multiple indexing (eg a[1][2]) * extend single indexing with improved RE (should probably use/have used _separate_at_paren()) * fix some cases that should have given undefined, not throwing * standardise RE group names * support length of objects, like {1: 2, 3: 4, length: 42} --- test/test_jsinterp.py | 10 ++++++++++ youtube_dl/jsinterp.py | 37 ++++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 86137d8e56a..d2fca29972a 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -366,6 +366,16 @@ def test_object(self): self._test('function f() { let a; return a?.qq; }', JS_Undefined) self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined) + def test_indexing(self): + self._test('function f() { return [1, 2, 3, 4][3]}', 4) + self._test('function f() { return [1, [2, [3, [4]]]][1][1][1][0]}', 4) + self._test('function f() { var o = {1: 2, 3: 4}; return o[3]}', 4) + self._test('function f() { var o = {1: 2, 3: 4}; return o["3"]}', 4) + self._test('function f() { return [1, [2, {3: [4]}]][1][1]["3"][0]}', 4) + self._test('function f() { return [1, 2, 3, 4].length}', 4) + self._test('function f() { var o = {1: 2, 3: 4}; return o.length}', JS_Undefined) + self._test('function f() { var o = {1: 2, 3: 4}; o["length"] = 42; return o.length}', 42) + def test_regex(self): self._test('function f() { let a=/,,[/,913,/](,)}/; }', None) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index b7a3ef69ab9..248210ce4a3 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -549,13 +549,14 @@ def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion) except Exception as e: raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e) - def _index(self, obj, idx, allow_undefined=False): - if idx == 'length': + def _index(self, obj, idx, allow_undefined=True): + if idx == 'length' and isinstance(obj, list): return len(obj) try: - return obj[int(idx)] if isinstance(obj, list) else obj[idx] - except Exception as e: + return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)] + except (TypeError, KeyError, IndexError) as e: if allow_undefined: + # when is not allowed? return JS_Undefined raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e) @@ -882,13 +883,13 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): m = re.match(r'''(?x) (?P - (?P{_NAME_RE})(?:\[(?P[^\]]+?)\])?\s* + (?P{_NAME_RE})(?:\[(?P(?:.+?\]\s*\[)*.+?)\])?\s* (?P{_OPERATOR_RE})? =(?!=)(?P.*)$ )|(?P (?!if|return|true|false|null|undefined|NaN|Infinity)(?P{_NAME_RE})$ )|(?P - (?P{_NAME_RE})\[(?P.+)\]$ + (?P{_NAME_RE})\[(?P(?:.+?\]\s*\[)*.+?)\]$ )|(?P (?P{_NAME_RE})(?:(?P\?)?\.(?P[^(]+)|\[(?P[^\]]+)\])\s* )|(?P @@ -898,19 +899,23 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): if md.get('assign'): left_val = local_vars.get(m.group('out')) - if not m.group('index'): + if not m.group('out_idx'): local_vars[m.group('out')] = self._operator( m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion) return local_vars[m.group('out')], should_return elif left_val in (None, JS_Undefined): raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr) - idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion) - if not isinstance(idx, (int, float)): - raise self.Exception('List index %s must be integer' % (idx, ), expr=expr) - idx = int(idx) + indexes = re.split(r'\]\s*\[', m.group('out_idx')) + for i, idx in enumerate(indexes, 1): + idx = self.interpret_expression(idx, local_vars, allow_recursion) + if i < len(indexes): + left_val = self._index(left_val, idx) + if isinstance(idx, float): + idx = int(idx) left_val[idx] = self._operator( - m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion) + m.group('op'), self._index(left_val, idx) if m.group('op') else None, + m.group('expr'), expr, local_vars, allow_recursion) return left_val[idx], should_return elif expr.isdigit(): @@ -939,8 +944,10 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): if md.get('indexing'): val = local_vars[m.group('in')] - idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion) - return self._index(val, idx), should_return + for idx in re.split(r'\]\s*\[', m.group('in_idx')): + idx = self.interpret_expression(idx, local_vars, allow_recursion) + val = self._index(val, idx) + return val, should_return op_result = self.handle_operators(expr, local_vars, allow_recursion) if op_result: @@ -989,7 +996,7 @@ def eval_method(variable, member): # Member access if arg_str is None: - return self._index(obj, member, nullish) + return self._index(obj, member) # Function call argvals = [ From 9993e8c5a90c879b944fcbf09bdc4eaf513cc75f Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 00:26:00 +0000 Subject: [PATCH 05/14] [jsinterp] Fix and improve arithmetic operations * addition becomes concat with a string operand * improve handling of edgier cases * arithmetic in float like JS (more places need cast to int?) * increase test coverage --- test/test_jsinterp.py | 20 ++++++++++++++++ youtube_dl/jsinterp.py | 53 +++++++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index d2fca29972a..52eaf1ed8a8 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -41,16 +41,27 @@ def test_add(self): self._test('function f(){return 42 + 7;}', 49) self._test('function f(){return 42 + undefined;}', NaN) self._test('function f(){return 42 + null;}', 42) + self._test('function f(){return 1 + "";}', '1') + self._test('function f(){return 42 + "7";}', '427') + self._test('function f(){return false + true;}', 1) + self._test('function f(){return "false" + true;}', 'falsetrue') + self._test('function f(){return ' + '1 + "2" + [3,4] + {k: 56} + null + undefined + Infinity;}', + '123,4[object Object]nullundefinedInfinity') def test_sub(self): self._test('function f(){return 42 - 7;}', 35) self._test('function f(){return 42 - undefined;}', NaN) self._test('function f(){return 42 - null;}', 42) + self._test('function f(){return 42 - "7";}', 35) + self._test('function f(){return 42 - "spam";}', NaN) def test_mul(self): self._test('function f(){return 42 * 7;}', 294) self._test('function f(){return 42 * undefined;}', NaN) self._test('function f(){return 42 * null;}', 0) + self._test('function f(){return 42 * "7";}', 294) + self._test('function f(){return 42 * "eggs";}', NaN) def test_div(self): jsi = JSInterpreter('function f(a, b){return a / b;}') @@ -58,17 +69,26 @@ def test_div(self): self._test(jsi, NaN, args=(JS_Undefined, 1)) self._test(jsi, float('inf'), args=(2, 0)) self._test(jsi, 0, args=(0, 3)) + self._test(jsi, 6, args=(42, 7)) + self._test(jsi, 0, args=(42, float('inf'))) + self._test(jsi, 6, args=("42", 7)) + self._test(jsi, NaN, args=("spam", 7)) def test_mod(self): self._test('function f(){return 42 % 7;}', 0) self._test('function f(){return 42 % 0;}', NaN) self._test('function f(){return 42 % undefined;}', NaN) + self._test('function f(){return 42 % "7";}', 0) + self._test('function f(){return 42 % "beans";}', NaN) def test_exp(self): self._test('function f(){return 42 ** 2;}', 1764) self._test('function f(){return 42 ** undefined;}', NaN) self._test('function f(){return 42 ** null;}', 1) + self._test('function f(){return undefined ** 0;}', 1) self._test('function f(){return undefined ** 42;}', NaN) + self._test('function f(){return 42 ** "2";}', 1764) + self._test('function f(){return 42 ** "spam";}', NaN) def test_calc(self): self._test('function f(a){return 2*a+1;}', 7, args=[3]) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 248210ce4a3..73934a6af76 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -11,6 +11,7 @@ from .utils import ( error_to_compat_str, ExtractorError, + float_or_none, js_to_json, remove_quotes, unified_timestamp, @@ -81,35 +82,47 @@ def wrapped(a, b): return wrapped -def _js_arith_op(op): +def _js_arith_op(op, div=False): @wraps_op(op) def wrapped(a, b): if JS_Undefined in (a, b): return _NaN - return op(a or 0, b or 0) + # null, "" --> 0 + a, b = (float_or_none( + (x.strip() if isinstance(x, compat_basestring) else x) or 0, + default=_NaN) for x in (a, b)) + if _NaN in (a, b): + return _NaN + try: + return op(a, b) + except ZeroDivisionError: + return _NaN if not (div and (a or b)) else _Infinity return wrapped -def _js_div(a, b): - if JS_Undefined in (a, b) or not (a or b): - return _NaN - return operator.truediv(a or 0, b) if b else _Infinity +_js_arith_add = _js_arith_op(operator.add) + + +def _js_add(a, b): + if not (isinstance(a, compat_basestring) or isinstance(b, compat_basestring)): + return _js_arith_add(a, b) + if not isinstance(a, compat_basestring): + a = _js_toString(a) + elif not isinstance(b, compat_basestring): + b = _js_toString(b) + return operator.concat(a, b) -def _js_mod(a, b): - if JS_Undefined in (a, b) or not b: - return _NaN - return (a or 0) % b +_js_mod = _js_arith_op(operator.mod) +__js_exp = _js_arith_op(operator.pow) def _js_exp(a, b): if not b: return 1 # even 0 ** 0 !! - elif JS_Undefined in (a, b): - return _NaN - return (a or 0) ** b + return __js_exp(a, b) def _js_to_primitive(v): @@ -117,7 +130,7 @@ def _js_to_primitive(v): ','.join(map(_js_toString, v)) if isinstance(v, list) else '[object Object]' if isinstance(v, dict) else compat_str(v) if not isinstance(v, ( - compat_numeric_types, compat_basestring, bool)) + compat_numeric_types, compat_basestring)) else v ) @@ -128,7 +141,9 @@ def _js_toString(v): else 'Infinity' if v == _Infinity else 'NaN' if v is _NaN else 'null' if v is None - else compat_str(v) if isinstance(v, compat_numeric_types) + # bool <= int: do this first + else ('false', 'true')[v] if isinstance(v, bool) + else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types) else _js_to_primitive(v)) @@ -240,11 +255,11 @@ def _js_typeof(expr): _OPERATORS = ( ('>>', _js_bit_op(operator.rshift)), ('<<', _js_bit_op(operator.lshift)), - ('+', _js_arith_op(operator.add)), + ('+', _js_add), ('-', _js_arith_op(operator.sub)), ('*', _js_arith_op(operator.mul)), ('%', _js_mod), - ('/', _js_div), + ('/', _js_arith_op(operator.truediv, div=True)), ('**', _js_exp), ) @@ -873,7 +888,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): start, end = m.span() sign = m.group('pre_sign') or m.group('post_sign') ret = local_vars[var] - local_vars[var] += 1 if sign[0] == '+' else -1 + local_vars[var] = _js_add(ret, 1 if sign[0] == '+' else -1) if m.group('pre_sign'): ret = local_vars[var] expr = expr[:start] + self._dump(ret, local_vars) + expr[end:] @@ -1023,7 +1038,7 @@ def eval_method(variable, member): if obj is compat_str: if member == 'fromCharCode': assertion(argvals, 'takes one or more arguments') - return ''.join(map(compat_chr, argvals)) + return ''.join(compat_chr(int(n)) for n in argvals) raise self.Exception('Unsupported string method ' + member, expr=expr) elif obj is float: if member == 'pow': From 4bf85ca5ba9dd7fb749a89c34fa3a56b1b848609 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:13:06 +0000 Subject: [PATCH 06/14] [jsinterp] Fix and improve split/join * improve split/join edge cases * correctly implement regex split (not like re.split) --- test/test_jsinterp.py | 19 +++++++++++++++++ youtube_dl/jsinterp.py | 48 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 52eaf1ed8a8..b6e87e9f14f 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -483,6 +483,13 @@ def test_join(self): self._test(jsi, 't-e-s-t', args=[test_input, '-']) self._test(jsi, '', args=[[], '-']) + self._test('function f(){return ' + '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join()}', + '1,1,abc,[object Object],,,Infinity,NaN') + self._test('function f(){return ' + '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join("~")}', + '1~1~abc~[object Object]~~~Infinity~NaN') + def test_split(self): test_result = list('test') tests = [ @@ -496,6 +503,18 @@ def test_split(self): self._test(jsi, test_result, args=['t-e-s-t', '-']) self._test(jsi, [''], args=['', '-']) self._test(jsi, [], args=['', '']) + # RegExp split + self._test('function f(){return "test".split(/(?:)/)}', + ['t', 'e', 's', 't']) + self._test('function f(){return "t-e-s-t".split(/[es-]+/)}', + ['t', 't']) + # from MDN: surrogate pairs aren't handled: case 1 fails + # self._test('function f(){return "😄😄".split(/(?:)/)}', + # ['\ud83d', '\ude04', '\ud83d', '\ude04']) + # case 2 beats Py3.2: it gets the case 1 result + if sys.version_info >= (2, 6) and not ((3, 0) <= sys.version_info < (3, 3)): + self._test('function f(){return "😄😄".split(/(?:)/u)}', + ['😄', '😄']) def test_slice(self): self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice()}', [0, 1, 2, 3, 4, 5, 6, 7, 8]) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 73934a6af76..bec959946a5 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -397,6 +397,9 @@ class JS_RegExp(object): RE_FLAGS = { # special knowledge: Python's re flags are bitmask values, current max 128 # invent new bitmask values well above that for literal parsing + # JS 'u' flag is effectively always set (surrogate pairs aren't seen), + # but \u{...} and \p{...} escapes aren't handled); no additional JS 'v' + # features are supported # TODO: execute matches with these flags (remaining: d, y) 'd': 1024, # Generate indices for substring matches 'g': 2048, # Global search @@ -404,6 +407,7 @@ class JS_RegExp(object): 'm': re.M, # Multi-line search 's': re.S, # Allows . to match newline characters 'u': re.U, # Treat a pattern as a sequence of unicode code points + 'v': re.U, # Like 'u' with extended character class and \p{} syntax 'y': 4096, # Perform a "sticky" search that matches starting at the current position in the target string } @@ -1047,13 +1051,47 @@ def eval_method(variable, member): raise self.Exception('Unsupported Math method ' + member, expr=expr) if member == 'split': - assertion(argvals, 'takes one or more arguments') - assertion(len(argvals) == 1, 'with limit argument is not implemented') - return obj.split(argvals[0]) if argvals[0] else list(obj) + assertion(len(argvals) <= 2, 'takes at most two arguments') + if len(argvals) > 1: + limit = argvals[1] + assertion(isinstance(limit, int) and limit >= 0, 'integer limit >= 0') + if limit == 0: + return [] + else: + limit = 0 + if len(argvals) == 0: + argvals = [JS_Undefined] + elif isinstance(argvals[0], self.JS_RegExp): + # avoid re.split(), similar but not enough + + def where(): + for m in argvals[0].finditer(obj): + yield m.span(0) + yield (None, None) + + def splits(limit=limit): + i = 0 + for j, jj in where(): + if j == jj == 0: + continue + if j is None and i >= len(obj): + break + yield obj[i:j] + if jj is None or limit == 1: + break + limit -= 1 + i = jj + + return list(splits()) + return ( + obj.split(argvals[0], limit - 1) if argvals[0] and argvals[0] != JS_Undefined + else list(obj)[:limit or None]) elif member == 'join': assertion(isinstance(obj, list), 'must be applied on a list') - assertion(len(argvals) == 1, 'takes exactly one argument') - return argvals[0].join(obj) + assertion(len(argvals) <= 1, 'takes at most one argument') + return (',' if len(argvals) == 0 else argvals[0]).join( + ('' if x in (None, JS_Undefined) else _js_toString(x)) + for x in obj) elif member == 'reverse': assertion(not argvals, 'does not take any arguments') obj.reverse() From 7ab85d88d67f557d4e38f84d4e7ca40b43c933ed Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:16:07 +0000 Subject: [PATCH 07/14] [jsinterp] Fix and improve "methods" * push, unshift return new length * impove edge cases for push/pop, shift/unshift, forEach, indexOf, charCodeAt * increase test coverage --- test/test_jsinterp.py | 34 ++++++++++++++++++++++++++++++++++ youtube_dl/jsinterp.py | 36 +++++++++++++++--------------------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index b6e87e9f14f..07ed481d8e9 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -544,6 +544,40 @@ def test_slice(self): self._test('function f(){return "012345678".slice(-1, 1)}', '') self._test('function f(){return "012345678".slice(-3, -1)}', '67') + def test_pop(self): + # pop + self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.pop(), a]}', + [8, [0, 1, 2, 3, 4, 5, 6, 7]]) + self._test('function f(){return [].pop()}', JS_Undefined) + # push + self._test('function f(){var a = [0, 1, 2]; return [a.push(3, 4), a]}', + [5, [0, 1, 2, 3, 4]]) + self._test('function f(){var a = [0, 1, 2]; return [a.push(), a]}', + [3, [0, 1, 2]]) + + def test_shift(self): + # shift + self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.shift(), a]}', + [0, [1, 2, 3, 4, 5, 6, 7, 8]]) + self._test('function f(){return [].shift()}', JS_Undefined) + # unshift + self._test('function f(){var a = [0, 1, 2]; return [a.unshift(3, 4), a]}', + [5, [3, 4, 0, 1, 2]]) + self._test('function f(){var a = [0, 1, 2]; return [a.unshift(), a]}', + [3, [0, 1, 2]]) + + def test_forEach(self): + self._test('function f(){var ret = []; var l = [4, 2]; ' + 'var log = function(e,i,a){ret.push([e,i,a]);}; ' + 'l.forEach(log); ' + 'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}', + [2, 4, 1, [4, 2]]) + self._test('function f(){var ret = []; var l = [4, 2]; ' + 'var log = function(e,i,a){this.push([e,i,a]);}; ' + 'l.forEach(log, ret); ' + 'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}', + [2, 4, 1, [4, 2]]) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index bec959946a5..0cfae4b283d 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -1113,37 +1113,31 @@ def splits(limit=limit): index, how_many = map(int, (argvals + [len(obj)])[:2]) if index < 0: index += len(obj) - add_items = argvals[2:] - res = [] - for _ in range(index, min(index + how_many, len(obj))): - res.append(obj.pop(index)) - for i, item in enumerate(add_items): - obj.insert(index + i, item) + res = [obj.pop(index) + for _ in range(index, min(index + how_many, len(obj)))] + obj[index:index] = argvals[2:] return res - elif member == 'unshift': - assertion(isinstance(obj, list), 'must be applied on a list') - assertion(argvals, 'takes one or more arguments') - for item in reversed(argvals): - obj.insert(0, item) - return obj - elif member == 'pop': + elif member in ('shift', 'pop'): assertion(isinstance(obj, list), 'must be applied on a list') assertion(not argvals, 'does not take any arguments') - if not obj: - return - return obj.pop() + return obj.pop(0 if member == 'shift' else -1) if len(obj) > 0 else JS_Undefined + elif member == 'unshift': + assertion(isinstance(obj, list), 'must be applied on a list') + # not enforced: assertion(argvals, 'takes one or more arguments') + obj[0:0] = argvals + return len(obj) elif member == 'push': - assertion(argvals, 'takes one or more arguments') + # not enforced: assertion(argvals, 'takes one or more arguments') obj.extend(argvals) - return obj + return len(obj) elif member == 'forEach': assertion(argvals, 'takes one or more arguments') - assertion(len(argvals) <= 2, 'takes at-most 2 arguments') + assertion(len(argvals) <= 2, 'takes at most 2 arguments') f, this = (argvals + [''])[:2] return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)] elif member == 'indexOf': assertion(argvals, 'takes one or more arguments') - assertion(len(argvals) <= 2, 'takes at-most 2 arguments') + assertion(len(argvals) <= 2, 'takes at most 2 arguments') idx, start = (argvals + [0])[:2] try: return obj.index(idx, start) @@ -1152,7 +1146,7 @@ def splits(limit=limit): elif member == 'charCodeAt': assertion(isinstance(obj, compat_str), 'must be applied on a string') # assertion(len(argvals) == 1, 'takes exactly one argument') # but not enforced - idx = argvals[0] if isinstance(argvals[0], int) else 0 + idx = argvals[0] if len(argvals) > 0 and isinstance(argvals[0], int) else 0 if idx >= len(obj): return None return ord(obj[idx]) From 067ad7439cd6c7908ea13d856cc92057a2459336 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:21:53 +0000 Subject: [PATCH 08/14] [jsinterp] Strip /* comments */ when parsing * NB: _separate() is looking creaky --- test/test_jsinterp.py | 10 +++++++++- youtube_dl/jsinterp.py | 20 ++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 07ed481d8e9..12e7b9b9485 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -160,7 +160,6 @@ def test_assignments(self): self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51) self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11) - @unittest.skip('Not yet fully implemented') def test_comments(self): self._test(''' function f() { @@ -179,6 +178,15 @@ def test_comments(self): } ''', 3) + self._test(''' + function f() { + var x = ( /* 1 + */ 2 + + /* 30 * 40 */ + 50); + return x; + } + ''', 52) + def test_precedence(self): self._test(''' function f() { diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 0cfae4b283d..ec8674936e4 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -488,9 +488,18 @@ def _separate(cls, expr, delim=',', max_split=None, skip_delims=None): skipping = 0 if skip_delims: skip_delims = variadic(skip_delims) + skip_txt = None for idx, char in enumerate(expr): + if skip_txt and idx <= skip_txt[1]: + continue paren_delta = 0 if not in_quote: + if char == '/' and expr[idx:idx + 2] == '/*': + # skip a comment + skip_txt = expr[idx:].find('*/', 2) + skip_txt = [idx, idx + skip_txt + 1] if skip_txt >= 2 else None + if skip_txt: + continue if char in _MATCHING_PARENS: counters[_MATCHING_PARENS[char]] += 1 paren_delta = 1 @@ -523,12 +532,19 @@ def _separate(cls, expr, delim=',', max_split=None, skip_delims=None): if pos < delim_len: pos += 1 continue - yield expr[start: idx - delim_len] + if skip_txt and skip_txt[0] >= start and skip_txt[1] <= idx - delim_len: + yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1: idx - delim_len] + else: + yield expr[start: idx - delim_len] + skip_txt = None start, pos = idx + 1, 0 splits += 1 if max_split and splits >= max_split: break - yield expr[start:] + if skip_txt and skip_txt[0] >= start: + yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1:] + else: + yield expr[start:] @classmethod def _separate_at_paren(cls, expr, delim=None): From 2968201ae039b7122ae95860dc5f9699e5db9b6e Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:29:13 +0000 Subject: [PATCH 09/14] [jsinterp] Add return hook for player `3bb1f723` * set var `_ytdl_do_not_return` to a specific value in the scope of a function * if an expression to be returned has that value, `return` becomes `void` --- youtube_dl/jsinterp.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index ec8674936e4..7835187f5fa 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -368,7 +368,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs raise if cls.ENABLED and stmt.strip(): if should_ret or repr(ret) != stmt: - cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion) + cls.write(['->', '=>'][bool(should_ret)], repr(ret), '<-|', stmt, level=allow_recursion) return ret, should_ret return interpret_statement @@ -603,7 +603,7 @@ def _dump(self, obj, namespace): # used below _VAR_RET_THROW_RE = re.compile(r'''(?x) - (?P(?:var|const|let)\s)|return(?:\s+|(?=["'])|$)|(?Pthrow\s+) + (?:(?Pvar|const|let)\s+|(?Preturn)(?:\s+|(?=["'])|$)|(?Pthrow)\s+) ''') _COMPOUND_RE = re.compile(r'''(?x) (?Ptry)\s*\{| @@ -683,7 +683,7 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): expr = stmt[len(m.group(0)):].strip() if m.group('throw'): raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion)) - should_return = not m.group('var') + should_return = 'return' if m.group('ret') else False if not expr: return None, should_return @@ -968,14 +968,20 @@ def interpret_statement(self, stmt, local_vars, allow_recursion=100): return _Infinity, should_return elif md.get('return'): - return local_vars[m.group('name')], should_return + ret = local_vars[m.group('name')] + # challenge may try to force returning the original value + # use an optional internal var to block this + if should_return == 'return': + if '_ytdl_do_not_return' not in local_vars: + return ret, True + return (ret, True) if ret != local_vars['_ytdl_do_not_return'] else (ret, False) + else: + return ret, should_return - try: + with compat_contextlib_suppress(ValueError): ret = json.loads(js_to_json(expr)) # strict=True) if not md.get('attribute'): return ret, should_return - except ValueError: - pass if md.get('indexing'): val = local_vars[m.group('in')] @@ -1213,7 +1219,7 @@ def interpret_iter(self, list_txt, local_vars, allow_recursion): yield self.interpret_expression(v, local_vars, allow_recursion) def extract_object(self, objname): - _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')''' + _FUNC_NAME_RE = r'''(?:{n}|"{n}"|'{n}')'''.format(n=_NAME_RE) obj = {} fields = next(filter(None, ( obj_m.group('fields') for obj_m in re.finditer( @@ -1272,6 +1278,7 @@ def extract_function(self, funcname): def extract_function_from_code(self, argnames, code, *global_stack): local_vars = {} + while True: mobj = re.search(r'function\((?P[^)]*)\)\s*{', code) if mobj is None: @@ -1282,10 +1289,11 @@ def extract_function_from_code(self, argnames, code, *global_stack): [x.strip() for x in mobj.group('args').split(',')], body, local_vars, *global_stack)) code = code[:start] + name + remaining + return self.build_function(argnames, code, local_vars, *global_stack) - def call_function(self, funcname, *args): - return self.extract_function(funcname)(args) + def call_function(self, funcname, *args, **kw_global_vars): + return self.extract_function(funcname)(args, kw_global_vars) @classmethod def build_arglist(cls, arg_text): @@ -1304,8 +1312,9 @@ def build_function(self, argnames, code, *global_stack): global_stack = list(global_stack) or [{}] argnames = tuple(argnames) - def resf(args, kwargs={}, allow_recursion=100): - global_stack[0].update(zip_longest(argnames, args, fillvalue=None)) + def resf(args, kwargs=None, allow_recursion=100): + kwargs = kwargs or {} + global_stack[0].update(zip_longest(argnames, args, fillvalue=JS_Undefined)) global_stack[0].update(kwargs) var_stack = LocalNameSpace(*global_stack) ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1) From a93fff30522a9c415471b74d79a8a54691de936b Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 7 Dec 2024 03:39:44 +0000 Subject: [PATCH 10/14] [YouTube] Handle player `3bb1f723` * fix signature code extraction * raise if n function returns input value * add new tests from yt-dlp Co-authored-by: bashonly --- test/test_youtube_signature.py | 37 +++++++++++++++++++++++++------- youtube_dl/extractor/youtube.py | 38 ++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 56e92fac5df..7d1ff90ba8b 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -12,6 +12,7 @@ import string from youtube_dl.compat import ( + compat_contextlib_suppress, compat_open as open, compat_str, compat_urlretrieve, @@ -50,23 +51,33 @@ ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js', 84, - '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>' + '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>', ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js', 83, - '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F' + '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F', ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js', '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288', - '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B' + '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B', ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12', '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3', - ) + ), + ( + 'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', + ), + ( + 'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + ), ] _NSIG_TESTS = [ @@ -142,6 +153,10 @@ 'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js', 'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ', ), + ( + 'https://www.youtube.com/s/player/7a062b77/player_ias.vflset/en_US/base.js', + 'NRcE3y3mVtm_cV-W', 'VbsCYUATvqlt5w', + ), ( 'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js', 'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A', @@ -154,6 +169,10 @@ 'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js', 'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ', ), + ( + 'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js', + '1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A', + ), ( 'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js', '_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ', @@ -182,6 +201,10 @@ 'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js', 'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw', ), + ( + 'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js', + 'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ', + ), ] @@ -216,11 +239,9 @@ def setUp(self): os.mkdir(self.TESTDATA_DIR) def tearDown(self): - try: + with compat_contextlib_suppress(OSError): for f in os.listdir(self.TESTDATA_DIR): os.remove(f) - except OSError: - pass def t_factory(name, sig_func, url_pattern): @@ -258,7 +279,7 @@ def n_sig(jscode, sig_input): make_sig_test = t_factory( - 'signature', signature, re.compile(r'.*-(?P[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$')) + 'signature', signature, re.compile(r'.*(?:-|/player/)(?P[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$')) for test_spec in _SIG_TESTS: make_sig_test(*test_spec) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 6fe520e9a44..d633032ae94 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1579,19 +1579,26 @@ def _genslice(start, end, step): self.to_screen('Extracted signature function:\n' + code) def _parse_sig_js(self, jscode): + # Examples where `sig` is funcname: + # sig=function(a){a=a.split(""); ... ;return a.join("")}; + # ;c&&(c=sig(decodeURIComponent(c)),a.set(b,encodeURIComponent(c)));return a}; + # {var l=f,m=h.sp,n=sig(decodeURIComponent(h.s));l.set(m,encodeURIComponent(n))} + # sig=function(J){J=J.split(""); ... ;return J.join("")}; + # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J}; + # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))} funcname = self._search_regex( - (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', + (r'\b(?P[a-zA-Z0-9$]+)&&\((?P=var)=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\((?P=var)\)\)', + r'(?P[a-zA-Z0-9$]+)\s*=\s*function\(\s*(?P[a-zA-Z0-9$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)', + r'(?:\b|[^a-zA-Z0-9$])(?P[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?', + # Old patterns + r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', r'\bm=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)', - r'\bc&&\(c=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)', - r'(?:\b|[^a-zA-Z0-9$])(?P[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?', - r'(?P[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # Obsolete patterns r'("|\')signature\1\s*,\s*(?P[a-zA-Z0-9$]+)\(', r'\.sig\|\|(?P[a-zA-Z0-9$]+)\(', r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[a-zA-Z0-9$]+)\(', r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', - r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\('), jscode, 'Initial JS player signature function name', group='sig') @@ -1658,6 +1665,7 @@ def _decrypt_nsig(self, n, video_id, player_url): def _extract_n_function_name(self, jscode): func_name, idx = self._search_regex( + # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}}; # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c) # or: (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c) # or: (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b) @@ -1666,7 +1674,7 @@ def _extract_n_function_name(self, jscode): # older: (b=a.get("n"))&&(b=nfunc(b) r'''(?x) \((?:[\w$()\s]+,)*?\s* # ( - (?P[a-z])\s*=\s* # b= + (?P[a-zA-Z])\s*=\s* # b=, R= (?: (?: # expect ,c=a.get(b) (etc) String\s*\.\s*fromCharCode\s*\(\s*110\s*\)| @@ -1679,7 +1687,7 @@ def _extract_n_function_name(self, jscode): (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*? (?:\.\s*n|\[\s*"n"\s*]|\.\s*get\s*\(\s*"n"\s*\)) | # ,c=a.get(b) - ,\s*(?P[a-z])\s*=\s*[a-z]\s* + ,\s*(?P[a-zA-Z])\s*=\s*[a-zA-Z]\s* (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*? (?:\[\s*(?P=b)\s*]|\.\s*get\s*\(\s*(?P=b)\s*\)) ) @@ -1697,15 +1705,19 @@ def _extract_n_function_name(self, jscode): r'''(?xs) (?:(?<=[^\w$])|^) # instead of \b, which ignores $ (?P(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\) - \s*\{(?:(?!};).)+?["']enhanced_except_ + \s*\{(?:(?!};).)+?(?: + ["']enhanced_except_ | + return\s*(?P"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[a-zA-Z0-9_$]+ + ) ''', jscode, 'Initial JS player n function name', group='name') if not idx: return func_name - return self._parse_json(self._search_regex( - r'var\s+{0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode, - 'Initial JS player n function list ({0}.{1})'.format(func_name, idx)), - func_name, transform_source=js_to_json)[int(idx)] + return self._search_json( + r'var\s+{0}\s*='.format(re.escape(func_name)), jscode, + 'Initial JS player n function list ({0}.{1})'.format(func_name, idx), + func_name, contains_pattern=r'\[[\s\S]+\]', end_pattern='[,;]', + transform_source=js_to_json)[int(idx)] def _extract_n_function_code(self, video_id, player_url): player_id = self._extract_player_info(player_url) @@ -1734,7 +1746,7 @@ def extract_nsig(s): except Exception as e: raise JSInterpreter.Exception(traceback.format_exc(), cause=e) - if ret.startswith('enhanced_except_'): + if ret.startswith('enhanced_except_') or ret.endswith(s): raise JSInterpreter.Exception('Signature function returned an exception') return ret From 9bc8fcf23c1d1133993a91d8ed5fd203a3f7f32d Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 12 Dec 2024 04:38:23 +0000 Subject: [PATCH 11/14] [YouTube] Pass nsig value as return hook, fixes player `3bb1f723` --- test/test_youtube_signature.py | 4 +++- youtube_dl/extractor/youtube.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 7d1ff90ba8b..1c2ba848e37 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# coding: utf-8 from __future__ import unicode_literals @@ -275,7 +276,8 @@ def signature(jscode, sig_input): def n_sig(jscode, sig_input): funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode) - return JSInterpreter(jscode).call_function(funcname, sig_input) + return JSInterpreter(jscode).call_function( + funcname, sig_input, _ytdl_do_not_return=sig_input) make_sig_test = t_factory( diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index d633032ae94..55881f4d1e8 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1740,7 +1740,7 @@ def _extract_n_function_from_code(self, jsi, func_code): def extract_nsig(s): try: - ret = func([s]) + ret = func([s], kwargs={'_ytdl_do_not_return': s}) except JSInterpreter.Exception: raise except Exception as e: From 3d649843fe97e6d9ceed91c4cf0f4f4eb07958f8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 13 Dec 2024 03:09:29 +0000 Subject: [PATCH 12/14] [YouTube] Simplify pattern for nsig function name extraction --- test/test_youtube_signature.py | 4 +++ youtube_dl/extractor/youtube.py | 46 ++++++++++++++------------------- 2 files changed, 23 insertions(+), 27 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 1c2ba848e37..1d6bce67c18 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -206,6 +206,10 @@ 'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js', 'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ', ), + ( + 'https://www.youtube.com/s/player/f8f53e1a/player_ias.vflset/en_US/base.js', + 'VTQOUOv0mCIeJ7i8kZB', 'kcfD8wy0sNLyNQ', + ), ] diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 55881f4d1e8..f59d1e36a1d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1665,37 +1665,29 @@ def _decrypt_nsig(self, n, video_id, player_url): def _extract_n_function_name(self, jscode): func_name, idx = self._search_regex( + # (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}}; # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}}; - # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c) - # or: (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c) - # or: (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b) + # or: (b=String.fromCharCode(110),c=a.get(b))&&c=narray[idx](c) + # or: (b="nn"[+a.D],c=a.get(b))&&(c=narray[idx](c) + # or: (PL(a),b=a.j.n||null)&&(b=narray[idx](b) # or: (b="nn"[+a.D],vL(a),c=a.j[b]||null)&&(c=narray[idx](c),a.set(b,c),narray.length||nfunc("") - # old: (b=a.get("n"))&&(b=nfunc[idx](b)(?P[a-z])\s*=\s*[a-z]\s* + # old: (b=a.get("n"))&&(b=narray[idx](b)(?P[a-z])\s*=\s*[a-z]\s* # older: (b=a.get("n"))&&(b=nfunc(b) r'''(?x) - \((?:[\w$()\s]+,)*?\s* # ( - (?P[a-zA-Z])\s*=\s* # b=, R= - (?: - (?: # expect ,c=a.get(b) (etc) - String\s*\.\s*fromCharCode\s*\(\s*110\s*\)| - "n+"\[\s*\+?s*[\w$.]+\s*] - )\s*(?:,[\w$()\s]+(?=,))*| - (?P[\w$]+) # a (old[er]) - )\s* - (?(old) - # b.get("n") - (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*? - (?:\.\s*n|\[\s*"n"\s*]|\.\s*get\s*\(\s*"n"\s*\)) - | # ,c=a.get(b) - ,\s*(?P[a-zA-Z])\s*=\s*[a-zA-Z]\s* - (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*? - (?:\[\s*(?P=b)\s*]|\.\s*get\s*\(\s*(?P=b)\s*\)) - ) - # interstitial junk - \s*(?:\|\|\s*null\s*)?(?:\)\s*)?&&\s*(?:\(\s*)? - (?(c)(?P=c)|(?P=b))\s*=\s* # [c|b]= - # nfunc|nfunc[idx] - (?P[a-zA-Z_$][\w$]*)(?:\s*\[(?P\d+)\])?\s*\(\s*[\w$]+\s*\) + # (expr, ..., + \((?:(?:\s*[\w$]+\s*=)?(?:[\w$"+\.\s(\[]+(?:[)\]]\s*)?),)* + # b=... + (?P[\w$]+)\s*=\s*(?!(?P=b)[^\w$])[\w$]+\s*(?:(?: + \.\s*[\w$]+ | + \[\s*[\w$]+\s*\] | + \.\s*get\s*\(\s*[\w$"]+\s*\) + )\s*){,2}(?:\s*\|\|\s*null(?=\s*\)))?\s* + \)\s*&&\s*\( # ...)&&( + # b = nfunc, b = narray[idx] + (?P=b)\s*=\s*(?P[\w$]+)\s* + (?:\[\s*(?P[\w$]+)\s*\]\s*)? + # (...) + \(\s*[\w$]+\s*\) ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'), default=(None, None)) # thx bashonly: yt-dlp/yt-dlp/pull/10611 From 82b525657a27f91b31b51fa4cf2135df6fc7481d Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 13 Dec 2024 12:16:31 +0000 Subject: [PATCH 13/14] [YouTube] Fix signature function extraction for `2f1832d2` * `_` was omitted from patterns * thx yt-dlp/yt-dlp#11801 Co-authored-by: bashonly --- test/test_youtube_signature.py | 9 +++++++++ youtube_dl/extractor/youtube.py | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 1d6bce67c18..fcbc9d7a813 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -79,6 +79,11 @@ '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', 'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', ), + ( + 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q', + ), ] _NSIG_TESTS = [ @@ -210,6 +215,10 @@ 'https://www.youtube.com/s/player/f8f53e1a/player_ias.vflset/en_US/base.js', 'VTQOUOv0mCIeJ7i8kZB', 'kcfD8wy0sNLyNQ', ), + ( + 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', + 'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP', + ), ] diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index f59d1e36a1d..7e0ba04c000 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1587,19 +1587,19 @@ def _parse_sig_js(self, jscode): # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J}; # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))} funcname = self._search_regex( - (r'\b(?P[a-zA-Z0-9$]+)&&\((?P=var)=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\((?P=var)\)\)', - r'(?P[a-zA-Z0-9$]+)\s*=\s*function\(\s*(?P[a-zA-Z0-9$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)', - r'(?:\b|[^a-zA-Z0-9$])(?P[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?', + (r'\b(?P[\w$]+)&&\((?P=var)=(?P[\w$]{2,})\(decodeURIComponent\((?P=var)\)\)', + r'(?P[\w$]+)\s*=\s*function\(\s*(?P[\w$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)', + r'(?:\b|[^\w$])(?P[\w$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[\w$]{2}\.[\w$]{2}\(a,\d+\))?', # Old patterns - r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', - r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', - r'\bm=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)', + r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[\w$]+)\(', + r'\b[\w]+\s*&&\s*[\w]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[\w$]+)\(', + r'\bm=(?P[\w$]{2,})\(decodeURIComponent\(h\.s\)\)', # Obsolete patterns - r'("|\')signature\1\s*,\s*(?P[a-zA-Z0-9$]+)\(', - r'\.sig\|\|(?P[a-zA-Z0-9$]+)\(', - r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[a-zA-Z0-9$]+)\(', - r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', - r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\('), + r'("|\')signature\1\s*,\s*(?P[\w$]+)\(', + r'\.sig\|\|(?P[\w$]+)\(', + r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[\w$]+)\(', + r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[\w$]+)\(', + r'\bc\s*&&\s*[\w]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[\w$]+)\('), jscode, 'Initial JS player signature function name', group='sig') jsi = JSInterpreter(jscode) @@ -1699,7 +1699,7 @@ def _extract_n_function_name(self, jscode): (?P(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\) \s*\{(?:(?!};).)+?(?: ["']enhanced_except_ | - return\s*(?P"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[a-zA-Z0-9_$]+ + return\s*(?P"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+ ) ''', jscode, 'Initial JS player n function name', group='name') if not idx: From 12bc3775d963039efaf4a36b0ea124a6b5e5ca8a Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 14 Dec 2024 11:18:34 +0000 Subject: [PATCH 14/14] [YouTube] Always extract using MWEB API client * temporary fix-up for 403 on download * MWEB parameters from yt-dlp 2024-12-06 --- youtube_dl/extractor/youtube.py | 84 ++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 7e0ba04c000..1f83acf7cbf 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3,11 +3,13 @@ from __future__ import unicode_literals import collections +import hashlib import itertools import json import os.path import random import re +import time import traceback from .common import InfoExtractor, SearchInfoExtractor @@ -290,6 +292,33 @@ def _real_initialize(self): _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;' _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|