diff --git a/jupyter-config/jupyter_server_config.d/notebook.json b/jupyter-config/jupyter_server_config.d/notebook.json new file mode 100644 index 0000000000..5a509c8e12 --- /dev/null +++ b/jupyter-config/jupyter_server_config.d/notebook.json @@ -0,0 +1,7 @@ +{ + "ServerApp": { + "jpserver_extensions": { + "notebook": true + } + } +} \ No newline at end of file diff --git a/notebook/__init__.py b/notebook/__init__.py index f406f3c6eb..cd11b1392d 100644 --- a/notebook/__init__.py +++ b/notebook/__init__.py @@ -24,3 +24,12 @@ from .nbextensions import install_nbextension from ._version import version_info, __version__ + +from .notebookapp import NotebookApp + +EXTENSION_NAME = "notebook" + +def _jupyter_server_extension_paths(): + return [{"module": EXTENSION_NAME}] + +load_jupyter_server_extension = NotebookApp.load_jupyter_server_extension \ No newline at end of file diff --git a/notebook/auth/__init__.py b/notebook/auth/__init__.py deleted file mode 100644 index 9f84fa2e94..0000000000 --- a/notebook/auth/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .security import passwd diff --git a/notebook/auth/__main__.py b/notebook/auth/__main__.py deleted file mode 100644 index ff413b534e..0000000000 --- a/notebook/auth/__main__.py +++ /dev/null @@ -1,42 +0,0 @@ -from notebook.auth import passwd -from getpass import getpass -from notebook.config_manager import BaseJSONConfigManager -from jupyter_core.paths import jupyter_config_dir -import argparse -import sys - -def set_password(args): - password = args.password - while not password : - password1 = getpass("" if args.quiet else "Provide password: ") - password_repeat = getpass("" if args.quiet else "Repeat password: ") - if password1 != password_repeat: - print("Passwords do not match, try again") - elif len(password1) < 4: - print("Please provide at least 4 characters") - else: - password = password1 - - password_hash = passwd(password) - cfg = BaseJSONConfigManager(config_dir=jupyter_config_dir()) - cfg.update('jupyter_notebook_config', { - 'NotebookApp': { - 'password': password_hash, - } - }) - if not args.quiet: - print("password stored in config dir: %s" % jupyter_config_dir()) - -def main(argv): - parser = argparse.ArgumentParser(argv[0]) - subparsers = parser.add_subparsers() - parser_password = subparsers.add_parser('password', help='sets a password for your notebook server') - parser_password.add_argument("password", help="password to set, if not given, a password will be queried for (NOTE: this may not be safe)", - nargs="?") - parser_password.add_argument("--quiet", help="suppress messages", action="store_true") - parser_password.set_defaults(function=set_password) - args = parser.parse_args(argv[1:]) - args.function(args) - -if __name__ == "__main__": - main(sys.argv) \ No newline at end of file diff --git a/notebook/auth/login.py b/notebook/auth/login.py deleted file mode 100644 index 8dbd6112fc..0000000000 --- a/notebook/auth/login.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Tornado handlers for logging into the notebook.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import re -import os - -try: - from urllib.parse import urlparse # Py 3 -except ImportError: - from urlparse import urlparse # Py 2 -import uuid - -from tornado.escape import url_escape - -from .security import passwd_check, set_password - -from ..base.handlers import IPythonHandler - - -class LoginHandler(IPythonHandler): - """The basic tornado login handler - - authenticates with a hashed password from the configuration. - """ - def _render(self, message=None): - self.write(self.render_template('login.html', - next=url_escape(self.get_argument('next', default=self.base_url)), - message=message, - )) - - def _redirect_safe(self, url, default=None): - """Redirect if url is on our PATH - - Full-domain redirects are allowed if they pass our CORS origin checks. - - Otherwise use default (self.base_url if unspecified). - """ - if default is None: - default = self.base_url - # protect chrome users from mishandling unescaped backslashes. - # \ is not valid in urls, but some browsers treat it as / - # instead of %5C, causing `\\` to behave as `//` - url = url.replace("\\", "%5C") - parsed = urlparse(url) - if parsed.netloc or not (parsed.path + '/').startswith(self.base_url): - # require that next_url be absolute path within our path - allow = False - # OR pass our cross-origin check - if parsed.netloc: - # if full URL, run our cross-origin check: - origin = '%s://%s' % (parsed.scheme, parsed.netloc) - origin = origin.lower() - if self.allow_origin: - allow = self.allow_origin == origin - elif self.allow_origin_pat: - allow = bool(self.allow_origin_pat.match(origin)) - if not allow: - # not allowed, use default - self.log.warning("Not allowing login redirect to %r" % url) - url = default - self.redirect(url) - - def get(self): - if self.current_user: - next_url = self.get_argument('next', default=self.base_url) - self._redirect_safe(next_url) - else: - self._render() - - @property - def hashed_password(self): - return self.password_from_settings(self.settings) - - def passwd_check(self, a, b): - return passwd_check(a, b) - - def post(self): - typed_password = self.get_argument('password', default=u'') - new_password = self.get_argument('new_password', default=u'') - - - - if self.get_login_available(self.settings): - if self.passwd_check(self.hashed_password, typed_password) and not new_password: - self.set_login_cookie(self, uuid.uuid4().hex) - elif self.token and self.token == typed_password: - self.set_login_cookie(self, uuid.uuid4().hex) - if new_password and self.settings.get('allow_password_change'): - config_dir = self.settings.get('config_dir') - config_file = os.path.join(config_dir, 'jupyter_notebook_config.json') - set_password(new_password, config_file=config_file) - self.log.info("Wrote hashed password to %s" % config_file) - else: - self.set_status(401) - self._render(message={'error': 'Invalid credentials'}) - return - - - next_url = self.get_argument('next', default=self.base_url) - self._redirect_safe(next_url) - - @classmethod - def set_login_cookie(cls, handler, user_id=None): - """Call this on handlers to set the login cookie for success""" - cookie_options = handler.settings.get('cookie_options', {}) - cookie_options.setdefault('httponly', True) - # tornado <4.2 has a bug that considers secure==True as soon as - # 'secure' kwarg is passed to set_secure_cookie - if handler.settings.get('secure_cookie', handler.request.protocol == 'https'): - cookie_options.setdefault('secure', True) - cookie_options.setdefault('path', handler.base_url) - handler.set_secure_cookie(handler.cookie_name, user_id, **cookie_options) - return user_id - - auth_header_pat = re.compile('token\s+(.+)', re.IGNORECASE) - - @classmethod - def get_token(cls, handler): - """Get the user token from a request - - Default: - - - in URL parameters: ?token= - - in header: Authorization: token - """ - - user_token = handler.get_argument('token', '') - if not user_token: - # get it from Authorization header - m = cls.auth_header_pat.match(handler.request.headers.get('Authorization', '')) - if m: - user_token = m.group(1) - return user_token - - @classmethod - def should_check_origin(cls, handler): - """Should the Handler check for CORS origin validation? - - Origin check should be skipped for token-authenticated requests. - - Returns: - - True, if Handler must check for valid CORS origin. - - False, if Handler should skip origin check since requests are token-authenticated. - """ - return not cls.is_token_authenticated(handler) - - @classmethod - def is_token_authenticated(cls, handler): - """Returns True if handler has been token authenticated. Otherwise, False. - - Login with a token is used to signal certain things, such as: - - - permit access to REST API - - xsrf protection - - skip origin-checks for scripts - """ - if getattr(handler, '_user_id', None) is None: - # ensure get_user has been called, so we know if we're token-authenticated - handler.get_current_user() - return getattr(handler, '_token_authenticated', False) - - @classmethod - def get_user(cls, handler): - """Called by handlers.get_current_user for identifying the current user. - - See tornado.web.RequestHandler.get_current_user for details. - """ - # Can't call this get_current_user because it will collide when - # called on LoginHandler itself. - if getattr(handler, '_user_id', None): - return handler._user_id - user_id = cls.get_user_token(handler) - if user_id is None: - get_secure_cookie_kwargs = handler.settings.get('get_secure_cookie_kwargs', {}) - user_id = handler.get_secure_cookie(handler.cookie_name, **get_secure_cookie_kwargs ) - else: - cls.set_login_cookie(handler, user_id) - # Record that the current request has been authenticated with a token. - # Used in is_token_authenticated above. - handler._token_authenticated = True - if user_id is None: - # If an invalid cookie was sent, clear it to prevent unnecessary - # extra warnings. But don't do this on a request with *no* cookie, - # because that can erroneously log you out (see gh-3365) - if handler.get_cookie(handler.cookie_name) is not None: - handler.log.warning("Clearing invalid/expired login cookie %s", handler.cookie_name) - handler.clear_login_cookie() - if not handler.login_available: - # Completely insecure! No authentication at all. - # No need to warn here, though; validate_security will have already done that. - user_id = 'anonymous' - - # cache value for future retrievals on the same request - handler._user_id = user_id - return user_id - - @classmethod - def get_user_token(cls, handler): - """Identify the user based on a token in the URL or Authorization header - - Returns: - - uuid if authenticated - - None if not - """ - token = handler.token - if not token: - return - # check login token from URL argument or Authorization header - user_token = cls.get_token(handler) - authenticated = False - if user_token == token: - # token-authenticated, set the login cookie - handler.log.debug("Accepting token-authenticated connection from %s", handler.request.remote_ip) - authenticated = True - - if authenticated: - return uuid.uuid4().hex - else: - return None - - - @classmethod - def validate_security(cls, app, ssl_options=None): - """Check the notebook application's security. - - Show messages, or abort if necessary, based on the security configuration. - """ - if not app.ip: - warning = "WARNING: The notebook server is listening on all IP addresses" - if ssl_options is None: - app.log.warning(warning + " and not using encryption. This " - "is not recommended.") - if not app.password and not app.token: - app.log.warning(warning + " and not using authentication. " - "This is highly insecure and not recommended.") - else: - if not app.password and not app.token: - app.log.warning( - "All authentication is disabled." - " Anyone who can connect to this server will be able to run code.") - - @classmethod - def password_from_settings(cls, settings): - """Return the hashed password from the tornado settings. - - If there is no configured password, an empty string will be returned. - """ - return settings.get('password', u'') - - @classmethod - def get_login_available(cls, settings): - """Whether this LoginHandler is needed - and therefore whether the login page should be displayed.""" - return bool(cls.password_from_settings(settings) or settings.get('token')) diff --git a/notebook/auth/logout.py b/notebook/auth/logout.py deleted file mode 100644 index 9eaee1f476..0000000000 --- a/notebook/auth/logout.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Tornado handlers for logging out of the notebook. -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from ..base.handlers import IPythonHandler - - -class LogoutHandler(IPythonHandler): - - def get(self): - self.clear_login_cookie() - if self.login_available: - message = {'info': 'Successfully logged out.'} - else: - message = {'warning': 'Cannot log out. Notebook authentication ' - 'is disabled.'} - self.write(self.render_template('logout.html', - message=message)) - - -default_handlers = [(r"/logout", LogoutHandler)] \ No newline at end of file diff --git a/notebook/auth/security.py b/notebook/auth/security.py deleted file mode 100644 index 8f69148ee3..0000000000 --- a/notebook/auth/security.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -Password generation for the Notebook. -""" - -from contextlib import contextmanager -import getpass -import hashlib -import io -import json -import os -import random -import traceback -import warnings - -from ipython_genutils.py3compat import cast_bytes, str_to_bytes, cast_unicode -from traitlets.config import Config, ConfigFileNotFound, JSONFileConfigLoader -from jupyter_core.paths import jupyter_config_dir - -# Length of the salt in nr of hex chars, which implies salt_len * 4 -# bits of randomness. -salt_len = 12 - - -def passwd(passphrase=None, algorithm='sha1'): - """Generate hashed password and salt for use in notebook configuration. - - In the notebook configuration, set `c.NotebookApp.password` to - the generated string. - - Parameters - ---------- - passphrase : str - Password to hash. If unspecified, the user is asked to input - and verify a password. - algorithm : str - Hashing algorithm to use (e.g, 'sha1' or any argument supported - by :func:`hashlib.new`). - - Returns - ------- - hashed_passphrase : str - Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'. - - Examples - -------- - >>> passwd('mypassword') - 'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12' - - """ - if passphrase is None: - for i in range(3): - p0 = getpass.getpass('Enter password: ') - p1 = getpass.getpass('Verify password: ') - if p0 == p1: - passphrase = p0 - break - else: - print('Passwords do not match.') - else: - raise ValueError('No matching passwords found. Giving up.') - - h = hashlib.new(algorithm) - salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len) - h.update(cast_bytes(passphrase, 'utf-8') + str_to_bytes(salt, 'ascii')) - - return ':'.join((algorithm, salt, h.hexdigest())) - - -def passwd_check(hashed_passphrase, passphrase): - """Verify that a given passphrase matches its hashed version. - - Parameters - ---------- - hashed_passphrase : str - Hashed password, in the format returned by `passwd`. - passphrase : str - Passphrase to validate. - - Returns - ------- - valid : bool - True if the passphrase matches the hash. - - Examples - -------- - >>> from notebook.auth.security import passwd_check - >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', - ... 'mypassword') - True - - >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a', - ... 'anotherpassword') - False - """ - try: - algorithm, salt, pw_digest = hashed_passphrase.split(':', 2) - except (ValueError, TypeError): - return False - - try: - h = hashlib.new(algorithm) - except ValueError: - return False - - if len(pw_digest) == 0: - return False - - h.update(cast_bytes(passphrase, 'utf-8') + cast_bytes(salt, 'ascii')) - - return h.hexdigest() == pw_digest - -@contextmanager -def persist_config(config_file=None, mode=0o600): - """Context manager that can be used to modify a config object - - On exit of the context manager, the config will be written back to disk, - by default with user-only (600) permissions. - """ - - if config_file is None: - config_file = os.path.join(jupyter_config_dir(), 'jupyter_notebook_config.json') - - loader = JSONFileConfigLoader(os.path.basename(config_file), os.path.dirname(config_file)) - try: - config = loader.load_config() - except ConfigFileNotFound: - config = Config() - - yield config - - with io.open(config_file, 'w', encoding='utf8') as f: - f.write(cast_unicode(json.dumps(config, indent=2))) - - try: - os.chmod(config_file, mode) - except Exception as e: - tb = traceback.format_exc() - warnings.warn("Failed to set permissions on %s:\n%s" % (config_file, tb), - RuntimeWarning) - - -def set_password(password=None, config_file=None): - """Ask user for password, store it in notebook json configuration file""" - - hashed_password = passwd(password) - - with persist_config(config_file) as config: - config.NotebookApp.password = hashed_password diff --git a/notebook/auth/tests/__init__.py b/notebook/auth/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/auth/tests/test_login.py b/notebook/auth/tests/test_login.py deleted file mode 100644 index 2b5574204a..0000000000 --- a/notebook/auth/tests/test_login.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Tests for login redirects""" - -import requests -from tornado.httputil import url_concat - -from notebook.tests.launchnotebook import NotebookTestBase - - -class LoginTest(NotebookTestBase): - def login(self, next): - first = requests.get(self.base_url() + "login") - first.raise_for_status() - resp = requests.post( - url_concat( - self.base_url() + "login", - {'next': next}, - ), - allow_redirects=False, - data={ - "password": self.token, - "_xsrf": first.cookies.get("_xsrf", ""), - }, - cookies=first.cookies, - ) - resp.raise_for_status() - return resp.headers['Location'] - - def test_next_bad(self): - for bad_next in ( - "//some-host", - "//host" + self.url_prefix + "tree", - "https://google.com", - "/absolute/not/base_url", - ): - url = self.login(next=bad_next) - self.assertEqual(url, self.url_prefix) - assert url - - def test_next_ok(self): - for next_path in ( - "tree/", - "//" + self.url_prefix + "tree", - "notebooks/notebook.ipynb", - "tree//something", - ): - expected = self.url_prefix + next_path - actual = self.login(next=expected) - self.assertEqual(actual, expected) diff --git a/notebook/auth/tests/test_security.py b/notebook/auth/tests/test_security.py deleted file mode 100644 index a17e80087e..0000000000 --- a/notebook/auth/tests/test_security.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding: utf-8 -from ..security import passwd, passwd_check, salt_len -import nose.tools as nt - -def test_passwd_structure(): - p = passwd('passphrase') - algorithm, salt, hashed = p.split(':') - nt.assert_equal(algorithm, 'sha1') - nt.assert_equal(len(salt), salt_len) - nt.assert_equal(len(hashed), 40) - -def test_roundtrip(): - p = passwd('passphrase') - nt.assert_equal(passwd_check(p, 'passphrase'), True) - -def test_bad(): - p = passwd('passphrase') - nt.assert_equal(passwd_check(p, p), False) - nt.assert_equal(passwd_check(p, 'a:b:c:d'), False) - nt.assert_equal(passwd_check(p, 'a:b'), False) - -def test_passwd_check_unicode(): - # GH issue #4524 - phash = u'sha1:23862bc21dd3:7a415a95ae4580582e314072143d9c382c491e4f' - assert passwd_check(phash, u"łe¶ŧ←↓→") \ No newline at end of file diff --git a/notebook/base/handlers.py b/notebook/base/handlers.py index cd801c9a10..26b5895b65 100755 --- a/notebook/base/handlers.py +++ b/notebook/base/handlers.py @@ -3,942 +3,19 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -import datetime -import functools -import ipaddress -import json -import mimetypes -import os -import re -import sys -import traceback -import types -import warnings -try: - # py3 - from http.client import responses - from http.cookies import Morsel -except ImportError: - from httplib import responses - from Cookie import Morsel -try: - from urllib.parse import urlparse # Py 3 -except ImportError: - from urlparse import urlparse # Py 2 +from jupyter_server.extension.handler import ExtensionHandler -from jinja2 import TemplateNotFound -from tornado import web, gen, escape, httputil -from tornado.log import app_log -import prometheus_client - -from notebook._sysinfo import get_sys_info - -from traitlets.config import Application -from ipython_genutils.path import filefind -from ipython_genutils.py3compat import string_types, PY3 - -import notebook -from notebook._tz import utcnow -from notebook.i18n import combine_translations -from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape -from notebook.services.security import csp_report_uri - -#----------------------------------------------------------------------------- -# Top-level handlers -#----------------------------------------------------------------------------- -non_alphanum = re.compile(r'[^A-Za-z0-9]') - -_sys_info_cache = None -def json_sys_info(): - global _sys_info_cache - if _sys_info_cache is None: - _sys_info_cache = json.dumps(get_sys_info()) - return _sys_info_cache - -def log(): - if Application.initialized(): - return Application.instance().log - else: - return app_log - -class AuthenticatedHandler(web.RequestHandler): - """A RequestHandler with an authenticated user.""" - - @property - def content_security_policy(self): - """The default Content-Security-Policy header - - Can be overridden by defining Content-Security-Policy in settings['headers'] - """ - if 'Content-Security-Policy' in self.settings.get('headers', {}): - # user-specified, don't override - return self.settings['headers']['Content-Security-Policy'] - - return '; '.join([ - "frame-ancestors 'self'", - # Make sure the report-uri is relative to the base_url - "report-uri " + self.settings.get('csp_report_uri', url_path_join(self.base_url, csp_report_uri)), - ]) - - def set_default_headers(self): - headers = {} - headers["X-Content-Type-Options"] = "nosniff" - headers.update(self.settings.get('headers', {})) - - headers["Content-Security-Policy"] = self.content_security_policy - - # Allow for overriding headers - for header_name, value in headers.items(): - try: - self.set_header(header_name, value) - except Exception as e: - # tornado raise Exception (not a subclass) - # if method is unsupported (websocket and Access-Control-Allow-Origin - # for example, so just ignore) - self.log.debug(e) - - def force_clear_cookie(self, name, path="/", domain=None): - """Deletes the cookie with the given name. - - Tornado's cookie handling currently (Jan 2018) stores cookies in a dict - keyed by name, so it can only modify one cookie with a given name per - response. The browser can store multiple cookies with the same name - but different domains and/or paths. This method lets us clear multiple - cookies with the same name. - - Due to limitations of the cookie protocol, you must pass the same - path and domain to clear a cookie as were used when that cookie - was set (but there is no way to find out on the server side - which values were used for a given cookie). - """ - name = escape.native_str(name) - expires = datetime.datetime.utcnow() - datetime.timedelta(days=365) - - morsel = Morsel() - morsel.set(name, '', '""') - morsel['expires'] = httputil.format_timestamp(expires) - morsel['path'] = path - if domain: - morsel['domain'] = domain - self.add_header("Set-Cookie", morsel.OutputString()) - - def clear_login_cookie(self): - cookie_options = self.settings.get('cookie_options', {}) - path = cookie_options.setdefault('path', self.base_url) - self.clear_cookie(self.cookie_name, path=path) - if path and path != '/': - # also clear cookie on / to ensure old cookies are cleared - # after the change in path behavior (changed in notebook 5.2.2). - # N.B. This bypasses the normal cookie handling, which can't update - # two cookies with the same name. See the method above. - self.force_clear_cookie(self.cookie_name) - - def get_current_user(self): - if self.login_handler is None: - return 'anonymous' - return self.login_handler.get_user(self) - - def skip_check_origin(self): - """Ask my login_handler if I should skip the origin_check - - For example: in the default LoginHandler, if a request is token-authenticated, - origin checking should be skipped. - """ - if self.request.method == 'OPTIONS': - # no origin-check on options requests, which are used to check origins! - return True - if self.login_handler is None or not hasattr(self.login_handler, 'should_check_origin'): - return False - return not self.login_handler.should_check_origin(self) - - @property - def token_authenticated(self): - """Have I been authenticated with a token?""" - if self.login_handler is None or not hasattr(self.login_handler, 'is_token_authenticated'): - return False - return self.login_handler.is_token_authenticated(self) - - @property - def cookie_name(self): - default_cookie_name = non_alphanum.sub('-', 'username-{}'.format( - self.request.host - )) - return self.settings.get('cookie_name', default_cookie_name) +class BaseHandler(ExtensionHandler): - @property - def logged_in(self): - """Is a user currently logged in?""" - user = self.get_current_user() - return (user and not user == 'anonymous') - - @property - def login_handler(self): - """Return the login handler for this application, if any.""" - return self.settings.get('login_handler_class', None) - - @property - def token(self): - """Return the login token for this application, if any.""" - return self.settings.get('token', None) - - @property - def login_available(self): - """May a user proceed to log in? - - This returns True if login capability is available, irrespective of - whether the user is already logged in or not. - - """ - if self.login_handler is None: - return False - return bool(self.login_handler.get_login_available(self.settings)) - - -class IPythonHandler(AuthenticatedHandler): - """IPython-specific extensions to authenticated handling + extension_name = "notebook" - Mostly property shortcuts to IPython-specific settings. - """ - - @property - def ignore_minified_js(self): - """Wether to user bundle in template. (*.min files) - - Mainly use for development and avoid file recompilation - """ - return self.settings.get('ignore_minified_js', False) - - @property - def config(self): - return self.settings.get('config', None) - - @property - def log(self): - """use the IPython log by default, falling back on tornado's logger""" - return log() - @property def jinja_template_vars(self): """User-supplied values to supply to jinja templates.""" - return self.settings.get('jinja_template_vars', {}) - - #--------------------------------------------------------------- - # URLs - #--------------------------------------------------------------- - - @property - def version_hash(self): - """The version hash to use for cache hints for static files""" - return self.settings.get('version_hash', '') - - @property - def mathjax_url(self): - url = self.settings.get('mathjax_url', '') - if not url or url_is_absolute(url): - return url - return url_path_join(self.base_url, url) - - @property - def mathjax_config(self): - return self.settings.get('mathjax_config', 'TeX-AMS-MML_HTMLorMML-full,Safe') - - @property - def base_url(self): - return self.settings.get('base_url', '/') - - @property - def default_url(self): - return self.settings.get('default_url', '') - - @property - def ws_url(self): - return self.settings.get('websocket_url', '') - - @property - def contents_js_source(self): - self.log.debug("Using contents: %s", self.settings.get('contents_js_source', - 'services/contents')) - return self.settings.get('contents_js_source', 'services/contents') - - #--------------------------------------------------------------- - # Manager objects - #--------------------------------------------------------------- - - @property - def kernel_manager(self): - return self.settings['kernel_manager'] - - @property - def contents_manager(self): - return self.settings['contents_manager'] - - @property - def session_manager(self): - return self.settings['session_manager'] - - @property - def terminal_manager(self): - return self.settings['terminal_manager'] - - @property - def kernel_spec_manager(self): - return self.settings['kernel_spec_manager'] - - @property - def config_manager(self): - return self.settings['config_manager'] - - #--------------------------------------------------------------- - # CORS - #--------------------------------------------------------------- - - @property - def allow_origin(self): - """Normal Access-Control-Allow-Origin""" - return self.settings.get('allow_origin', '') - - @property - def allow_origin_pat(self): - """Regular expression version of allow_origin""" - return self.settings.get('allow_origin_pat', None) - - @property - def allow_credentials(self): - """Whether to set Access-Control-Allow-Credentials""" - return self.settings.get('allow_credentials', False) - - def set_default_headers(self): - """Add CORS headers, if defined""" - super(IPythonHandler, self).set_default_headers() - if self.allow_origin: - self.set_header("Access-Control-Allow-Origin", self.allow_origin) - elif self.allow_origin_pat: - origin = self.get_origin() - if origin and self.allow_origin_pat.match(origin): - self.set_header("Access-Control-Allow-Origin", origin) - elif ( - self.token_authenticated - and "Access-Control-Allow-Origin" not in - self.settings.get('headers', {}) - ): - # allow token-authenticated requests cross-origin by default. - # only apply this exception if allow-origin has not been specified. - self.set_header('Access-Control-Allow-Origin', - self.request.headers.get('Origin', '')) - - if self.allow_credentials: - self.set_header("Access-Control-Allow-Credentials", 'true') - - def set_attachment_header(self, filename): - """Set Content-Disposition: attachment header - - As a method to ensure handling of filename encoding - """ - escaped_filename = url_escape(filename) - self.set_header('Content-Disposition', - 'attachment;' - " filename*=utf-8''{utf8}" - .format( - utf8=escaped_filename, - ) - ) - - def get_origin(self): - # Handle WebSocket Origin naming convention differences - # The difference between version 8 and 13 is that in 8 the - # client sends a "Sec-Websocket-Origin" header and in 13 it's - # simply "Origin". - if "Origin" in self.request.headers: - origin = self.request.headers.get("Origin") - else: - origin = self.request.headers.get("Sec-Websocket-Origin", None) - return origin - - # origin_to_satisfy_tornado is present because tornado requires - # check_origin to take an origin argument, but we don't use it - def check_origin(self, origin_to_satisfy_tornado=""): - """Check Origin for cross-site API requests, including websockets - - Copied from WebSocket with changes: - - - allow unspecified host/origin (e.g. scripts) - - allow token-authenticated requests - """ - if self.allow_origin == '*' or self.skip_check_origin(): - return True - - host = self.request.headers.get("Host") - origin = self.request.headers.get("Origin") - - # If no header is provided, let the request through. - # Origin can be None for: - # - same-origin (IE, Firefox) - # - Cross-site POST form (IE, Firefox) - # - Scripts - # The cross-site POST (XSRF) case is handled by tornado's xsrf_token - if origin is None or host is None: - return True - - origin = origin.lower() - origin_host = urlparse(origin).netloc - - # OK if origin matches host - if origin_host == host: - return True - - # Check CORS headers - if self.allow_origin: - allow = self.allow_origin == origin - elif self.allow_origin_pat: - allow = bool(self.allow_origin_pat.match(origin)) - else: - # No CORS headers deny the request - allow = False - if not allow: - self.log.warning("Blocking Cross Origin API request for %s. Origin: %s, Host: %s", - self.request.path, origin, host, - ) - return allow - - def check_referer(self): - """Check Referer for cross-site requests. - - Disables requests to certain endpoints with - external or missing Referer. - - If set, allow_origin settings are applied to the Referer - to whitelist specific cross-origin sites. - - Used on GET for api endpoints and /files/ - to block cross-site inclusion (XSSI). - """ - host = self.request.headers.get("Host") - referer = self.request.headers.get("Referer") - - if not host: - self.log.warning("Blocking request with no host") - return False - if not referer: - self.log.warning("Blocking request with no referer") - return False - - referer_url = urlparse(referer) - referer_host = referer_url.netloc - if referer_host == host: - return True - - # apply cross-origin checks to Referer: - origin = "{}://{}".format(referer_url.scheme, referer_url.netloc) - if self.allow_origin: - allow = self.allow_origin == origin - elif self.allow_origin_pat: - allow = bool(self.allow_origin_pat.match(origin)) - else: - # No CORS settings, deny the request - allow = False - - if not allow: - self.log.warning("Blocking Cross Origin request for %s. Referer: %s, Host: %s", - self.request.path, origin, host, - ) - return allow - - def check_xsrf_cookie(self): - """Bypass xsrf cookie checks when token-authenticated""" - if self.token_authenticated or self.settings.get('disable_check_xsrf', False): - # Token-authenticated requests do not need additional XSRF-check - # Servers without authentication are vulnerable to XSRF - return - try: - return super(IPythonHandler, self).check_xsrf_cookie() - except web.HTTPError as e: - if self.request.method in {'GET', 'HEAD'}: - # Consider Referer a sufficient cross-origin check for GET requests - if not self.check_referer(): - referer = self.request.headers.get('Referer') - if referer: - msg = "Blocking Cross Origin request from {}.".format(referer) - else: - msg = "Blocking request from unknown origin" - raise web.HTTPError(403, msg) - else: - raise - - def check_host(self): - """Check the host header if remote access disallowed. - - Returns True if the request should continue, False otherwise. - """ - if self.settings.get('allow_remote_access', False): - return True - - # Remove port (e.g. ':8888') from host - host = re.match(r'^(.*?)(:\d+)?$', self.request.host).group(1) - - # Browsers format IPv6 addresses like [::1]; we need to remove the [] - if host.startswith('[') and host.endswith(']'): - host = host[1:-1] - - if not PY3: - # ip_address only accepts unicode on Python 2 - host = host.decode('utf8', 'replace') - - try: - addr = ipaddress.ip_address(host) - except ValueError: - # Not an IP address: check against hostnames - allow = host in self.settings.get('local_hostnames', ['localhost']) - else: - allow = addr.is_loopback - - if not allow: - self.log.warning( - ("Blocking request with non-local 'Host' %s (%s). " - "If the notebook should be accessible at that name, " - "set NotebookApp.allow_remote_access to disable the check."), - host, self.request.host - ) - return allow - - def prepare(self): - if not self.check_host(): - raise web.HTTPError(403) - return super(IPythonHandler, self).prepare() - - #--------------------------------------------------------------- - # template rendering - #--------------------------------------------------------------- + key = '{extension_name}_jinja_template_vars'.format(extension_name=self.extension_name) + return self.settings.get(key, {}) def get_template(self, name): """Return the jinja template object for a given name""" - return self.settings['jinja2_env'].get_template(name) - - def render_template(self, name, **ns): - ns.update(self.template_namespace) - template = self.get_template(name) - return template.render(**ns) - - @property - def template_namespace(self): - return dict( - base_url=self.base_url, - default_url=self.default_url, - ws_url=self.ws_url, - logged_in=self.logged_in, - allow_password_change=self.settings.get('allow_password_change'), - login_available=self.login_available, - token_available=bool(self.token), - static_url=self.static_url, - sys_info=json_sys_info(), - contents_js_source=self.contents_js_source, - version_hash=self.version_hash, - ignore_minified_js=self.ignore_minified_js, - xsrf_form_html=self.xsrf_form_html, - token=self.token, - xsrf_token=self.xsrf_token.decode('utf8'), - nbjs_translations=json.dumps(combine_translations( - self.request.headers.get('Accept-Language', ''))), - **self.jinja_template_vars - ) - - def get_json_body(self): - """Return the body of the request as JSON data.""" - if not self.request.body: - return None - # Do we need to call body.decode('utf-8') here? - body = self.request.body.strip().decode(u'utf-8') - try: - model = json.loads(body) - except Exception: - self.log.debug("Bad JSON: %r", body) - self.log.error("Couldn't parse JSON", exc_info=True) - raise web.HTTPError(400, u'Invalid JSON in body of request') - return model - - def write_error(self, status_code, **kwargs): - """render custom error pages""" - exc_info = kwargs.get('exc_info') - message = '' - status_message = responses.get(status_code, 'Unknown HTTP Error') - exception = '(unknown)' - if exc_info: - exception = exc_info[1] - # get the custom message, if defined - try: - message = exception.log_message % exception.args - except Exception: - pass - - # construct the custom reason, if defined - reason = getattr(exception, 'reason', '') - if reason: - status_message = reason - - # build template namespace - ns = dict( - status_code=status_code, - status_message=status_message, - message=message, - exception=exception, - ) - - self.set_header('Content-Type', 'text/html') - # render the template - try: - html = self.render_template('%s.html' % status_code, **ns) - except TemplateNotFound: - html = self.render_template('error.html', **ns) - - self.write(html) - - -class APIHandler(IPythonHandler): - """Base class for API handlers""" - - def prepare(self): - if not self.check_origin(): - raise web.HTTPError(404) - return super(APIHandler, self).prepare() - - def write_error(self, status_code, **kwargs): - """APIHandler errors are JSON, not human pages""" - self.set_header('Content-Type', 'application/json') - message = responses.get(status_code, 'Unknown HTTP Error') - reply = { - 'message': message, - } - exc_info = kwargs.get('exc_info') - if exc_info: - e = exc_info[1] - if isinstance(e, HTTPError): - reply['message'] = e.log_message or message - reply['reason'] = e.reason - else: - reply['message'] = 'Unhandled error' - reply['reason'] = None - reply['traceback'] = ''.join(traceback.format_exception(*exc_info)) - self.log.warning(reply['message']) - self.finish(json.dumps(reply)) - - def get_current_user(self): - """Raise 403 on API handlers instead of redirecting to human login page""" - # preserve _user_cache so we don't raise more than once - if hasattr(self, '_user_cache'): - return self._user_cache - self._user_cache = user = super(APIHandler, self).get_current_user() - return user - - def get_login_url(self): - # if get_login_url is invoked in an API handler, - # that means @web.authenticated is trying to trigger a redirect. - # instead of redirecting, raise 403 instead. - if not self.current_user: - raise web.HTTPError(403) - return super(APIHandler, self).get_login_url() - - @property - def content_security_policy(self): - csp = '; '.join([ - super(APIHandler, self).content_security_policy, - "default-src 'none'", - ]) - return csp - - # set _track_activity = False on API handlers that shouldn't track activity - _track_activity = True - - def update_api_activity(self): - """Update last_activity of API requests""" - # record activity of authenticated requests - if self._track_activity and getattr(self, '_user_cache', None): - self.settings['api_last_activity'] = utcnow() - - def finish(self, *args, **kwargs): - self.update_api_activity() - self.set_header('Content-Type', 'application/json') - return super(APIHandler, self).finish(*args, **kwargs) - - def options(self, *args, **kwargs): - if 'Access-Control-Allow-Headers' in self.settings.get('headers', {}): - self.set_header('Access-Control-Allow-Headers', self.settings['headers']['Access-Control-Allow-Headers']) - else: - self.set_header('Access-Control-Allow-Headers', - 'accept, content-type, authorization, x-xsrftoken') - self.set_header('Access-Control-Allow-Methods', - 'GET, PUT, POST, PATCH, DELETE, OPTIONS') - - # if authorization header is requested, - # that means the request is token-authenticated. - # avoid browser-side rejection of the preflight request. - # only allow this exception if allow_origin has not been specified - # and notebook authentication is enabled. - # If the token is not valid, the 'real' request will still be rejected. - requested_headers = self.request.headers.get('Access-Control-Request-Headers', '').split(',') - if requested_headers and any( - h.strip().lower() == 'authorization' - for h in requested_headers - ) and ( - # FIXME: it would be even better to check specifically for token-auth, - # but there is currently no API for this. - self.login_available - ) and ( - self.allow_origin - or self.allow_origin_pat - or 'Access-Control-Allow-Origin' in self.settings.get('headers', {}) - ): - self.set_header('Access-Control-Allow-Origin', - self.request.headers.get('Origin', '')) - - -class Template404(IPythonHandler): - """Render our 404 template""" - def prepare(self): - raise web.HTTPError(404) - - -class AuthenticatedFileHandler(IPythonHandler, web.StaticFileHandler): - """static files should only be accessible when logged in""" - - @property - def content_security_policy(self): - # In case we're serving HTML/SVG, confine any Javascript to a unique - # origin so it can't interact with the notebook server. - return super(AuthenticatedFileHandler, self).content_security_policy + \ - "; sandbox allow-scripts" - - @web.authenticated - def head(self, path): - self.check_xsrf_cookie() - return super(AuthenticatedFileHandler, self).head(path) - - @web.authenticated - def get(self, path): - self.check_xsrf_cookie() - - if os.path.splitext(path)[1] == '.ipynb' or self.get_argument("download", False): - name = path.rsplit('/', 1)[-1] - self.set_attachment_header(name) - - return web.StaticFileHandler.get(self, path) - - def get_content_type(self): - path = self.absolute_path.strip('/') - if '/' in path: - _, name = path.rsplit('/', 1) - else: - name = path - if name.endswith('.ipynb'): - return 'application/x-ipynb+json' - else: - cur_mime = mimetypes.guess_type(name)[0] - if cur_mime == 'text/plain': - return 'text/plain; charset=UTF-8' - else: - return super(AuthenticatedFileHandler, self).get_content_type() - - def set_headers(self): - super(AuthenticatedFileHandler, self).set_headers() - # disable browser caching, rely on 304 replies for savings - if "v" not in self.request.arguments: - self.add_header("Cache-Control", "no-cache") - - def compute_etag(self): - return None - - def validate_absolute_path(self, root, absolute_path): - """Validate and return the absolute path. - - Requires tornado 3.1 - - Adding to tornado's own handling, forbids the serving of hidden files. - """ - abs_path = super(AuthenticatedFileHandler, self).validate_absolute_path(root, absolute_path) - abs_root = os.path.abspath(root) - if is_hidden(abs_path, abs_root) and not self.contents_manager.allow_hidden: - self.log.info("Refusing to serve hidden file, via 404 Error, use flag 'ContentsManager.allow_hidden' to enable") - raise web.HTTPError(404) - return abs_path - - -def json_errors(method): - """Decorate methods with this to return GitHub style JSON errors. - - This should be used on any JSON API on any handler method that can raise HTTPErrors. - - This will grab the latest HTTPError exception using sys.exc_info - and then: - - 1. Set the HTTP status code based on the HTTPError - 2. Create and return a JSON body with a message field describing - the error in a human readable form. - """ - warnings.warn('@json_errors is deprecated in notebook 5.2.0. Subclass APIHandler instead.', - DeprecationWarning, - stacklevel=2, - ) - @functools.wraps(method) - def wrapper(self, *args, **kwargs): - self.write_error = types.MethodType(APIHandler.write_error, self) - return method(self, *args, **kwargs) - return wrapper - - - -#----------------------------------------------------------------------------- -# File handler -#----------------------------------------------------------------------------- - -# to minimize subclass changes: -HTTPError = web.HTTPError - -class FileFindHandler(IPythonHandler, web.StaticFileHandler): - """subclass of StaticFileHandler for serving files from a search path""" - - # cache search results, don't search for files more than once - _static_paths = {} - - def set_headers(self): - super(FileFindHandler, self).set_headers() - # disable browser caching, rely on 304 replies for savings - if "v" not in self.request.arguments or \ - any(self.request.path.startswith(path) for path in self.no_cache_paths): - self.set_header("Cache-Control", "no-cache") - - def initialize(self, path, default_filename=None, no_cache_paths=None): - self.no_cache_paths = no_cache_paths or [] - - if isinstance(path, string_types): - path = [path] - - self.root = tuple( - os.path.abspath(os.path.expanduser(p)) + os.sep for p in path - ) - self.default_filename = default_filename - - def compute_etag(self): - return None - - @classmethod - def get_absolute_path(cls, roots, path): - """locate a file to serve on our static file search path""" - with cls._lock: - if path in cls._static_paths: - return cls._static_paths[path] - try: - abspath = os.path.abspath(filefind(path, roots)) - except IOError: - # IOError means not found - return '' - - cls._static_paths[path] = abspath - - - log().debug("Path %s served from %s"%(path, abspath)) - return abspath - - def validate_absolute_path(self, root, absolute_path): - """check if the file should be served (raises 404, 403, etc.)""" - if absolute_path == '': - raise web.HTTPError(404) - - for root in self.root: - if (absolute_path + os.sep).startswith(root): - break - - return super(FileFindHandler, self).validate_absolute_path(root, absolute_path) - - -class APIVersionHandler(APIHandler): - - def get(self): - # not authenticated, so give as few info as possible - self.finish(json.dumps({"version":notebook.__version__})) - - -class TrailingSlashHandler(web.RequestHandler): - """Simple redirect handler that strips trailing slashes - - This should be the first, highest priority handler. - """ - - def get(self): - self.redirect(self.request.uri.rstrip('/')) - - post = put = get - - -class FilesRedirectHandler(IPythonHandler): - """Handler for redirecting relative URLs to the /files/ handler""" - - @staticmethod - def redirect_to_files(self, path): - """make redirect logic a reusable static method - - so it can be called from other handlers. - """ - cm = self.contents_manager - if cm.dir_exists(path): - # it's a *directory*, redirect to /tree - url = url_path_join(self.base_url, 'tree', url_escape(path)) - else: - orig_path = path - # otherwise, redirect to /files - parts = path.split('/') - - if not cm.file_exists(path=path) and 'files' in parts: - # redirect without files/ iff it would 404 - # this preserves pre-2.0-style 'files/' links - self.log.warning("Deprecated files/ URL: %s", orig_path) - parts.remove('files') - path = '/'.join(parts) - - if not cm.file_exists(path=path): - raise web.HTTPError(404) - - url = url_path_join(self.base_url, 'files', url_escape(path)) - self.log.debug("Redirecting %s to %s", self.request.path, url) - self.redirect(url) - - def get(self, path=''): - return self.redirect_to_files(self, path) - - -class RedirectWithParams(web.RequestHandler): - """Sam as web.RedirectHandler, but preserves URL parameters""" - def initialize(self, url, permanent=True): - self._url = url - self._permanent = permanent - - def get(self): - sep = '&' if '?' in self._url else '?' - url = sep.join([self._url, self.request.query]) - self.redirect(url, permanent=self._permanent) - -class PrometheusMetricsHandler(IPythonHandler): - """ - Return prometheus metrics for this notebook server - """ - @web.authenticated - def get(self): - self.set_header('Content-Type', prometheus_client.CONTENT_TYPE_LATEST) - self.write(prometheus_client.generate_latest(prometheus_client.REGISTRY)) - - -#----------------------------------------------------------------------------- -# URL pattern fragments for re-use -#----------------------------------------------------------------------------- - -# path matches any number of `/foo[/bar...]` or just `/` or '' -path_regex = r"(?P(?:(?:/[^/]+)+|/?))" - -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - - -default_handlers = [ - (r".*/", TrailingSlashHandler), - (r"api", APIVersionHandler), - (r'/(robots\.txt|favicon\.ico)', web.StaticFileHandler), - (r'/metrics', PrometheusMetricsHandler) -] + key = '{extension_name}_jinja2_env'.format(extension_name=self.extension_name) + return self.settings[key].get_template(name) diff --git a/notebook/base/zmqhandlers.py b/notebook/base/zmqhandlers.py deleted file mode 100644 index 327fdd5598..0000000000 --- a/notebook/base/zmqhandlers.py +++ /dev/null @@ -1,300 +0,0 @@ -# coding: utf-8 -"""Tornado handlers for WebSocket <-> ZMQ sockets.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json -import struct -import sys -from urllib.parse import urlparse - -import tornado -from tornado import gen, ioloop, web -from tornado.iostream import StreamClosedError -from tornado.websocket import WebSocketHandler, WebSocketClosedError - -from jupyter_client.session import Session -from jupyter_client.jsonutil import date_default, extract_dates -from ipython_genutils.py3compat import cast_unicode - -from notebook.utils import maybe_future -from .handlers import IPythonHandler - - -def serialize_binary_message(msg): - """serialize a message as a binary blob - - Header: - - 4 bytes: number of msg parts (nbufs) as 32b int - 4 * nbufs bytes: offset for each buffer as integer as 32b int - - Offsets are from the start of the buffer, including the header. - - Returns - ------- - - The message serialized to bytes. - - """ - # don't modify msg or buffer list in-place - msg = msg.copy() - buffers = list(msg.pop('buffers')) - if sys.version_info < (3, 4): - buffers = [x.tobytes() for x in buffers] - bmsg = json.dumps(msg, default=date_default).encode('utf8') - buffers.insert(0, bmsg) - nbufs = len(buffers) - offsets = [4 * (nbufs + 1)] - for buf in buffers[:-1]: - offsets.append(offsets[-1] + len(buf)) - offsets_buf = struct.pack('!' + 'I' * (nbufs + 1), nbufs, *offsets) - buffers.insert(0, offsets_buf) - return b''.join(buffers) - - -def deserialize_binary_message(bmsg): - """deserialize a message from a binary blog - - Header: - - 4 bytes: number of msg parts (nbufs) as 32b int - 4 * nbufs bytes: offset for each buffer as integer as 32b int - - Offsets are from the start of the buffer, including the header. - - Returns - ------- - - message dictionary - """ - nbufs = struct.unpack('!i', bmsg[:4])[0] - offsets = list(struct.unpack('!' + 'I' * nbufs, bmsg[4:4*(nbufs+1)])) - offsets.append(None) - bufs = [] - for start, stop in zip(offsets[:-1], offsets[1:]): - bufs.append(bmsg[start:stop]) - msg = json.loads(bufs[0].decode('utf8')) - msg['header'] = extract_dates(msg['header']) - msg['parent_header'] = extract_dates(msg['parent_header']) - msg['buffers'] = bufs[1:] - return msg - -# ping interval for keeping websockets alive (30 seconds) -WS_PING_INTERVAL = 30000 - - -class WebSocketMixin(object): - """Mixin for common websocket options""" - ping_callback = None - last_ping = 0 - last_pong = 0 - stream = None - - @property - def ping_interval(self): - """The interval for websocket keep-alive pings. - - Set ws_ping_interval = 0 to disable pings. - """ - return self.settings.get('ws_ping_interval', WS_PING_INTERVAL) - - @property - def ping_timeout(self): - """If no ping is received in this many milliseconds, - close the websocket connection (VPNs, etc. can fail to cleanly close ws connections). - Default is max of 3 pings or 30 seconds. - """ - return self.settings.get('ws_ping_timeout', - max(3 * self.ping_interval, WS_PING_INTERVAL) - ) - - def check_origin(self, origin=None): - """Check Origin == Host or Access-Control-Allow-Origin. - - Tornado >= 4 calls this method automatically, raising 403 if it returns False. - """ - - if self.allow_origin == '*' or ( - hasattr(self, 'skip_check_origin') and self.skip_check_origin()): - return True - - host = self.request.headers.get("Host") - if origin is None: - origin = self.get_origin() - - # If no origin or host header is provided, assume from script - if origin is None or host is None: - return True - - origin = origin.lower() - origin_host = urlparse(origin).netloc - - # OK if origin matches host - if origin_host == host: - return True - - # Check CORS headers - if self.allow_origin: - allow = self.allow_origin == origin - elif self.allow_origin_pat: - allow = bool(self.allow_origin_pat.match(origin)) - else: - # No CORS headers deny the request - allow = False - if not allow: - self.log.warning("Blocking Cross Origin WebSocket Attempt. Origin: %s, Host: %s", - origin, host, - ) - return allow - - def clear_cookie(self, *args, **kwargs): - """meaningless for websockets""" - pass - - def open(self, *args, **kwargs): - self.log.debug("Opening websocket %s", self.request.path) - - # start the pinging - if self.ping_interval > 0: - loop = ioloop.IOLoop.current() - self.last_ping = loop.time() # Remember time of last ping - self.last_pong = self.last_ping - self.ping_callback = ioloop.PeriodicCallback( - self.send_ping, self.ping_interval, - ) - self.ping_callback.start() - return super(WebSocketMixin, self).open(*args, **kwargs) - - def send_ping(self): - """send a ping to keep the websocket alive""" - if self.ws_connection is None and self.ping_callback is not None: - self.ping_callback.stop() - return - - # check for timeout on pong. Make sure that we really have sent a recent ping in - # case the machine with both server and client has been suspended since the last ping. - now = ioloop.IOLoop.current().time() - since_last_pong = 1e3 * (now - self.last_pong) - since_last_ping = 1e3 * (now - self.last_ping) - if since_last_ping < 2*self.ping_interval and since_last_pong > self.ping_timeout: - self.log.warning("WebSocket ping timeout after %i ms.", since_last_pong) - self.close() - return - try: - self.ping(b'') - except (StreamClosedError, WebSocketClosedError): - # websocket has been closed, stop pinging - self.ping_callback.stop() - return - - self.last_ping = now - - def on_pong(self, data): - self.last_pong = ioloop.IOLoop.current().time() - - -class ZMQStreamHandler(WebSocketMixin, WebSocketHandler): - - if tornado.version_info < (4,1): - """Backport send_error from tornado 4.1 to 4.0""" - def send_error(self, *args, **kwargs): - if self.stream is None: - super(WebSocketHandler, self).send_error(*args, **kwargs) - else: - # If we get an uncaught exception during the handshake, - # we have no choice but to abruptly close the connection. - # TODO: for uncaught exceptions after the handshake, - # we can close the connection more gracefully. - self.stream.close() - - - def _reserialize_reply(self, msg_or_list, channel=None): - """Reserialize a reply message using JSON. - - msg_or_list can be an already-deserialized msg dict or the zmq buffer list. - If it is the zmq list, it will be deserialized with self.session. - - This takes the msg list from the ZMQ socket and serializes the result for the websocket. - This method should be used by self._on_zmq_reply to build messages that can - be sent back to the browser. - - """ - if isinstance(msg_or_list, dict): - # already unpacked - msg = msg_or_list - else: - idents, msg_list = self.session.feed_identities(msg_or_list) - msg = self.session.deserialize(msg_list) - if channel: - msg['channel'] = channel - if msg['buffers']: - buf = serialize_binary_message(msg) - return buf - else: - smsg = json.dumps(msg, default=date_default) - return cast_unicode(smsg) - - def _on_zmq_reply(self, stream, msg_list): - # Sometimes this gets triggered when the on_close method is scheduled in the - # eventloop but hasn't been called. - if self.ws_connection is None or stream.closed(): - self.log.warning("zmq message arrived on closed channel") - self.close() - return - channel = getattr(stream, 'channel', None) - try: - msg = self._reserialize_reply(msg_list, channel=channel) - except Exception: - self.log.critical("Malformed message: %r" % msg_list, exc_info=True) - return - - try: - self.write_message(msg, binary=isinstance(msg, bytes)) - except (StreamClosedError, WebSocketClosedError): - self.log.warning("zmq message arrived on closed channel") - self.close() - return - - -class AuthenticatedZMQStreamHandler(ZMQStreamHandler, IPythonHandler): - - def set_default_headers(self): - """Undo the set_default_headers in IPythonHandler - - which doesn't make sense for websockets - """ - pass - - def pre_get(self): - """Run before finishing the GET request - - Extend this method to add logic that should fire before - the websocket finishes completing. - """ - # authenticate the request before opening the websocket - if self.get_current_user() is None: - self.log.warning("Couldn't authenticate WebSocket connection") - raise web.HTTPError(403) - - if self.get_argument('session_id', False): - self.session.session = cast_unicode(self.get_argument('session_id')) - else: - self.log.warning("No session ID specified") - - @gen.coroutine - def get(self, *args, **kwargs): - # pre_get can be a coroutine in subclasses - # assign and yield in two step to avoid tornado 3 issues - res = self.pre_get() - yield maybe_future(res) - res = super(AuthenticatedZMQStreamHandler, self).get(*args, **kwargs) - yield maybe_future(res) - - def initialize(self): - self.log.debug("Initializing websocket connection %s", self.request.path) - self.session = Session(config=self.config) - - def get_compression_options(self): - return self.settings.get('websocket_compression_options', None) diff --git a/notebook/bundler/__init__.py b/notebook/bundler/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/bundler/__main__.py b/notebook/bundler/__main__.py deleted file mode 100644 index cde186dbb9..0000000000 --- a/notebook/bundler/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from .bundlerextensions import main - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/notebook/bundler/bundlerextensions.py b/notebook/bundler/bundlerextensions.py deleted file mode 100644 index 576336c3ad..0000000000 --- a/notebook/bundler/bundlerextensions.py +++ /dev/null @@ -1,307 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import sys -import os - -from ..extensions import BaseExtensionApp, _get_config_dir, GREEN_ENABLED, RED_DISABLED -from .._version import __version__ -from notebook.config_manager import BaseJSONConfigManager - -from jupyter_core.paths import jupyter_config_path - -from traitlets.utils.importstring import import_item -from traitlets import Bool - -BUNDLER_SECTION = "notebook" -BUNDLER_SUBSECTION = "bundlerextensions" - -def _get_bundler_metadata(module): - """Gets the list of bundlers associated with a Python package. - - Returns a tuple of (the module, [{ - 'name': 'unique name of the bundler', - 'label': 'file menu item label for the bundler', - 'module_name': 'dotted package/module name containing the bundler', - 'group': 'download or deploy parent menu item' - }]) - - Parameters - ---------- - - module : str - Importable Python module exposing the - magic-named `_jupyter_bundlerextension_paths` function - """ - m = import_item(module) - if not hasattr(m, '_jupyter_bundlerextension_paths'): - raise KeyError('The Python module {} does not contain a valid bundlerextension'.format(module)) - bundlers = m._jupyter_bundlerextension_paths() - return m, bundlers - -def _set_bundler_state(name, label, module_name, group, state, - user=True, sys_prefix=False, logger=None): - """Set whether a bundler is enabled or disabled. - - Returns True if the final state is the one requested. - - Parameters - ---------- - name : string - Unique name of the bundler - label : string - Human-readable label for the bundler menu item in the notebook UI - module_name : string - Dotted module/package name containing the bundler - group : string - 'download' or 'deploy' indicating the parent menu containing the label - state : bool - The state in which to leave the extension - user : bool [default: True] - Whether to update the user's .jupyter/nbconfig directory - sys_prefix : bool [default: False] - Whether to update the sys.prefix, i.e. environment. Will override - `user`. - logger : Jupyter logger [optional] - Logger instance to use - """ - user = False if sys_prefix else user - config_dir = os.path.join( - _get_config_dir(user=user, sys_prefix=sys_prefix), 'nbconfig') - cm = BaseJSONConfigManager(config_dir=config_dir) - - if logger: - logger.info("{} {} bundler {}...".format( - "Enabling" if state else "Disabling", - name, - module_name - )) - - if state: - cm.update(BUNDLER_SECTION, { - BUNDLER_SUBSECTION: { - name: { - "label": label, - "module_name": module_name, - "group" : group - } - } - }) - else: - cm.update(BUNDLER_SECTION, { - BUNDLER_SUBSECTION: { - name: None - } - }) - - return (cm.get(BUNDLER_SECTION) - .get(BUNDLER_SUBSECTION, {}) - .get(name) is not None) == state - - -def _set_bundler_state_python(state, module, user, sys_prefix, logger=None): - """Enables or disables bundlers defined in a Python package. - - Returns a list of whether the state was achieved for each bundler. - - Parameters - ---------- - state : Bool - Whether the extensions should be enabled - module : str - Importable Python module exposing the - magic-named `_jupyter_bundlerextension_paths` function - user : bool - Whether to enable in the user's nbconfig directory. - sys_prefix : bool - Enable/disable in the sys.prefix, i.e. environment - logger : Jupyter logger [optional] - Logger instance to use - """ - m, bundlers = _get_bundler_metadata(module) - return [_set_bundler_state(name=bundler["name"], - label=bundler["label"], - module_name=bundler["module_name"], - group=bundler["group"], - state=state, - user=user, sys_prefix=sys_prefix, - logger=logger) - for bundler in bundlers] - -def enable_bundler_python(module, user=True, sys_prefix=False, logger=None): - """Enables bundlers defined in a Python package. - - Returns whether each bundle defined in the packaged was enabled or not. - - Parameters - ---------- - module : str - Importable Python module exposing the - magic-named `_jupyter_bundlerextension_paths` function - user : bool [default: True] - Whether to enable in the user's nbconfig directory. - sys_prefix : bool [default: False] - Whether to enable in the sys.prefix, i.e. environment. Will override - `user` - logger : Jupyter logger [optional] - Logger instance to use - """ - return _set_bundler_state_python(True, module, user, sys_prefix, - logger=logger) - -def disable_bundler_python(module, user=True, sys_prefix=False, logger=None): - """Disables bundlers defined in a Python package. - - Returns whether each bundle defined in the packaged was enabled or not. - - Parameters - ---------- - module : str - Importable Python module exposing the - magic-named `_jupyter_bundlerextension_paths` function - user : bool [default: True] - Whether to enable in the user's nbconfig directory. - sys_prefix : bool [default: False] - Whether to enable in the sys.prefix, i.e. environment. Will override - `user` - logger : Jupyter logger [optional] - Logger instance to use - """ - return _set_bundler_state_python(False, module, user, sys_prefix, - logger=logger) - -class ToggleBundlerExtensionApp(BaseExtensionApp): - """A base class for apps that enable/disable bundlerextensions""" - name = "jupyter bundlerextension enable/disable" - version = __version__ - description = "Enable/disable a bundlerextension in configuration." - - user = Bool(True, config=True, help="Apply the configuration only for the current user (default)") - - _toggle_value = None - - def _config_file_name_default(self): - """The default config file name.""" - return 'jupyter_notebook_config' - - def toggle_bundler_python(self, module): - """Toggle some extensions in an importable Python module. - - Returns a list of booleans indicating whether the state was changed as - requested. - - Parameters - ---------- - module : str - Importable Python module exposing the - magic-named `_jupyter_bundlerextension_paths` function - """ - toggle = (enable_bundler_python if self._toggle_value - else disable_bundler_python) - return toggle(module, - user=self.user, - sys_prefix=self.sys_prefix, - logger=self.log) - - def start(self): - if not self.extra_args: - sys.exit('Please specify an bundlerextension/package to enable or disable') - elif len(self.extra_args) > 1: - sys.exit('Please specify one bundlerextension/package at a time') - if self.python: - self.toggle_bundler_python(self.extra_args[0]) - else: - raise NotImplementedError('Cannot install bundlers from non-Python packages') - -class EnableBundlerExtensionApp(ToggleBundlerExtensionApp): - """An App that enables bundlerextensions""" - name = "jupyter bundlerextension enable" - description = """ - Enable a bundlerextension in frontend configuration. - - Usage - jupyter bundlerextension enable [--system|--sys-prefix] - """ - _toggle_value = True - -class DisableBundlerExtensionApp(ToggleBundlerExtensionApp): - """An App that disables bundlerextensions""" - name = "jupyter bundlerextension disable" - description = """ - Disable a bundlerextension in frontend configuration. - - Usage - jupyter bundlerextension disable [--system|--sys-prefix] - """ - _toggle_value = None - - -class ListBundlerExtensionApp(BaseExtensionApp): - """An App that lists and validates nbextensions""" - name = "jupyter nbextension list" - version = __version__ - description = "List all nbextensions known by the configuration system" - - def list_nbextensions(self): - """List all the nbextensions""" - config_dirs = [os.path.join(p, 'nbconfig') for p in jupyter_config_path()] - - print("Known bundlerextensions:") - - for config_dir in config_dirs: - head = u' config dir: {}'.format(config_dir) - head_shown = False - - cm = BaseJSONConfigManager(parent=self, config_dir=config_dir) - data = cm.get('notebook') - if 'bundlerextensions' in data: - if not head_shown: - # only show heading if there is an nbextension here - print(head) - head_shown = True - - for bundler_id, info in data['bundlerextensions'].items(): - label = info.get('label') - module = info.get('module_name') - if label is None or module is None: - msg = u' {} {}'.format(bundler_id, RED_DISABLED) - else: - msg = u' "{}" from {} {}'.format( - label, module, GREEN_ENABLED - ) - print(msg) - - def start(self): - """Perform the App's functions as configured""" - self.list_nbextensions() - - -class BundlerExtensionApp(BaseExtensionApp): - """Base jupyter bundlerextension command entry point""" - name = "jupyter bundlerextension" - version = __version__ - description = "Work with Jupyter bundler extensions" - examples = """ -jupyter bundlerextension list # list all configured bundlers -jupyter bundlerextension enable --py # enable all bundlers in a Python package -jupyter bundlerextension disable --py # disable all bundlers in a Python package -""" - - subcommands = dict( - enable=(EnableBundlerExtensionApp, "Enable a bundler extension"), - disable=(DisableBundlerExtensionApp, "Disable a bundler extension"), - list=(ListBundlerExtensionApp, "List bundler extensions") - ) - - def start(self): - """Perform the App's functions as configured""" - super(BundlerExtensionApp, self).start() - - # The above should have called a subcommand and raised NoStart; if we - # get here, it didn't, so we should self.log.info a message. - subcmds = ", ".join(sorted(self.subcommands)) - sys.exit("Please supply at least one subcommand: %s" % subcmds) - -main = BundlerExtensionApp.launch_instance - -if __name__ == '__main__': - main() diff --git a/notebook/bundler/handlers.py b/notebook/bundler/handlers.py deleted file mode 100644 index 9b74abefd7..0000000000 --- a/notebook/bundler/handlers.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Tornado handler for bundling notebooks.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from ipython_genutils.importstring import import_item -from tornado import web, gen - -from notebook.utils import maybe_future, url2path -from notebook.base.handlers import IPythonHandler -from notebook.services.config import ConfigManager - -from . import tools - - -class BundlerHandler(IPythonHandler): - def initialize(self): - """Make tools module available on the handler instance for compatibility - with existing bundler API and ease of reference.""" - self.tools = tools - - def get_bundler(self, bundler_id): - """ - Get bundler metadata from config given a bundler ID. - - Parameters - ---------- - bundler_id: str - Unique bundler ID within the notebook/bundlerextensions config section - - Returns - ------- - dict - Bundler metadata with label, group, and module_name attributes - - - Raises - ------ - KeyError - If the bundler ID is unknown - """ - cm = ConfigManager() - return cm.get('notebook').get('bundlerextensions', {})[bundler_id] - - @web.authenticated - @gen.coroutine - def get(self, path): - """Bundle the given notebook. - - Parameters - ---------- - path: str - Path to the notebook (path parameter) - bundler: str - Bundler ID to use (query parameter) - """ - bundler_id = self.get_query_argument('bundler') - model = self.contents_manager.get(path=url2path(path)) - - try: - bundler = self.get_bundler(bundler_id) - except KeyError: - raise web.HTTPError(400, 'Bundler %s not enabled' % bundler_id) - - module_name = bundler['module_name'] - try: - # no-op in python3, decode error in python2 - module_name = str(module_name) - except UnicodeEncodeError: - # Encode unicode as utf-8 in python2 else import_item fails - module_name = module_name.encode('utf-8') - - try: - bundler_mod = import_item(module_name) - except ImportError: - raise web.HTTPError(500, 'Could not import bundler %s ' % bundler_id) - - # Let the bundler respond in any way it sees fit and assume it will - # finish the request - yield maybe_future(bundler_mod.bundle(self, model)) - -_bundler_id_regex = r'(?P[A-Za-z0-9_]+)' - -default_handlers = [ - (r"/bundle/(.*)", BundlerHandler) -] diff --git a/notebook/bundler/tarball_bundler.py b/notebook/bundler/tarball_bundler.py deleted file mode 100644 index 854ab67881..0000000000 --- a/notebook/bundler/tarball_bundler.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import os -import io -import tarfile -import nbformat - -def _jupyter_bundlerextension_paths(): - """Metadata for notebook bundlerextension""" - return [{ - # unique bundler name - "name": "tarball_bundler", - # module containing bundle function - "module_name": "notebook.bundler.tarball_bundler", - # human-redable menu item label - "label" : "Notebook Tarball (tar.gz)", - # group under 'deploy' or 'download' menu - "group" : "download", - }] - -def bundle(handler, model): - """Create a compressed tarball containing the notebook document. - - Parameters - ---------- - handler : tornado.web.RequestHandler - Handler that serviced the bundle request - model : dict - Notebook model from the configured ContentManager - """ - notebook_filename = model['name'] - notebook_content = nbformat.writes(model['content']).encode('utf-8') - notebook_name = os.path.splitext(notebook_filename)[0] - tar_filename = '{}.tar.gz'.format(notebook_name) - - info = tarfile.TarInfo(notebook_filename) - info.size = len(notebook_content) - - with io.BytesIO() as tar_buffer: - with tarfile.open(tar_filename, "w:gz", fileobj=tar_buffer) as tar: - tar.addfile(info, io.BytesIO(notebook_content)) - - handler.set_attachment_header(tar_filename) - handler.set_header('Content-Type', 'application/gzip') - - # Return the buffer value as the response - handler.finish(tar_buffer.getvalue()) diff --git a/notebook/bundler/tests/__init__.py b/notebook/bundler/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/bundler/tests/resources/another_subdir/test_file.txt b/notebook/bundler/tests/resources/another_subdir/test_file.txt deleted file mode 100644 index 597cd83d4d..0000000000 --- a/notebook/bundler/tests/resources/another_subdir/test_file.txt +++ /dev/null @@ -1 +0,0 @@ -Used to test globbing. \ No newline at end of file diff --git a/notebook/bundler/tests/resources/empty.ipynb b/notebook/bundler/tests/resources/empty.ipynb deleted file mode 100644 index bbdd6febfc..0000000000 --- a/notebook/bundler/tests/resources/empty.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nbformat_minor": 0, - "cells": [], - "nbformat": 4, - "metadata": {} -} \ No newline at end of file diff --git a/notebook/bundler/tests/resources/subdir/subsubdir/.gitkeep b/notebook/bundler/tests/resources/subdir/subsubdir/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/bundler/tests/resources/subdir/test_file.txt b/notebook/bundler/tests/resources/subdir/test_file.txt deleted file mode 100644 index 597cd83d4d..0000000000 --- a/notebook/bundler/tests/resources/subdir/test_file.txt +++ /dev/null @@ -1 +0,0 @@ -Used to test globbing. \ No newline at end of file diff --git a/notebook/bundler/tests/test_bundler_api.py b/notebook/bundler/tests/test_bundler_api.py deleted file mode 100644 index 1aad738107..0000000000 --- a/notebook/bundler/tests/test_bundler_api.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Test the bundlers API.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import io -from os.path import join as pjoin - -from notebook.tests.launchnotebook import NotebookTestBase -from nbformat import write -from nbformat.v4 import ( - new_notebook, new_markdown_cell, new_code_cell, new_output, -) - -try: - from unittest.mock import patch -except ImportError: - from mock import patch # py3 - -def bundle(handler, model): - """Bundler test stub. Echo the notebook path.""" - handler.finish(model['path']) - -class BundleAPITest(NotebookTestBase): - """Test the bundlers web service API""" - @classmethod - def setup_class(cls): - """Make a test notebook. Borrowed from nbconvert test. Assumes the class - teardown will clean it up in the end.""" - super(BundleAPITest, cls).setup_class() - nbdir = cls.notebook_dir - - nb = new_notebook() - - nb.cells.append(new_markdown_cell(u'Created by test')) - cc1 = new_code_cell(source=u'print(2*6)') - cc1.outputs.append(new_output(output_type="stream", text=u'12')) - nb.cells.append(cc1) - - with io.open(pjoin(nbdir, 'testnb.ipynb'), 'w', - encoding='utf-8') as f: - write(nb, f, version=4) - - def test_missing_bundler_arg(self): - """Should respond with 400 error about missing bundler arg""" - resp = self.request('GET', 'bundle/fake.ipynb') - self.assertEqual(resp.status_code, 400) - self.assertIn('Missing argument bundler', resp.text) - - def test_notebook_not_found(self): - """Shoudl respond with 404 error about missing notebook""" - resp = self.request('GET', 'bundle/fake.ipynb', - params={'bundler': 'fake_bundler'}) - self.assertEqual(resp.status_code, 404) - self.assertIn('Not Found', resp.text) - - def test_bundler_not_enabled(self): - """Should respond with 400 error about disabled bundler""" - resp = self.request('GET', 'bundle/testnb.ipynb', - params={'bundler': 'fake_bundler'}) - self.assertEqual(resp.status_code, 400) - self.assertIn('Bundler fake_bundler not enabled', resp.text) - - def test_bundler_import_error(self): - """Should respond with 500 error about failure to load bundler module""" - with patch('notebook.bundler.handlers.BundlerHandler.get_bundler') as mock: - mock.return_value = {'module_name': 'fake_module'} - resp = self.request('GET', 'bundle/testnb.ipynb', - params={'bundler': 'fake_bundler'}) - mock.assert_called_with('fake_bundler') - self.assertEqual(resp.status_code, 500) - self.assertIn('Could not import bundler fake_bundler', resp.text) - - def test_bundler_invoke(self): - """Should respond with 200 and output from test bundler stub""" - with patch('notebook.bundler.handlers.BundlerHandler.get_bundler') as mock: - mock.return_value = {'module_name': 'notebook.bundler.tests.test_bundler_api'} - resp = self.request('GET', 'bundle/testnb.ipynb', - params={'bundler': 'stub_bundler'}) - mock.assert_called_with('stub_bundler') - self.assertEqual(resp.status_code, 200) - self.assertIn('testnb.ipynb', resp.text) \ No newline at end of file diff --git a/notebook/bundler/tests/test_bundler_tools.py b/notebook/bundler/tests/test_bundler_tools.py deleted file mode 100644 index 855cf978fc..0000000000 --- a/notebook/bundler/tests/test_bundler_tools.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Test the bundler tools.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import unittest -import os -import shutil -import tempfile -import notebook.bundler.tools as tools - -HERE = os.path.abspath(os.path.dirname(__file__)) - -class TestBundlerTools(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmp, ignore_errors=True) - - def test_get_no_cell_references(self): - '''Should find no references in a regular HTML comment.''' - no_references = tools.get_cell_reference_patterns({'source':'''!<-- -a -b -c --->''', 'cell_type':'markdown'}) - self.assertEqual(len(no_references), 0) - - def test_get_cell_reference_patterns_comment_multiline(self): - '''Should find two references and ignore a comment within an HTML comment.''' - cell = {'cell_type':'markdown', 'source':''''''} - references = tools.get_cell_reference_patterns(cell) - self.assertTrue('a' in references and 'b/' in references, str(references)) - self.assertEqual(len(references), 2, str(references)) - - def test_get_cell_reference_patterns_comment_trailing_filename(self): - '''Should find three references within an HTML comment.''' - cell = {'cell_type':'markdown', 'source':''''''} - references = tools.get_cell_reference_patterns(cell) - self.assertTrue('a' in references and 'b/' in references and 'c' in references, str(references)) - self.assertEqual(len(references), 3, str(references)) - - def test_get_cell_reference_patterns_precode(self): - '''Should find no references in a fenced code block in a *code* cell.''' - self.assertTrue(tools.get_cell_reference_patterns) - no_references = tools.get_cell_reference_patterns({'source':'''``` -foo -bar -baz -``` -''', 'cell_type':'code'}) - self.assertEqual(len(no_references), 0) - - def test_get_cell_reference_patterns_precode_mdcomment(self): - '''Should find two references and ignore a comment in a fenced code block.''' - cell = {'cell_type':'markdown', 'source':'''``` -a -b/ -#comment -```'''} - references = tools.get_cell_reference_patterns(cell) - self.assertTrue('a' in references and 'b/' in references, str(references)) - self.assertEqual(len(references), 2, str(references)) - - def test_get_cell_reference_patterns_precode_backticks(self): - '''Should find three references in a fenced code block.''' - cell = {'cell_type':'markdown', 'source':'''```c -a -b/ -#comment -```'''} - references = tools.get_cell_reference_patterns(cell) - self.assertTrue('a' in references and 'b/' in references and 'c' in references, str(references)) - self.assertEqual(len(references), 3, str(references)) - - def test_glob_dir(self): - '''Should expand to single file in the resources/ subfolder.''' - self.assertIn(os.path.join('resources', 'empty.ipynb'), - tools.expand_references(HERE, ['resources/empty.ipynb'])) - - def test_glob_subdir(self): - '''Should expand to all files in the resources/ subfolder.''' - self.assertIn(os.path.join('resources', 'empty.ipynb'), - tools.expand_references(HERE, ['resources/'])) - - def test_glob_splat(self): - '''Should expand to all contents under this test/ directory.''' - globs = tools.expand_references(HERE, ['*']) - self.assertIn('test_bundler_tools.py', globs, globs) - self.assertIn('resources', globs, globs) - - def test_glob_splatsplat_in_middle(self): - '''Should expand to test_file.txt deep under this test/ directory.''' - globs = tools.expand_references(HERE, ['resources/**/test_file.txt']) - self.assertIn(os.path.join('resources', 'subdir', 'test_file.txt'), globs, globs) - - def test_glob_splatsplat_trailing(self): - '''Should expand to all descendants of this test/ directory.''' - globs = tools.expand_references(HERE, ['resources/**']) - self.assertIn(os.path.join('resources', 'empty.ipynb'), globs, globs) - self.assertIn(os.path.join('resources', 'subdir', 'test_file.txt'), globs, globs) - - def test_glob_splatsplat_leading(self): - '''Should expand to test_file.txt under any path.''' - globs = tools.expand_references(HERE, ['**/test_file.txt']) - self.assertIn(os.path.join('resources', 'subdir', 'test_file.txt'), globs, globs) - self.assertIn(os.path.join('resources', 'another_subdir', 'test_file.txt'), globs, globs) - - def test_copy_filelist(self): - '''Should copy select files from source to destination''' - globs = tools.expand_references(HERE, ['**/test_file.txt']) - tools.copy_filelist(HERE, self.tmp, globs) - self.assertTrue(os.path.isfile(os.path.join(self.tmp, 'resources', 'subdir', 'test_file.txt'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp, 'resources', 'another_subdir', 'test_file.txt'))) - self.assertFalse(os.path.isfile(os.path.join(self.tmp, 'resources', 'empty.ipynb'))) diff --git a/notebook/bundler/tests/test_bundlerextension.py b/notebook/bundler/tests/test_bundlerextension.py deleted file mode 100644 index 98e9bf0844..0000000000 --- a/notebook/bundler/tests/test_bundlerextension.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Test the bundlerextension CLI.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import os -import shutil -import unittest - -try: - from unittest.mock import patch -except ImportError: - from mock import patch # py2 - -from ipython_genutils.tempdir import TemporaryDirectory -from ipython_genutils import py3compat - -from traitlets.tests.utils import check_help_all_output - -import notebook.nbextensions as nbextensions -from notebook.config_manager import BaseJSONConfigManager -from ..bundlerextensions import (_get_config_dir, enable_bundler_python, - disable_bundler_python) - -def test_help_output(): - check_help_all_output('notebook.bundler.bundlerextensions') - check_help_all_output('notebook.bundler.bundlerextensions', ['enable']) - check_help_all_output('notebook.bundler.bundlerextensions', ['disable']) - -class TestBundlerExtensionCLI(unittest.TestCase): - """Tests the bundlerextension CLI against the example zip_bundler.""" - def setUp(self): - """Build an isolated config environment.""" - td = TemporaryDirectory() - - self.test_dir = py3compat.cast_unicode(td.name) - self.data_dir = os.path.join(self.test_dir, 'data') - self.config_dir = os.path.join(self.test_dir, 'config') - self.system_data_dir = os.path.join(self.test_dir, 'system_data') - self.system_path = [self.system_data_dir] - - # Use temp directory, not real user or system config paths - self.patch_env = patch.dict('os.environ', { - 'JUPYTER_CONFIG_DIR': self.config_dir, - 'JUPYTER_DATA_DIR': self.data_dir, - }) - self.patch_env.start() - self.patch_system_path = patch.object(nbextensions, - 'SYSTEM_JUPYTER_PATH', self.system_path) - self.patch_system_path.start() - - def tearDown(self): - """Remove the test config environment.""" - shutil.rmtree(self.test_dir, ignore_errors=True) - self.patch_env.stop() - self.patch_system_path.stop() - - def test_enable(self): - """Should add the bundler to the notebook configuration.""" - enable_bundler_python('notebook.bundler.zip_bundler') - - config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig') - cm = BaseJSONConfigManager(config_dir=config_dir) - bundlers = cm.get('notebook').get('bundlerextensions', {}) - self.assertEqual(len(bundlers), 1) - self.assertIn('notebook_zip_download', bundlers) - - def test_disable(self): - """Should remove the bundler from the notebook configuration.""" - self.test_enable() - disable_bundler_python('notebook.bundler.zip_bundler') - - config_dir = os.path.join(_get_config_dir(user=True), 'nbconfig') - cm = BaseJSONConfigManager(config_dir=config_dir) - bundlers = cm.get('notebook').get('bundlerextensions', {}) - self.assertEqual(len(bundlers), 0) diff --git a/notebook/bundler/tools.py b/notebook/bundler/tools.py deleted file mode 100644 index cdf3556ace..0000000000 --- a/notebook/bundler/tools.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Set of common tools to aid bundler implementations.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import os -import shutil -import errno -import nbformat -import fnmatch -import glob - -def get_file_references(abs_nb_path, version): - """Gets a list of files referenced either in Markdown fenced code blocks - or in HTML comments from the notebook. Expands patterns expressed in - gitignore syntax (https://git-scm.com/docs/gitignore). Returns the - fully expanded list of filenames relative to the notebook dirname. - - Parameters - ---------- - abs_nb_path: str - Absolute path of the notebook on disk - version: int - Version of the notebook document format to use - - Returns - ------- - list - Filename strings relative to the notebook path - """ - ref_patterns = get_reference_patterns(abs_nb_path, version) - expanded = expand_references(os.path.dirname(abs_nb_path), ref_patterns) - return expanded - -def get_reference_patterns(abs_nb_path, version): - """Gets a list of reference patterns either in Markdown fenced code blocks - or in HTML comments from the notebook. - - Parameters - ---------- - abs_nb_path: str - Absolute path of the notebook on disk - version: int - Version of the notebook document format to use - - Returns - ------- - list - Pattern strings from the notebook - """ - notebook = nbformat.read(abs_nb_path, version) - referenced_list = [] - for cell in notebook.cells: - references = get_cell_reference_patterns(cell) - if references: - referenced_list = referenced_list + references - return referenced_list - -def get_cell_reference_patterns(cell): - ''' - Retrieves the list of references from a single notebook cell. Looks for - fenced code blocks or HTML comments in Markdown cells, e.g., - - ``` - some.csv - foo/ - !foo/bar - ``` - - or - - - - Parameters - ---------- - cell: dict - Notebook cell object - - Returns - ------- - list - Reference patterns found in the cell - ''' - referenced = [] - # invisible after execution: unrendered HTML comment - if cell.get('cell_type').startswith('markdown') and cell.get('source').startswith(''): - break - # Trying to go out of the current directory leads to - # trouble when deploying - if line.find('../') < 0 and not line.startswith('#'): - referenced.append(line) - # visible after execution: rendered as a code element within a pre element - elif cell.get('cell_type').startswith('markdown') and cell.get('source').find('```') >= 0: - source = cell.get('source') - offset = source.find('```') - lines = source[offset + len('```'):].splitlines() - for line in lines: - if line.startswith('```'): - break - # Trying to go out of the current directory leads to - # trouble when deploying - if line.find('../') < 0 and not line.startswith('#'): - referenced.append(line) - - # Clean out blank references - return [ref for ref in referenced if ref.strip()] - -def expand_references(root_path, references): - """Expands a set of reference patterns by evaluating them against the - given root directory. Expansions are performed against patterns - expressed in the same manner as in gitignore - (https://git-scm.com/docs/gitignore). - - NOTE: Temporarily changes the current working directory when called. - - Parameters - ---------- - root_path: str - Assumed root directory for the patterns - references: list - Reference patterns from get_reference_patterns expressed with - forward-slash directory separators - - Returns - ------- - list - Filename strings relative to the root path - """ - # Use normpath to convert to platform specific slashes, but be sure - # to retain a trailing slash which normpath pulls off - normalized_references = [] - for ref in references: - normalized_ref = os.path.normpath(ref) - # un-normalized separator - if ref.endswith('/'): - normalized_ref += os.sep - normalized_references.append(normalized_ref) - references = normalized_references - - globbed = [] - negations = [] - must_walk = [] - for pattern in references: - if pattern and pattern.find(os.sep) < 0: - # simple shell glob - cwd = os.getcwd() - os.chdir(root_path) - if pattern.startswith('!'): - negations = negations + glob.glob(pattern[1:]) - else: - globbed = globbed + glob.glob(pattern) - os.chdir(cwd) - elif pattern: - must_walk.append(pattern) - - for pattern in must_walk: - pattern_is_negation = pattern.startswith('!') - if pattern_is_negation: - testpattern = pattern[1:] - else: - testpattern = pattern - for root, _, filenames in os.walk(root_path): - for filename in filenames: - joined = os.path.join(root[len(root_path) + 1:], filename) - if testpattern.endswith(os.sep): - if joined.startswith(testpattern): - if pattern_is_negation: - negations.append(joined) - else: - globbed.append(joined) - elif testpattern.find('**') >= 0: - # path wildcard - ends = testpattern.split('**') - if len(ends) == 2: - if joined.startswith(ends[0]) and joined.endswith(ends[1]): - if pattern_is_negation: - negations.append(joined) - else: - globbed.append(joined) - else: - # segments should be respected - if fnmatch.fnmatch(joined, testpattern): - if pattern_is_negation: - negations.append(joined) - else: - globbed.append(joined) - - for negated in negations: - try: - globbed.remove(negated) - except ValueError as err: - pass - return set(globbed) - -def copy_filelist(src, dst, src_relative_filenames): - """Copies the given list of files, relative to src, into dst, creating - directories along the way as needed and ignore existence errors. - Skips any files that do not exist. Does not create empty directories - from src in dst. - - Parameters - ---------- - src: str - Root of the source directory - dst: str - Root of the destination directory - src_relative_filenames: list - Filenames relative to src - """ - for filename in src_relative_filenames: - # Only consider the file if it exists in src - if os.path.isfile(os.path.join(src, filename)): - parent_relative = os.path.dirname(filename) - if parent_relative: - # Make sure the parent directory exists - parent_dst = os.path.join(dst, parent_relative) - try: - os.makedirs(parent_dst) - except OSError as exc: - if exc.errno == errno.EEXIST: - pass - else: - raise exc - shutil.copy2(os.path.join(src, filename), os.path.join(dst, filename)) diff --git a/notebook/bundler/zip_bundler.py b/notebook/bundler/zip_bundler.py deleted file mode 100644 index f7bd5cc7a6..0000000000 --- a/notebook/bundler/zip_bundler.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import os -import io -import zipfile -import notebook.bundler.tools as tools - -def _jupyter_bundlerextension_paths(): - """Metadata for notebook bundlerextension""" - return [{ - 'name': 'notebook_zip_download', - 'label': 'IPython Notebook bundle (.zip)', - 'module_name': 'notebook.bundler.zip_bundler', - 'group': 'download' - }] - -def bundle(handler, model): - """Create a zip file containing the original notebook and files referenced - from it. Retain the referenced files in paths relative to the notebook. - Return the zip as a file download. - - Assumes the notebook and other files are all on local disk. - - Parameters - ---------- - handler : tornado.web.RequestHandler - Handler that serviced the bundle request - model : dict - Notebook model from the configured ContentManager - """ - abs_nb_path = os.path.join(handler.settings['contents_manager'].root_dir, - model['path']) - notebook_filename = model['name'] - notebook_name = os.path.splitext(notebook_filename)[0] - - # Headers - zip_filename = os.path.splitext(notebook_name)[0] + '.zip' - handler.set_attachment_header(zip_filename) - handler.set_header('Content-Type', 'application/zip') - - # Get associated files - ref_filenames = tools.get_file_references(abs_nb_path, 4) - - # Prepare the zip file - zip_buffer = io.BytesIO() - zipf = zipfile.ZipFile(zip_buffer, mode='w', compression=zipfile.ZIP_DEFLATED) - zipf.write(abs_nb_path, notebook_filename) - - notebook_dir = os.path.dirname(abs_nb_path) - for nb_relative_filename in ref_filenames: - # Build absolute path to file on disk - abs_fn = os.path.join(notebook_dir, nb_relative_filename) - # Store file under path relative to notebook - zipf.write(abs_fn, nb_relative_filename) - - zipf.close() - - # Return the buffer value as the response - handler.finish(zip_buffer.getvalue()) \ No newline at end of file diff --git a/notebook/edit/__init__.py b/notebook/edit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/edit/handlers.py b/notebook/edit/handlers.py deleted file mode 100644 index c0864ab5e3..0000000000 --- a/notebook/edit/handlers.py +++ /dev/null @@ -1,29 +0,0 @@ -#encoding: utf-8 -"""Tornado handlers for the terminal emulator.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from tornado import web -from ..base.handlers import IPythonHandler, path_regex -from ..utils import url_escape - -class EditorHandler(IPythonHandler): - """Render the text editor interface.""" - @web.authenticated - def get(self, path): - path = path.strip('/') - if not self.contents_manager.file_exists(path): - raise web.HTTPError(404, u'File does not exist: %s' % path) - - basename = path.rsplit('/', 1)[-1] - self.write(self.render_template('edit.html', - file_path=url_escape(path), - basename=basename, - page_title=basename + " (editing)", - ) - ) - -default_handlers = [ - (r"/edit%s" % path_regex, EditorHandler), -] \ No newline at end of file diff --git a/notebook/files/__init__.py b/notebook/files/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/files/handlers.py b/notebook/files/handlers.py deleted file mode 100644 index 192de1f923..0000000000 --- a/notebook/files/handlers.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Serve files directly from the ContentsManager.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import mimetypes -import json -from base64 import decodebytes - -from tornado import web - -from notebook.base.handlers import IPythonHandler -from notebook.utils import maybe_future - - -class FilesHandler(IPythonHandler): - """serve files via ContentsManager - - Normally used when ContentsManager is not a FileContentsManager. - - FileContentsManager subclasses use AuthenticatedFilesHandler by default, - a subclass of StaticFileHandler. - """ - - @property - def content_security_policy(self): - # In case we're serving HTML/SVG, confine any Javascript to a unique - # origin so it can't interact with the notebook server. - return super(FilesHandler, self).content_security_policy + \ - "; sandbox allow-scripts" - - @web.authenticated - def head(self, path): - self.check_xsrf_cookie() - return self.get(path, include_body=False) - - @web.authenticated - def get(self, path, include_body=True): - # /files/ requests must originate from the same site - self.check_xsrf_cookie() - cm = self.contents_manager - - if cm.is_hidden(path) and not cm.allow_hidden: - self.log.info("Refusing to serve hidden file, via 404 Error") - raise web.HTTPError(404) - - path = path.strip('/') - if '/' in path: - _, name = path.rsplit('/', 1) - else: - name = path - - model = yield maybe_future(cm.get(path, type='file', content=include_body)) - - if self.get_argument("download", False): - self.set_attachment_header(name) - - # get mimetype from filename - if name.lower().endswith('.ipynb'): - self.set_header('Content-Type', 'application/x-ipynb+json') - else: - cur_mime = mimetypes.guess_type(name)[0] - if cur_mime == 'text/plain': - self.set_header('Content-Type', 'text/plain; charset=UTF-8') - elif cur_mime is not None: - self.set_header('Content-Type', cur_mime) - else: - if model['format'] == 'base64': - self.set_header('Content-Type', 'application/octet-stream') - else: - self.set_header('Content-Type', 'text/plain; charset=UTF-8') - - if include_body: - if model['format'] == 'base64': - b64_bytes = model['content'].encode('ascii') - self.write(decodebytes(b64_bytes)) - elif model['format'] == 'json': - self.write(json.dumps(model['content'])) - else: - self.write(model['content']) - self.flush() - - -default_handlers = [] diff --git a/notebook/gateway/__init__.py b/notebook/gateway/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/gateway/handlers.py b/notebook/gateway/handlers.py deleted file mode 100644 index 0fbbfb18b9..0000000000 --- a/notebook/gateway/handlers.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import os -import logging -import mimetypes - -from ..base.handlers import APIHandler, IPythonHandler -from ..utils import url_path_join - -from tornado import gen, web -from tornado.concurrent import Future -from tornado.ioloop import IOLoop -from tornado.websocket import WebSocketHandler, websocket_connect -from tornado.httpclient import HTTPRequest -from tornado.escape import url_escape, json_decode, utf8 - -from ipython_genutils.py3compat import cast_unicode -from jupyter_client.session import Session -from traitlets.config.configurable import LoggingConfigurable - -from .managers import GatewayClient - - -class WebSocketChannelsHandler(WebSocketHandler, IPythonHandler): - - session = None - gateway = None - kernel_id = None - - def set_default_headers(self): - """Undo the set_default_headers in IPythonHandler which doesn't make sense for websockets""" - pass - - def get_compression_options(self): - # use deflate compress websocket - return {} - - def authenticate(self): - """Run before finishing the GET request - - Extend this method to add logic that should fire before - the websocket finishes completing. - """ - # authenticate the request before opening the websocket - if self.get_current_user() is None: - self.log.warning("Couldn't authenticate WebSocket connection") - raise web.HTTPError(403) - - if self.get_argument('session_id', False): - self.session.session = cast_unicode(self.get_argument('session_id')) - else: - self.log.warning("No session ID specified") - - def initialize(self): - self.log.debug("Initializing websocket connection %s", self.request.path) - self.session = Session(config=self.config) - self.gateway = GatewayWebSocketClient(gateway_url=GatewayClient.instance().url) - - @gen.coroutine - def get(self, kernel_id, *args, **kwargs): - self.authenticate() - self.kernel_id = cast_unicode(kernel_id, 'ascii') - super(WebSocketChannelsHandler, self).get(kernel_id=kernel_id, *args, **kwargs) - - def open(self, kernel_id, *args, **kwargs): - """Handle web socket connection open to notebook server and delegate to gateway web socket handler """ - self.gateway.on_open( - kernel_id=kernel_id, - message_callback=self.write_message, - compression_options=self.get_compression_options() - ) - - def on_message(self, message): - """Forward message to gateway web socket handler.""" - self.log.debug("Sending message to gateway: {}".format(message)) - self.gateway.on_message(message) - - def write_message(self, message, binary=False): - """Send message back to notebook client. This is called via callback from self.gateway._read_messages.""" - self.log.debug("Receiving message from gateway: {}".format(message)) - if self.ws_connection: # prevent WebSocketClosedError - super(WebSocketChannelsHandler, self).write_message(message, binary=binary) - elif self.log.isEnabledFor(logging.DEBUG): - msg_summary = WebSocketChannelsHandler._get_message_summary(json_decode(utf8(message))) - self.log.debug("Notebook client closed websocket connection - message dropped: {}".format(msg_summary)) - - def on_close(self): - self.log.debug("Closing websocket connection %s", self.request.path) - self.gateway.on_close() - super(WebSocketChannelsHandler, self).on_close() - - @staticmethod - def _get_message_summary(message): - summary = [] - message_type = message['msg_type'] - summary.append('type: {}'.format(message_type)) - - if message_type == 'status': - summary.append(', state: {}'.format(message['content']['execution_state'])) - elif message_type == 'error': - summary.append(', {}:{}:{}'.format(message['content']['ename'], - message['content']['evalue'], - message['content']['traceback'])) - else: - summary.append(', ...') # don't display potentially sensitive data - - return ''.join(summary) - - -class GatewayWebSocketClient(LoggingConfigurable): - """Proxy web socket connection to a kernel/enterprise gateway.""" - - def __init__(self, **kwargs): - super(GatewayWebSocketClient, self).__init__(**kwargs) - self.kernel_id = None - self.ws = None - self.ws_future = Future() - self.ws_future_cancelled = False - - @gen.coroutine - def _connect(self, kernel_id): - self.kernel_id = kernel_id - ws_url = url_path_join( - GatewayClient.instance().ws_url, - GatewayClient.instance().kernels_endpoint, url_escape(kernel_id), 'channels' - ) - self.log.info('Connecting to {}'.format(ws_url)) - kwargs = {} - kwargs = GatewayClient.instance().load_connection_args(**kwargs) - - request = HTTPRequest(ws_url, **kwargs) - self.ws_future = websocket_connect(request) - self.ws_future.add_done_callback(self._connection_done) - - def _connection_done(self, fut): - if not self.ws_future_cancelled: # prevent concurrent.futures._base.CancelledError - self.ws = fut.result() - self.log.debug("Connection is ready: ws: {}".format(self.ws)) - else: - self.log.warning("Websocket connection has been cancelled via client disconnect before its establishment. " - "Kernel with ID '{}' may not be terminated on GatewayClient: {}". - format(self.kernel_id, GatewayClient.instance().url)) - - def _disconnect(self): - if self.ws is not None: - # Close connection - self.ws.close() - elif not self.ws_future.done(): - # Cancel pending connection. Since future.cancel() is a noop on tornado, we'll track cancellation locally - self.ws_future.cancel() - self.ws_future_cancelled = True - self.log.debug("_disconnect: ws_future_cancelled: {}".format(self.ws_future_cancelled)) - - @gen.coroutine - def _read_messages(self, callback): - """Read messages from gateway server.""" - while True: - message = None - if not self.ws_future_cancelled: - try: - message = yield self.ws.read_message() - except Exception as e: - self.log.error("Exception reading message from websocket: {}".format(e)) # , exc_info=True) - if message is None: - break - callback(message) # pass back to notebook client (see self.on_open and WebSocketChannelsHandler.open) - else: # ws cancelled - stop reading - break - - def on_open(self, kernel_id, message_callback, **kwargs): - """Web socket connection open against gateway server.""" - self._connect(kernel_id) - loop = IOLoop.current() - loop.add_future( - self.ws_future, - lambda future: self._read_messages(message_callback) - ) - - def on_message(self, message): - """Send message to gateway server.""" - if self.ws is None: - loop = IOLoop.current() - loop.add_future( - self.ws_future, - lambda future: self._write_message(message) - ) - else: - self._write_message(message) - - def _write_message(self, message): - """Send message to gateway server.""" - try: - if not self.ws_future_cancelled: - self.ws.write_message(message) - except Exception as e: - self.log.error("Exception writing message to websocket: {}".format(e)) # , exc_info=True) - - def on_close(self): - """Web socket closed event.""" - self._disconnect() - - -class GatewayResourceHandler(APIHandler): - """Retrieves resources for specific kernelspec definitions from kernel/enterprise gateway.""" - - @web.authenticated - @gen.coroutine - def get(self, kernel_name, path, include_body=True): - ksm = self.kernel_spec_manager - kernel_spec_res = yield ksm.get_kernel_spec_resource(kernel_name, path) - if kernel_spec_res is None: - self.log.warning("Kernelspec resource '{}' for '{}' not found. Gateway may not support" - " resource serving.".format(path, kernel_name)) - else: - self.set_header("Content-Type", mimetypes.guess_type(path)[0]) - self.finish(kernel_spec_res) - - -from ..services.kernels.handlers import _kernel_id_regex -from ..services.kernelspecs.handlers import kernel_name_regex - -default_handlers = [ - (r"/api/kernels/%s/channels" % _kernel_id_regex, WebSocketChannelsHandler), - (r"/kernelspecs/%s/(?P.*)" % kernel_name_regex, GatewayResourceHandler), -] diff --git a/notebook/gateway/managers.py b/notebook/gateway/managers.py deleted file mode 100644 index 6c97e57a9a..0000000000 --- a/notebook/gateway/managers.py +++ /dev/null @@ -1,588 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import os -import json - -from socket import gaierror -from tornado import gen, web -from tornado.escape import json_encode, json_decode, url_escape -from tornado.httpclient import HTTPClient, AsyncHTTPClient, HTTPError - -from ..services.kernels.kernelmanager import MappingKernelManager -from ..services.sessions.sessionmanager import SessionManager - -from jupyter_client.kernelspec import KernelSpecManager -from ..utils import url_path_join - -from traitlets import Instance, Unicode, Float, Bool, default, validate, TraitError -from traitlets.config import SingletonConfigurable - - -class GatewayClient(SingletonConfigurable): - """This class manages the configuration. It's its own singleton class so that we - can share these values across all objects. It also contains some helper methods - to build request arguments out of the various config options. - - """ - - url = Unicode(default_value=None, allow_none=True, config=True, - help="""The url of the Kernel or Enterprise Gateway server where - kernel specifications are defined and kernel management takes place. - If defined, this Notebook server acts as a proxy for all kernel - management and kernel specification retrieval. (JUPYTER_GATEWAY_URL env var) - """ - ) - - url_env = 'JUPYTER_GATEWAY_URL' - @default('url') - def _url_default(self): - return os.environ.get(self.url_env) - - @validate('url') - def _url_validate(self, proposal): - value = proposal['value'] - # Ensure value, if present, starts with 'http' - if value is not None and len(value) > 0: - if not str(value).lower().startswith('http'): - raise TraitError("GatewayClient url must start with 'http': '%r'" % value) - return value - - ws_url = Unicode(default_value=None, allow_none=True, config=True, - help="""The websocket url of the Kernel or Enterprise Gateway server. If not provided, this value - will correspond to the value of the Gateway url with 'ws' in place of 'http'. (JUPYTER_GATEWAY_WS_URL env var) - """ - ) - - ws_url_env = 'JUPYTER_GATEWAY_WS_URL' - @default('ws_url') - def _ws_url_default(self): - default_value = os.environ.get(self.ws_url_env) - if default_value is None: - if self.gateway_enabled: - default_value = self.url.lower().replace('http', 'ws') - return default_value - - @validate('ws_url') - def _ws_url_validate(self, proposal): - value = proposal['value'] - # Ensure value, if present, starts with 'ws' - if value is not None and len(value) > 0: - if not str(value).lower().startswith('ws'): - raise TraitError("GatewayClient ws_url must start with 'ws': '%r'" % value) - return value - - kernels_endpoint_default_value = '/api/kernels' - kernels_endpoint_env = 'JUPYTER_GATEWAY_KERNELS_ENDPOINT' - kernels_endpoint = Unicode(default_value=kernels_endpoint_default_value, config=True, - help="""The gateway API endpoint for accessing kernel resources (JUPYTER_GATEWAY_KERNELS_ENDPOINT env var)""") - - @default('kernels_endpoint') - def _kernels_endpoint_default(self): - return os.environ.get(self.kernels_endpoint_env, self.kernels_endpoint_default_value) - - kernelspecs_endpoint_default_value = '/api/kernelspecs' - kernelspecs_endpoint_env = 'JUPYTER_GATEWAY_KERNELSPECS_ENDPOINT' - kernelspecs_endpoint = Unicode(default_value=kernelspecs_endpoint_default_value, config=True, - help="""The gateway API endpoint for accessing kernelspecs (JUPYTER_GATEWAY_KERNELSPECS_ENDPOINT env var)""") - - @default('kernelspecs_endpoint') - def _kernelspecs_endpoint_default(self): - return os.environ.get(self.kernelspecs_endpoint_env, self.kernelspecs_endpoint_default_value) - - kernelspecs_resource_endpoint_default_value = '/kernelspecs' - kernelspecs_resource_endpoint_env = 'JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT' - kernelspecs_resource_endpoint = Unicode(default_value=kernelspecs_resource_endpoint_default_value, config=True, - help="""The gateway endpoint for accessing kernelspecs resources - (JUPYTER_GATEWAY_KERNELSPECS_RESOURCE_ENDPOINT env var)""") - - @default('kernelspecs_resource_endpoint') - def _kernelspecs_resource_endpoint_default(self): - return os.environ.get(self.kernelspecs_resource_endpoint_env, self.kernelspecs_resource_endpoint_default_value) - - connect_timeout_default_value = 20.0 - connect_timeout_env = 'JUPYTER_GATEWAY_CONNECT_TIMEOUT' - connect_timeout = Float(default_value=connect_timeout_default_value, config=True, - help="""The time allowed for HTTP connection establishment with the Gateway server. - (JUPYTER_GATEWAY_CONNECT_TIMEOUT env var)""") - - @default('connect_timeout') - def connect_timeout_default(self): - return float(os.environ.get('JUPYTER_GATEWAY_CONNECT_TIMEOUT', self.connect_timeout_default_value)) - - request_timeout_default_value = 20.0 - request_timeout_env = 'JUPYTER_GATEWAY_REQUEST_TIMEOUT' - request_timeout = Float(default_value=request_timeout_default_value, config=True, - help="""The time allowed for HTTP request completion. (JUPYTER_GATEWAY_REQUEST_TIMEOUT env var)""") - - @default('request_timeout') - def request_timeout_default(self): - return float(os.environ.get('JUPYTER_GATEWAY_REQUEST_TIMEOUT', self.request_timeout_default_value)) - - client_key = Unicode(default_value=None, allow_none=True, config=True, - help="""The filename for client SSL key, if any. (JUPYTER_GATEWAY_CLIENT_KEY env var) - """ - ) - client_key_env = 'JUPYTER_GATEWAY_CLIENT_KEY' - - @default('client_key') - def _client_key_default(self): - return os.environ.get(self.client_key_env) - - client_cert = Unicode(default_value=None, allow_none=True, config=True, - help="""The filename for client SSL certificate, if any. (JUPYTER_GATEWAY_CLIENT_CERT env var) - """ - ) - client_cert_env = 'JUPYTER_GATEWAY_CLIENT_CERT' - - @default('client_cert') - def _client_cert_default(self): - return os.environ.get(self.client_cert_env) - - ca_certs = Unicode(default_value=None, allow_none=True, config=True, - help="""The filename of CA certificates or None to use defaults. (JUPYTER_GATEWAY_CA_CERTS env var) - """ - ) - ca_certs_env = 'JUPYTER_GATEWAY_CA_CERTS' - - @default('ca_certs') - def _ca_certs_default(self): - return os.environ.get(self.ca_certs_env) - - http_user = Unicode(default_value=None, allow_none=True, config=True, - help="""The username for HTTP authentication. (JUPYTER_GATEWAY_HTTP_USER env var) - """ - ) - http_user_env = 'JUPYTER_GATEWAY_HTTP_USER' - - @default('http_user') - def _http_user_default(self): - return os.environ.get(self.http_user_env) - - http_pwd = Unicode(default_value=None, allow_none=True, config=True, - help="""The password for HTTP authentication. (JUPYTER_GATEWAY_HTTP_PWD env var) - """ - ) - http_pwd_env = 'JUPYTER_GATEWAY_HTTP_PWD' - - @default('http_pwd') - def _http_pwd_default(self): - return os.environ.get(self.http_pwd_env) - - headers_default_value = '{}' - headers_env = 'JUPYTER_GATEWAY_HEADERS' - headers = Unicode(default_value=headers_default_value, allow_none=True,config=True, - help="""Additional HTTP headers to pass on the request. This value will be converted to a dict. - (JUPYTER_GATEWAY_HEADERS env var) - """ - ) - - @default('headers') - def _headers_default(self): - return os.environ.get(self.headers_env, self.headers_default_value) - - auth_token = Unicode(default_value=None, allow_none=True, config=True, - help="""The authorization token used in the HTTP headers. (JUPYTER_GATEWAY_AUTH_TOKEN env var) - """ - ) - auth_token_env = 'JUPYTER_GATEWAY_AUTH_TOKEN' - - @default('auth_token') - def _auth_token_default(self): - return os.environ.get(self.auth_token_env) - - validate_cert_default_value = True - validate_cert_env = 'JUPYTER_GATEWAY_VALIDATE_CERT' - validate_cert = Bool(default_value=validate_cert_default_value, config=True, - help="""For HTTPS requests, determines if server's certificate should be validated or not. - (JUPYTER_GATEWAY_VALIDATE_CERT env var)""" - ) - - @default('validate_cert') - def validate_cert_default(self): - return bool(os.environ.get(self.validate_cert_env, str(self.validate_cert_default_value)) not in ['no', 'false']) - - def __init__(self, **kwargs): - super(GatewayClient, self).__init__(**kwargs) - self._static_args = {} # initialized on first use - - env_whitelist_default_value = '' - env_whitelist_env = 'JUPYTER_GATEWAY_ENV_WHITELIST' - env_whitelist = Unicode(default_value=env_whitelist_default_value, config=True, - help="""A comma-separated list of environment variable names that will be included, along with - their values, in the kernel startup request. The corresponding `env_whitelist` configuration - value must also be set on the Gateway server - since that configuration value indicates which - environmental values to make available to the kernel. (JUPYTER_GATEWAY_ENV_WHITELIST env var)""") - - @default('env_whitelist') - def _env_whitelist_default(self): - return os.environ.get(self.env_whitelist_env, self.env_whitelist_default_value) - - @property - def gateway_enabled(self): - return bool(self.url is not None and len(self.url) > 0) - - def init_static_args(self): - """Initialize arguments used on every request. Since these are static values, we'll - perform this operation once. - - """ - self._static_args['headers'] = json.loads(self.headers) - self._static_args['headers'].update({'Authorization': 'token {}'.format(self.auth_token)}) - self._static_args['connect_timeout'] = self.connect_timeout - self._static_args['request_timeout'] = self.request_timeout - self._static_args['validate_cert'] = self.validate_cert - if self.client_cert: - self._static_args['client_cert'] = self.client_cert - self._static_args['client_key'] = self.client_key - if self.ca_certs: - self._static_args['ca_certs'] = self.ca_certs - if self.http_user: - self._static_args['auth_username'] = self.http_user - if self.http_pwd: - self._static_args['auth_password'] = self.http_pwd - - def load_connection_args(self, **kwargs): - """Merges the static args relative to the connection, with the given keyword arguments. If statics - have yet to be initialized, we'll do that here. - - """ - if len(self._static_args) == 0: - self.init_static_args() - - kwargs.update(self._static_args) - return kwargs - - -@gen.coroutine -def gateway_request(endpoint, **kwargs): - """Make an async request to kernel gateway endpoint, returns a response """ - client = AsyncHTTPClient() - kwargs = GatewayClient.instance().load_connection_args(**kwargs) - try: - response = yield client.fetch(endpoint, **kwargs) - # Trap a set of common exceptions so that we can inform the user that their Gateway url is incorrect - # or the server is not running. - # NOTE: We do this here since this handler is called during the Notebook's startup and subsequent refreshes - # of the tree view. - except ConnectionRefusedError: - raise web.HTTPError(503, "Connection refused from Gateway server url '{}'. " - "Check to be sure the Gateway instance is running.".format(GatewayClient.instance().url)) - except HTTPError: - # This can occur if the host is valid (e.g., foo.com) but there's nothing there. - raise web.HTTPError(504, "Error attempting to connect to Gateway server url '{}'. " \ - "Ensure gateway url is valid and the Gateway instance is running.".format( - GatewayClient.instance().url)) - except gaierror as e: - raise web.HTTPError(404, "The Gateway server specified in the gateway_url '{}' doesn't appear to be valid. " - "Ensure gateway url is valid and the Gateway instance is running.".format( - GatewayClient.instance().url)) - - raise gen.Return(response) - - -class GatewayKernelManager(MappingKernelManager): - """Kernel manager that supports remote kernels hosted by Jupyter Kernel or Enterprise Gateway.""" - - # We'll maintain our own set of kernel ids - _kernels = {} - - def __init__(self, **kwargs): - super(GatewayKernelManager, self).__init__(**kwargs) - self.base_endpoint = url_path_join(GatewayClient.instance().url, GatewayClient.instance().kernels_endpoint) - - def __contains__(self, kernel_id): - return kernel_id in self._kernels - - def remove_kernel(self, kernel_id): - """Complete override since we want to be more tolerant of missing keys """ - try: - return self._kernels.pop(kernel_id) - except KeyError: - pass - - def _get_kernel_endpoint_url(self, kernel_id=None): - """Builds a url for the kernels endpoint - - Parameters - ---------- - kernel_id: kernel UUID (optional) - """ - if kernel_id: - return url_path_join(self.base_endpoint, url_escape(str(kernel_id))) - - return self.base_endpoint - - @gen.coroutine - def start_kernel(self, kernel_id=None, path=None, **kwargs): - """Start a kernel for a session and return its kernel_id. - - Parameters - ---------- - kernel_id : uuid - The uuid to associate the new kernel with. If this - is not None, this kernel will be persistent whenever it is - requested. - path : API path - The API path (unicode, '/' delimited) for the cwd. - Will be transformed to an OS path relative to root_dir. - """ - self.log.info('Request start kernel: kernel_id=%s, path="%s"', kernel_id, path) - - if kernel_id is None: - if path is not None: - kwargs['cwd'] = self.cwd_for_path(path) - kernel_name = kwargs.get('kernel_name', 'python3') - kernel_url = self._get_kernel_endpoint_url() - self.log.debug("Request new kernel at: %s" % kernel_url) - - # Let KERNEL_USERNAME take precedent over http_user config option. - if os.environ.get('KERNEL_USERNAME') is None and GatewayClient.instance().http_user: - os.environ['KERNEL_USERNAME'] = GatewayClient.instance().http_user - - kernel_env = {k: v for (k, v) in dict(os.environ).items() if k.startswith('KERNEL_') - or k in GatewayClient.instance().env_whitelist.split(",")} - - # Convey the full path to where this notebook file is located. - if path is not None and kernel_env.get('KERNEL_WORKING_DIR') is None: - kernel_env['KERNEL_WORKING_DIR'] = kwargs['cwd'] - - json_body = json_encode({'name': kernel_name, 'env': kernel_env}) - - response = yield gateway_request(kernel_url, method='POST', body=json_body) - kernel = json_decode(response.body) - kernel_id = kernel['id'] - self.log.info("Kernel started: %s" % kernel_id) - self.log.debug("Kernel args: %r" % kwargs) - else: - kernel = yield self.get_kernel(kernel_id) - kernel_id = kernel['id'] - self.log.info("Using existing kernel: %s" % kernel_id) - - self._kernels[kernel_id] = kernel - raise gen.Return(kernel_id) - - @gen.coroutine - def get_kernel(self, kernel_id=None, **kwargs): - """Get kernel for kernel_id. - - Parameters - ---------- - kernel_id : uuid - The uuid of the kernel. - """ - kernel_url = self._get_kernel_endpoint_url(kernel_id) - self.log.debug("Request kernel at: %s" % kernel_url) - try: - response = yield gateway_request(kernel_url, method='GET') - except HTTPError as error: - if error.code == 404: - self.log.warn("Kernel not found at: %s" % kernel_url) - self.remove_kernel(kernel_id) - kernel = None - else: - raise - else: - kernel = json_decode(response.body) - self._kernels[kernel_id] = kernel - self.log.debug("Kernel retrieved: %s" % kernel) - raise gen.Return(kernel) - - @gen.coroutine - def kernel_model(self, kernel_id): - """Return a dictionary of kernel information described in the - JSON standard model. - - Parameters - ---------- - kernel_id : uuid - The uuid of the kernel. - """ - self.log.debug("RemoteKernelManager.kernel_model: %s", kernel_id) - model = yield self.get_kernel(kernel_id) - raise gen.Return(model) - - @gen.coroutine - def list_kernels(self, **kwargs): - """Get a list of kernels.""" - kernel_url = self._get_kernel_endpoint_url() - self.log.debug("Request list kernels: %s", kernel_url) - response = yield gateway_request(kernel_url, method='GET') - kernels = json_decode(response.body) - self._kernels = {x['id']:x for x in kernels} - raise gen.Return(kernels) - - @gen.coroutine - def shutdown_kernel(self, kernel_id, now=False, restart=False): - """Shutdown a kernel by its kernel uuid. - - Parameters - ========== - kernel_id : uuid - The id of the kernel to shutdown. - """ - kernel_url = self._get_kernel_endpoint_url(kernel_id) - self.log.debug("Request shutdown kernel at: %s", kernel_url) - response = yield gateway_request(kernel_url, method='DELETE') - self.log.debug("Shutdown kernel response: %d %s", response.code, response.reason) - self.remove_kernel(kernel_id) - - @gen.coroutine - def restart_kernel(self, kernel_id, now=False, **kwargs): - """Restart a kernel by its kernel uuid. - - Parameters - ========== - kernel_id : uuid - The id of the kernel to restart. - """ - kernel_url = self._get_kernel_endpoint_url(kernel_id) + '/restart' - self.log.debug("Request restart kernel at: %s", kernel_url) - response = yield gateway_request(kernel_url, method='POST', body=json_encode({})) - self.log.debug("Restart kernel response: %d %s", response.code, response.reason) - - @gen.coroutine - def interrupt_kernel(self, kernel_id, **kwargs): - """Interrupt a kernel by its kernel uuid. - - Parameters - ========== - kernel_id : uuid - The id of the kernel to interrupt. - """ - kernel_url = self._get_kernel_endpoint_url(kernel_id) + '/interrupt' - self.log.debug("Request interrupt kernel at: %s", kernel_url) - response = yield gateway_request(kernel_url, method='POST', body=json_encode({})) - self.log.debug("Interrupt kernel response: %d %s", response.code, response.reason) - - def shutdown_all(self, now=False): - """Shutdown all kernels.""" - # Note: We have to make this sync because the NotebookApp does not wait for async. - shutdown_kernels = [] - kwargs = {'method': 'DELETE'} - kwargs = GatewayClient.instance().load_connection_args(**kwargs) - client = HTTPClient() - for kernel_id in self._kernels.keys(): - kernel_url = self._get_kernel_endpoint_url(kernel_id) - self.log.debug("Request delete kernel at: %s", kernel_url) - try: - response = client.fetch(kernel_url, **kwargs) - except HTTPError: - pass - else: - self.log.debug("Delete kernel response: %d %s", response.code, response.reason) - shutdown_kernels.append(kernel_id) # avoid changing dict size during iteration - client.close() - for kernel_id in shutdown_kernels: - self.remove_kernel(kernel_id) - - -class GatewayKernelSpecManager(KernelSpecManager): - - def __init__(self, **kwargs): - super(GatewayKernelSpecManager, self).__init__(**kwargs) - self.base_endpoint = url_path_join(GatewayClient.instance().url, - GatewayClient.instance().kernelspecs_endpoint) - self.base_resource_endpoint = url_path_join(GatewayClient.instance().url, - GatewayClient.instance().kernelspecs_resource_endpoint) - - def _get_kernelspecs_endpoint_url(self, kernel_name=None): - """Builds a url for the kernels endpoint - - Parameters - ---------- - kernel_name: kernel name (optional) - """ - if kernel_name: - return url_path_join(self.base_endpoint, url_escape(kernel_name)) - - return self.base_endpoint - - @gen.coroutine - def get_all_specs(self): - fetched_kspecs = yield self.list_kernel_specs() - - # get the default kernel name and compare to that of this server. - # If different log a warning and reset the default. However, the - # caller of this method will still return this server's value until - # the next fetch of kernelspecs - at which time they'll match. - km = self.parent.kernel_manager - remote_default_kernel_name = fetched_kspecs.get('default') - if remote_default_kernel_name != km.default_kernel_name: - self.log.info("Default kernel name on Gateway server ({gateway_default}) differs from " - "Notebook server ({notebook_default}). Updating to Gateway server's value.". - format(gateway_default=remote_default_kernel_name, - notebook_default=km.default_kernel_name)) - km.default_kernel_name = remote_default_kernel_name - - remote_kspecs = fetched_kspecs.get('kernelspecs') - raise gen.Return(remote_kspecs) - - @gen.coroutine - def list_kernel_specs(self): - """Get a list of kernel specs.""" - kernel_spec_url = self._get_kernelspecs_endpoint_url() - self.log.debug("Request list kernel specs at: %s", kernel_spec_url) - response = yield gateway_request(kernel_spec_url, method='GET') - kernel_specs = json_decode(response.body) - raise gen.Return(kernel_specs) - - @gen.coroutine - def get_kernel_spec(self, kernel_name, **kwargs): - """Get kernel spec for kernel_name. - - Parameters - ---------- - kernel_name : str - The name of the kernel. - """ - kernel_spec_url = self._get_kernelspecs_endpoint_url(kernel_name=str(kernel_name)) - self.log.debug("Request kernel spec at: %s" % kernel_spec_url) - try: - response = yield gateway_request(kernel_spec_url, method='GET') - except HTTPError as error: - if error.code == 404: - # Convert not found to KeyError since that's what the Notebook handler expects - # message is not used, but might as well make it useful for troubleshooting - raise KeyError('kernelspec {kernel_name} not found on Gateway server at: {gateway_url}'. - format(kernel_name=kernel_name, gateway_url=GatewayClient.instance().url)) - else: - raise - else: - kernel_spec = json_decode(response.body) - - raise gen.Return(kernel_spec) - - @gen.coroutine - def get_kernel_spec_resource(self, kernel_name, path): - """Get kernel spec for kernel_name. - - Parameters - ---------- - kernel_name : str - The name of the kernel. - path : str - The name of the desired resource - """ - kernel_spec_resource_url = url_path_join(self.base_resource_endpoint, str(kernel_name), str(path)) - self.log.debug("Request kernel spec resource '{}' at: {}".format(path, kernel_spec_resource_url)) - try: - response = yield gateway_request(kernel_spec_resource_url, method='GET') - except HTTPError as error: - if error.code == 404: - kernel_spec_resource = None - else: - raise - else: - kernel_spec_resource = response.body - raise gen.Return(kernel_spec_resource) - - -class GatewaySessionManager(SessionManager): - kernel_manager = Instance('notebook.gateway.managers.GatewayKernelManager') - - @gen.coroutine - def kernel_culled(self, kernel_id): - """Checks if the kernel is still considered alive and returns true if its not found. """ - kernel = yield self.kernel_manager.get_kernel(kernel_id) - raise gen.Return(kernel is None) diff --git a/notebook/kernelspecs/__init__.py b/notebook/kernelspecs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/kernelspecs/handlers.py b/notebook/kernelspecs/handlers.py deleted file mode 100644 index 9ec642a82b..0000000000 --- a/notebook/kernelspecs/handlers.py +++ /dev/null @@ -1,27 +0,0 @@ -from tornado import web -from ..base.handlers import IPythonHandler -from ..services.kernelspecs.handlers import kernel_name_regex - -class KernelSpecResourceHandler(web.StaticFileHandler, IPythonHandler): - SUPPORTED_METHODS = ('GET', 'HEAD') - - def initialize(self): - web.StaticFileHandler.initialize(self, path='') - - @web.authenticated - def get(self, kernel_name, path, include_body=True): - ksm = self.kernel_spec_manager - try: - self.root = ksm.get_kernel_spec(kernel_name).resource_dir - except KeyError: - raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) - self.log.debug("Serving kernel resource from: %s", self.root) - return web.StaticFileHandler.get(self, path, include_body=include_body) - - @web.authenticated - def head(self, kernel_name, path): - return self.get(kernel_name, path, include_body=False) - -default_handlers = [ - (r"/kernelspecs/%s/(?P.*)" % kernel_name_regex, KernelSpecResourceHandler), -] \ No newline at end of file diff --git a/notebook/nbconvert/__init__.py b/notebook/nbconvert/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/nbconvert/handlers.py b/notebook/nbconvert/handlers.py deleted file mode 100644 index bf0a4bfba8..0000000000 --- a/notebook/nbconvert/handlers.py +++ /dev/null @@ -1,200 +0,0 @@ -"""Tornado handlers for nbconvert.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import io -import os -import zipfile - -from tornado import web, escape -from tornado.log import app_log - -from ..base.handlers import ( - IPythonHandler, FilesRedirectHandler, - path_regex, -) -from nbformat import from_dict - -from ipython_genutils.py3compat import cast_bytes -from ipython_genutils import text - -def find_resource_files(output_files_dir): - files = [] - for dirpath, dirnames, filenames in os.walk(output_files_dir): - files.extend([os.path.join(dirpath, f) for f in filenames]) - return files - -def respond_zip(handler, name, output, resources): - """Zip up the output and resource files and respond with the zip file. - - Returns True if it has served a zip file, False if there are no resource - files, in which case we serve the plain output file. - """ - # Check if we have resource files we need to zip - output_files = resources.get('outputs', None) - if not output_files: - return False - - # Headers - zip_filename = os.path.splitext(name)[0] + '.zip' - handler.set_attachment_header(zip_filename) - handler.set_header('Content-Type', 'application/zip') - handler.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - - # Prepare the zip file - buffer = io.BytesIO() - zipf = zipfile.ZipFile(buffer, mode='w', compression=zipfile.ZIP_DEFLATED) - output_filename = os.path.splitext(name)[0] + resources['output_extension'] - zipf.writestr(output_filename, cast_bytes(output, 'utf-8')) - for filename, data in output_files.items(): - zipf.writestr(os.path.basename(filename), data) - zipf.close() - - handler.finish(buffer.getvalue()) - return True - -def get_exporter(format, **kwargs): - """get an exporter, raising appropriate errors""" - # if this fails, will raise 500 - try: - from nbconvert.exporters.base import get_exporter - except ImportError as e: - raise web.HTTPError(500, "Could not import nbconvert: %s" % e) - - try: - Exporter = get_exporter(format) - except KeyError: - # should this be 400? - raise web.HTTPError(404, u"No exporter for format: %s" % format) - - try: - return Exporter(**kwargs) - except Exception as e: - app_log.exception("Could not construct Exporter: %s", Exporter) - raise web.HTTPError(500, "Could not construct Exporter: %s" % e) - -class NbconvertFileHandler(IPythonHandler): - - SUPPORTED_METHODS = ('GET',) - - @property - def content_security_policy(self): - # In case we're serving HTML/SVG, confine any Javascript to a unique - # origin so it can't interact with the notebook server. - return super(NbconvertFileHandler, self).content_security_policy + \ - "; sandbox allow-scripts" - - @web.authenticated - def get(self, format, path): - - exporter = get_exporter(format, config=self.config, log=self.log) - - path = path.strip('/') - # If the notebook relates to a real file (default contents manager), - # give its path to nbconvert. - if hasattr(self.contents_manager, '_get_os_path'): - os_path = self.contents_manager._get_os_path(path) - ext_resources_dir, basename = os.path.split(os_path) - else: - ext_resources_dir = None - - model = self.contents_manager.get(path=path) - name = model['name'] - if model['type'] != 'notebook': - # not a notebook, redirect to files - return FilesRedirectHandler.redirect_to_files(self, path) - - nb = model['content'] - - self.set_header('Last-Modified', model['last_modified']) - - # create resources dictionary - mod_date = model['last_modified'].strftime(text.date_format) - nb_title = os.path.splitext(name)[0] - - resource_dict = { - "metadata": { - "name": nb_title, - "modified_date": mod_date - }, - "config_dir": self.application.settings['config_dir'] - } - - if ext_resources_dir: - resource_dict['metadata']['path'] = ext_resources_dir - - try: - output, resources = exporter.from_notebook_node( - nb, - resources=resource_dict - ) - except Exception as e: - self.log.exception("nbconvert failed: %s", e) - raise web.HTTPError(500, "nbconvert failed: %s" % e) - - if respond_zip(self, name, output, resources): - return - - # Force download if requested - if self.get_argument('download', 'false').lower() == 'true': - filename = os.path.splitext(name)[0] + resources['output_extension'] - self.set_attachment_header(filename) - - # MIME type - if exporter.output_mimetype: - self.set_header('Content-Type', - '%s; charset=utf-8' % exporter.output_mimetype) - - self.set_header('Cache-Control', 'no-store, no-cache, must-revalidate, max-age=0') - self.finish(output) - -class NbconvertPostHandler(IPythonHandler): - SUPPORTED_METHODS = ('POST',) - - @property - def content_security_policy(self): - # In case we're serving HTML/SVG, confine any Javascript to a unique - # origin so it can't interact with the notebook server. - return super(NbconvertPostHandler, self).content_security_policy + \ - "; sandbox allow-scripts" - - @web.authenticated - def post(self, format): - exporter = get_exporter(format, config=self.config) - - model = self.get_json_body() - name = model.get('name', 'notebook.ipynb') - nbnode = from_dict(model['content']) - - try: - output, resources = exporter.from_notebook_node(nbnode, resources={ - "metadata": {"name": name[:name.rfind('.')],}, - "config_dir": self.application.settings['config_dir'], - }) - except Exception as e: - raise web.HTTPError(500, "nbconvert failed: %s" % e) - - if respond_zip(self, name, output, resources): - return - - # MIME type - if exporter.output_mimetype: - self.set_header('Content-Type', - '%s; charset=utf-8' % exporter.output_mimetype) - - self.finish(output) - - -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - -_format_regex = r"(?P\w+)" - - -default_handlers = [ - (r"/nbconvert/%s" % _format_regex, NbconvertPostHandler), - (r"/nbconvert/%s%s" % (_format_regex, path_regex), - NbconvertFileHandler), -] diff --git a/notebook/nbconvert/tests/__init__.py b/notebook/nbconvert/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/nbconvert/tests/test_nbconvert_handlers.py b/notebook/nbconvert/tests/test_nbconvert_handlers.py deleted file mode 100644 index ebcfb4e9b8..0000000000 --- a/notebook/nbconvert/tests/test_nbconvert_handlers.py +++ /dev/null @@ -1,139 +0,0 @@ -# coding: utf-8 -import io -import json -import os -from os.path import join as pjoin -import shutil - -import requests - -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase, assert_http_error -from nbformat import write -from nbformat.v4 import ( - new_notebook, new_markdown_cell, new_code_cell, new_output, -) - -from ipython_genutils.testing.decorators import onlyif_cmds_exist - -try: #PY3 - from base64 import encodebytes -except ImportError: #PY2 - from base64 import encodestring as encodebytes - - - - -class NbconvertAPI(object): - """Wrapper for nbconvert API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, path, body=None, params=None): - response = self.request(verb, - url_path_join('nbconvert', path), - data=body, params=params, - ) - response.raise_for_status() - return response - - def from_file(self, format, path, name, download=False): - return self._req('GET', url_path_join(format, path, name), - params={'download':download}) - - def from_post(self, format, nbmodel): - body = json.dumps(nbmodel) - return self._req('POST', format, body) - - def list_formats(self): - return self._req('GET', '') - -png_green_pixel = encodebytes(b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00' -b'\x00\x00\x01\x00\x00x00\x01\x08\x02\x00\x00\x00\x90wS\xde\x00\x00\x00\x0cIDAT' -b'\x08\xd7c\x90\xfb\xcf\x00\x00\x02\\\x01\x1e.~d\x87\x00\x00\x00\x00IEND\xaeB`\x82' -).decode('ascii') - -class APITest(NotebookTestBase): - def setUp(self): - nbdir = self.notebook_dir - - if not os.path.isdir(pjoin(nbdir, 'foo')): - subdir = pjoin(nbdir, 'foo') - - os.mkdir(subdir) - - # Make sure that we clean this up when we're done. - # By using addCleanup this will happen correctly even if we fail - # later in setUp. - @self.addCleanup - def cleanup_dir(): - shutil.rmtree(subdir, ignore_errors=True) - - nb = new_notebook() - - nb.cells.append(new_markdown_cell(u'Created by test ³')) - cc1 = new_code_cell(source=u'print(2*6)') - cc1.outputs.append(new_output(output_type="stream", text=u'12')) - cc1.outputs.append(new_output(output_type="execute_result", - data={'image/png' : png_green_pixel}, - execution_count=1, - )) - nb.cells.append(cc1) - - with io.open(pjoin(nbdir, 'foo', 'testnb.ipynb'), 'w', - encoding='utf-8') as f: - write(nb, f, version=4) - - self.nbconvert_api = NbconvertAPI(self.request) - - @onlyif_cmds_exist('pandoc') - def test_from_file(self): - r = self.nbconvert_api.from_file('html', 'foo', 'testnb.ipynb') - self.assertEqual(r.status_code, 200) - self.assertIn(u'text/html', r.headers['Content-Type']) - self.assertIn(u'Created by test', r.text) - self.assertIn(u'print', r.text) - - r = self.nbconvert_api.from_file('python', 'foo', 'testnb.ipynb') - self.assertIn(u'text/x-python', r.headers['Content-Type']) - self.assertIn(u'print(2*6)', r.text) - - @onlyif_cmds_exist('pandoc') - def test_from_file_404(self): - with assert_http_error(404): - self.nbconvert_api.from_file('html', 'foo', 'thisdoesntexist.ipynb') - - @onlyif_cmds_exist('pandoc') - def test_from_file_download(self): - r = self.nbconvert_api.from_file('python', 'foo', 'testnb.ipynb', download=True) - content_disposition = r.headers['Content-Disposition'] - self.assertIn('attachment', content_disposition) - self.assertIn('testnb.py', content_disposition) - - @onlyif_cmds_exist('pandoc') - def test_from_file_zip(self): - r = self.nbconvert_api.from_file('latex', 'foo', 'testnb.ipynb', download=True) - self.assertIn(u'application/zip', r.headers['Content-Type']) - self.assertIn(u'.zip', r.headers['Content-Disposition']) - - @onlyif_cmds_exist('pandoc') - def test_from_post(self): - nbmodel = self.request('GET', 'api/contents/foo/testnb.ipynb').json() - - r = self.nbconvert_api.from_post(format='html', nbmodel=nbmodel) - self.assertEqual(r.status_code, 200) - self.assertIn(u'text/html', r.headers['Content-Type']) - self.assertIn(u'Created by test', r.text) - self.assertIn(u'print', r.text) - - r = self.nbconvert_api.from_post(format='python', nbmodel=nbmodel) - self.assertIn(u'text/x-python', r.headers['Content-Type']) - self.assertIn(u'print(2*6)', r.text) - - @onlyif_cmds_exist('pandoc') - def test_from_post_zip(self): - nbmodel = self.request('GET', 'api/contents/foo/testnb.ipynb').json() - - r = self.nbconvert_api.from_post(format='latex', nbmodel=nbmodel) - self.assertIn(u'application/zip', r.headers['Content-Type']) - self.assertIn(u'.zip', r.headers['Content-Disposition']) diff --git a/notebook/notebook/handlers.py b/notebook/notebook/handlers.py index 14f927bef0..9b107b38e6 100644 --- a/notebook/notebook/handlers.py +++ b/notebook/notebook/handlers.py @@ -8,11 +8,13 @@ from tornado import web HTTPError = web.HTTPError -from ..base.handlers import ( - IPythonHandler, FilesRedirectHandler, path_regex, +from jupyter_server.base.handlers import ( + FilesRedirectHandler, path_regex, ) -from ..utils import url_escape -from ..transutils import _ +from jupyter_server.utils import url_escape +from jupyter_server.transutils import _ + +from ..base.handlers import BaseHandler def get_frontend_exporters(): @@ -65,7 +67,7 @@ def get_frontend_exporters(): return sorted(frontend_exporters) -class NotebookHandler(IPythonHandler): +class NotebookHandler(BaseHandler): @web.authenticated def get(self, path): diff --git a/notebook/notebookapp.py b/notebook/notebookapp.py index a1a2108267..d3d68a6b5d 100755 --- a/notebook/notebookapp.py +++ b/notebook/notebookapp.py @@ -7,110 +7,53 @@ from __future__ import absolute_import, print_function import notebook -import binascii -import datetime -import errno import gettext -import hashlib -import hmac -import importlib import io -import ipaddress import json -import logging import mimetypes import os import random import re import select -import signal -import socket import sys -import tempfile -import threading import time -import warnings -import webbrowser - -try: #PY3 - from base64 import encodebytes -except ImportError: #PY2 - from base64 import encodestring as encodebytes - - -from jinja2 import Environment, FileSystemLoader from notebook.transutils import trans, _ -# Install the pyzmq ioloop. This has to be done before anything else from -# tornado is imported. -from zmq.eventloop import ioloop -ioloop.install() - # check for tornado 3.1.0 -try: - import tornado -except ImportError: - raise ImportError(_("The Jupyter Notebook requires tornado >= 4.0")) -try: - version_info = tornado.version_info -except AttributeError: - raise ImportError(_("The Jupyter Notebook requires tornado >= 4.0, but you have < 1.1.0")) -if version_info < (4,0): - raise ImportError(_("The Jupyter Notebook requires tornado >= 4.0, but you have %s") % tornado.version) - -from tornado import httpserver -from tornado import web -from tornado.httputil import url_concat -from tornado.log import LogFormatter, app_log, access_log, gen_log - from notebook import ( DEFAULT_STATIC_FILES_PATH, DEFAULT_TEMPLATE_PATH_LIST, __version__, ) -# py23 compatibility -try: - raw_input = raw_input -except NameError: - raw_input = input - -from .base.handlers import Template404, RedirectWithParams -from .log import log_request -from .services.kernels.kernelmanager import MappingKernelManager -from .services.config import ConfigManager -from .services.contents.manager import ContentsManager -from .services.contents.filemanager import FileContentsManager -from .services.contents.largefilemanager import LargeFileManager -from .services.sessions.sessionmanager import SessionManager -from .gateway.managers import GatewayKernelManager, GatewayKernelSpecManager, GatewaySessionManager, GatewayClient - -from .auth.login import LoginHandler -from .auth.logout import LogoutHandler -from .base.handlers import FileFindHandler - -from traitlets.config import Config -from traitlets.config.application import catch_config_error, boolean_flag +from jinja2 import Environment, FileSystemLoader + + +from jupyter_server.services.kernels.kernelmanager import MappingKernelManager +from jupyter_server.services.contents.manager import ContentsManager +from jupyter_server.services.contents.filemanager import FileContentsManager + +from traitlets.config.application import boolean_flag from jupyter_core.application import ( - JupyterApp, base_flags, base_aliases, + base_flags, base_aliases, ) -from jupyter_core.paths import jupyter_config_path + +from jupyter_server.base.handlers import FileFindHandler + from jupyter_client import KernelManager -from jupyter_client.kernelspec import KernelSpecManager -from jupyter_client.session import Session -from nbformat.sign import NotebookNotary from traitlets import ( Any, Dict, Unicode, Integer, List, Bool, Bytes, Instance, TraitError, Type, Float, observe, default, validate ) from ipython_genutils import py3compat from jupyter_core.paths import jupyter_runtime_dir, jupyter_path -from notebook._sysinfo import get_sys_info -from ._tz import utcnow, utcfromtimestamp -from .utils import url_path_join, check_pid, url_escape, urljoin, pathname2url +from jupyter_server.utils import url_path_join +# Try to load Notebook as an extension of the Jupyter Server +from jupyter_server.extension.application import ExtensionApp + #----------------------------------------------------------------------------- # Module globals #----------------------------------------------------------------------------- @@ -141,377 +84,6 @@ def load_handlers(name): mod = __import__(name, fromlist=['default_handlers']) return mod.default_handlers -#----------------------------------------------------------------------------- -# The Tornado web application -#----------------------------------------------------------------------------- - - -class NotebookWebApplication(web.Application): - - def __init__(self, jupyter_app, kernel_manager, contents_manager, - session_manager, kernel_spec_manager, - config_manager, extra_services, log, - base_url, default_url, settings_overrides, jinja_env_options): - - settings = self.init_settings( - jupyter_app, kernel_manager, contents_manager, - session_manager, kernel_spec_manager, config_manager, - extra_services, log, base_url, - default_url, settings_overrides, jinja_env_options) - handlers = self.init_handlers(settings) - - super(NotebookWebApplication, self).__init__(handlers, **settings) - - def init_settings(self, jupyter_app, kernel_manager, contents_manager, - session_manager, kernel_spec_manager, - config_manager, extra_services, - log, base_url, default_url, settings_overrides, - jinja_env_options=None): - - _template_path = settings_overrides.get( - "template_path", - jupyter_app.template_file_path, - ) - if isinstance(_template_path, py3compat.string_types): - _template_path = (_template_path,) - template_path = [os.path.expanduser(path) for path in _template_path] - - jenv_opt = {"autoescape": True} - jenv_opt.update(jinja_env_options if jinja_env_options else {}) - - env = Environment(loader=FileSystemLoader(template_path), extensions=['jinja2.ext.i18n'], **jenv_opt) - sys_info = get_sys_info() - - # If the user is running the notebook in a git directory, make the assumption - # that this is a dev install and suggest to the developer `npm run build:watch`. - base_dir = os.path.realpath(os.path.join(__file__, '..', '..')) - dev_mode = os.path.exists(os.path.join(base_dir, '.git')) - - nbui = gettext.translation('nbui', localedir=os.path.join(base_dir, 'notebook/i18n'), fallback=True) - env.install_gettext_translations(nbui, newstyle=False) - - if dev_mode: - DEV_NOTE_NPM = """It looks like you're running the notebook from source. - If you're working on the Javascript of the notebook, try running - - %s - - in another terminal window to have the system incrementally - watch and build the notebook's JavaScript for you, as you make changes.""" % 'npm run build:watch' - log.info(DEV_NOTE_NPM) - - if sys_info['commit_source'] == 'repository': - # don't cache (rely on 304) when working from master - version_hash = '' - else: - # reset the cache on server restart - version_hash = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - - if jupyter_app.ignore_minified_js: - log.warning(_("""The `ignore_minified_js` flag is deprecated and no longer works.""")) - log.warning(_("""Alternatively use `%s` when working on the notebook's Javascript and LESS""") % 'npm run build:watch') - warnings.warn(_("The `ignore_minified_js` flag is deprecated and will be removed in Notebook 6.0"), DeprecationWarning) - - now = utcnow() - - root_dir = contents_manager.root_dir - home = py3compat.str_to_unicode(os.path.expanduser('~'), encoding=sys.getfilesystemencoding()) - if root_dir.startswith(home + os.path.sep): - # collapse $HOME to ~ - root_dir = '~' + root_dir[len(home):] - - settings = dict( - # basics - log_function=log_request, - base_url=base_url, - default_url=default_url, - template_path=template_path, - static_path=jupyter_app.static_file_path, - static_custom_path=jupyter_app.static_custom_path, - static_handler_class = FileFindHandler, - static_url_prefix = url_path_join(base_url,'/static/'), - static_handler_args = { - # don't cache custom.js - 'no_cache_paths': [url_path_join(base_url, 'static', 'custom')], - }, - version_hash=version_hash, - ignore_minified_js=jupyter_app.ignore_minified_js, - - # rate limits - iopub_msg_rate_limit=jupyter_app.iopub_msg_rate_limit, - iopub_data_rate_limit=jupyter_app.iopub_data_rate_limit, - rate_limit_window=jupyter_app.rate_limit_window, - - # authentication - cookie_secret=jupyter_app.cookie_secret, - login_url=url_path_join(base_url,'/login'), - login_handler_class=jupyter_app.login_handler_class, - logout_handler_class=jupyter_app.logout_handler_class, - password=jupyter_app.password, - xsrf_cookies=True, - disable_check_xsrf=jupyter_app.disable_check_xsrf, - allow_remote_access=jupyter_app.allow_remote_access, - local_hostnames=jupyter_app.local_hostnames, - - # managers - kernel_manager=kernel_manager, - contents_manager=contents_manager, - session_manager=session_manager, - kernel_spec_manager=kernel_spec_manager, - config_manager=config_manager, - - # handlers - extra_services=extra_services, - - # Jupyter stuff - started=now, - # place for extensions to register activity - # so that they can prevent idle-shutdown - last_activity_times={}, - jinja_template_vars=jupyter_app.jinja_template_vars, - nbextensions_path=jupyter_app.nbextensions_path, - websocket_url=jupyter_app.websocket_url, - mathjax_url=jupyter_app.mathjax_url, - mathjax_config=jupyter_app.mathjax_config, - shutdown_button=jupyter_app.quit_button, - config=jupyter_app.config, - config_dir=jupyter_app.config_dir, - allow_password_change=jupyter_app.allow_password_change, - server_root_dir=root_dir, - jinja2_env=env, - terminals_available=False, # Set later if terminals are available - ) - - # allow custom overrides for the tornado web app. - settings.update(settings_overrides) - return settings - - def init_handlers(self, settings): - """Load the (URL pattern, handler) tuples for each component.""" - - # Order matters. The first handler to match the URL will handle the request. - handlers = [] - # load extra services specified by users before default handlers - for service in settings['extra_services']: - handlers.extend(load_handlers(service)) - handlers.extend(load_handlers('notebook.tree.handlers')) - handlers.extend([(r"/login", settings['login_handler_class'])]) - handlers.extend([(r"/logout", settings['logout_handler_class'])]) - handlers.extend(load_handlers('notebook.files.handlers')) - handlers.extend(load_handlers('notebook.view.handlers')) - handlers.extend(load_handlers('notebook.notebook.handlers')) - handlers.extend(load_handlers('notebook.nbconvert.handlers')) - handlers.extend(load_handlers('notebook.bundler.handlers')) - handlers.extend(load_handlers('notebook.kernelspecs.handlers')) - handlers.extend(load_handlers('notebook.edit.handlers')) - handlers.extend(load_handlers('notebook.services.api.handlers')) - handlers.extend(load_handlers('notebook.services.config.handlers')) - handlers.extend(load_handlers('notebook.services.contents.handlers')) - handlers.extend(load_handlers('notebook.services.sessions.handlers')) - handlers.extend(load_handlers('notebook.services.nbconvert.handlers')) - handlers.extend(load_handlers('notebook.services.security.handlers')) - handlers.extend(load_handlers('notebook.services.shutdown')) - handlers.extend(load_handlers('notebook.services.kernels.handlers')) - handlers.extend(load_handlers('notebook.services.kernelspecs.handlers')) - - handlers.extend(settings['contents_manager'].get_extra_handlers()) - - # If gateway mode is enabled, replace appropriate handlers to perform redirection - if GatewayClient.instance().gateway_enabled: - # for each handler required for gateway, locate its pattern - # in the current list and replace that entry... - gateway_handlers = load_handlers('notebook.gateway.handlers') - for i, gwh in enumerate(gateway_handlers): - for j, h in enumerate(handlers): - if gwh[0] == h[0]: - handlers[j] = (gwh[0], gwh[1]) - break - - handlers.append( - (r"/nbextensions/(.*)", FileFindHandler, { - 'path': settings['nbextensions_path'], - 'no_cache_paths': ['/'], # don't cache anything in nbextensions - }), - ) - handlers.append( - (r"/custom/(.*)", FileFindHandler, { - 'path': settings['static_custom_path'], - 'no_cache_paths': ['/'], # don't cache anything in custom - }) - ) - # register base handlers last - handlers.extend(load_handlers('notebook.base.handlers')) - # set the URL that will be redirected from `/` - handlers.append( - (r'/?', RedirectWithParams, { - 'url' : settings['default_url'], - 'permanent': False, # want 302, not 301 - }) - ) - - # prepend base_url onto the patterns that we match - new_handlers = [] - for handler in handlers: - pattern = url_path_join(settings['base_url'], handler[0]) - new_handler = tuple([pattern] + list(handler[1:])) - new_handlers.append(new_handler) - # add 404 on the end, which will catch everything that falls through - new_handlers.append((r'(.*)', Template404)) - return new_handlers - - def last_activity(self): - """Get a UTC timestamp for when the server last did something. - - Includes: API activity, kernel activity, kernel shutdown, and terminal - activity. - """ - sources = [ - self.settings['started'], - self.settings['kernel_manager'].last_kernel_activity, - ] - try: - sources.append(self.settings['api_last_activity']) - except KeyError: - pass - try: - sources.append(self.settings['terminal_last_activity']) - except KeyError: - pass - sources.extend(self.settings['last_activity_times'].values()) - return max(sources) - - -class NotebookPasswordApp(JupyterApp): - """Set a password for the notebook server. - - Setting a password secures the notebook server - and removes the need for token-based authentication. - """ - - description = __doc__ - - def _config_file_default(self): - return os.path.join(self.config_dir, 'jupyter_notebook_config.json') - - def start(self): - from .auth.security import set_password - set_password(config_file=self.config_file) - self.log.info("Wrote hashed password to %s" % self.config_file) - -def shutdown_server(server_info, timeout=5, log=None): - """Shutdown a notebook server in a separate process. - - *server_info* should be a dictionary as produced by list_running_servers(). - - Will first try to request shutdown using /api/shutdown . - On Unix, if the server is still running after *timeout* seconds, it will - send SIGTERM. After another timeout, it escalates to SIGKILL. - - Returns True if the server was stopped by any means, False if stopping it - failed (on Windows). - """ - from tornado.httpclient import HTTPClient, HTTPRequest - url = server_info['url'] - pid = server_info['pid'] - req = HTTPRequest(url + 'api/shutdown', method='POST', body=b'', headers={ - 'Authorization': 'token ' + server_info['token'] - }) - if log: log.debug("POST request to %sapi/shutdown", url) - HTTPClient().fetch(req) - - # Poll to see if it shut down. - for _ in range(timeout*10): - if check_pid(pid): - if log: log.debug("Server PID %s is gone", pid) - return True - time.sleep(0.1) - - if sys.platform.startswith('win'): - return False - - if log: log.debug("SIGTERM to PID %s", pid) - os.kill(pid, signal.SIGTERM) - - # Poll to see if it shut down. - for _ in range(timeout * 10): - if check_pid(pid): - if log: log.debug("Server PID %s is gone", pid) - return True - time.sleep(0.1) - - if log: log.debug("SIGKILL to PID %s", pid) - os.kill(pid, signal.SIGKILL) - return True # SIGKILL cannot be caught - - -class NbserverStopApp(JupyterApp): - version = __version__ - description="Stop currently running notebook server for a given port" - - port = Integer(8888, config=True, - help="Port of the server to be killed. Default 8888") - - def parse_command_line(self, argv=None): - super(NbserverStopApp, self).parse_command_line(argv) - if self.extra_args: - self.port=int(self.extra_args[0]) - - def shutdown_server(self, server): - return shutdown_server(server, log=self.log) - - def start(self): - servers = list(list_running_servers(self.runtime_dir)) - if not servers: - self.exit("There are no running servers") - for server in servers: - if server['port'] == self.port: - print("Shutting down server on port", self.port, "...") - if not self.shutdown_server(server): - sys.exit("Could not stop server") - return - else: - print("There is currently no server running on port {}".format(self.port), file=sys.stderr) - print("Ports currently in use:", file=sys.stderr) - for server in servers: - print(" - {}".format(server['port']), file=sys.stderr) - self.exit(1) - - -class NbserverListApp(JupyterApp): - version = __version__ - description=_("List currently running notebook servers.") - - flags = dict( - jsonlist=({'NbserverListApp': {'jsonlist': True}}, - _("Produce machine-readable JSON list output.")), - json=({'NbserverListApp': {'json': True}}, - _("Produce machine-readable JSON object on each line of output.")), - ) - - jsonlist = Bool(False, config=True, - help=_("If True, the output will be a JSON list of objects, one per " - "active notebook server, each with the details from the " - "relevant server info file.")) - json = Bool(False, config=True, - help=_("If True, each line of output will be a JSON object with the " - "details from the server info file. For a JSON list output, " - "see the NbserverListApp.jsonlist configuration value")) - - def start(self): - serverinfo_list = list(list_running_servers(self.runtime_dir)) - if self.jsonlist: - print(json.dumps(serverinfo_list, indent=2)) - elif self.json: - for serverinfo in serverinfo_list: - print(json.dumps(serverinfo)) - else: - print("Currently running servers:") - for serverinfo in serverinfo_list: - url = serverinfo['url'] - if serverinfo.get('token'): - url = url + '?token=%s' % serverinfo['token'] - print(url, "::", serverinfo['notebook_dir']) - #----------------------------------------------------------------------------- # Aliases and Flags #----------------------------------------------------------------------------- @@ -567,223 +139,21 @@ def start(self): # NotebookApp #----------------------------------------------------------------------------- -class NotebookApp(JupyterApp): - +class NotebookApp(ExtensionApp): + name = 'jupyter-notebook' + extension_name = 'notebook' + version = __version__ description = _("""The Jupyter HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client.""") - examples = _examples - aliases = aliases - flags = flags - - classes = [ - KernelManager, Session, MappingKernelManager, KernelSpecManager, - ContentsManager, FileContentsManager, NotebookNotary, - GatewayKernelManager, GatewayKernelSpecManager, GatewaySessionManager, GatewayClient, - ] - flags = Dict(flags) - aliases = Dict(aliases) - - subcommands = dict( - list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), - stop=(NbserverStopApp, NbserverStopApp.description.splitlines()[0]), - password=(NotebookPasswordApp, NotebookPasswordApp.description.splitlines()[0]), - ) - - _log_formatter_cls = LogFormatter - - @default('log_level') - def _default_log_level(self): - return logging.INFO - - @default('log_datefmt') - def _default_log_datefmt(self): - """Exclude date from default date format""" - return "%H:%M:%S" - - @default('log_format') - def _default_log_format(self): - """override default log format to include time""" - return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s" ignore_minified_js = Bool(False, config=True, help=_('Deprecated: Use minified JS file or not, mainly use during dev to avoid JS recompilation'), ) - # file to be opened in the notebook server - file_to_run = Unicode('', config=True) - - # Network related information - - allow_origin = Unicode('', config=True, - help="""Set the Access-Control-Allow-Origin header - - Use '*' to allow any origin to access your server. - - Takes precedence over allow_origin_pat. - """ - ) - - allow_origin_pat = Unicode('', config=True, - help="""Use a regular expression for the Access-Control-Allow-Origin header - - Requests from an origin matching the expression will get replies with: - - Access-Control-Allow-Origin: origin - - where `origin` is the origin of the request. - - Ignored if allow_origin is set. - """ - ) - - allow_credentials = Bool(False, config=True, - help=_("Set the Access-Control-Allow-Credentials: true header") - ) - - allow_root = Bool(False, config=True, - help=_("Whether to allow the user to run the notebook as root.") - ) - - default_url = Unicode('/tree', config=True, - help=_("The default URL to redirect to from `/`") - ) - - ip = Unicode('localhost', config=True, - help=_("The IP address the notebook server will listen on.") - ) - - @default('ip') - def _default_ip(self): - """Return localhost if available, 127.0.0.1 otherwise. - - On some (horribly broken) systems, localhost cannot be bound. - """ - s = socket.socket() - try: - s.bind(('localhost', 0)) - except socket.error as e: - self.log.warning(_("Cannot bind to localhost, using 127.0.0.1 as default ip\n%s"), e) - return '127.0.0.1' - else: - s.close() - return 'localhost' - - @validate('ip') - def _valdate_ip(self, proposal): - value = proposal['value'] - if value == u'*': - value = u'' - return value - - custom_display_url = Unicode(u'', config=True, - help=_("""Override URL shown to users. - - Replace actual URL, including protocol, address, port and base URL, - with the given value when displaying URL to the users. Do not change - the actual connection URL. If authentication token is enabled, the - token is added to the custom URL automatically. - - This option is intended to be used when the URL to display to the user - cannot be determined reliably by the Jupyter notebook server (proxified - or containerized setups for example).""") - ) - - port = Integer(8888, config=True, - help=_("The port the notebook server will listen on.") - ) - - port_retries = Integer(50, config=True, - help=_("The number of additional ports to try if the specified port is not available.") - ) - - certfile = Unicode(u'', config=True, - help=_("""The full path to an SSL/TLS certificate file.""") - ) - - keyfile = Unicode(u'', config=True, - help=_("""The full path to a private key file for usage with SSL/TLS.""") - ) - - client_ca = Unicode(u'', config=True, - help=_("""The full path to a certificate authority certificate for SSL/TLS client authentication.""") - ) - - cookie_secret_file = Unicode(config=True, - help=_("""The file where the cookie secret is stored.""") - ) - - @default('cookie_secret_file') - def _default_cookie_secret_file(self): - return os.path.join(self.runtime_dir, 'notebook_cookie_secret') - - cookie_secret = Bytes(b'', config=True, - help="""The random bytes used to secure cookies. - By default this is a new random number every time you start the Notebook. - Set it to a value in a config file to enable logins to persist across server sessions. - - Note: Cookie secrets should be kept private, do not share config files with - cookie_secret stored in plaintext (you can read the value from a file). - """ - ) - - @default('cookie_secret') - def _default_cookie_secret(self): - if os.path.exists(self.cookie_secret_file): - with io.open(self.cookie_secret_file, 'rb') as f: - key = f.read() - else: - key = encodebytes(os.urandom(32)) - self._write_cookie_secret_file(key) - h = hmac.new(key, digestmod=hashlib.sha256) - h.update(self.password.encode()) - return h.digest() - - def _write_cookie_secret_file(self, secret): - """write my secret to my secret_file""" - self.log.info(_("Writing notebook server cookie secret to %s"), self.cookie_secret_file) - try: - with io.open(self.cookie_secret_file, 'wb') as f: - f.write(secret) - except OSError as e: - self.log.error(_("Failed to write cookie secret to %s: %s"), - self.cookie_secret_file, e) - try: - os.chmod(self.cookie_secret_file, 0o600) - except OSError: - self.log.warning( - _("Could not set permissions on %s"), - self.cookie_secret_file - ) - - token = Unicode('', - help=_("""Token used for authenticating first-time connections to the server. - - When no password is enabled, - the default is to generate a new, random token. - - Setting to an empty string disables authentication altogether, which is NOT RECOMMENDED. - """) - ).tag(config=True) - - _token_generated = True - - @default('token') - def _token_default(self): - if os.getenv('JUPYTER_TOKEN'): - self._token_generated = False - return os.getenv('JUPYTER_TOKEN') - if self.password: - # no token if password is enabled - self._token_generated = False - return u'' - else: - self._token_generated = True - return binascii.hexlify(os.urandom(24)).decode('ascii') - max_body_size = Integer(512 * 1024 * 1024, config=True, help=""" Sets the maximum allowed size of the client request body, specified in @@ -802,193 +172,15 @@ def _token_default(self): """ ) - @observe('token') - def _token_changed(self, change): - self._token_generated = False - - password = Unicode(u'', config=True, - help="""Hashed password to use for web authentication. - - To generate, type in a python/IPython shell: - - from notebook.auth import passwd; passwd() - - The string should be of the form type:salt:hashed-password. - """ - ) - - password_required = Bool(False, config=True, - help="""Forces users to use a password for the Notebook server. - This is useful in a multi user environment, for instance when - everybody in the LAN can access each other's machine through ssh. - - In such a case, server the notebook server on localhost is not secure - since any user can connect to the notebook server via ssh. - - """ - ) - - allow_password_change = Bool(True, config=True, - help="""Allow password to be changed at login for the notebook server. - - While loggin in with a token, the notebook server UI will give the opportunity to - the user to enter a new password at the same time that will replace - the token login mechanism. - - This can be set to false to prevent changing password from the UI/API. - """ - ) - - - disable_check_xsrf = Bool(False, config=True, - help="""Disable cross-site-request-forgery protection - - Jupyter notebook 4.3.1 introduces protection from cross-site request forgeries, - requiring API requests to either: - - - originate from pages served by this server (validated with XSRF cookie and token), or - - authenticate with a token - - Some anonymous compute resources still desire the ability to run code, - completely without authentication. - These services can disable all authentication and security checks, - with the full knowledge of what that implies. - """ - ) - - allow_remote_access = Bool(config=True, - help="""Allow requests where the Host header doesn't point to a local server - - By default, requests get a 403 forbidden response if the 'Host' header - shows that the browser thinks it's on a non-local domain. - Setting this option to True disables this check. - - This protects against 'DNS rebinding' attacks, where a remote web server - serves you a page and then changes its DNS to send later requests to a - local IP, bypassing same-origin checks. - - Local IP addresses (such as 127.0.0.1 and ::1) are allowed as local, - along with hostnames configured in local_hostnames. - """) - - @default('allow_remote_access') - def _default_allow_remote(self): - """Disallow remote access if we're listening only on loopback addresses""" - - # if blank, self.ip was configured to "*" meaning bind to all interfaces, - # see _valdate_ip - if self.ip == "": - return True - - try: - addr = ipaddress.ip_address(self.ip) - except ValueError: - # Address is a hostname - for info in socket.getaddrinfo(self.ip, self.port, 0, socket.SOCK_STREAM): - addr = info[4][0] - if not py3compat.PY3: - addr = addr.decode('ascii') - - try: - parsed = ipaddress.ip_address(addr.split('%')[0]) - except ValueError: - self.log.warning("Unrecognised IP address: %r", addr) - continue - - # Macs map localhost to 'fe80::1%lo0', a link local address - # scoped to the loopback interface. For now, we'll assume that - # any scoped link-local address is effectively local. - if not (parsed.is_loopback - or (('%' in addr) and parsed.is_link_local)): - return True - return False - else: - return not addr.is_loopback - - local_hostnames = List(Unicode(), ['localhost'], config=True, - help="""Hostnames to allow as local when allow_remote_access is False. - - Local IP addresses (such as 127.0.0.1 and ::1) are automatically accepted - as local as well. - """ - ) - - open_browser = Bool(True, config=True, - help="""Whether to open in a browser after starting. - The specific browser used is platform dependent and - determined by the python standard library `webbrowser` - module, unless it is overridden using the --browser - (NotebookApp.browser) configuration option. - """) - - browser = Unicode(u'', config=True, - help="""Specify what command to use to invoke a web - browser when opening the notebook. If not specified, the - default browser will be determined by the `webbrowser` - standard library module, which allows setting of the - BROWSER environment variable to override it. - """) - - webbrowser_open_new = Integer(2, config=True, - help=_("""Specify Where to open the notebook on startup. This is the - `new` argument passed to the standard library method `webbrowser.open`. - The behaviour is not guaranteed, but depends on browser support. Valid - values are: - - - 2 opens a new tab, - - 1 opens a new window, - - 0 opens in an existing window. - - See the `webbrowser.open` documentation for details. - """)) - - webapp_settings = Dict(config=True, - help=_("DEPRECATED, use tornado_settings") - ) - - @observe('webapp_settings') - def _update_webapp_settings(self, change): - self.log.warning(_("\n webapp_settings is deprecated, use tornado_settings.\n")) - self.tornado_settings = change['new'] - - tornado_settings = Dict(config=True, - help=_("Supply overrides for the tornado.web.Application that the " - "Jupyter notebook uses.")) - - websocket_compression_options = Any(None, config=True, - help=_(""" - Set the tornado compression options for websocket connections. - - This value will be returned from :meth:`WebSocketHandler.get_compression_options`. - None (default) will disable compression. - A dict (even an empty one) will enable compression. - - See the tornado docs for WebSocketHandler.get_compression_options for details. - """) - ) - terminado_settings = Dict(config=True, - help=_('Supply overrides for terminado. Currently only supports "shell_command".')) - - cookie_options = Dict(config=True, - help=_("Extra keyword arguments to pass to `set_secure_cookie`." - " See tornado's set_secure_cookie docs for details.") - ) - get_secure_cookie_kwargs = Dict(config=True, - help=_("Extra keyword arguments to pass to `get_secure_cookie`." - " See tornado's get_secure_cookie docs for details.") - ) - ssl_options = Dict(config=True, - help=_("""Supply SSL options for the tornado HTTPServer. - See the tornado docs for details.""")) - jinja_environment_options = Dict(config=True, - help=_("Supply extra arguments that will be passed to Jinja environment.")) + help=_("Supply extra arguments that will be passed to Jinja environment.") + ) jinja_template_vars = Dict( config=True, help=_("Extra variables to supply to jinja templates when rendering."), ) - + enable_mathjax = Bool(True, config=True, help="""Whether to enable MathJax for typesetting math/TeX @@ -1006,29 +198,6 @@ def _update_enable_mathjax(self, change): if not change['new']: self.mathjax_url = u'' - base_url = Unicode('/', config=True, - help='''The base URL for the notebook server. - - Leading and trailing slashes can be omitted, - and will automatically be added. - ''') - - @validate('base_url') - def _update_base_url(self, proposal): - value = proposal['value'] - if not value.startswith('/'): - value = '/' + value - if not value.endswith('/'): - value = value + '/' - return value - - base_project_url = Unicode('/', config=True, help=_("""DEPRECATED use base_url""")) - - @observe('base_project_url') - def _update_base_project_url(self, change): - self.log.warning(_("base_project_url is deprecated, use base_url")) - self.base_url = change['new'] - extra_static_paths = List(Unicode(), config=True, help="""Extra paths to search for serving static files. @@ -1085,14 +254,6 @@ def nbextensions_path(self): path.append(os.path.join(get_ipython_dir(), 'nbextensions')) return path - websocket_url = Unicode("", config=True, - help="""The base URL for websockets, - if it differs from the HTTP server (hint: it almost certainly doesn't). - - Should be in the form of an HTTP origin: ws[s]://hostname[:port] - """ - ) - mathjax_url = Unicode("", config=True, help="""A custom url for MathJax.js. Should be in the form of a case-sensitive url to MathJax, @@ -1100,11 +261,20 @@ def nbextensions_path(self): """ ) + @property + def static_url_prefix(self): + """Get the static url prefix for serving static files.""" + return super(NotebookApp, self).static_url_prefix + try: + return super(NotebookApp, self).static_url_prefix + except AttributeError: + return self.tornado_settings.get("static_url_prefix", "static") + @default('mathjax_url') def _default_mathjax_url(self): if not self.enable_mathjax: return u'' - static_url_prefix = self.tornado_settings.get("static_url_prefix", "static") + static_url_prefix = self.static_url_prefix return url_path_join(static_url_prefix, 'components', 'MathJax', 'MathJax.js') @observe('mathjax_url') @@ -1123,71 +293,6 @@ def _update_mathjax_url(self, change): @observe('mathjax_config') def _update_mathjax_config(self, change): self.log.info(_("Using MathJax configuration file: %s"), change['new']) - - quit_button = Bool(True, config=True, - help="""If True, display a button in the dashboard to quit - (shutdown the notebook server).""" - ) - - contents_manager_class = Type( - default_value=LargeFileManager, - klass=ContentsManager, - config=True, - help=_('The notebook manager class to use.') - ) - - kernel_manager_class = Type( - default_value=MappingKernelManager, - config=True, - help=_('The kernel manager class to use.') - ) - - session_manager_class = Type( - default_value=SessionManager, - config=True, - help=_('The session manager class to use.') - ) - - config_manager_class = Type( - default_value=ConfigManager, - config = True, - help=_('The config manager class to use') - ) - - kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) - - kernel_spec_manager_class = Type( - default_value=KernelSpecManager, - config=True, - help=""" - The kernel spec manager class to use. Should be a subclass - of `jupyter_client.kernelspec.KernelSpecManager`. - - The Api of KernelSpecManager is provisional and might change - without warning between this version of Jupyter and the next stable one. - """ - ) - - login_handler_class = Type( - default_value=LoginHandler, - klass=web.RequestHandler, - config=True, - help=_('The login handler class to use.'), - ) - - logout_handler_class = Type( - default_value=LogoutHandler, - klass=web.RequestHandler, - config=True, - help=_('The logout handler class to use.'), - ) - - trust_xheaders = Bool(False, config=True, - help=(_("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" - "sent by the upstream reverse proxy. Necessary if the proxy handles SSL")) - ) - - info_file = Unicode() @default('info_file') def _default_info_file(self): @@ -1200,71 +305,42 @@ def _default_info_file(self): def _default_browser_open_file(self): basename = "nbserver-%s-open.html" % os.getpid() return os.path.join(self.runtime_dir, basename) - - pylab = Unicode('disabled', config=True, - help=_(""" - DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. - """) - ) - - @observe('pylab') - def _update_pylab(self, change): - """when --pylab is specified, display a warning and exit""" - if change['new'] != 'warn': - backend = ' %s' % change['new'] - else: - backend = '' - self.log.error(_("Support for specifying --pylab on the command line has been removed.")) - self.log.error( - _("Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.").format(backend) - ) - self.exit(1) - notebook_dir = Unicode(config=True, - help=_("The directory to use for notebooks and kernels.") - ) + # notebook_dir = Unicode(config=True, + # help=_("The directory to use for notebooks and kernels.") + # ) + + # @default('notebook_dir') + # def _default_notebook_dir(self): + # if self.file_to_run: + # return os.path.dirname(os.path.abspath(self.file_to_run)) + # else: + # return py3compat.getcwd() + + # @validate('notebook_dir') + # def _notebook_dir_validate(self, proposal): + # value = proposal['value'] + # # Strip any trailing slashes + # # *except* if it's root + # _, path = os.path.splitdrive(value) + # if path == os.sep: + # return value + # value = value.rstrip(os.sep) + # if not os.path.isabs(value): + # # If we receive a non-absolute path, make it absolute. + # value = os.path.abspath(value) + # if not os.path.isdir(value): + # raise TraitError(trans.gettext("No such notebook dir: '%r'") % value) + # return value + + # @observe('notebook_dir') + # def _update_notebook_dir(self, change): + # """Do a bit of validation of the notebook dir.""" + # # setting App.notebook_dir implies setting notebook and kernel dirs as well + # new = change['new'] + # self.config.FileContentsManager.root_dir = new + # self.config.MappingKernelManager.root_dir = new - @default('notebook_dir') - def _default_notebook_dir(self): - if self.file_to_run: - return os.path.dirname(os.path.abspath(self.file_to_run)) - else: - return py3compat.getcwd() - - @validate('notebook_dir') - def _notebook_dir_validate(self, proposal): - value = proposal['value'] - # Strip any trailing slashes - # *except* if it's root - _, path = os.path.splitdrive(value) - if path == os.sep: - return value - value = value.rstrip(os.sep) - if not os.path.isabs(value): - # If we receive a non-absolute path, make it absolute. - value = os.path.abspath(value) - if not os.path.isdir(value): - raise TraitError(trans.gettext("No such notebook dir: '%r'") % value) - return value - - @observe('notebook_dir') - def _update_notebook_dir(self, change): - """Do a bit of validation of the notebook dir.""" - # setting App.notebook_dir implies setting notebook and kernel dirs as well - new = change['new'] - self.config.FileContentsManager.root_dir = new - self.config.MappingKernelManager.root_dir = new - - # TODO: Remove me in notebook 5.0 - server_extensions = List(Unicode(), config=True, - help=(_("DEPRECATED use the nbserver_extensions dict instead")) - ) - - @observe('server_extensions') - def _update_server_extensions(self, change): - self.log.warning(_("server_extensions is deprecated, use nbserver_extensions")) - self.server_extensions = change['new'] - nbserver_extensions = Dict({}, config=True, help=(_("Dict of Python modules to load as notebook server extensions." "Entry values can be used to enable and disable the loading of" @@ -1272,632 +348,100 @@ def _update_server_extensions(self, change): "order.")) ) - reraise_server_extension_failures = Bool( - False, - config=True, - help=_("Reraise exceptions encountered loading server extensions?"), - ) + # ------------------------------------------------------------------------ + # traits and methods for Jupyter Server + # ------------------------------------------------------------------------ - iopub_msg_rate_limit = Float(1000, config=True, help=_("""(msgs/sec) - Maximum rate at which messages can be sent on iopub before they are - limited.""")) - - iopub_data_rate_limit = Float(1000000, config=True, help=_("""(bytes/sec) - Maximum rate at which stream output can be sent on iopub before they are - limited.""")) - - rate_limit_window = Float(3, config=True, help=_("""(sec) Time window used to - check the message and data rate limits.""")) - - shutdown_no_activity_timeout = Integer(0, config=True, - help=("Shut down the server after N seconds with no kernels or " - "terminals running and no activity. " - "This can be used together with culling idle kernels " - "(MappingKernelManager.cull_idle_timeout) to " - "shutdown the notebook server when it's not in use. This is not " - "precisely timed: it may shut down up to a minute later. " - "0 (the default) disables this automatic shutdown.") - ) + default_url = Unicode("/tree", config=True) - terminals_enabled = Bool(True, config=True, - help=_("""Set to False to disable terminals. - - This does *not* make the notebook server more secure by itself. - Anything the user can in a terminal, they can also do in a notebook. - - Terminals may also be automatically disabled if the terminado package - is not available. - """)) - - def parse_command_line(self, argv=None): - super(NotebookApp, self).parse_command_line(argv) - - if self.extra_args: - arg0 = self.extra_args[0] - f = os.path.abspath(arg0) - self.argv.remove(arg0) - if not os.path.exists(f): - self.log.critical(_("No such file or directory: %s"), f) - self.exit(1) - - # Use config here, to ensure that it takes higher priority than - # anything that comes from the config dirs. - c = Config() - if os.path.isdir(f): - c.NotebookApp.notebook_dir = f - elif os.path.isfile(f): - c.NotebookApp.file_to_run = f - self.update_config(c) - - def init_configurables(self): - - # If gateway server is configured, replace appropriate managers to perform redirection. To make - # this determination, instantiate the GatewayClient config singleton. - self.gateway_config = GatewayClient.instance(parent=self) - - if self.gateway_config.gateway_enabled: - self.kernel_manager_class = 'notebook.gateway.managers.GatewayKernelManager' - self.session_manager_class = 'notebook.gateway.managers.GatewaySessionManager' - self.kernel_spec_manager_class = 'notebook.gateway.managers.GatewayKernelSpecManager' - - self.kernel_spec_manager = self.kernel_spec_manager_class( - parent=self, - ) - self.kernel_manager = self.kernel_manager_class( - parent=self, - log=self.log, - connection_dir=self.runtime_dir, - kernel_spec_manager=self.kernel_spec_manager, - ) - self.contents_manager = self.contents_manager_class( - parent=self, - log=self.log, - ) - self.session_manager = self.session_manager_class( - parent=self, - log=self.log, - kernel_manager=self.kernel_manager, - contents_manager=self.contents_manager, - ) - self.config_manager = self.config_manager_class( - parent=self, - log=self.log, - ) - - def init_logging(self): - # This prevents double log messages because tornado use a root logger that - # self.log is a child of. The logging module dipatches log messages to a log - # and all of its ancenstors until propagate is set to False. - self.log.propagate = False - - for log in app_log, access_log, gen_log: - # consistent log output name (NotebookApp instead of tornado.access, etc.) - log.name = self.log.name - # hook up tornado 3's loggers to our app handlers - logger = logging.getLogger('tornado') - logger.propagate = True - logger.parent = self.log - logger.setLevel(self.log.level) - - def init_webapp(self): - """initialize tornado webapp and httpserver""" - self.tornado_settings['allow_origin'] = self.allow_origin - self.tornado_settings['websocket_compression_options'] = self.websocket_compression_options - if self.allow_origin_pat: - self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat) - self.tornado_settings['allow_credentials'] = self.allow_credentials - self.tornado_settings['cookie_options'] = self.cookie_options - self.tornado_settings['get_secure_cookie_kwargs'] = self.get_secure_cookie_kwargs - self.tornado_settings['token'] = self.token - - # ensure default_url starts with base_url - if not self.default_url.startswith(self.base_url): - self.default_url = url_path_join(self.base_url, self.default_url) - - if self.password_required and (not self.password): - self.log.critical(_("Notebook servers are configured to only be run with a password.")) - self.log.critical(_("Hint: run the following command to set a password")) - self.log.critical(_("\t$ python -m notebook.auth password")) - sys.exit(1) - - self.web_app = NotebookWebApplication( - self, self.kernel_manager, self.contents_manager, - self.session_manager, self.kernel_spec_manager, - self.config_manager, self.extra_services, - self.log, self.base_url, self.default_url, self.tornado_settings, - self.jinja_environment_options, - ) - ssl_options = self.ssl_options - if self.certfile: - ssl_options['certfile'] = self.certfile - if self.keyfile: - ssl_options['keyfile'] = self.keyfile - if self.client_ca: - ssl_options['ca_certs'] = self.client_ca - if not ssl_options: - # None indicates no SSL config - ssl_options = None - else: - # SSL may be missing, so only import it if it's to be used - import ssl - # PROTOCOL_TLS selects the highest ssl/tls protocol version that both the client and - # server support. When PROTOCOL_TLS is not available use PROTOCOL_SSLv23. - # PROTOCOL_TLS is new in version 2.7.13, 3.5.3 and 3.6 - ssl_options.setdefault( - 'ssl_version', - getattr(ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_SSLv23) - ) - if ssl_options.get('ca_certs', False): - ssl_options.setdefault('cert_reqs', ssl.CERT_REQUIRED) - - self.login_handler_class.validate_security(self, ssl_options=ssl_options) - self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, - xheaders=self.trust_xheaders, - max_body_size=self.max_body_size, - max_buffer_size=self.max_buffer_size) - - success = None - for port in random_ports(self.port, self.port_retries+1): - try: - self.http_server.listen(port, self.ip) - except socket.error as e: - if e.errno == errno.EADDRINUSE: - self.log.info(_('The port %i is already in use, trying another port.') % port) - continue - elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): - self.log.warning(_("Permission to listen on port %i denied") % port) - continue - else: - raise - else: - self.port = port - success = True - break - if not success: - self.log.critical(_('ERROR: the notebook server could not be started because ' - 'no available port could be found.')) - self.exit(1) - @property - def display_url(self): - if self.custom_display_url: - url = self.custom_display_url - if not url.endswith('/'): - url += '/' - else: - if self.ip in ('', '0.0.0.0'): - ip = "%s" % socket.gethostname() - else: - ip = self.ip - url = self._url(ip) - if self.token: - # Don't log full token if it came from config - token = self.token if self._token_generated else '...' - url = (url_concat(url, {'token': token}) - + '\n or ' - + url_concat(self._url('127.0.0.1'), {'token': token})) - return url + def static_paths(self): + """Rename trait in jupyter_server.""" + return self.static_file_path @property - def connection_url(self): - ip = self.ip if self.ip else 'localhost' - return self._url(ip) - - def _url(self, ip): - proto = 'https' if self.certfile else 'http' - return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) - - def init_terminals(self): - if not self.terminals_enabled: - return - - try: - from .terminal import initialize - initialize(self.web_app, self.notebook_dir, self.connection_url, self.terminado_settings) - self.web_app.settings['terminals_available'] = True - except ImportError as e: - self.log.warning(_("Terminals not available (error was %s)"), e) - - def init_signal(self): - if not sys.platform.startswith('win') and sys.stdin and sys.stdin.isatty(): - signal.signal(signal.SIGINT, self._handle_sigint) - signal.signal(signal.SIGTERM, self._signal_stop) - if hasattr(signal, 'SIGUSR1'): - # Windows doesn't support SIGUSR1 - signal.signal(signal.SIGUSR1, self._signal_info) - if hasattr(signal, 'SIGINFO'): - # only on BSD-based systems - signal.signal(signal.SIGINFO, self._signal_info) - - def _handle_sigint(self, sig, frame): - """SIGINT handler spawns confirmation dialog""" - # register more forceful signal handler for ^C^C case - signal.signal(signal.SIGINT, self._signal_stop) - # request confirmation dialog in bg thread, to avoid - # blocking the App - thread = threading.Thread(target=self._confirm_exit) - thread.daemon = True - thread.start() - - def _restore_sigint_handler(self): - """callback for restoring original SIGINT handler""" - signal.signal(signal.SIGINT, self._handle_sigint) - - def _confirm_exit(self): - """confirm shutdown on ^C - - A second ^C, or answering 'y' within 5s will cause shutdown, - otherwise original SIGINT handler will be restored. - - This doesn't work on Windows. - """ - info = self.log.info - info(_('interrupted')) - print(self.notebook_info()) - yes = _('y') - no = _('n') - sys.stdout.write(_("Shutdown this notebook server (%s/[%s])? ") % (yes, no)) - sys.stdout.flush() - r,w,x = select.select([sys.stdin], [], [], 5) - if r: - line = sys.stdin.readline() - if line.lower().startswith(yes) and no not in line.lower(): - self.log.critical(_("Shutdown confirmed")) - # schedule stop on the main thread, - # since this might be called from a signal handler - self.io_loop.add_callback_from_signal(self.io_loop.stop) - return - else: - print(_("No answer for 5s:"), end=' ') - print(_("resuming operation...")) - # no answer, or answer is no: - # set it back to original SIGINT handler - # use IOLoop.add_callback because signal.signal must be called - # from main thread - self.io_loop.add_callback_from_signal(self._restore_sigint_handler) - - def _signal_stop(self, sig, frame): - self.log.critical(_("received signal %s, stopping"), sig) - self.io_loop.add_callback_from_signal(self.io_loop.stop) + def template_paths(self): + """Rename trait for Jupyter Server.""" + return self.template_file_path - def _signal_info(self, sig, frame): - print(self.notebook_info()) - - def init_components(self): - """Check the components submodule, and warn if it's unclean""" - # TODO: this should still check, but now we use bower, not git submodule - pass - - def init_server_extension_config(self): - """Consolidate server extensions specified by all configs. - - The resulting list is stored on self.nbserver_extensions and updates config object. - - The extension API is experimental, and may change in future releases. - """ - # TODO: Remove me in notebook 5.0 - for modulename in self.server_extensions: - # Don't override disable state of the extension if it already exist - # in the new traitlet - if not modulename in self.nbserver_extensions: - self.nbserver_extensions[modulename] = True - - # Load server extensions with ConfigManager. - # This enables merging on keys, which we want for extension enabling. - # Regular config loading only merges at the class level, - # so each level (user > env > system) clobbers the previous. - config_path = jupyter_config_path() - if self.config_dir not in config_path: - # add self.config_dir to the front, if set manually - config_path.insert(0, self.config_dir) - manager = ConfigManager(read_config_path=config_path) - section = manager.get(self.config_file_name) - extensions = section.get('NotebookApp', {}).get('nbserver_extensions', {}) - - for modulename, enabled in sorted(extensions.items()): - if modulename not in self.nbserver_extensions: - self.config.NotebookApp.nbserver_extensions.update({modulename: enabled}) - self.nbserver_extensions.update({modulename: enabled}) - - def init_server_extensions(self): - """Load any extensions specified by config. - - Import the module, then call the load_jupyter_server_extension function, - if one exists. - - The extension API is experimental, and may change in future releases. - """ - - - for modulename, enabled in sorted(self.nbserver_extensions.items()): - if enabled: - try: - mod = importlib.import_module(modulename) - func = getattr(mod, 'load_jupyter_server_extension', None) - if func is not None: - func(self) - except Exception: - if self.reraise_server_extension_failures: - raise - self.log.warning(_("Error loading server extension %s"), modulename, - exc_info=True) - - def init_mime_overrides(self): - # On some Windows machines, an application has registered incorrect - # mimetypes in the registry. - # Tornado uses this when serving .css and .js files, causing browsers to - # reject these files. We know the mimetype always needs to be text/css for css - # and application/javascript for JS, so we override it here - # and explicitly tell the mimetypes to not trust the Windows registry - if os.name == 'nt': - # do not trust windows registry, which regularly has bad info - mimetypes.init(files=[]) - # ensure css, js are correct, which are required for pages to function - mimetypes.add_type('text/css', '.css') - mimetypes.add_type('application/javascript', '.js') - - - def shutdown_no_activity(self): - """Shutdown server on timeout when there are no kernels or terminals.""" - km = self.kernel_manager - if len(km) != 0: - return # Kernels still running + def initialize_templates(self): + """Initialize the jinja templates for the notebook application.""" + _template_path = self.template_paths + if isinstance(_template_path, py3compat.string_types): + _template_path = (_template_path,) + template_path = [os.path.expanduser(path) for path in _template_path] - try: - term_mgr = self.web_app.settings['terminal_manager'] - except KeyError: - pass # Terminals not enabled - else: - if term_mgr.terminals: - return # Terminals still running - - seconds_since_active = \ - (utcnow() - self.web_app.last_activity()).total_seconds() - self.log.debug("No activity for %d seconds.", - seconds_since_active) - if seconds_since_active > self.shutdown_no_activity_timeout: - self.log.info("No kernels or terminals for %d seconds; shutting down.", - seconds_since_active) - self.stop() - - def init_shutdown_no_activity(self): - if self.shutdown_no_activity_timeout > 0: - self.log.info("Will shut down after %d seconds with no kernels or terminals.", - self.shutdown_no_activity_timeout) - pc = ioloop.PeriodicCallback(self.shutdown_no_activity, 60000) - pc.start() - - @catch_config_error - def initialize(self, argv=None): - super(NotebookApp, self).initialize(argv) - self.init_logging() - if self._dispatching: - return - self.init_configurables() - self.init_server_extension_config() - self.init_components() - self.init_webapp() - self.init_terminals() - self.init_signal() - self.init_server_extensions() - self.init_mime_overrides() - self.init_shutdown_no_activity() - - def cleanup_kernels(self): - """Shutdown all kernels. - - The kernels will shutdown themselves when this process no longer exists, - but explicit shutdown allows the KernelManagers to cleanup the connection files. - """ - n_kernels = len(self.kernel_manager.list_kernel_ids()) - kernel_msg = trans.ngettext('Shutting down %d kernel', 'Shutting down %d kernels', n_kernels) - self.log.info(kernel_msg % n_kernels) - self.kernel_manager.shutdown_all() - - def notebook_info(self, kernel_count=True): - "Return the current working directory and the server url information" - info = self.contents_manager.info_string() + "\n" - if kernel_count: - n_kernels = len(self.kernel_manager.list_kernel_ids()) - kernel_msg = trans.ngettext("%d active kernel", "%d active kernels", n_kernels) - info += kernel_msg % n_kernels - info += "\n" - # Format the info so that the URL fits on a single line in 80 char display - info += _("The Jupyter Notebook is running at:\n%s") % self.display_url - if self.gateway_config.gateway_enabled: - info += _("\nKernels will be managed by the Gateway server running at:\n%s") % self.gateway_config.url - return info - - def server_info(self): - """Return a JSONable dict of information about this server.""" - return {'url': self.connection_url, - 'hostname': self.ip if self.ip else 'localhost', - 'port': self.port, - 'secure': bool(self.certfile), - 'base_url': self.base_url, - 'token': self.token, - 'notebook_dir': os.path.abspath(self.notebook_dir), - 'password': bool(self.password), - 'pid': os.getpid(), - } - - def write_server_info_file(self): - """Write the result of server_info() to the JSON file info_file.""" - try: - with open(self.info_file, 'w') as f: - json.dump(self.server_info(), f, indent=2, sort_keys=True) - except OSError as e: - self.log.error(_("Failed to write server-info to %s: %s"), - self.info_file, e) + jenv_opt = {"autoescape": True} + jenv_opt.update(self.jinja_environment_options if self.jinja_environment_options else {}) - def remove_server_info_file(self): - """Remove the nbserver-.json file created for this server. + env = Environment(loader=FileSystemLoader(template_path), extensions=['jinja2.ext.i18n'], **jenv_opt) - Ignores the error raised when the file has already been removed. - """ - try: - os.unlink(self.info_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise + # If the user is running the notebook in a git directory, make the assumption + # that this is a dev install and suggest to the developer `npm run build:watch`. + base_dir = os.path.realpath(os.path.join(__file__, '..', '..')) + dev_mode = os.path.exists(os.path.join(base_dir, '.git')) - def write_browser_open_file(self): - """Write an nbserver--open.html file + nbui = gettext.translation('nbui', localedir=os.path.join(base_dir, 'notebook/i18n'), fallback=True) + env.install_gettext_translations(nbui, newstyle=False) - This can be used to open the notebook in a browser - """ - # default_url contains base_url, but so does connection_url - open_url = self.default_url[len(self.base_url):] + if dev_mode: + DEV_NOTE_NPM = """It looks like you're running the notebook from source. + If you're working on the Javascript of the notebook, try running + %s + in another terminal window to have the system incrementally + watch and build the notebook's JavaScript for you, as you make changes.""" % 'npm run build:watch' + self.log.info(DEV_NOTE_NPM) - with open(self.browser_open_file, 'w', encoding='utf-8') as f: - self._write_browser_open_file(open_url, f) + template_settings = dict( + notebook_template_paths=template_path, + notebook_jinja_template_vars=self.jinja_template_vars, + notebook_jinja2_env=env, + ) + self.settings.update(**template_settings) - def _write_browser_open_file(self, url, fh): - if self.token: - url = url_concat(url, {'token': self.token}) - url = url_path_join(self.connection_url, url) - jinja2_env = self.web_app.settings['jinja2_env'] - template = jinja2_env.get_template('browser-open.html') - fh.write(template.render(open_url=url)) + def initialize_settings(self): + """Add settings to the tornado app.""" + if self.ignore_minified_js: + log.warning(_("""The `ignore_minified_js` flag is deprecated and no longer works.""")) + log.warning(_("""Alternatively use `%s` when working on the notebook's Javascript and LESS""") % 'npm run build:watch') + warnings.warn(_("The `ignore_minified_js` flag is deprecated and will be removed in Notebook 6.0"), DeprecationWarning) - def remove_browser_open_file(self): - """Remove the nbserver--open.html file created for this server. + settings = dict( + ignore_minified_js=self.ignore_minified_js, + mathjax_url=self.mathjax_url, + mathjax_config=self.mathjax_config, + nbextensions_path=self.nbextensions_path, + ) + self.settings.update(**settings) - Ignores the error raised when the file has already been removed. - """ - try: - os.unlink(self.browser_open_file) - except OSError as e: - if e.errno != errno.ENOENT: - raise + def initialize_handlers(self): + """Load the (URL pattern, handler) tuples for each component.""" + # Order matters. The first handler to match the URL will handle the request. + handlers = [] + # load extra services specified by users before default handlers + for service in self.settings['extra_services']: + handlers.extend(load_handlers(service)) + handlers.extend(load_handlers('notebook.tree.handlers')) + handlers.extend(load_handlers('notebook.notebook.handlers')) - def launch_browser(self): - try: - browser = webbrowser.get(self.browser or None) - except webbrowser.Error as e: - self.log.warning(_('No web browser found: %s.') % e) - browser = None - - if not browser: - return - - if self.file_to_run: - if not os.path.exists(self.file_to_run): - self.log.critical(_("%s does not exist") % self.file_to_run) - self.exit(1) - - relpath = os.path.relpath(self.file_to_run, self.notebook_dir) - uri = url_escape(url_path_join('notebooks', *relpath.split(os.sep))) - - # Write a temporary file to open in the browser - fd, open_file = tempfile.mkstemp(suffix='.html') - with open(fd, 'w', encoding='utf-8') as fh: - self._write_browser_open_file(uri, fh) - else: - open_file = self.browser_open_file + handlers.append( + (r"/nbextensions/(.*)", FileFindHandler, { + 'path': self.settings['nbextensions_path'], + 'no_cache_paths': ['/'], # don't cache anything in nbextensions + }), + ) + handlers.append( + (r"/custom/(.*)", FileFindHandler, { + 'path': self.settings['static_custom_path'], + 'no_cache_paths': ['/'], # don't cache anything in custom + }) + ) + + # Add new handlers to Jupyter server handlers. + self.handlers.extend(handlers) - b = lambda: browser.open( - urljoin('file:', pathname2url(open_file)), - new=self.webbrowser_open_new) - threading.Thread(target=b).start() - def start(self): - """ Start the Notebook server app, after initialization - - This method takes no arguments so all configuration and initialization - must be done prior to calling this method.""" - - super(NotebookApp, self).start() - - if not self.allow_root: - # check if we are running as root, and abort if it's not allowed - try: - uid = os.geteuid() - except AttributeError: - uid = -1 # anything nonzero here, since we can't check UID assume non-root - if uid == 0: - self.log.critical(_("Running as root is not recommended. Use --allow-root to bypass.")) - self.exit(1) - - info = self.log.info - for line in self.notebook_info(kernel_count=False).split("\n"): - info(line) - info(_("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")) - if 'dev' in notebook.__version__: - info(_("Welcome to Project Jupyter! Explore the various tools available" - " and their corresponding documentation. If you are interested" - " in contributing to the platform, please visit the community" - "resources section at https://jupyter.org/community.html.")) - - self.write_server_info_file() - self.write_browser_open_file() - - if self.open_browser or self.file_to_run: - self.launch_browser() - - if self.token and self._token_generated: - # log full URL with generated token, so there's a copy/pasteable link - # with auth info. - self.log.critical('\n'.join([ - '\n', - 'To access the notebook, open this file in a browser:', - ' %s' % urljoin('file:', pathname2url(self.browser_open_file)), - 'Or copy and paste one of these URLs:', - ' %s' % self.display_url, - ])) - - self.io_loop = ioloop.IOLoop.current() - if sys.platform.startswith('win'): - # add no-op to wake every 5s - # to handle signals that may be ignored by the inner loop - pc = ioloop.PeriodicCallback(lambda : None, 5000) - pc.start() - try: - self.io_loop.start() - except KeyboardInterrupt: - info(_("Interrupted...")) - finally: - self.remove_server_info_file() - self.remove_browser_open_file() - self.cleanup_kernels() - - def stop(self): - def _stop(): - self.http_server.stop() - self.io_loop.stop() - self.io_loop.add_callback(_stop) - - -def list_running_servers(runtime_dir=None): - """Iterate over the server info files of running notebook servers. - - Given a runtime directory, find nbserver-* files in the security directory, - and yield dicts of their information, each one pertaining to - a currently running notebook server instance. - """ - if runtime_dir is None: - runtime_dir = jupyter_runtime_dir() - - # The runtime dir might not exist - if not os.path.isdir(runtime_dir): - return - - for file_name in os.listdir(runtime_dir): - if re.match('nbserver-(.+).json', file_name): - with io.open(os.path.join(runtime_dir, file_name), encoding='utf-8') as f: - info = json.load(f) - - # Simple check whether that process is really still running - # Also remove leftover files from IPython 2.x without a pid field - if ('pid' in info) and check_pid(info['pid']): - yield info - else: - # If the process has died, try to delete its info file - try: - os.unlink(os.path.join(runtime_dir, file_name)) - except OSError: - pass # TODO: This should warn or log or something #----------------------------------------------------------------------------- # Main entry point #----------------------------------------------------------------------------- diff --git a/notebook/prometheus/__init__.py b/notebook/prometheus/__init__.py deleted file mode 100644 index c63c130a1c..0000000000 --- a/notebook/prometheus/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -A package containing all the functionality and -configuration connected to the prometheus metrics -""" \ No newline at end of file diff --git a/notebook/prometheus/log_functions.py b/notebook/prometheus/log_functions.py deleted file mode 100644 index a67a252ade..0000000000 --- a/notebook/prometheus/log_functions.py +++ /dev/null @@ -1,24 +0,0 @@ -from ..prometheus.metrics import HTTP_REQUEST_DURATION_SECONDS - - -def prometheus_log_method(handler): - """ - Tornado log handler for recording RED metrics. - - We record the following metrics: - Rate - the number of requests, per second, your services are serving. - Errors - the number of failed requests per second. - Duration - The amount of time each request takes expressed as a time interval. - - We use a fully qualified name of the handler as a label, - rather than every url path to reduce cardinality. - - This function should be either the value of or called from a function - that is the 'log_function' tornado setting. This makes it get called - at the end of every request, allowing us to record the metrics we need. - """ - HTTP_REQUEST_DURATION_SECONDS.labels( - method=handler.request.method, - handler='{}.{}'.format(handler.__class__.__module__, type(handler).__name__), - status_code=handler.get_status() - ).observe(handler.request.request_time()) diff --git a/notebook/prometheus/metrics.py b/notebook/prometheus/metrics.py deleted file mode 100644 index abc9d0e16b..0000000000 --- a/notebook/prometheus/metrics.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Prometheus metrics exported by Jupyter Notebook Server - -Read https://prometheus.io/docs/practices/naming/ for naming -conventions for metrics & labels. -""" - - -from prometheus_client import Histogram, Gauge - - -HTTP_REQUEST_DURATION_SECONDS = Histogram( - 'http_request_duration_seconds', - 'duration in seconds for all HTTP requests', - ['method', 'handler', 'status_code'], -) - -TERMINAL_CURRENTLY_RUNNING_TOTAL = Gauge( - 'terminal_currently_running_total', - 'counter for how many terminals are running', -) - -KERNEL_CURRENTLY_RUNNING_TOTAL = Gauge( - 'kernel_currently_running_total', - 'counter for how many kernels are running labeled by type', - ['type'] -) diff --git a/notebook/serverextensions.py b/notebook/serverextensions.py deleted file mode 100644 index 7ca1fb03b2..0000000000 --- a/notebook/serverextensions.py +++ /dev/null @@ -1,334 +0,0 @@ -# coding: utf-8 -"""Utilities for installing server extensions for the notebook""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from __future__ import print_function - -import importlib -import sys - -from jupyter_core.paths import jupyter_config_path -from ._version import __version__ -from .config_manager import BaseJSONConfigManager -from .extensions import ( - BaseExtensionApp, _get_config_dir, GREEN_ENABLED, RED_DISABLED, GREEN_OK, RED_X -) -from traitlets import Bool -from traitlets.utils.importstring import import_item - - -# ------------------------------------------------------------------------------ -# Public API -# ------------------------------------------------------------------------------ - -def toggle_serverextension_python(import_name, enabled=None, parent=None, - user=True, sys_prefix=False, logger=None): - """Toggle a server extension. - - By default, toggles the extension in the system-wide Jupyter configuration - location (e.g. /usr/local/etc/jupyter). - - Parameters - ---------- - - import_name : str - Importable Python module (dotted-notation) exposing the magic-named - `load_jupyter_server_extension` function - enabled : bool [default: None] - Toggle state for the extension. Set to None to toggle, True to enable, - and False to disable the extension. - parent : Configurable [default: None] - user : bool [default: True] - Toggle in the user's configuration location (e.g. ~/.jupyter). - sys_prefix : bool [default: False] - Toggle in the current Python environment's configuration location - (e.g. ~/.envs/my-env/etc/jupyter). Will override `user`. - logger : Jupyter logger [optional] - Logger instance to use - """ - user = False if sys_prefix else user - config_dir = _get_config_dir(user=user, sys_prefix=sys_prefix) - cm = BaseJSONConfigManager(parent=parent, config_dir=config_dir) - cfg = cm.get("jupyter_notebook_config") - server_extensions = ( - cfg.setdefault("NotebookApp", {}) - .setdefault("nbserver_extensions", {}) - ) - - old_enabled = server_extensions.get(import_name, None) - new_enabled = enabled if enabled is not None else not old_enabled - - if logger: - if new_enabled: - logger.info(u"Enabling: %s" % (import_name)) - else: - logger.info(u"Disabling: %s" % (import_name)) - - server_extensions[import_name] = new_enabled - - if logger: - logger.info(u"- Writing config: {}".format(config_dir)) - - cm.update("jupyter_notebook_config", cfg) - - if new_enabled: - validate_serverextension(import_name, logger) - - -def validate_serverextension(import_name, logger=None): - """Assess the health of an installed server extension - - Returns a list of validation warnings. - - Parameters - ---------- - - import_name : str - Importable Python module (dotted-notation) exposing the magic-named - `load_jupyter_server_extension` function - logger : Jupyter logger [optional] - Logger instance to use - """ - - warnings = [] - infos = [] - - func = None - - if logger: - logger.info(" - Validating...") - - try: - mod = importlib.import_module(import_name) - func = getattr(mod, 'load_jupyter_server_extension', None) - version = getattr(mod, '__version__', '') - except Exception: - logger.warning("Error loading server extension %s", import_name) - - import_msg = u" {} is {} importable?" - if func is not None: - infos.append(import_msg.format(GREEN_OK, import_name)) - else: - warnings.append(import_msg.format(RED_X, import_name)) - - post_mortem = u" {} {} {}" - if logger: - if warnings: - [logger.info(info) for info in infos] - [logger.warn(warning) for warning in warnings] - else: - logger.info(post_mortem.format(import_name, version, GREEN_OK)) - - return warnings - - -# ---------------------------------------------------------------------- -# Applications -# ---------------------------------------------------------------------- - -flags = {} -flags.update(BaseExtensionApp.flags) -flags.pop("y", None) -flags.pop("generate-config", None) -flags.update({ - "user" : ({ - "ToggleServerExtensionApp" : { - "user" : True, - }}, "Perform the operation for the current user" - ), - "system" : ({ - "ToggleServerExtensionApp" : { - "user" : False, - "sys_prefix": False, - }}, "Perform the operation system-wide" - ), - "sys-prefix" : ({ - "ToggleServerExtensionApp" : { - "sys_prefix" : True, - }}, "Use sys.prefix as the prefix for installing server extensions" - ), - "py" : ({ - "ToggleServerExtensionApp" : { - "python" : True, - }}, "Install from a Python package" - ), -}) -flags['python'] = flags['py'] - - -class ToggleServerExtensionApp(BaseExtensionApp): - """A base class for enabling/disabling extensions""" - name = "jupyter serverextension enable/disable" - description = "Enable/disable a server extension using frontend configuration files." - - flags = flags - - user = Bool(True, config=True, help="Whether to do a user install") - sys_prefix = Bool(False, config=True, help="Use the sys.prefix as the prefix") - python = Bool(False, config=True, help="Install from a Python package") - - def toggle_server_extension(self, import_name): - """Change the status of a named server extension. - - Uses the value of `self._toggle_value`. - - Parameters - --------- - - import_name : str - Importable Python module (dotted-notation) exposing the magic-named - `load_jupyter_server_extension` function - """ - toggle_serverextension_python( - import_name, self._toggle_value, parent=self, user=self.user, - sys_prefix=self.sys_prefix, logger=self.log) - - def toggle_server_extension_python(self, package): - """Change the status of some server extensions in a Python package. - - Uses the value of `self._toggle_value`. - - Parameters - --------- - - package : str - Importable Python module exposing the - magic-named `_jupyter_server_extension_paths` function - """ - m, server_exts = _get_server_extension_metadata(package) - for server_ext in server_exts: - module = server_ext['module'] - self.toggle_server_extension(module) - - def start(self): - """Perform the App's actions as configured""" - if not self.extra_args: - sys.exit('Please specify a server extension/package to enable or disable') - for arg in self.extra_args: - if self.python: - self.toggle_server_extension_python(arg) - else: - self.toggle_server_extension(arg) - - -class EnableServerExtensionApp(ToggleServerExtensionApp): - """An App that enables (and validates) Server Extensions""" - name = "jupyter serverextension enable" - description = """ - Enable a serverextension in configuration. - - Usage - jupyter serverextension enable [--system|--sys-prefix] - """ - _toggle_value = True - - -class DisableServerExtensionApp(ToggleServerExtensionApp): - """An App that disables Server Extensions""" - name = "jupyter serverextension disable" - description = """ - Disable a serverextension in configuration. - - Usage - jupyter serverextension disable [--system|--sys-prefix] - """ - _toggle_value = False - - -class ListServerExtensionsApp(BaseExtensionApp): - """An App that lists (and validates) Server Extensions""" - name = "jupyter serverextension list" - version = __version__ - description = "List all server extensions known by the configuration system" - - def list_server_extensions(self): - """List all enabled and disabled server extensions, by config path - - Enabled extensions are validated, potentially generating warnings. - """ - config_dirs = jupyter_config_path() - for config_dir in config_dirs: - cm = BaseJSONConfigManager(parent=self, config_dir=config_dir) - data = cm.get("jupyter_notebook_config") - server_extensions = ( - data.setdefault("NotebookApp", {}) - .setdefault("nbserver_extensions", {}) - ) - if server_extensions: - print(u'config dir: {}'.format(config_dir)) - for import_name, enabled in server_extensions.items(): - print(u' {} {}'.format( - import_name, - GREEN_ENABLED if enabled else RED_DISABLED)) - validate_serverextension(import_name, self.log) - - def start(self): - """Perform the App's actions as configured""" - self.list_server_extensions() - - -_examples = """ -jupyter serverextension list # list all configured server extensions -jupyter serverextension enable --py # enable all server extensions in a Python package -jupyter serverextension disable --py # disable all server extensions in a Python package -""" - - -class ServerExtensionApp(BaseExtensionApp): - """Root level server extension app""" - name = "jupyter serverextension" - version = __version__ - description = "Work with Jupyter server extensions" - examples = _examples - - subcommands = dict( - enable=(EnableServerExtensionApp, "Enable a server extension"), - disable=(DisableServerExtensionApp, "Disable a server extension"), - list=(ListServerExtensionsApp, "List server extensions") - ) - - def start(self): - """Perform the App's actions as configured""" - super(ServerExtensionApp, self).start() - - # The above should have called a subcommand and raised NoStart; if we - # get here, it didn't, so we should self.log.info a message. - subcmds = ", ".join(sorted(self.subcommands)) - sys.exit("Please supply at least one subcommand: %s" % subcmds) - - -main = ServerExtensionApp.launch_instance - -# ------------------------------------------------------------------------------ -# Private API -# ------------------------------------------------------------------------------ - - -def _get_server_extension_metadata(module): - """Load server extension metadata from a module. - - Returns a tuple of ( - the package as loaded - a list of server extension specs: [ - { - "module": "mockextension" - } - ] - ) - - Parameters - ---------- - - module : str - Importable Python module exposing the - magic-named `_jupyter_server_extension_paths` function - """ - m = import_item(module) - if not hasattr(m, '_jupyter_server_extension_paths'): - raise KeyError(u'The Python module {} does not include any valid server extensions'.format(module)) - return m, m._jupyter_server_extension_paths() - -if __name__ == '__main__': - main() diff --git a/notebook/services/__init__.py b/notebook/services/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/api/__init__.py b/notebook/services/api/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/api/api.yaml b/notebook/services/api/api.yaml deleted file mode 100644 index 90dd85d8b0..0000000000 --- a/notebook/services/api/api.yaml +++ /dev/null @@ -1,851 +0,0 @@ -swagger: '2.0' -info: - title: Jupyter Notebook API - description: Notebook API - version: "5" - contact: - name: Jupyter Project - url: https://jupyter.org -# will be prefixed to all paths -basePath: / -produces: - - application/json -consumes: - - application/json -parameters: - kernel: - name: kernel_id - required: true - in: path - description: kernel uuid - type: string - format: uuid - session: - name: session - required: true - in: path - description: session uuid - type: string - format: uuid - path: - name: path - required: true - in: path - description: file path - type: string - checkpoint_id: - name: checkpoint_id - required: true - in: path - description: Checkpoint id for a file - type: string - section_name: - name: section_name - required: true - in: path - description: Name of config section - type: string - terminal_id: - name: terminal_id - required: true - in: path - description: ID of terminal session - type: string - -paths: - - - /api/contents/{path}: - parameters: - - $ref: '#/parameters/path' - get: - summary: Get contents of file or directory - description: "A client can optionally specify a type and/or format argument via URL parameter. When given, the Contents service shall return a model in the requested type and/or format. If the request cannot be satisfied, e.g. type=text is requested, but the file is binary, then the request shall fail with 400 and have a JSON response containing a 'reason' field, with the value 'bad format' or 'bad type', depending on what was requested." - tags: - - contents - parameters: - - name: type - in: query - description: File type ('file', 'directory') - type: string - enum: - - file - - directory - - name: format - in: query - description: "How file content should be returned ('text', 'base64')" - type: string - enum: - - text - - base64 - - name: content - in: query - description: "Return content (0 for no content, 1 for return content)" - type: integer - responses: - 404: - description: No item found - 400: - description: Bad request - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - 200: - description: Contents of file or directory - headers: - Last-Modified: - description: Last modified date for file - type: string - format: dateTime - schema: - $ref: '#/definitions/Contents' - 500: - description: Model key error - post: - summary: Create a new file in the specified path - description: "A POST to /api/contents/path creates a New untitled, empty file or directory. A POST to /api/contents/path with body {'copy_from': '/path/to/OtherNotebook.ipynb'} creates a new copy of OtherNotebook in path." - tags: - - contents - parameters: - - name: model - in: body - description: Path of file to copy - schema: - type: object - properties: - copy_from: - type: string - ext: - type: string - type: - type: string - responses: - 201: - description: File created - headers: - Location: - description: URL for the new file - type: string - format: url - schema: - $ref: '#/definitions/Contents' - 404: - description: No item found - 400: - description: Bad request - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - patch: - summary: Rename a file or directory without re-uploading content - tags: - - contents - parameters: - - name: path - in: body - required: true - description: New path for file or directory. - schema: - type: object - properties: - path: - type: string - format: path - description: New path for file or directory - responses: - 200: - description: Path updated - headers: - Location: - description: Updated URL for the file or directory - type: string - format: url - schema: - $ref: '#/definitions/Contents' - 400: - description: No data provided - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - put: - summary: Save or upload file. - description: "Saves the file in the location specified by name and path. PUT is very similar to POST, but the requester specifies the name, whereas with POST, the server picks the name." - tags: - - contents - parameters: - - name: model - in: body - description: New path for file or directory - schema: - type: object - properties: - name: - type: string - description: The new filename if changed - path: - type: string - description: New path for file or directory - type: - type: string - description: Path dtype ('notebook', 'file', 'directory') - format: - type: string - description: File format ('json', 'text', 'base64') - content: - type: string - description: The actual body of the document excluding directory type - responses: - 200: - description: File saved - headers: - Location: - description: Updated URL for the file or directory - type: string - format: url - schema: - $ref: '#/definitions/Contents' - 201: - description: Path created - headers: - Location: - description: URL for the file or directory - type: string - format: url - schema: - $ref: '#/definitions/Contents' - 400: - description: No data provided - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - delete: - summary: Delete a file in the given path - tags: - - contents - responses: - 204: - description: File deleted - headers: - Location: - description: URL for the removed file - type: string - format: url - /api/contents/{path}/checkpoints: - parameters: - - $ref: '#/parameters/path' - get: - summary: Get a list of checkpoints for a file - description: List checkpoints for a given file. There will typically be zero or one results. - tags: - - contents - responses: - 404: - description: No item found - 400: - description: Bad request - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - 200: - description: List of checkpoints for a file - schema: - type: array - items: - $ref: '#/definitions/Checkpoints' - 500: - description: Model key error - post: - summary: Create a new checkpoint for a file - description: "Create a new checkpoint with the current state of a file. With the default FileContentsManager, only one checkpoint is supported, so creating new checkpoints clobbers existing ones." - tags: - - contents - responses: - 201: - description: Checkpoint created - headers: - Location: - description: URL for the checkpoint - type: string - format: url - schema: - $ref: '#/definitions/Checkpoints' - 404: - description: No item found - 400: - description: Bad request - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - /api/contents/{path}/checkpoints/{checkpoint_id}: - post: - summary: Restore a file to a particular checkpointed state - parameters: - - $ref: "#/parameters/path" - - $ref: "#/parameters/checkpoint_id" - tags: - - contents - responses: - 204: - description: Checkpoint restored - 400: - description: Bad request - schema: - type: object - properties: - error: - type: string - description: Error condition - reason: - type: string - description: Explanation of error reason - delete: - summary: Delete a checkpoint - parameters: - - $ref: "#/parameters/path" - - $ref: "#/parameters/checkpoint_id" - tags: - - contents - responses: - 204: - description: Checkpoint deleted - /api/sessions/{session}: - parameters: - - $ref: '#/parameters/session' - get: - summary: Get session - tags: - - sessions - responses: - 200: - description: Session - schema: - $ref: '#/definitions/Session' - patch: - summary: "This can be used to rename the session." - tags: - - sessions - parameters: - - name: model - in: body - required: true - schema: - $ref: '#/definitions/Session' - responses: - 200: - description: Session - schema: - $ref: '#/definitions/Session' - 400: - description: No data provided - delete: - summary: Delete a session - tags: - - sessions - responses: - 204: - description: Session (and kernel) were deleted - 410: - description: "Kernel was deleted before the session, and the session was *not* deleted (TODO - check to make sure session wasn't deleted)" - /api/sessions: - get: - summary: List available sessions - tags: - - sessions - responses: - 200: - description: List of current sessions - schema: - type: array - items: - $ref: '#/definitions/Session' - post: - summary: "Create a new session, or return an existing session if a session of the same name already exists" - tags: - - sessions - parameters: - - name: session - in: body - schema: - $ref: '#/definitions/Session' - responses: - 201: - description: Session created or returned - schema: - $ref: '#/definitions/Session' - headers: - Location: - description: URL for session commands - type: string - format: url - 501: - description: Session not available - schema: - type: object - description: error message - properties: - message: - type: string - short_message: - type: string - - /api/kernels: - get: - summary: List the JSON data for all kernels that are currently running - tags: - - kernels - responses: - 200: - description: List of currently-running kernel uuids - schema: - type: array - items: - $ref: '#/definitions/Kernel' - post: - summary: Start a kernel and return the uuid - tags: - - kernels - parameters: - - name: name - in: body - description: Kernel spec name (defaults to default kernel spec for server) - schema: - type: object - properties: - name: - type: string - responses: - 201: - description: Kernel started - schema: - $ref: '#/definitions/Kernel' - headers: - Location: - description: Model for started kernel - type: string - format: url - /api/kernels/{kernel_id}: - parameters: - - $ref: '#/parameters/kernel' - get: - summary: Get kernel information - tags: - - kernels - responses: - 200: - description: Kernel information - schema: - $ref: '#/definitions/Kernel' - delete: - summary: Kill a kernel and delete the kernel id - tags: - - kernels - responses: - 204: - description: Kernel deleted - /api/kernels/{kernel_id}/interrupt: - parameters: - - $ref: '#/parameters/kernel' - post: - summary: Interrupt a kernel - tags: - - kernels - responses: - 204: - description: Kernel interrupted - /api/kernels/{kernel_id}/restart: - parameters: - - $ref: '#/parameters/kernel' - post: - summary: Restart a kernel - tags: - - kernels - responses: - 200: - description: Kernel interrupted - headers: - Location: - description: URL for kernel commands - type: string - format: url - schema: - $ref: '#/definitions/Kernel' - - /api/kernelspecs: - get: - summary: Get kernel specs - tags: - - kernelspecs - responses: - 200: - description: Kernel specs - schema: - type: object - properties: - default: - type: string - description: Default kernel name - kernelspecs: - type: object - additionalProperties: - $ref: '#/definitions/KernelSpec' - /api/config/{section_name}: - get: - summary: Get a configuration section by name - parameters: - - $ref: "#/parameters/section_name" - tags: - - config - responses: - 200: - description: Configuration object - schema: - type: object - patch: - summary: Update a configuration section by name - tags: - - config - parameters: - - $ref: "#/parameters/section_name" - - name: configuration - in: body - schema: - type: object - responses: - 200: - description: Configuration object - schema: - type: object - - /api/terminals: - get: - summary: Get available terminals - tags: - - terminals - responses: - 200: - description: A list of all available terminal ids. - schema: - type: array - items: - $ref: '#/definitions/Terminal_ID' - 403: - description: Forbidden to access - 404: - description: Not found - - post: - summary: Create a new terminal - tags: - - terminals - responses: - 200: - description: Succesfully created a new terminal - schema: - $ref: '#/definitions/Terminal_ID' - 403: - description: Forbidden to access - 404: - description: Not found - - /api/terminals/{terminal_id}: - get: - summary: Get a terminal session corresponding to an id. - tags: - - terminals - parameters: - - $ref: '#/parameters/terminal_id' - responses: - 200: - description: Terminal session with given id - schema: - $ref: '#/definitions/Terminal_ID' - 403: - description: Forbidden to access - 404: - description: Not found - - delete: - summary: Delete a terminal session corresponding to an id. - tags: - - terminals - parameters: - - $ref: '#/parameters/terminal_id' - responses: - 204: - description: Succesfully deleted terminal session - 403: - description: Forbidden to access - 404: - description: Not found - - - - - /api/status: - get: - summary: Get the current status/activity of the server. - tags: - - status - responses: - 200: - description: The current status of the server - schema: - $ref: '#/definitions/APIStatus' - - /api/spec.yaml: - get: - summary: Get the current spec for the notebook server's APIs. - tags: - - api-spec - produces: - - text/x-yaml - responses: - 200: - description: The current spec for the notebook server's APIs. - schema: - type: file -definitions: - APIStatus: - description: | - Notebook server API status. - Added in notebook 5.0. - properties: - started: - type: string - description: | - ISO8601 timestamp indicating when the notebook server started. - last_activity: - type: string - description: | - ISO8601 timestamp indicating the last activity on the server, - either on the REST API or kernel activity. - connections: - type: number - description: | - The total number of currently open connections to kernels. - kernels: - type: number - description: | - The total number of running kernels. - KernelSpec: - description: Kernel spec (contents of kernel.json) - properties: - name: - type: string - description: Unique name for kernel - KernelSpecFile: - $ref: '#/definitions/KernelSpecFile' - resources: - type: object - properties: - kernel.js: - type: string - format: filename - description: path for kernel.js file - kernel.css: - type: string - format: filename - description: path for kernel.css file - logo-*: - type: string - format: filename - description: path for logo file. Logo filenames are of the form `logo-widthxheight` - KernelSpecFile: - description: Kernel spec json file - required: - - argv - - display_name - - language - properties: - language: - type: string - description: The programming language which this kernel runs. This will be stored in notebook metadata. - argv: - type: array - description: "A list of command line arguments used to start the kernel. The text `{connection_file}` in any argument will be replaced with the path to the connection file." - items: - type: string - display_name: - type: string - description: "The kernel's name as it should be displayed in the UI. Unlike the kernel name used in the API, this can contain arbitrary unicode characters." - codemirror_mode: - type: string - description: Codemirror mode. Can be a string *or* an valid Codemirror mode object. This defaults to the string from the `language` property. - env: - type: object - description: A dictionary of environment variables to set for the kernel. These will be added to the current environment variables. - additionalProperties: - type: string - help_links: - type: array - description: Help items to be displayed in the help menu in the notebook UI. - items: - type: object - required: - - text - - url - properties: - text: - type: string - description: menu item link text - url: - type: string - format: URL - description: menu item link url - Kernel: - description: Kernel information - required: - - id - - name - properties: - id: - type: string - format: uuid - description: uuid of kernel - name: - type: string - description: kernel spec name - last_activity: - type: string - description: | - ISO 8601 timestamp for the last-seen activity on this kernel. - Use this in combination with execution_state == 'idle' to identify - which kernels have been idle since a given time. - Timestamps will be UTC, indicated 'Z' suffix. - Added in notebook server 5.0. - connections: - type: number - description: | - The number of active connections to this kernel. - execution_state: - type: string - description: | - Current execution state of the kernel (typically 'idle' or 'busy', but may be other values, such as 'starting'). - Added in notebook server 5.0. - Session: - description: A session - type: object - properties: - id: - type: string - format: uuid - path: - type: string - description: path to the session - name: - type: string - description: name of the session - type: - type: string - description: session type - kernel: - $ref: '#/definitions/Kernel' - Contents: - description: "A contents object. The content and format keys may be null if content is not contained. If type is 'file', then the mimetype will be null." - type: object - required: - - type - - name - - path - - writable - - created - - last_modified - - mimetype - - format - - content - properties: - name: - type: string - description: "Name of file or directory, equivalent to the last part of the path" - path: - type: string - description: Full path for file or directory - type: - type: string - description: Type of content - enum: - - directory - - file - - notebook - writable: - type: boolean - description: indicates whether the requester has permission to edit the file - created: - type: string - description: Creation timestamp - format: dateTime - last_modified: - type: string - description: Last modified timestamp - format: dateTime - size: - type: integer - description: "The size of the file or notebook in bytes. If no size is provided, defaults to null." - mimetype: - type: string - description: "The mimetype of a file. If content is not null, and type is 'file', this will contain the mimetype of the file, otherwise this will be null." - content: - type: string - description: "The content, if requested (otherwise null). Will be an array if type is 'directory'" - format: - type: string - description: Format of content (one of null, 'text', 'base64', 'json') - Checkpoints: - description: A checkpoint object. - type: object - required: - - id - - last_modified - properties: - id: - type: string - description: Unique id for the checkpoint. - last_modified: - type: string - description: Last modified timestamp - format: dateTime - Terminal_ID: - description: A Terminal_ID object - type: object - required: - - name - properties: - name: - type: string - description: name of terminal ID diff --git a/notebook/services/api/handlers.py b/notebook/services/api/handlers.py deleted file mode 100644 index 23446e005d..0000000000 --- a/notebook/services/api/handlers.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Tornado handlers for api specifications.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json -import os - -from tornado import gen, web - -from ...base.handlers import IPythonHandler, APIHandler -from notebook._tz import utcfromtimestamp, isoformat -from notebook.utils import maybe_future - - -class APISpecHandler(web.StaticFileHandler, IPythonHandler): - - def initialize(self): - web.StaticFileHandler.initialize(self, path=os.path.dirname(__file__)) - - @web.authenticated - def get(self): - self.log.warning("Serving api spec (experimental, incomplete)") - return web.StaticFileHandler.get(self, 'api.yaml') - - def get_content_type(self): - return 'text/x-yaml' - - -class APIStatusHandler(APIHandler): - - _track_activity = False - - @web.authenticated - @gen.coroutine - def get(self): - # if started was missing, use unix epoch - started = self.settings.get('started', utcfromtimestamp(0)) - started = isoformat(started) - - kernels = yield maybe_future(self.kernel_manager.list_kernels()) - total_connections = sum(k['connections'] for k in kernels) - last_activity = isoformat(self.application.last_activity()) - model = { - 'started': started, - 'last_activity': last_activity, - 'kernels': len(kernels), - 'connections': total_connections, - } - self.finish(json.dumps(model, sort_keys=True)) - - -default_handlers = [ - (r"/api/spec.yaml", APISpecHandler), - (r"/api/status", APIStatusHandler), -] diff --git a/notebook/services/api/tests/__init__.py b/notebook/services/api/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/api/tests/test_api.py b/notebook/services/api/tests/test_api.py deleted file mode 100644 index 0a48b793e6..0000000000 --- a/notebook/services/api/tests/test_api.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Test the basic /api endpoints""" - -import requests - -from notebook._tz import isoformat -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase - - -class KernelAPITest(NotebookTestBase): - """Test the kernels web service API""" - - def _req(self, verb, path, **kwargs): - r = self.request(verb, url_path_join('api', path)) - r.raise_for_status() - return r - - def get(self, path, **kwargs): - return self._req('GET', path) - - def test_get_spec(self): - r = self.get('spec.yaml') - assert r.text - - def test_get_status(self): - r = self.get('status') - data = r.json() - assert data['connections'] == 0 - assert data['kernels'] == 0 - assert data['last_activity'].endswith('Z') - assert data['started'].endswith('Z') - assert data['started'] == isoformat(self.notebook.web_app.settings['started']) diff --git a/notebook/services/config/__init__.py b/notebook/services/config/__init__.py deleted file mode 100644 index d8d9380206..0000000000 --- a/notebook/services/config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .manager import ConfigManager diff --git a/notebook/services/config/handlers.py b/notebook/services/config/handlers.py deleted file mode 100644 index 76c1bd3e56..0000000000 --- a/notebook/services/config/handlers.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Tornado handlers for frontend config storage.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -import json -import os -import io -import errno -from tornado import web - -from ipython_genutils.py3compat import PY3 -from ...base.handlers import APIHandler - -class ConfigHandler(APIHandler): - - @web.authenticated - def get(self, section_name): - self.set_header("Content-Type", 'application/json') - self.finish(json.dumps(self.config_manager.get(section_name))) - - @web.authenticated - def put(self, section_name): - data = self.get_json_body() # Will raise 400 if content is not valid JSON - self.config_manager.set(section_name, data) - self.set_status(204) - - @web.authenticated - def patch(self, section_name): - new_data = self.get_json_body() - section = self.config_manager.update(section_name, new_data) - self.finish(json.dumps(section)) - - -# URL to handler mappings - -section_name_regex = r"(?P\w+)" - -default_handlers = [ - (r"/api/config/%s" % section_name_regex, ConfigHandler), -] diff --git a/notebook/services/config/manager.py b/notebook/services/config/manager.py deleted file mode 100644 index 59f267dd96..0000000000 --- a/notebook/services/config/manager.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Manager to read and modify frontend config data in JSON files. -""" -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import os.path - -from notebook.config_manager import BaseJSONConfigManager, recursive_update -from jupyter_core.paths import jupyter_config_dir, jupyter_config_path -from traitlets import Unicode, Instance, List, observe, default -from traitlets.config import LoggingConfigurable - - -class ConfigManager(LoggingConfigurable): - """Config Manager used for storing notebook frontend config""" - - # Public API - - def get(self, section_name): - """Get the config from all config sections.""" - config = {} - # step through back to front, to ensure front of the list is top priority - for p in self.read_config_path[::-1]: - cm = BaseJSONConfigManager(config_dir=p) - recursive_update(config, cm.get(section_name)) - return config - - def set(self, section_name, data): - """Set the config only to the user's config.""" - return self.write_config_manager.set(section_name, data) - - def update(self, section_name, new_data): - """Update the config only to the user's config.""" - return self.write_config_manager.update(section_name, new_data) - - # Private API - - read_config_path = List(Unicode()) - - @default('read_config_path') - def _default_read_config_path(self): - return [os.path.join(p, 'nbconfig') for p in jupyter_config_path()] - - write_config_dir = Unicode() - - @default('write_config_dir') - def _default_write_config_dir(self): - return os.path.join(jupyter_config_dir(), 'nbconfig') - - write_config_manager = Instance(BaseJSONConfigManager) - - @default('write_config_manager') - def _default_write_config_manager(self): - return BaseJSONConfigManager(config_dir=self.write_config_dir) - - @observe('write_config_dir') - def _update_write_config_dir(self, change): - self.write_config_manager = BaseJSONConfigManager(config_dir=self.write_config_dir) diff --git a/notebook/services/config/tests/__init__.py b/notebook/services/config/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/config/tests/test_config_api.py b/notebook/services/config/tests/test_config_api.py deleted file mode 100644 index 28b931b17e..0000000000 --- a/notebook/services/config/tests/test_config_api.py +++ /dev/null @@ -1,68 +0,0 @@ -# coding: utf-8 -"""Test the config webservice API.""" - -import json - -import requests - -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase - - -class ConfigAPI(object): - """Wrapper for notebook API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, section, body=None): - response = self.request(verb, - url_path_join('api/config', section), - data=body, - ) - response.raise_for_status() - return response - - def get(self, section): - return self._req('GET', section) - - def set(self, section, values): - return self._req('PUT', section, json.dumps(values)) - - def modify(self, section, values): - return self._req('PATCH', section, json.dumps(values)) - -class APITest(NotebookTestBase): - """Test the config web service API""" - def setUp(self): - self.config_api = ConfigAPI(self.request) - - def test_create_retrieve_config(self): - sample = {'foo': 'bar', 'baz': 73} - r = self.config_api.set('example', sample) - self.assertEqual(r.status_code, 204) - - r = self.config_api.get('example') - self.assertEqual(r.status_code, 200) - self.assertEqual(r.json(), sample) - - def test_modify(self): - sample = {'foo': 'bar', 'baz': 73, - 'sub': {'a': 6, 'b': 7}, 'sub2': {'c': 8}} - self.config_api.set('example', sample) - - r = self.config_api.modify('example', {'foo': None, # should delete foo - 'baz': 75, - 'wib': [1,2,3], - 'sub': {'a': 8, 'b': None, 'd': 9}, - 'sub2': {'c': None} # should delete sub2 - }) - self.assertEqual(r.status_code, 200) - self.assertEqual(r.json(), {'baz': 75, 'wib': [1,2,3], - 'sub': {'a': 8, 'd': 9}}) - - def test_get_unknown(self): - # We should get an empty config dictionary instead of a 404 - r = self.config_api.get('nonexistant') - self.assertEqual(r.status_code, 200) - self.assertEqual(r.json(), {}) - diff --git a/notebook/services/contents/__init__.py b/notebook/services/contents/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/contents/checkpoints.py b/notebook/services/contents/checkpoints.py deleted file mode 100644 index c29a669c22..0000000000 --- a/notebook/services/contents/checkpoints.py +++ /dev/null @@ -1,142 +0,0 @@ -""" -Classes for managing Checkpoints. -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from tornado.web import HTTPError - -from traitlets.config.configurable import LoggingConfigurable - - -class Checkpoints(LoggingConfigurable): - """ - Base class for managing checkpoints for a ContentsManager. - - Subclasses are required to implement: - - create_checkpoint(self, contents_mgr, path) - restore_checkpoint(self, contents_mgr, checkpoint_id, path) - rename_checkpoint(self, checkpoint_id, old_path, new_path) - delete_checkpoint(self, checkpoint_id, path) - list_checkpoints(self, path) - """ - def create_checkpoint(self, contents_mgr, path): - """Create a checkpoint.""" - raise NotImplementedError("must be implemented in a subclass") - - def restore_checkpoint(self, contents_mgr, checkpoint_id, path): - """Restore a checkpoint""" - raise NotImplementedError("must be implemented in a subclass") - - def rename_checkpoint(self, checkpoint_id, old_path, new_path): - """Rename a single checkpoint from old_path to new_path.""" - raise NotImplementedError("must be implemented in a subclass") - - def delete_checkpoint(self, checkpoint_id, path): - """delete a checkpoint for a file""" - raise NotImplementedError("must be implemented in a subclass") - - def list_checkpoints(self, path): - """Return a list of checkpoints for a given file""" - raise NotImplementedError("must be implemented in a subclass") - - def rename_all_checkpoints(self, old_path, new_path): - """Rename all checkpoints for old_path to new_path.""" - for cp in self.list_checkpoints(old_path): - self.rename_checkpoint(cp['id'], old_path, new_path) - - def delete_all_checkpoints(self, path): - """Delete all checkpoints for the given path.""" - for checkpoint in self.list_checkpoints(path): - self.delete_checkpoint(checkpoint['id'], path) - - -class GenericCheckpointsMixin(object): - """ - Helper for creating Checkpoints subclasses that can be used with any - ContentsManager. - - Provides a ContentsManager-agnostic implementation of `create_checkpoint` - and `restore_checkpoint` in terms of the following operations: - - - create_file_checkpoint(self, content, format, path) - - create_notebook_checkpoint(self, nb, path) - - get_file_checkpoint(self, checkpoint_id, path) - - get_notebook_checkpoint(self, checkpoint_id, path) - - To create a generic CheckpointManager, add this mixin to a class that - implement the above four methods plus the remaining Checkpoints API - methods: - - - delete_checkpoint(self, checkpoint_id, path) - - list_checkpoints(self, path) - - rename_checkpoint(self, checkpoint_id, old_path, new_path) - """ - - def create_checkpoint(self, contents_mgr, path): - model = contents_mgr.get(path, content=True) - type = model['type'] - if type == 'notebook': - return self.create_notebook_checkpoint( - model['content'], - path, - ) - elif type == 'file': - return self.create_file_checkpoint( - model['content'], - model['format'], - path, - ) - else: - raise HTTPError(500, u'Unexpected type %s' % type) - - def restore_checkpoint(self, contents_mgr, checkpoint_id, path): - """Restore a checkpoint.""" - type = contents_mgr.get(path, content=False)['type'] - if type == 'notebook': - model = self.get_notebook_checkpoint(checkpoint_id, path) - elif type == 'file': - model = self.get_file_checkpoint(checkpoint_id, path) - else: - raise HTTPError(500, u'Unexpected type %s' % type) - contents_mgr.save(model, path) - - # Required Methods - def create_file_checkpoint(self, content, format, path): - """Create a checkpoint of the current state of a file - - Returns a checkpoint model for the new checkpoint. - """ - raise NotImplementedError("must be implemented in a subclass") - - def create_notebook_checkpoint(self, nb, path): - """Create a checkpoint of the current state of a file - - Returns a checkpoint model for the new checkpoint. - """ - raise NotImplementedError("must be implemented in a subclass") - - def get_file_checkpoint(self, checkpoint_id, path): - """Get the content of a checkpoint for a non-notebook file. - - Returns a dict of the form: - { - 'type': 'file', - 'content': , - 'format': {'text','base64'}, - } - """ - raise NotImplementedError("must be implemented in a subclass") - - def get_notebook_checkpoint(self, checkpoint_id, path): - """Get the content of a checkpoint for a notebook. - - Returns a dict of the form: - { - 'type': 'notebook', - 'content': , - } - """ - raise NotImplementedError("must be implemented in a subclass") diff --git a/notebook/services/contents/filecheckpoints.py b/notebook/services/contents/filecheckpoints.py deleted file mode 100644 index 5a9c835749..0000000000 --- a/notebook/services/contents/filecheckpoints.py +++ /dev/null @@ -1,202 +0,0 @@ -""" -File-based Checkpoints implementations. -""" -import os -import shutil - -from tornado.web import HTTPError - -from .checkpoints import ( - Checkpoints, - GenericCheckpointsMixin, -) -from .fileio import FileManagerMixin - -from jupyter_core.utils import ensure_dir_exists -from ipython_genutils.py3compat import getcwd -from traitlets import Unicode - -from notebook import _tz as tz - - -class FileCheckpoints(FileManagerMixin, Checkpoints): - """ - A Checkpoints that caches checkpoints for files in adjacent - directories. - - Only works with FileContentsManager. Use GenericFileCheckpoints if - you want file-based checkpoints with another ContentsManager. - """ - - checkpoint_dir = Unicode( - '.ipynb_checkpoints', - config=True, - help="""The directory name in which to keep file checkpoints - - This is a path relative to the file's own directory. - - By default, it is .ipynb_checkpoints - """, - ) - - root_dir = Unicode(config=True) - - def _root_dir_default(self): - try: - return self.parent.root_dir - except AttributeError: - return getcwd() - - # ContentsManager-dependent checkpoint API - def create_checkpoint(self, contents_mgr, path): - """Create a checkpoint.""" - checkpoint_id = u'checkpoint' - src_path = contents_mgr._get_os_path(path) - dest_path = self.checkpoint_path(checkpoint_id, path) - self._copy(src_path, dest_path) - return self.checkpoint_model(checkpoint_id, dest_path) - - def restore_checkpoint(self, contents_mgr, checkpoint_id, path): - """Restore a checkpoint.""" - src_path = self.checkpoint_path(checkpoint_id, path) - dest_path = contents_mgr._get_os_path(path) - self._copy(src_path, dest_path) - - # ContentsManager-independent checkpoint API - def rename_checkpoint(self, checkpoint_id, old_path, new_path): - """Rename a checkpoint from old_path to new_path.""" - old_cp_path = self.checkpoint_path(checkpoint_id, old_path) - new_cp_path = self.checkpoint_path(checkpoint_id, new_path) - if os.path.isfile(old_cp_path): - self.log.debug( - "Renaming checkpoint %s -> %s", - old_cp_path, - new_cp_path, - ) - with self.perm_to_403(): - shutil.move(old_cp_path, new_cp_path) - - def delete_checkpoint(self, checkpoint_id, path): - """delete a file's checkpoint""" - path = path.strip('/') - cp_path = self.checkpoint_path(checkpoint_id, path) - if not os.path.isfile(cp_path): - self.no_such_checkpoint(path, checkpoint_id) - - self.log.debug("unlinking %s", cp_path) - with self.perm_to_403(): - os.unlink(cp_path) - - def list_checkpoints(self, path): - """list the checkpoints for a given file - - This contents manager currently only supports one checkpoint per file. - """ - path = path.strip('/') - checkpoint_id = "checkpoint" - os_path = self.checkpoint_path(checkpoint_id, path) - if not os.path.isfile(os_path): - return [] - else: - return [self.checkpoint_model(checkpoint_id, os_path)] - - # Checkpoint-related utilities - def checkpoint_path(self, checkpoint_id, path): - """find the path to a checkpoint""" - path = path.strip('/') - parent, name = ('/' + path).rsplit('/', 1) - parent = parent.strip('/') - basename, ext = os.path.splitext(name) - filename = u"{name}-{checkpoint_id}{ext}".format( - name=basename, - checkpoint_id=checkpoint_id, - ext=ext, - ) - os_path = self._get_os_path(path=parent) - cp_dir = os.path.join(os_path, self.checkpoint_dir) - with self.perm_to_403(): - ensure_dir_exists(cp_dir) - cp_path = os.path.join(cp_dir, filename) - return cp_path - - def checkpoint_model(self, checkpoint_id, os_path): - """construct the info dict for a given checkpoint""" - stats = os.stat(os_path) - last_modified = tz.utcfromtimestamp(stats.st_mtime) - info = dict( - id=checkpoint_id, - last_modified=last_modified, - ) - return info - - # Error Handling - def no_such_checkpoint(self, path, checkpoint_id): - raise HTTPError( - 404, - u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id) - ) - - -class GenericFileCheckpoints(GenericCheckpointsMixin, FileCheckpoints): - """ - Local filesystem Checkpoints that works with any conforming - ContentsManager. - """ - def create_file_checkpoint(self, content, format, path): - """Create a checkpoint from the current content of a file.""" - path = path.strip('/') - # only the one checkpoint ID: - checkpoint_id = u"checkpoint" - os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) - self.log.debug("creating checkpoint for %s", path) - with self.perm_to_403(): - self._save_file(os_checkpoint_path, content, format=format) - - # return the checkpoint info - return self.checkpoint_model(checkpoint_id, os_checkpoint_path) - - def create_notebook_checkpoint(self, nb, path): - """Create a checkpoint from the current content of a notebook.""" - path = path.strip('/') - # only the one checkpoint ID: - checkpoint_id = u"checkpoint" - os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) - self.log.debug("creating checkpoint for %s", path) - with self.perm_to_403(): - self._save_notebook(os_checkpoint_path, nb) - - # return the checkpoint info - return self.checkpoint_model(checkpoint_id, os_checkpoint_path) - - def get_notebook_checkpoint(self, checkpoint_id, path): - """Get a checkpoint for a notebook.""" - path = path.strip('/') - self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) - os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) - - if not os.path.isfile(os_checkpoint_path): - self.no_such_checkpoint(path, checkpoint_id) - - return { - 'type': 'notebook', - 'content': self._read_notebook( - os_checkpoint_path, - as_version=4, - ), - } - - def get_file_checkpoint(self, checkpoint_id, path): - """Get a checkpoint for a file.""" - path = path.strip('/') - self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) - os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) - - if not os.path.isfile(os_checkpoint_path): - self.no_such_checkpoint(path, checkpoint_id) - - content, format = self._read_file(os_checkpoint_path, format=None) - return { - 'type': 'file', - 'content': content, - 'format': format, - } diff --git a/notebook/services/contents/fileio.py b/notebook/services/contents/fileio.py deleted file mode 100644 index 42b518c54e..0000000000 --- a/notebook/services/contents/fileio.py +++ /dev/null @@ -1,344 +0,0 @@ -""" -Utilities for file-based Contents/Checkpoints managers. -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from contextlib import contextmanager -import errno -import io -import os -import shutil - -from tornado.web import HTTPError - -from notebook.utils import ( - to_api_path, - to_os_path, -) -import nbformat - -from ipython_genutils.py3compat import str_to_unicode - -from traitlets.config import Configurable -from traitlets import Bool - -try: #PY3 - from base64 import encodebytes, decodebytes -except ImportError: #PY2 - from base64 import encodestring as encodebytes, decodestring as decodebytes - - -def replace_file(src, dst): - """ replace dst with src - - switches between os.replace or os.rename based on python 2.7 or python 3 - """ - if hasattr(os, 'replace'): # PY3 - os.replace(src, dst) - else: - if os.name == 'nt' and os.path.exists(dst): - # Rename over existing file doesn't work on Windows - os.remove(dst) - os.rename(src, dst) - -def copy2_safe(src, dst, log=None): - """copy src to dst - - like shutil.copy2, but log errors in copystat instead of raising - """ - shutil.copyfile(src, dst) - try: - shutil.copystat(src, dst) - except OSError: - if log: - log.debug("copystat on %s failed", dst, exc_info=True) - -def path_to_intermediate(path): - '''Name of the intermediate file used in atomic writes. - - The .~ prefix will make Dropbox ignore the temporary file.''' - dirname, basename = os.path.split(path) - return os.path.join(dirname, '.~'+basename) - -def path_to_invalid(path): - '''Name of invalid file after a failed atomic write and subsequent read.''' - dirname, basename = os.path.split(path) - return os.path.join(dirname, basename+'.invalid') - -@contextmanager -def atomic_writing(path, text=True, encoding='utf-8', log=None, **kwargs): - """Context manager to write to a file only if the entire write is successful. - - This works by copying the previous file contents to a temporary file in the - same directory, and renaming that file back to the target if the context - exits with an error. If the context is successful, the new data is synced to - disk and the temporary file is removed. - - Parameters - ---------- - path : str - The target file to write to. - - text : bool, optional - Whether to open the file in text mode (i.e. to write unicode). Default is - True. - - encoding : str, optional - The encoding to use for files opened in text mode. Default is UTF-8. - - **kwargs - Passed to :func:`io.open`. - """ - # realpath doesn't work on Windows: https://bugs.python.org/issue9949 - # Luckily, we only need to resolve the file itself being a symlink, not - # any of its directories, so this will suffice: - if os.path.islink(path): - path = os.path.join(os.path.dirname(path), os.readlink(path)) - - tmp_path = path_to_intermediate(path) - - if os.path.isfile(path): - copy2_safe(path, tmp_path, log=log) - - if text: - # Make sure that text files have Unix linefeeds by default - kwargs.setdefault('newline', '\n') - fileobj = io.open(path, 'w', encoding=encoding, **kwargs) - else: - fileobj = io.open(path, 'wb', **kwargs) - - try: - yield fileobj - except: - # Failed! Move the backup file back to the real path to avoid corruption - fileobj.close() - replace_file(tmp_path, path) - raise - - # Flush to disk - fileobj.flush() - os.fsync(fileobj.fileno()) - fileobj.close() - - # Written successfully, now remove the backup copy - if os.path.isfile(tmp_path): - os.remove(tmp_path) - - - -@contextmanager -def _simple_writing(path, text=True, encoding='utf-8', log=None, **kwargs): - """Context manager to write file without doing atomic writing - ( for weird filesystem eg: nfs). - - Parameters - ---------- - path : str - The target file to write to. - - text : bool, optional - Whether to open the file in text mode (i.e. to write unicode). Default is - True. - - encoding : str, optional - The encoding to use for files opened in text mode. Default is UTF-8. - - **kwargs - Passed to :func:`io.open`. - """ - # realpath doesn't work on Windows: https://bugs.python.org/issue9949 - # Luckily, we only need to resolve the file itself being a symlink, not - # any of its directories, so this will suffice: - if os.path.islink(path): - path = os.path.join(os.path.dirname(path), os.readlink(path)) - - if text: - # Make sure that text files have Unix linefeeds by default - kwargs.setdefault('newline', '\n') - fileobj = io.open(path, 'w', encoding=encoding, **kwargs) - else: - fileobj = io.open(path, 'wb', **kwargs) - - try: - yield fileobj - except: - fileobj.close() - raise - - fileobj.close() - - - - -class FileManagerMixin(Configurable): - """ - Mixin for ContentsAPI classes that interact with the filesystem. - - Provides facilities for reading, writing, and copying both notebooks and - generic files. - - Shared by FileContentsManager and FileCheckpoints. - - Note - ---- - Classes using this mixin must provide the following attributes: - - root_dir : unicode - A directory against against which API-style paths are to be resolved. - - log : logging.Logger - """ - - use_atomic_writing = Bool(True, config=True, help= - """By default notebooks are saved on disk on a temporary file and then if succefully written, it replaces the old ones. - This procedure, namely 'atomic_writing', causes some bugs on file system whitout operation order enforcement (like some networked fs). - If set to False, the new notebook is written directly on the old one which could fail (eg: full filesystem or quota )""") - - @contextmanager - def open(self, os_path, *args, **kwargs): - """wrapper around io.open that turns permission errors into 403""" - with self.perm_to_403(os_path): - with io.open(os_path, *args, **kwargs) as f: - yield f - - @contextmanager - def atomic_writing(self, os_path, *args, **kwargs): - """wrapper around atomic_writing that turns permission errors to 403. - Depending on flag 'use_atomic_writing', the wrapper perform an actual atomic writing or - simply writes the file (whatever an old exists or not)""" - with self.perm_to_403(os_path): - if self.use_atomic_writing: - with atomic_writing(os_path, *args, log=self.log, **kwargs) as f: - yield f - else: - with _simple_writing(os_path, *args, log=self.log, **kwargs) as f: - yield f - - @contextmanager - def perm_to_403(self, os_path=''): - """context manager for turning permission errors into 403.""" - try: - yield - except (OSError, IOError) as e: - if e.errno in {errno.EPERM, errno.EACCES}: - # make 403 error message without root prefix - # this may not work perfectly on unicode paths on Python 2, - # but nobody should be doing that anyway. - if not os_path: - os_path = str_to_unicode(e.filename or 'unknown file') - path = to_api_path(os_path, root=self.root_dir) - raise HTTPError(403, u'Permission denied: %s' % path) - else: - raise - - def _copy(self, src, dest): - """copy src to dest - - like shutil.copy2, but log errors in copystat - """ - copy2_safe(src, dest, log=self.log) - - def _get_os_path(self, path): - """Given an API path, return its file system path. - - Parameters - ---------- - path : string - The relative API path to the named file. - - Returns - ------- - path : string - Native, absolute OS path to for a file. - - Raises - ------ - 404: if path is outside root - """ - root = os.path.abspath(self.root_dir) - os_path = to_os_path(path, root) - if not (os.path.abspath(os_path) + os.path.sep).startswith(root): - raise HTTPError(404, "%s is outside root contents directory" % path) - return os_path - - def _read_notebook(self, os_path, as_version=4): - """Read a notebook from an os path.""" - with self.open(os_path, 'r', encoding='utf-8') as f: - try: - return nbformat.read(f, as_version=as_version) - except Exception as e: - e_orig = e - - # If use_atomic_writing is enabled, we'll guess that it was also - # enabled when this notebook was written and look for a valid - # atomic intermediate. - tmp_path = path_to_intermediate(os_path) - - if not self.use_atomic_writing or not os.path.exists(tmp_path): - raise HTTPError( - 400, - u"Unreadable Notebook: %s %r" % (os_path, e_orig), - ) - - # Move the bad file aside, restore the intermediate, and try again. - invalid_file = path_to_invalid(os_path) - replace_file(os_path, invalid_file) - replace_file(tmp_path, os_path) - return self._read_notebook(os_path, as_version) - - def _save_notebook(self, os_path, nb): - """Save a notebook to an os_path.""" - with self.atomic_writing(os_path, encoding='utf-8') as f: - nbformat.write(nb, f, version=nbformat.NO_CONVERT) - - def _read_file(self, os_path, format): - """Read a non-notebook file. - - os_path: The path to be read. - format: - If 'text', the contents will be decoded as UTF-8. - If 'base64', the raw bytes contents will be encoded as base64. - If not specified, try to decode as UTF-8, and fall back to base64 - """ - if not os.path.isfile(os_path): - raise HTTPError(400, "Cannot read non-file %s" % os_path) - - with self.open(os_path, 'rb') as f: - bcontent = f.read() - - if format is None or format == 'text': - # Try to interpret as unicode if format is unknown or if unicode - # was explicitly requested. - try: - return bcontent.decode('utf8'), 'text' - except UnicodeError: - if format == 'text': - raise HTTPError( - 400, - "%s is not UTF-8 encoded" % os_path, - reason='bad format', - ) - return encodebytes(bcontent).decode('ascii'), 'base64' - - def _save_file(self, os_path, content, format): - """Save content of a generic file.""" - if format not in {'text', 'base64'}: - raise HTTPError( - 400, - "Must specify format of file contents as 'text' or 'base64'", - ) - try: - if format == 'text': - bcontent = content.encode('utf8') - else: - b64_bytes = content.encode('ascii') - bcontent = decodebytes(b64_bytes) - except Exception as e: - raise HTTPError( - 400, u'Encoding error saving %s: %s' % (os_path, e) - ) - - with self.atomic_writing(os_path, text=False) as f: - f.write(bcontent) diff --git a/notebook/services/contents/filemanager.py b/notebook/services/contents/filemanager.py deleted file mode 100644 index 7f445a8f71..0000000000 --- a/notebook/services/contents/filemanager.py +++ /dev/null @@ -1,592 +0,0 @@ -"""A contents manager that uses the local file system for storage.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from datetime import datetime -import errno -import io -import os -import shutil -import stat -import sys -import warnings -import mimetypes -import nbformat - -from send2trash import send2trash -from tornado import web - -from .filecheckpoints import FileCheckpoints -from .fileio import FileManagerMixin -from .manager import ContentsManager -from ...utils import exists - -from ipython_genutils.importstring import import_item -from traitlets import Any, Unicode, Bool, TraitError, observe, default, validate -from ipython_genutils.py3compat import getcwd, string_types - -from notebook import _tz as tz -from notebook.utils import ( - is_hidden, is_file_hidden, - to_api_path, -) -from notebook.base.handlers import AuthenticatedFileHandler -from notebook.transutils import _ - -try: - from os.path import samefile -except ImportError: - # windows + py2 - from notebook.utils import samefile_simple as samefile - -_script_exporter = None - - -def _post_save_script(model, os_path, contents_manager, **kwargs): - """convert notebooks to Python script after save with nbconvert - - replaces `jupyter notebook --script` - """ - from nbconvert.exporters.script import ScriptExporter - warnings.warn("`_post_save_script` is deprecated and will be removed in Notebook 5.0", DeprecationWarning) - - if model['type'] != 'notebook': - return - - global _script_exporter - if _script_exporter is None: - _script_exporter = ScriptExporter(parent=contents_manager) - log = contents_manager.log - - base, ext = os.path.splitext(os_path) - script, resources = _script_exporter.from_filename(os_path) - script_fname = base + resources.get('output_extension', '.txt') - log.info("Saving script /%s", to_api_path(script_fname, contents_manager.root_dir)) - with io.open(script_fname, 'w', encoding='utf-8') as f: - f.write(script) - - -class FileContentsManager(FileManagerMixin, ContentsManager): - - root_dir = Unicode(config=True) - - @default('root_dir') - def _default_root_dir(self): - try: - return self.parent.notebook_dir - except AttributeError: - return getcwd() - - save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0') - @observe('save_script') - def _update_save_script(self, change): - if not change['new']: - return - self.log.warning(""" - `--script` is deprecated and will be removed in notebook 5.0. - - You can trigger nbconvert via pre- or post-save hooks: - - ContentsManager.pre_save_hook - FileContentsManager.post_save_hook - - A post-save hook has been registered that calls: - - jupyter nbconvert --to script [notebook] - - which behaves similarly to `--script`. - """) - - self.post_save_hook = _post_save_script - - post_save_hook = Any(None, config=True, allow_none=True, - help="""Python callable or importstring thereof - - to be called on the path of a file just saved. - - This can be used to process the file on disk, - such as converting the notebook to a script or HTML via nbconvert. - - It will be called as (all arguments passed by keyword):: - - hook(os_path=os_path, model=model, contents_manager=instance) - - - path: the filesystem path to the file just written - - model: the model representing the file - - contents_manager: this ContentsManager instance - """ - ) - - @validate('post_save_hook') - def _validate_post_save_hook(self, proposal): - value = proposal['value'] - if isinstance(value, string_types): - value = import_item(value) - if not callable(value): - raise TraitError("post_save_hook must be callable") - return value - - def run_post_save_hook(self, model, os_path): - """Run the post-save hook if defined, and log errors""" - if self.post_save_hook: - try: - self.log.debug("Running post-save hook on %s", os_path) - self.post_save_hook(os_path=os_path, model=model, contents_manager=self) - except Exception as e: - self.log.error("Post-save hook failed o-n %s", os_path, exc_info=True) - raise web.HTTPError(500, u'Unexpected error while running post hook save: %s' % e) - - @validate('root_dir') - def _validate_root_dir(self, proposal): - """Do a bit of validation of the root_dir.""" - value = proposal['value'] - if not os.path.isabs(value): - # If we receive a non-absolute path, make it absolute. - value = os.path.abspath(value) - if not os.path.isdir(value): - raise TraitError("%r is not a directory" % value) - return value - - @default('checkpoints_class') - def _checkpoints_class_default(self): - return FileCheckpoints - - delete_to_trash = Bool(True, config=True, - help="""If True (default), deleting files will send them to the - platform's trash/recycle bin, where they can be recovered. If False, - deleting files really deletes them.""") - - @default('files_handler_class') - def _files_handler_class_default(self): - return AuthenticatedFileHandler - - @default('files_handler_params') - def _files_handler_params_default(self): - return {'path': self.root_dir} - - def is_hidden(self, path): - """Does the API style path correspond to a hidden directory or file? - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to root_dir). - - Returns - ------- - hidden : bool - Whether the path exists and is hidden. - """ - path = path.strip('/') - os_path = self._get_os_path(path=path) - return is_hidden(os_path, self.root_dir) - - def file_exists(self, path): - """Returns True if the file exists, else returns False. - - API-style wrapper for os.path.isfile - - Parameters - ---------- - path : string - The relative path to the file (with '/' as separator) - - Returns - ------- - exists : bool - Whether the file exists. - """ - path = path.strip('/') - os_path = self._get_os_path(path) - return os.path.isfile(os_path) - - def dir_exists(self, path): - """Does the API-style path refer to an extant directory? - - API-style wrapper for os.path.isdir - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to root_dir). - - Returns - ------- - exists : bool - Whether the path is indeed a directory. - """ - path = path.strip('/') - os_path = self._get_os_path(path=path) - return os.path.isdir(os_path) - - def exists(self, path): - """Returns True if the path exists, else returns False. - - API-style wrapper for os.path.exists - - Parameters - ---------- - path : string - The API path to the file (with '/' as separator) - - Returns - ------- - exists : bool - Whether the target exists. - """ - path = path.strip('/') - os_path = self._get_os_path(path=path) - return exists(os_path) - - def _base_model(self, path): - """Build the common base of a contents model""" - os_path = self._get_os_path(path) - info = os.lstat(os_path) - - try: - # size of file - size = info.st_size - except (ValueError, OSError): - self.log.warning('Unable to get size.') - size = None - - try: - last_modified = tz.utcfromtimestamp(info.st_mtime) - except (ValueError, OSError): - # Files can rarely have an invalid timestamp - # https://github.com/jupyter/notebook/issues/2539 - # https://github.com/jupyter/notebook/issues/2757 - # Use the Unix epoch as a fallback so we don't crash. - self.log.warning('Invalid mtime %s for %s', info.st_mtime, os_path) - last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) - - try: - created = tz.utcfromtimestamp(info.st_ctime) - except (ValueError, OSError): # See above - self.log.warning('Invalid ctime %s for %s', info.st_ctime, os_path) - created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) - - # Create the base model. - model = {} - model['name'] = path.rsplit('/', 1)[-1] - model['path'] = path - model['last_modified'] = last_modified - model['created'] = created - model['content'] = None - model['format'] = None - model['mimetype'] = None - model['size'] = size - - try: - model['writable'] = os.access(os_path, os.W_OK) - except OSError: - self.log.error("Failed to check write permissions on %s", os_path) - model['writable'] = False - return model - - def _dir_model(self, path, content=True): - """Build a model for a directory - - if content is requested, will include a listing of the directory - """ - os_path = self._get_os_path(path) - - four_o_four = u'directory does not exist: %r' % path - - if not os.path.isdir(os_path): - raise web.HTTPError(404, four_o_four) - elif is_hidden(os_path, self.root_dir) and not self.allow_hidden: - self.log.info("Refusing to serve hidden directory %r, via 404 Error", - os_path - ) - raise web.HTTPError(404, four_o_four) - - model = self._base_model(path) - model['type'] = 'directory' - model['size'] = None - if content: - model['content'] = contents = [] - os_dir = self._get_os_path(path) - for name in os.listdir(os_dir): - try: - os_path = os.path.join(os_dir, name) - except UnicodeDecodeError as e: - self.log.warning( - "failed to decode filename '%s': %s", name, e) - continue - - try: - st = os.lstat(os_path) - except OSError as e: - # skip over broken symlinks in listing - if e.errno == errno.ENOENT: - self.log.warning("%s doesn't exist", os_path) - else: - self.log.warning("Error stat-ing %s: %s", os_path, e) - continue - - if (not stat.S_ISLNK(st.st_mode) - and not stat.S_ISREG(st.st_mode) - and not stat.S_ISDIR(st.st_mode)): - self.log.debug("%s not a regular file", os_path) - continue - - if self.should_list(name): - if self.allow_hidden or not is_file_hidden(os_path, stat_res=st): - contents.append( - self.get(path='%s/%s' % (path, name), content=False) - ) - - model['format'] = 'json' - - return model - - - def _file_model(self, path, content=True, format=None): - """Build a model for a file - - if content is requested, include the file contents. - - format: - If 'text', the contents will be decoded as UTF-8. - If 'base64', the raw bytes contents will be encoded as base64. - If not specified, try to decode as UTF-8, and fall back to base64 - """ - model = self._base_model(path) - model['type'] = 'file' - - os_path = self._get_os_path(path) - model['mimetype'] = mimetypes.guess_type(os_path)[0] - - if content: - content, format = self._read_file(os_path, format) - if model['mimetype'] is None: - default_mime = { - 'text': 'text/plain', - 'base64': 'application/octet-stream' - }[format] - model['mimetype'] = default_mime - - model.update( - content=content, - format=format, - ) - - return model - - def _notebook_model(self, path, content=True): - """Build a notebook model - - if content is requested, the notebook content will be populated - as a JSON structure (not double-serialized) - """ - model = self._base_model(path) - model['type'] = 'notebook' - os_path = self._get_os_path(path) - - if content: - nb = self._read_notebook(os_path, as_version=4) - self.mark_trusted_cells(nb, path) - model['content'] = nb - model['format'] = 'json' - self.validate_notebook_model(model) - - return model - - def get(self, path, content=True, type=None, format=None): - """ Takes a path for an entity and returns its model - - Parameters - ---------- - path : str - the API path that describes the relative path for the target - content : bool - Whether to include the contents in the reply - type : str, optional - The requested type - 'file', 'notebook', or 'directory'. - Will raise HTTPError 400 if the content doesn't match. - format : str, optional - The requested format for file contents. 'text' or 'base64'. - Ignored if this returns a notebook or directory model. - - Returns - ------- - model : dict - the contents model. If content=True, returns the contents - of the file or directory as well. - """ - path = path.strip('/') - - if not self.exists(path): - raise web.HTTPError(404, u'No such file or directory: %s' % path) - - os_path = self._get_os_path(path) - if os.path.isdir(os_path): - if type not in (None, 'directory'): - raise web.HTTPError(400, - u'%s is a directory, not a %s' % (path, type), reason='bad type') - model = self._dir_model(path, content=content) - elif type == 'notebook' or (type is None and path.endswith('.ipynb')): - model = self._notebook_model(path, content=content) - else: - if type == 'directory': - raise web.HTTPError(400, - u'%s is not a directory' % path, reason='bad type') - model = self._file_model(path, content=content, format=format) - return model - - def _save_directory(self, os_path, model, path=''): - """create a directory""" - if is_hidden(os_path, self.root_dir) and not self.allow_hidden: - raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path) - if not os.path.exists(os_path): - with self.perm_to_403(): - os.mkdir(os_path) - elif not os.path.isdir(os_path): - raise web.HTTPError(400, u'Not a directory: %s' % (os_path)) - else: - self.log.debug("Directory %r already exists", os_path) - - def save(self, model, path=''): - """Save the file model and return the model with no content.""" - path = path.strip('/') - - if 'type' not in model: - raise web.HTTPError(400, u'No file type provided') - if 'content' not in model and model['type'] != 'directory': - raise web.HTTPError(400, u'No file content provided') - - os_path = self._get_os_path(path) - self.log.debug("Saving %s", os_path) - - self.run_pre_save_hook(model=model, path=path) - - try: - if model['type'] == 'notebook': - nb = nbformat.from_dict(model['content']) - self.check_and_sign(nb, path) - self._save_notebook(os_path, nb) - # One checkpoint should always exist for notebooks. - if not self.checkpoints.list_checkpoints(path): - self.create_checkpoint(path) - elif model['type'] == 'file': - # Missing format will be handled internally by _save_file. - self._save_file(os_path, model['content'], model.get('format')) - elif model['type'] == 'directory': - self._save_directory(os_path, model, path) - else: - raise web.HTTPError(400, "Unhandled contents type: %s" % model['type']) - except web.HTTPError: - raise - except Exception as e: - self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True) - raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e)) - - validation_message = None - if model['type'] == 'notebook': - self.validate_notebook_model(model) - validation_message = model.get('message', None) - - model = self.get(path, content=False) - if validation_message: - model['message'] = validation_message - - self.run_post_save_hook(model=model, os_path=os_path) - - return model - - def delete_file(self, path): - """Delete file at path.""" - path = path.strip('/') - os_path = self._get_os_path(path) - rm = os.unlink - if not os.path.exists(os_path): - raise web.HTTPError(404, u'File or directory does not exist: %s' % os_path) - - def _check_trash(os_path): - if sys.platform in {'win32', 'darwin'}: - return True - - # It's a bit more nuanced than this, but until we can better - # distinguish errors from send2trash, assume that we can only trash - # files on the same partition as the home directory. - file_dev = os.stat(os_path).st_dev - home_dev = os.stat(os.path.expanduser('~')).st_dev - return file_dev == home_dev - - def is_non_empty_dir(os_path): - if os.path.isdir(os_path): - # A directory containing only leftover checkpoints is - # considered empty. - cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None) - if set(os.listdir(os_path)) - {cp_dir}: - return True - - return False - - if self.delete_to_trash: - if sys.platform == 'win32' and is_non_empty_dir(os_path): - # send2trash can really delete files on Windows, so disallow - # deleting non-empty files. See Github issue 3631. - raise web.HTTPError(400, u'Directory %s not empty' % os_path) - if _check_trash(os_path): - self.log.debug("Sending %s to trash", os_path) - # Looking at the code in send2trash, I don't think the errors it - # raises let us distinguish permission errors from other errors in - # code. So for now, just let them all get logged as server errors. - send2trash(os_path) - return - else: - self.log.warning("Skipping trash for %s, on different device " - "to home directory", os_path) - - if os.path.isdir(os_path): - # Don't permanently delete non-empty directories. - if is_non_empty_dir(os_path): - raise web.HTTPError(400, u'Directory %s not empty' % os_path) - self.log.debug("Removing directory %s", os_path) - with self.perm_to_403(): - shutil.rmtree(os_path) - else: - self.log.debug("Unlinking file %s", os_path) - with self.perm_to_403(): - rm(os_path) - - def rename_file(self, old_path, new_path): - """Rename a file.""" - old_path = old_path.strip('/') - new_path = new_path.strip('/') - if new_path == old_path: - return - - new_os_path = self._get_os_path(new_path) - old_os_path = self._get_os_path(old_path) - - # Should we proceed with the move? - if os.path.exists(new_os_path) and not samefile(old_os_path, new_os_path): - raise web.HTTPError(409, u'File already exists: %s' % new_path) - - # Move the file - try: - with self.perm_to_403(): - shutil.move(old_os_path, new_os_path) - except web.HTTPError: - raise - except Exception as e: - raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e)) - - def info_string(self): - return _("Serving notebooks from local directory: %s") % self.root_dir - - def get_kernel_path(self, path, model=None): - """Return the initial API path of a kernel associated with a given notebook""" - if self.dir_exists(path): - return path - if '/' in path: - parent_dir = path.rsplit('/', 1)[0] - else: - parent_dir = '' - return parent_dir diff --git a/notebook/services/contents/handlers.py b/notebook/services/contents/handlers.py deleted file mode 100644 index 9f782add50..0000000000 --- a/notebook/services/contents/handlers.py +++ /dev/null @@ -1,327 +0,0 @@ -"""Tornado handlers for the contents web service. - -Preliminary documentation at https://github.com/ipython/ipython/wiki/IPEP-27%3A-Contents-Service -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json - -from tornado import gen, web - -from notebook.utils import maybe_future, url_path_join, url_escape -from jupyter_client.jsonutil import date_default - -from notebook.base.handlers import ( - IPythonHandler, APIHandler, path_regex, -) - - -def validate_model(model, expect_content): - """ - Validate a model returned by a ContentsManager method. - - If expect_content is True, then we expect non-null entries for 'content' - and 'format'. - """ - required_keys = { - "name", - "path", - "type", - "writable", - "created", - "last_modified", - "mimetype", - "content", - "format", - } - missing = required_keys - set(model.keys()) - if missing: - raise web.HTTPError( - 500, - u"Missing Model Keys: {missing}".format(missing=missing), - ) - - maybe_none_keys = ['content', 'format'] - if expect_content: - errors = [key for key in maybe_none_keys if model[key] is None] - if errors: - raise web.HTTPError( - 500, - u"Keys unexpectedly None: {keys}".format(keys=errors), - ) - else: - errors = { - key: model[key] - for key in maybe_none_keys - if model[key] is not None - } - if errors: - raise web.HTTPError( - 500, - u"Keys unexpectedly not None: {keys}".format(keys=errors), - ) - - -class ContentsHandler(APIHandler): - - def location_url(self, path): - """Return the full URL location of a file. - - Parameters - ---------- - path : unicode - The API path of the file, such as "foo/bar.txt". - """ - return url_path_join( - self.base_url, 'api', 'contents', url_escape(path) - ) - - def _finish_model(self, model, location=True): - """Finish a JSON request with a model, setting relevant headers, etc.""" - if location: - location = self.location_url(model['path']) - self.set_header('Location', location) - self.set_header('Last-Modified', model['last_modified']) - self.set_header('Content-Type', 'application/json') - self.finish(json.dumps(model, default=date_default)) - - @web.authenticated - @gen.coroutine - def get(self, path=''): - """Return a model for a file or directory. - - A directory model contains a list of models (without content) - of the files and directories it contains. - """ - path = path or '' - type = self.get_query_argument('type', default=None) - if type not in {None, 'directory', 'file', 'notebook'}: - raise web.HTTPError(400, u'Type %r is invalid' % type) - - format = self.get_query_argument('format', default=None) - if format not in {None, 'text', 'base64'}: - raise web.HTTPError(400, u'Format %r is invalid' % format) - content = self.get_query_argument('content', default='1') - if content not in {'0', '1'}: - raise web.HTTPError(400, u'Content %r is invalid' % content) - content = int(content) - - model = yield maybe_future(self.contents_manager.get( - path=path, type=type, format=format, content=content, - )) - validate_model(model, expect_content=content) - self._finish_model(model, location=False) - - @web.authenticated - @gen.coroutine - def patch(self, path=''): - """PATCH renames a file or directory without re-uploading content.""" - cm = self.contents_manager - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, u'JSON body missing') - model = yield maybe_future(cm.update(model, path)) - validate_model(model, expect_content=False) - self._finish_model(model) - - @gen.coroutine - def _copy(self, copy_from, copy_to=None): - """Copy a file, optionally specifying a target directory.""" - self.log.info(u"Copying {copy_from} to {copy_to}".format( - copy_from=copy_from, - copy_to=copy_to or '', - )) - model = yield maybe_future(self.contents_manager.copy(copy_from, copy_to)) - self.set_status(201) - validate_model(model, expect_content=False) - self._finish_model(model) - - @gen.coroutine - def _upload(self, model, path): - """Handle upload of a new file to path""" - self.log.info(u"Uploading file to %s", path) - model = yield maybe_future(self.contents_manager.new(model, path)) - self.set_status(201) - validate_model(model, expect_content=False) - self._finish_model(model) - - @gen.coroutine - def _new_untitled(self, path, type='', ext=''): - """Create a new, empty untitled entity""" - self.log.info(u"Creating new %s in %s", type or 'file', path) - model = yield maybe_future(self.contents_manager.new_untitled(path=path, type=type, ext=ext)) - self.set_status(201) - validate_model(model, expect_content=False) - self._finish_model(model) - - @gen.coroutine - def _save(self, model, path): - """Save an existing file.""" - chunk = model.get("chunk", None) - if not chunk or chunk == -1: # Avoid tedious log information - self.log.info(u"Saving file at %s", path) - model = yield maybe_future(self.contents_manager.save(model, path)) - validate_model(model, expect_content=False) - self._finish_model(model) - - @web.authenticated - @gen.coroutine - def post(self, path=''): - """Create a new file in the specified path. - - POST creates new files. The server always decides on the name. - - POST /api/contents/path - New untitled, empty file or directory. - POST /api/contents/path - with body {"copy_from" : "/path/to/OtherNotebook.ipynb"} - New copy of OtherNotebook in path - """ - - cm = self.contents_manager - - file_exists = yield maybe_future(cm.file_exists(path)) - if file_exists: - raise web.HTTPError(400, "Cannot POST to files, use PUT instead.") - - dir_exists = yield maybe_future(cm.dir_exists(path)) - if not dir_exists: - raise web.HTTPError(404, "No such directory: %s" % path) - - model = self.get_json_body() - - if model is not None: - copy_from = model.get('copy_from') - ext = model.get('ext', '') - type = model.get('type', '') - if copy_from: - yield self._copy(copy_from, path) - else: - yield self._new_untitled(path, type=type, ext=ext) - else: - yield self._new_untitled(path) - - @web.authenticated - @gen.coroutine - def put(self, path=''): - """Saves the file in the location specified by name and path. - - PUT is very similar to POST, but the requester specifies the name, - whereas with POST, the server picks the name. - - PUT /api/contents/path/Name.ipynb - Save notebook at ``path/Name.ipynb``. Notebook structure is specified - in `content` key of JSON request body. If content is not specified, - create a new empty notebook. - """ - model = self.get_json_body() - if model: - if model.get('copy_from'): - raise web.HTTPError(400, "Cannot copy with PUT, only POST") - exists = yield maybe_future(self.contents_manager.file_exists(path)) - if exists: - yield maybe_future(self._save(model, path)) - else: - yield maybe_future(self._upload(model, path)) - else: - yield maybe_future(self._new_untitled(path)) - - @web.authenticated - @gen.coroutine - def delete(self, path=''): - """delete a file in the given path""" - cm = self.contents_manager - self.log.warning('delete %s', path) - yield maybe_future(cm.delete(path)) - self.set_status(204) - self.finish() - - -class CheckpointsHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self, path=''): - """get lists checkpoints for a file""" - cm = self.contents_manager - checkpoints = yield maybe_future(cm.list_checkpoints(path)) - data = json.dumps(checkpoints, default=date_default) - self.finish(data) - - @web.authenticated - @gen.coroutine - def post(self, path=''): - """post creates a new checkpoint""" - cm = self.contents_manager - checkpoint = yield maybe_future(cm.create_checkpoint(path)) - data = json.dumps(checkpoint, default=date_default) - location = url_path_join(self.base_url, 'api/contents', - url_escape(path), 'checkpoints', url_escape(checkpoint['id'])) - self.set_header('Location', location) - self.set_status(201) - self.finish(data) - - -class ModifyCheckpointsHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def post(self, path, checkpoint_id): - """post restores a file from a checkpoint""" - cm = self.contents_manager - yield maybe_future(cm.restore_checkpoint(checkpoint_id, path)) - self.set_status(204) - self.finish() - - @web.authenticated - @gen.coroutine - def delete(self, path, checkpoint_id): - """delete clears a checkpoint for a given file""" - cm = self.contents_manager - yield maybe_future(cm.delete_checkpoint(checkpoint_id, path)) - self.set_status(204) - self.finish() - - -class NotebooksRedirectHandler(IPythonHandler): - """Redirect /api/notebooks to /api/contents""" - SUPPORTED_METHODS = ('GET', 'PUT', 'PATCH', 'POST', 'DELETE') - - def get(self, path): - self.log.warning("/api/notebooks is deprecated, use /api/contents") - self.redirect(url_path_join( - self.base_url, - 'api/contents', - path - )) - - put = patch = post = delete = get - - -class TrustNotebooksHandler(IPythonHandler): - """ Handles trust/signing of notebooks """ - - @web.authenticated - @gen.coroutine - def post(self,path=''): - cm = self.contents_manager - yield maybe_future(cm.trust_notebook(path)) - self.set_status(201) - self.finish() -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - - -_checkpoint_id_regex = r"(?P[\w-]+)" - -default_handlers = [ - (r"/api/contents%s/checkpoints" % path_regex, CheckpointsHandler), - (r"/api/contents%s/checkpoints/%s" % (path_regex, _checkpoint_id_regex), - ModifyCheckpointsHandler), - (r"/api/contents%s/trust" % path_regex, TrustNotebooksHandler), - (r"/api/contents%s" % path_regex, ContentsHandler), - (r"/api/notebooks/?(.*)", NotebooksRedirectHandler), -] diff --git a/notebook/services/contents/largefilemanager.py b/notebook/services/contents/largefilemanager.py deleted file mode 100644 index 10808ba83f..0000000000 --- a/notebook/services/contents/largefilemanager.py +++ /dev/null @@ -1,70 +0,0 @@ -from notebook.services.contents.filemanager import FileContentsManager -from contextlib import contextmanager -from tornado import web -import nbformat -import base64 -import os, io - -class LargeFileManager(FileContentsManager): - """Handle large file upload.""" - - def save(self, model, path=''): - """Save the file model and return the model with no content.""" - chunk = model.get('chunk', None) - if chunk is not None: - path = path.strip('/') - - if 'type' not in model: - raise web.HTTPError(400, u'No file type provided') - if model['type'] != 'file': - raise web.HTTPError(400, u'File type "{}" is not supported for large file transfer'.format(model['type'])) - if 'content' not in model and model['type'] != 'directory': - raise web.HTTPError(400, u'No file content provided') - - os_path = self._get_os_path(path) - - try: - if chunk == 1: - self.log.debug("Saving %s", os_path) - self.run_pre_save_hook(model=model, path=path) - super(LargeFileManager, self)._save_file(os_path, model['content'], model.get('format')) - else: - self._save_large_file(os_path, model['content'], model.get('format')) - except web.HTTPError: - raise - except Exception as e: - self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True) - raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e)) - - model = self.get(path, content=False) - - # Last chunk - if chunk == -1: - self.run_post_save_hook(model=model, os_path=os_path) - return model - else: - return super(LargeFileManager, self).save(model, path) - - def _save_large_file(self, os_path, content, format): - """Save content of a generic file.""" - if format not in {'text', 'base64'}: - raise web.HTTPError( - 400, - "Must specify format of file contents as 'text' or 'base64'", - ) - try: - if format == 'text': - bcontent = content.encode('utf8') - else: - b64_bytes = content.encode('ascii') - bcontent = base64.b64decode(b64_bytes) - except Exception as e: - raise web.HTTPError( - 400, u'Encoding error saving %s: %s' % (os_path, e) - ) - - with self.perm_to_403(os_path): - if os.path.islink(os_path): - os_path = os.path.join(os.path.dirname(os_path), os.readlink(os_path)) - with io.open(os_path, 'ab') as f: - f.write(bcontent) diff --git a/notebook/services/contents/manager.py b/notebook/services/contents/manager.py deleted file mode 100644 index df59ae7c05..0000000000 --- a/notebook/services/contents/manager.py +++ /dev/null @@ -1,527 +0,0 @@ -"""A base class for contents managers.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from fnmatch import fnmatch -import itertools -import json -import os -import re - -from tornado.web import HTTPError, RequestHandler - -from ...files.handlers import FilesHandler -from .checkpoints import Checkpoints -from traitlets.config.configurable import LoggingConfigurable -from nbformat import sign, validate as validate_nb, ValidationError -from nbformat.v4 import new_notebook -from ipython_genutils.importstring import import_item -from traitlets import ( - Any, - Bool, - Dict, - Instance, - List, - TraitError, - Type, - Unicode, - validate, - default, -) -from ipython_genutils.py3compat import string_types -from notebook.base.handlers import IPythonHandler -from notebook.transutils import _ - - -copy_pat = re.compile(r'\-Copy\d*\.') - - -class ContentsManager(LoggingConfigurable): - """Base class for serving files and directories. - - This serves any text or binary file, - as well as directories, - with special handling for JSON notebook documents. - - Most APIs take a path argument, - which is always an API-style unicode path, - and always refers to a directory. - - - unicode, not url-escaped - - '/'-separated - - leading and trailing '/' will be stripped - - if unspecified, path defaults to '', - indicating the root path. - - """ - - root_dir = Unicode('/', config=True) - - allow_hidden = Bool(False, config=True, help="Allow access to hidden files") - - notary = Instance(sign.NotebookNotary) - def _notary_default(self): - return sign.NotebookNotary(parent=self) - - hide_globs = List(Unicode(), [ - u'__pycache__', '*.pyc', '*.pyo', - '.DS_Store', '*.so', '*.dylib', '*~', - ], config=True, help=""" - Glob patterns to hide in file and directory listings. - """) - - untitled_notebook = Unicode(_("Untitled"), config=True, - help="The base name used when creating untitled notebooks." - ) - - untitled_file = Unicode("untitled", config=True, - help="The base name used when creating untitled files." - ) - - untitled_directory = Unicode("Untitled Folder", config=True, - help="The base name used when creating untitled directories." - ) - - pre_save_hook = Any(None, config=True, allow_none=True, - help="""Python callable or importstring thereof - - To be called on a contents model prior to save. - - This can be used to process the structure, - such as removing notebook outputs or other side effects that - should not be saved. - - It will be called as (all arguments passed by keyword):: - - hook(path=path, model=model, contents_manager=self) - - - model: the model to be saved. Includes file contents. - Modifying this dict will affect the file that is stored. - - path: the API path of the save destination - - contents_manager: this ContentsManager instance - """ - ) - - @validate('pre_save_hook') - def _validate_pre_save_hook(self, proposal): - value = proposal['value'] - if isinstance(value, string_types): - value = import_item(self.pre_save_hook) - if not callable(value): - raise TraitError("pre_save_hook must be callable") - return value - - def run_pre_save_hook(self, model, path, **kwargs): - """Run the pre-save hook if defined, and log errors""" - if self.pre_save_hook: - try: - self.log.debug("Running pre-save hook on %s", path) - self.pre_save_hook(model=model, path=path, contents_manager=self, **kwargs) - except Exception: - self.log.error("Pre-save hook failed on %s", path, exc_info=True) - - checkpoints_class = Type(Checkpoints, config=True) - checkpoints = Instance(Checkpoints, config=True) - checkpoints_kwargs = Dict(config=True) - - @default('checkpoints') - def _default_checkpoints(self): - return self.checkpoints_class(**self.checkpoints_kwargs) - - @default('checkpoints_kwargs') - def _default_checkpoints_kwargs(self): - return dict( - parent=self, - log=self.log, - ) - - files_handler_class = Type( - FilesHandler, klass=RequestHandler, allow_none=True, config=True, - help="""handler class to use when serving raw file requests. - - Default is a fallback that talks to the ContentsManager API, - which may be inefficient, especially for large files. - - Local files-based ContentsManagers can use a StaticFileHandler subclass, - which will be much more efficient. - - Access to these files should be Authenticated. - """ - ) - - files_handler_params = Dict( - config=True, - help="""Extra parameters to pass to files_handler_class. - - For example, StaticFileHandlers generally expect a `path` argument - specifying the root directory from which to serve files. - """ - ) - - def get_extra_handlers(self): - """Return additional handlers - - Default: self.files_handler_class on /files/.* - """ - handlers = [] - if self.files_handler_class: - handlers.append( - (r"/files/(.*)", self.files_handler_class, self.files_handler_params) - ) - return handlers - - # ContentsManager API part 1: methods that must be - # implemented in subclasses. - - def dir_exists(self, path): - """Does a directory exist at the given path? - - Like os.path.isdir - - Override this method in subclasses. - - Parameters - ---------- - path : string - The path to check - - Returns - ------- - exists : bool - Whether the path does indeed exist. - """ - raise NotImplementedError - - def is_hidden(self, path): - """Is path a hidden directory or file? - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to root dir). - - Returns - ------- - hidden : bool - Whether the path is hidden. - - """ - raise NotImplementedError - - def file_exists(self, path=''): - """Does a file exist at the given path? - - Like os.path.isfile - - Override this method in subclasses. - - Parameters - ---------- - path : string - The API path of a file to check for. - - Returns - ------- - exists : bool - Whether the file exists. - """ - raise NotImplementedError('must be implemented in a subclass') - - def exists(self, path): - """Does a file or directory exist at the given path? - - Like os.path.exists - - Parameters - ---------- - path : string - The API path of a file or directory to check for. - - Returns - ------- - exists : bool - Whether the target exists. - """ - return self.file_exists(path) or self.dir_exists(path) - - def get(self, path, content=True, type=None, format=None): - """Get a file or directory model.""" - raise NotImplementedError('must be implemented in a subclass') - - def save(self, model, path): - """ - Save a file or directory model to path. - - Should return the saved model with no content. Save implementations - should call self.run_pre_save_hook(model=model, path=path) prior to - writing any data. - """ - raise NotImplementedError('must be implemented in a subclass') - - def delete_file(self, path): - """Delete the file or directory at path.""" - raise NotImplementedError('must be implemented in a subclass') - - def rename_file(self, old_path, new_path): - """Rename a file or directory.""" - raise NotImplementedError('must be implemented in a subclass') - - # ContentsManager API part 2: methods that have useable default - # implementations, but can be overridden in subclasses. - - def delete(self, path): - """Delete a file/directory and any associated checkpoints.""" - path = path.strip('/') - if not path: - raise HTTPError(400, "Can't delete root") - self.delete_file(path) - self.checkpoints.delete_all_checkpoints(path) - - def rename(self, old_path, new_path): - """Rename a file and any checkpoints associated with that file.""" - self.rename_file(old_path, new_path) - self.checkpoints.rename_all_checkpoints(old_path, new_path) - - def update(self, model, path): - """Update the file's path - - For use in PATCH requests, to enable renaming a file without - re-uploading its contents. Only used for renaming at the moment. - """ - path = path.strip('/') - new_path = model.get('path', path).strip('/') - if path != new_path: - self.rename(path, new_path) - model = self.get(new_path, content=False) - return model - - def info_string(self): - return "Serving contents" - - def get_kernel_path(self, path, model=None): - """Return the API path for the kernel - - KernelManagers can turn this value into a filesystem path, - or ignore it altogether. - - The default value here will start kernels in the directory of the - notebook server. FileContentsManager overrides this to use the - directory containing the notebook. - """ - return '' - - def increment_filename(self, filename, path='', insert=''): - """Increment a filename until it is unique. - - Parameters - ---------- - filename : unicode - The name of a file, including extension - path : unicode - The API path of the target's directory - insert: unicode - The characters to insert after the base filename - - Returns - ------- - name : unicode - A filename that is unique, based on the input filename. - """ - # Extract the full suffix from the filename (e.g. .tar.gz) - path = path.strip('/') - basename, dot, ext = filename.partition('.') - suffix = dot + ext - - for i in itertools.count(): - if i: - insert_i = '{}{}'.format(insert, i) - else: - insert_i = '' - name = u'{basename}{insert}{suffix}'.format(basename=basename, - insert=insert_i, suffix=suffix) - if not self.exists(u'{}/{}'.format(path, name)): - break - return name - - def validate_notebook_model(self, model): - """Add failed-validation message to model""" - try: - validate_nb(model['content']) - except ValidationError as e: - model['message'] = u'Notebook validation failed: {}:\n{}'.format( - e.message, json.dumps(e.instance, indent=1, default=lambda obj: ''), - ) - return model - - def new_untitled(self, path='', type='', ext=''): - """Create a new untitled file or directory in path - - path must be a directory - - File extension can be specified. - - Use `new` to create files with a fully specified path (including filename). - """ - path = path.strip('/') - if not self.dir_exists(path): - raise HTTPError(404, 'No such directory: %s' % path) - - model = {} - if type: - model['type'] = type - - if ext == '.ipynb': - model.setdefault('type', 'notebook') - else: - model.setdefault('type', 'file') - - insert = '' - if model['type'] == 'directory': - untitled = self.untitled_directory - insert = ' ' - elif model['type'] == 'notebook': - untitled = self.untitled_notebook - ext = '.ipynb' - elif model['type'] == 'file': - untitled = self.untitled_file - else: - raise HTTPError(400, "Unexpected model type: %r" % model['type']) - - name = self.increment_filename(untitled + ext, path, insert=insert) - path = u'{0}/{1}'.format(path, name) - return self.new(model, path) - - def new(self, model=None, path=''): - """Create a new file or directory and return its model with no content. - - To create a new untitled entity in a directory, use `new_untitled`. - """ - path = path.strip('/') - if model is None: - model = {} - - if path.endswith('.ipynb'): - model.setdefault('type', 'notebook') - else: - model.setdefault('type', 'file') - - # no content, not a directory, so fill out new-file model - if 'content' not in model and model['type'] != 'directory': - if model['type'] == 'notebook': - model['content'] = new_notebook() - model['format'] = 'json' - else: - model['content'] = '' - model['type'] = 'file' - model['format'] = 'text' - - model = self.save(model, path) - return model - - def copy(self, from_path, to_path=None): - """Copy an existing file and return its new model. - - If to_path not specified, it will be the parent directory of from_path. - If to_path is a directory, filename will increment `from_path-Copy#.ext`. - - from_path must be a full path to a file. - """ - path = from_path.strip('/') - if to_path is not None: - to_path = to_path.strip('/') - - if '/' in path: - from_dir, from_name = path.rsplit('/', 1) - else: - from_dir = '' - from_name = path - - model = self.get(path) - model.pop('path', None) - model.pop('name', None) - if model['type'] == 'directory': - raise HTTPError(400, "Can't copy directories") - - if to_path is None: - to_path = from_dir - if self.dir_exists(to_path): - name = copy_pat.sub(u'.', from_name) - to_name = self.increment_filename(name, to_path, insert='-Copy') - to_path = u'{0}/{1}'.format(to_path, to_name) - - model = self.save(model, to_path) - return model - - def log_info(self): - self.log.info(self.info_string()) - - def trust_notebook(self, path): - """Explicitly trust a notebook - - Parameters - ---------- - path : string - The path of a notebook - """ - model = self.get(path) - nb = model['content'] - self.log.warning("Trusting notebook %s", path) - self.notary.mark_cells(nb, True) - self.check_and_sign(nb, path) - - def check_and_sign(self, nb, path=''): - """Check for trusted cells, and sign the notebook. - - Called as a part of saving notebooks. - - Parameters - ---------- - nb : dict - The notebook dict - path : string - The notebook's path (for logging) - """ - if self.notary.check_cells(nb): - self.notary.sign(nb) - else: - self.log.warning("Notebook %s is not trusted", path) - - def mark_trusted_cells(self, nb, path=''): - """Mark cells as trusted if the notebook signature matches. - - Called as a part of loading notebooks. - - Parameters - ---------- - nb : dict - The notebook object (in current nbformat) - path : string - The notebook's path (for logging) - """ - trusted = self.notary.check_signature(nb) - if not trusted: - self.log.warning("Notebook %s is not trusted", path) - self.notary.mark_cells(nb, trusted) - - def should_list(self, name): - """Should this file/directory name be displayed in a listing?""" - return not any(fnmatch(name, glob) for glob in self.hide_globs) - - # Part 3: Checkpoints API - def create_checkpoint(self, path): - """Create a checkpoint.""" - return self.checkpoints.create_checkpoint(self, path) - - def restore_checkpoint(self, checkpoint_id, path): - """ - Restore a checkpoint. - """ - self.checkpoints.restore_checkpoint(self, checkpoint_id, path) - - def list_checkpoints(self, path): - return self.checkpoints.list_checkpoints(path) - - def delete_checkpoint(self, checkpoint_id, path): - return self.checkpoints.delete_checkpoint(checkpoint_id, path) diff --git a/notebook/services/contents/tests/__init__.py b/notebook/services/contents/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/contents/tests/test_contents_api.py b/notebook/services/contents/tests/test_contents_api.py deleted file mode 100644 index 5502e67b22..0000000000 --- a/notebook/services/contents/tests/test_contents_api.py +++ /dev/null @@ -1,723 +0,0 @@ -# coding: utf-8 -"""Test the contents webservice API.""" - -from contextlib import contextmanager -from functools import partial -import io -import json -import os -import shutil -import sys -from unicodedata import normalize - -pjoin = os.path.join - -import requests - -from ..filecheckpoints import GenericFileCheckpoints - -from traitlets.config import Config -from notebook.utils import url_path_join, url_escape, to_os_path -from notebook.tests.launchnotebook import NotebookTestBase, assert_http_error -from nbformat import write, from_dict -from nbformat.v4 import ( - new_notebook, new_markdown_cell, -) -from nbformat import v2 -from ipython_genutils import py3compat -from ipython_genutils.tempdir import TemporaryDirectory - -try: #PY3 - from base64 import encodebytes, decodebytes -except ImportError: #PY2 - from base64 import encodestring as encodebytes, decodestring as decodebytes - - -def uniq_stable(elems): - """uniq_stable(elems) -> list - - Return from an iterable, a list of all the unique elements in the input, - maintaining the order in which they first appear. - """ - seen = set() - return [x for x in elems if x not in seen and not seen.add(x)] - -def notebooks_only(dir_model): - return [nb for nb in dir_model['content'] if nb['type']=='notebook'] - -def dirs_only(dir_model): - return [x for x in dir_model['content'] if x['type']=='directory'] - - -class API(object): - """Wrapper for contents API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, path, body=None, params=None): - response = self.request(verb, - url_path_join('api/contents', path), - data=body, params=params, - ) - response.raise_for_status() - return response - - def list(self, path='/'): - return self._req('GET', path) - - def read(self, path, type=None, format=None, content=None): - params = {} - if type is not None: - params['type'] = type - if format is not None: - params['format'] = format - if content == False: - params['content'] = '0' - return self._req('GET', path, params=params) - - def create_untitled(self, path='/', ext='.ipynb'): - body = None - if ext: - body = json.dumps({'ext': ext}) - return self._req('POST', path, body) - - def mkdir_untitled(self, path='/'): - return self._req('POST', path, json.dumps({'type': 'directory'})) - - def copy(self, copy_from, path='/'): - body = json.dumps({'copy_from':copy_from}) - return self._req('POST', path, body) - - def create(self, path='/'): - return self._req('PUT', path) - - def upload(self, path, body): - return self._req('PUT', path, body) - - def mkdir(self, path='/'): - return self._req('PUT', path, json.dumps({'type': 'directory'})) - - def copy_put(self, copy_from, path='/'): - body = json.dumps({'copy_from':copy_from}) - return self._req('PUT', path, body) - - def save(self, path, body): - return self._req('PUT', path, body) - - def delete(self, path='/'): - return self._req('DELETE', path) - - def rename(self, path, new_path): - body = json.dumps({'path': new_path}) - return self._req('PATCH', path, body) - - def get_checkpoints(self, path): - return self._req('GET', url_path_join(path, 'checkpoints')) - - def new_checkpoint(self, path): - return self._req('POST', url_path_join(path, 'checkpoints')) - - def restore_checkpoint(self, path, checkpoint_id): - return self._req('POST', url_path_join(path, 'checkpoints', checkpoint_id)) - - def delete_checkpoint(self, path, checkpoint_id): - return self._req('DELETE', url_path_join(path, 'checkpoints', checkpoint_id)) - -class APITest(NotebookTestBase): - """Test the kernels web service API""" - dirs_nbs = [('', 'inroot'), - ('Directory with spaces in', 'inspace'), - (u'unicodé', 'innonascii'), - ('foo', 'a'), - ('foo', 'b'), - ('foo', 'name with spaces'), - ('foo', u'unicodé'), - ('foo/bar', 'baz'), - ('ordering', 'A'), - ('ordering', 'b'), - ('ordering', 'C'), - (u'å b', u'ç d'), - ] - hidden_dirs = ['.hidden', '__pycache__'] - - # Don't include root dir. - dirs = uniq_stable([py3compat.cast_unicode(d) for (d,n) in dirs_nbs[1:]]) - top_level_dirs = {normalize('NFC', d.split('/')[0]) for d in dirs} - - @staticmethod - def _blob_for_name(name): - return name.encode('utf-8') + b'\xFF' - - @staticmethod - def _txt_for_name(name): - return u'%s text file' % name - - def to_os_path(self, api_path): - return to_os_path(api_path, root=self.notebook_dir) - - def make_dir(self, api_path): - """Create a directory at api_path""" - os_path = self.to_os_path(api_path) - try: - os.makedirs(os_path) - except OSError: - print("Directory already exists: %r" % os_path) - - def make_txt(self, api_path, txt): - """Make a text file at a given api_path""" - os_path = self.to_os_path(api_path) - with io.open(os_path, 'w', encoding='utf-8') as f: - f.write(txt) - - def make_blob(self, api_path, blob): - """Make a binary file at a given api_path""" - os_path = self.to_os_path(api_path) - with io.open(os_path, 'wb') as f: - f.write(blob) - - def make_nb(self, api_path, nb): - """Make a notebook file at a given api_path""" - os_path = self.to_os_path(api_path) - - with io.open(os_path, 'w', encoding='utf-8') as f: - write(nb, f, version=4) - - def delete_dir(self, api_path): - """Delete a directory at api_path, removing any contents.""" - os_path = self.to_os_path(api_path) - shutil.rmtree(os_path, ignore_errors=True) - - def delete_file(self, api_path): - """Delete a file at the given path if it exists.""" - if self.isfile(api_path): - os.unlink(self.to_os_path(api_path)) - - def isfile(self, api_path): - return os.path.isfile(self.to_os_path(api_path)) - - def isdir(self, api_path): - return os.path.isdir(self.to_os_path(api_path)) - - def setUp(self): - for d in (self.dirs + self.hidden_dirs): - self.make_dir(d) - self.addCleanup(partial(self.delete_dir, d)) - - for d, name in self.dirs_nbs: - # create a notebook - nb = new_notebook() - nbname = u'{}/{}.ipynb'.format(d, name) - self.make_nb(nbname, nb) - self.addCleanup(partial(self.delete_file, nbname)) - - # create a text file - txt = self._txt_for_name(name) - txtname = u'{}/{}.txt'.format(d, name) - self.make_txt(txtname, txt) - self.addCleanup(partial(self.delete_file, txtname)) - - blob = self._blob_for_name(name) - blobname = u'{}/{}.blob'.format(d, name) - self.make_blob(blobname, blob) - self.addCleanup(partial(self.delete_file, blobname)) - - self.api = API(self.request) - - def test_list_notebooks(self): - nbs = notebooks_only(self.api.list().json()) - self.assertEqual(len(nbs), 1) - self.assertEqual(nbs[0]['name'], 'inroot.ipynb') - - nbs = notebooks_only(self.api.list('/Directory with spaces in/').json()) - self.assertEqual(len(nbs), 1) - self.assertEqual(nbs[0]['name'], 'inspace.ipynb') - - nbs = notebooks_only(self.api.list(u'/unicodé/').json()) - self.assertEqual(len(nbs), 1) - self.assertEqual(nbs[0]['name'], 'innonascii.ipynb') - self.assertEqual(nbs[0]['path'], u'unicodé/innonascii.ipynb') - - nbs = notebooks_only(self.api.list('/foo/bar/').json()) - self.assertEqual(len(nbs), 1) - self.assertEqual(nbs[0]['name'], 'baz.ipynb') - self.assertEqual(nbs[0]['path'], 'foo/bar/baz.ipynb') - - nbs = notebooks_only(self.api.list('foo').json()) - self.assertEqual(len(nbs), 4) - nbnames = { normalize('NFC', n['name']) for n in nbs } - expected = [ u'a.ipynb', u'b.ipynb', u'name with spaces.ipynb', u'unicodé.ipynb'] - expected = { normalize('NFC', name) for name in expected } - self.assertEqual(nbnames, expected) - - nbs = notebooks_only(self.api.list('ordering').json()) - nbnames = {n['name'] for n in nbs} - expected = {'A.ipynb', 'b.ipynb', 'C.ipynb'} - self.assertEqual(nbnames, expected) - - def test_list_dirs(self): - dirs = dirs_only(self.api.list().json()) - dir_names = {normalize('NFC', d['name']) for d in dirs} - self.assertEqual(dir_names, self.top_level_dirs) # Excluding hidden dirs - - def test_get_dir_no_content(self): - for d in self.dirs: - model = self.api.read(d, content=False).json() - self.assertEqual(model['path'], d) - self.assertEqual(model['type'], 'directory') - self.assertIn('content', model) - self.assertEqual(model['content'], None) - - def test_list_nonexistant_dir(self): - with assert_http_error(404): - self.api.list('nonexistant') - - def test_get_nb_contents(self): - for d, name in self.dirs_nbs: - path = url_path_join(d, name + '.ipynb') - nb = self.api.read(path).json() - self.assertEqual(nb['name'], u'%s.ipynb' % name) - self.assertEqual(nb['path'], path) - self.assertEqual(nb['type'], 'notebook') - self.assertIn('content', nb) - self.assertEqual(nb['format'], 'json') - self.assertIn('metadata', nb['content']) - self.assertIsInstance(nb['content']['metadata'], dict) - - def test_get_nb_no_content(self): - for d, name in self.dirs_nbs: - path = url_path_join(d, name + '.ipynb') - nb = self.api.read(path, content=False).json() - self.assertEqual(nb['name'], u'%s.ipynb' % name) - self.assertEqual(nb['path'], path) - self.assertEqual(nb['type'], 'notebook') - self.assertIn('content', nb) - self.assertEqual(nb['content'], None) - - def test_get_nb_invalid(self): - nb = { - 'nbformat': 4, - 'metadata': {}, - 'cells': [{ - 'cell_type': 'wrong', - 'metadata': {}, - }], - } - path = u'å b/Validate tést.ipynb' - self.make_txt(path, py3compat.cast_unicode(json.dumps(nb))) - model = self.api.read(path).json() - self.assertEqual(model['path'], path) - self.assertEqual(model['type'], 'notebook') - self.assertIn('content', model) - self.assertIn('message', model) - self.assertIn("validation failed", model['message'].lower()) - - def test_get_contents_no_such_file(self): - # Name that doesn't exist - should be a 404 - with assert_http_error(404): - self.api.read('foo/q.ipynb') - - def test_get_text_file_contents(self): - for d, name in self.dirs_nbs: - path = url_path_join(d, name + '.txt') - model = self.api.read(path).json() - self.assertEqual(model['name'], u'%s.txt' % name) - self.assertEqual(model['path'], path) - self.assertIn('content', model) - self.assertEqual(model['format'], 'text') - self.assertEqual(model['type'], 'file') - self.assertEqual(model['content'], self._txt_for_name(name)) - - # Name that doesn't exist - should be a 404 - with assert_http_error(404): - self.api.read('foo/q.txt') - - # Specifying format=text should fail on a non-UTF-8 file - with assert_http_error(400): - self.api.read('foo/bar/baz.blob', type='file', format='text') - - def test_get_binary_file_contents(self): - for d, name in self.dirs_nbs: - path = url_path_join(d, name + '.blob') - model = self.api.read(path).json() - self.assertEqual(model['name'], u'%s.blob' % name) - self.assertEqual(model['path'], path) - self.assertIn('content', model) - self.assertEqual(model['format'], 'base64') - self.assertEqual(model['type'], 'file') - self.assertEqual( - decodebytes(model['content'].encode('ascii')), - self._blob_for_name(name), - ) - - # Name that doesn't exist - should be a 404 - with assert_http_error(404): - self.api.read('foo/q.txt') - - def test_get_bad_type(self): - with assert_http_error(400): - self.api.read(u'unicodé', type='file') # this is a directory - - with assert_http_error(400): - self.api.read(u'unicodé/innonascii.ipynb', type='directory') - - def _check_created(self, resp, path, type='notebook'): - self.assertEqual(resp.status_code, 201) - location_header = py3compat.str_to_unicode(resp.headers['Location']) - self.assertEqual(location_header, url_path_join(self.url_prefix, u'api/contents', url_escape(path))) - rjson = resp.json() - self.assertEqual(rjson['name'], path.rsplit('/', 1)[-1]) - self.assertEqual(rjson['path'], path) - self.assertEqual(rjson['type'], type) - isright = self.isdir if type == 'directory' else self.isfile - assert isright(path) - - def test_create_untitled(self): - resp = self.api.create_untitled(path=u'å b') - self._check_created(resp, u'å b/Untitled.ipynb') - - # Second time - resp = self.api.create_untitled(path=u'å b') - self._check_created(resp, u'å b/Untitled1.ipynb') - - # And two directories down - resp = self.api.create_untitled(path='foo/bar') - self._check_created(resp, 'foo/bar/Untitled.ipynb') - - def test_create_untitled_txt(self): - resp = self.api.create_untitled(path='foo/bar', ext='.txt') - self._check_created(resp, 'foo/bar/untitled.txt', type='file') - - resp = self.api.read(path='foo/bar/untitled.txt') - model = resp.json() - self.assertEqual(model['type'], 'file') - self.assertEqual(model['format'], 'text') - self.assertEqual(model['content'], '') - - def test_upload(self): - nb = new_notebook() - nbmodel = {'content': nb, 'type': 'notebook'} - path = u'å b/Upload tést.ipynb' - resp = self.api.upload(path, body=json.dumps(nbmodel)) - self._check_created(resp, path) - - def test_mkdir_untitled(self): - resp = self.api.mkdir_untitled(path=u'å b') - self._check_created(resp, u'å b/Untitled Folder', type='directory') - - # Second time - resp = self.api.mkdir_untitled(path=u'å b') - self._check_created(resp, u'å b/Untitled Folder 1', type='directory') - - # And two directories down - resp = self.api.mkdir_untitled(path='foo/bar') - self._check_created(resp, 'foo/bar/Untitled Folder', type='directory') - - def test_mkdir(self): - path = u'å b/New ∂ir' - resp = self.api.mkdir(path) - self._check_created(resp, path, type='directory') - - def test_mkdir_hidden_400(self): - with assert_http_error(400): - resp = self.api.mkdir(u'å b/.hidden') - - def test_upload_txt(self): - body = u'ünicode téxt' - model = { - 'content' : body, - 'format' : 'text', - 'type' : 'file', - } - path = u'å b/Upload tést.txt' - resp = self.api.upload(path, body=json.dumps(model)) - - # check roundtrip - resp = self.api.read(path) - model = resp.json() - self.assertEqual(model['type'], 'file') - self.assertEqual(model['format'], 'text') - self.assertEqual(model['content'], body) - - def test_upload_b64(self): - body = b'\xFFblob' - b64body = encodebytes(body).decode('ascii') - model = { - 'content' : b64body, - 'format' : 'base64', - 'type' : 'file', - } - path = u'å b/Upload tést.blob' - resp = self.api.upload(path, body=json.dumps(model)) - - # check roundtrip - resp = self.api.read(path) - model = resp.json() - self.assertEqual(model['type'], 'file') - self.assertEqual(model['path'], path) - self.assertEqual(model['format'], 'base64') - decoded = decodebytes(model['content'].encode('ascii')) - self.assertEqual(decoded, body) - - def test_upload_v2(self): - nb = v2.new_notebook() - ws = v2.new_worksheet() - nb.worksheets.append(ws) - ws.cells.append(v2.new_code_cell(input='print("hi")')) - nbmodel = {'content': nb, 'type': 'notebook'} - path = u'å b/Upload tést.ipynb' - resp = self.api.upload(path, body=json.dumps(nbmodel)) - self._check_created(resp, path) - resp = self.api.read(path) - data = resp.json() - self.assertEqual(data['content']['nbformat'], 4) - - def test_copy(self): - resp = self.api.copy(u'å b/ç d.ipynb', u'å b') - self._check_created(resp, u'å b/ç d-Copy1.ipynb') - - resp = self.api.copy(u'å b/ç d.ipynb', u'å b') - self._check_created(resp, u'å b/ç d-Copy2.ipynb') - - def test_copy_copy(self): - resp = self.api.copy(u'å b/ç d.ipynb', u'å b') - self._check_created(resp, u'å b/ç d-Copy1.ipynb') - - resp = self.api.copy(u'å b/ç d-Copy1.ipynb', u'å b') - self._check_created(resp, u'å b/ç d-Copy2.ipynb') - - def test_copy_path(self): - resp = self.api.copy(u'foo/a.ipynb', u'å b') - self._check_created(resp, u'å b/a.ipynb') - - resp = self.api.copy(u'foo/a.ipynb', u'å b') - self._check_created(resp, u'å b/a-Copy1.ipynb') - - def test_copy_put_400(self): - with assert_http_error(400): - resp = self.api.copy_put(u'å b/ç d.ipynb', u'å b/cøpy.ipynb') - - def test_copy_dir_400(self): - # can't copy directories - with assert_http_error(400): - resp = self.api.copy(u'å b', u'foo') - - def test_delete(self): - for d, name in self.dirs_nbs: - print('%r, %r' % (d, name)) - resp = self.api.delete(url_path_join(d, name + '.ipynb')) - self.assertEqual(resp.status_code, 204) - - for d in self.dirs + ['/']: - nbs = notebooks_only(self.api.list(d).json()) - print('------') - print(d) - print(nbs) - self.assertEqual(nbs, []) - - def test_delete_dirs(self): - # depth-first delete everything, so we don't try to delete empty directories - for name in sorted(self.dirs + ['/'], key=len, reverse=True): - listing = self.api.list(name).json()['content'] - for model in listing: - self.api.delete(model['path']) - listing = self.api.list('/').json()['content'] - self.assertEqual(listing, []) - - def test_delete_non_empty_dir(self): - if sys.platform == 'win32': - self.skipTest("Disabled deleting non-empty dirs on Windows") - # Test that non empty directory can be deleted - self.api.delete(u'å b') - # Check if directory has actually been deleted - with assert_http_error(404): - self.api.list(u'å b') - - def test_rename(self): - resp = self.api.rename('foo/a.ipynb', 'foo/z.ipynb') - self.assertEqual(resp.headers['Location'].split('/')[-1], 'z.ipynb') - self.assertEqual(resp.json()['name'], 'z.ipynb') - self.assertEqual(resp.json()['path'], 'foo/z.ipynb') - assert self.isfile('foo/z.ipynb') - - nbs = notebooks_only(self.api.list('foo').json()) - nbnames = set(n['name'] for n in nbs) - self.assertIn('z.ipynb', nbnames) - self.assertNotIn('a.ipynb', nbnames) - - def test_checkpoints_follow_file(self): - - # Read initial file state - orig = self.api.read('foo/a.ipynb') - - # Create a checkpoint of initial state - r = self.api.new_checkpoint('foo/a.ipynb') - cp1 = r.json() - - # Modify file and save - nbcontent = json.loads(orig.text)['content'] - nb = from_dict(nbcontent) - hcell = new_markdown_cell('Created by test') - nb.cells.append(hcell) - nbmodel = {'content': nb, 'type': 'notebook'} - self.api.save('foo/a.ipynb', body=json.dumps(nbmodel)) - - # Rename the file. - self.api.rename('foo/a.ipynb', 'foo/z.ipynb') - - # Looking for checkpoints in the old location should yield no results. - self.assertEqual(self.api.get_checkpoints('foo/a.ipynb').json(), []) - - # Looking for checkpoints in the new location should work. - cps = self.api.get_checkpoints('foo/z.ipynb').json() - self.assertEqual(cps, [cp1]) - - # Delete the file. The checkpoint should be deleted as well. - self.api.delete('foo/z.ipynb') - cps = self.api.get_checkpoints('foo/z.ipynb').json() - self.assertEqual(cps, []) - - def test_rename_existing(self): - with assert_http_error(409): - self.api.rename('foo/a.ipynb', 'foo/b.ipynb') - - def test_save(self): - resp = self.api.read('foo/a.ipynb') - nbcontent = json.loads(resp.text)['content'] - nb = from_dict(nbcontent) - nb.cells.append(new_markdown_cell(u'Created by test ³')) - - nbmodel = {'content': nb, 'type': 'notebook'} - resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel)) - - nbcontent = self.api.read('foo/a.ipynb').json()['content'] - newnb = from_dict(nbcontent) - self.assertEqual(newnb.cells[0].source, - u'Created by test ³') - - def test_checkpoints(self): - resp = self.api.read('foo/a.ipynb') - r = self.api.new_checkpoint('foo/a.ipynb') - self.assertEqual(r.status_code, 201) - cp1 = r.json() - self.assertEqual(set(cp1), {'id', 'last_modified'}) - self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id']) - - # Modify it - nbcontent = json.loads(resp.text)['content'] - nb = from_dict(nbcontent) - hcell = new_markdown_cell('Created by test') - nb.cells.append(hcell) - # Save - nbmodel= {'content': nb, 'type': 'notebook'} - resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel)) - - # List checkpoints - cps = self.api.get_checkpoints('foo/a.ipynb').json() - self.assertEqual(cps, [cp1]) - - nbcontent = self.api.read('foo/a.ipynb').json()['content'] - nb = from_dict(nbcontent) - self.assertEqual(nb.cells[0].source, 'Created by test') - - # Restore cp1 - r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id']) - self.assertEqual(r.status_code, 204) - nbcontent = self.api.read('foo/a.ipynb').json()['content'] - nb = from_dict(nbcontent) - self.assertEqual(nb.cells, []) - - # Delete cp1 - r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id']) - self.assertEqual(r.status_code, 204) - cps = self.api.get_checkpoints('foo/a.ipynb').json() - self.assertEqual(cps, []) - - def test_file_checkpoints(self): - """ - Test checkpointing of non-notebook files. - """ - filename = 'foo/a.txt' - resp = self.api.read(filename) - orig_content = json.loads(resp.text)['content'] - - # Create a checkpoint. - r = self.api.new_checkpoint(filename) - self.assertEqual(r.status_code, 201) - cp1 = r.json() - self.assertEqual(set(cp1), {'id', 'last_modified'}) - self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id']) - - # Modify the file and save. - new_content = orig_content + '\nsecond line' - model = { - 'content': new_content, - 'type': 'file', - 'format': 'text', - } - resp = self.api.save(filename, body=json.dumps(model)) - - # List checkpoints - cps = self.api.get_checkpoints(filename).json() - self.assertEqual(cps, [cp1]) - - content = self.api.read(filename).json()['content'] - self.assertEqual(content, new_content) - - # Restore cp1 - r = self.api.restore_checkpoint(filename, cp1['id']) - self.assertEqual(r.status_code, 204) - restored_content = self.api.read(filename).json()['content'] - self.assertEqual(restored_content, orig_content) - - # Delete cp1 - r = self.api.delete_checkpoint(filename, cp1['id']) - self.assertEqual(r.status_code, 204) - cps = self.api.get_checkpoints(filename).json() - self.assertEqual(cps, []) - - @contextmanager - def patch_cp_root(self, dirname): - """ - Temporarily patch the root dir of our checkpoint manager. - """ - cpm = self.notebook.contents_manager.checkpoints - old_dirname = cpm.root_dir - cpm.root_dir = dirname - try: - yield - finally: - cpm.root_dir = old_dirname - - def test_checkpoints_separate_root(self): - """ - Test that FileCheckpoints functions correctly even when it's - using a different root dir from FileContentsManager. This also keeps - the implementation honest for use with ContentsManagers that don't map - models to the filesystem - - Override this method to a no-op when testing other managers. - """ - with TemporaryDirectory() as td: - with self.patch_cp_root(td): - self.test_checkpoints() - - with TemporaryDirectory() as td: - with self.patch_cp_root(td): - self.test_file_checkpoints() - - -class GenericFileCheckpointsAPITest(APITest): - """ - Run the tests from APITest with GenericFileCheckpoints. - """ - config = Config() - config.FileContentsManager.checkpoints_class = GenericFileCheckpoints - - def test_config_did_something(self): - - self.assertIsInstance( - self.notebook.contents_manager.checkpoints, - GenericFileCheckpoints, - ) - - diff --git a/notebook/services/contents/tests/test_fileio.py b/notebook/services/contents/tests/test_fileio.py deleted file mode 100644 index 256c664ae8..0000000000 --- a/notebook/services/contents/tests/test_fileio.py +++ /dev/null @@ -1,131 +0,0 @@ -# encoding: utf-8 -"""Tests for file IO""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import io as stdlib_io -import os.path -import stat - -import nose.tools as nt - -from ipython_genutils.testing.decorators import skip_win32 -from ..fileio import atomic_writing - -from ipython_genutils.tempdir import TemporaryDirectory - -umask = 0 - -def test_atomic_writing(): - class CustomExc(Exception): pass - - with TemporaryDirectory() as td: - f1 = os.path.join(td, 'penguin') - with stdlib_io.open(f1, 'w') as f: - f.write(u'Before') - - if os.name != 'nt': - os.chmod(f1, 0o701) - orig_mode = stat.S_IMODE(os.stat(f1).st_mode) - - f2 = os.path.join(td, 'flamingo') - try: - os.symlink(f1, f2) - have_symlink = True - except (AttributeError, NotImplementedError, OSError): - # AttributeError: Python doesn't support it - # NotImplementedError: The system doesn't support it - # OSError: The user lacks the privilege (Windows) - have_symlink = False - - with nt.assert_raises(CustomExc): - with atomic_writing(f1) as f: - f.write(u'Failing write') - raise CustomExc - - # Because of the exception, the file should not have been modified - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'Before') - - with atomic_writing(f1) as f: - f.write(u'Overwritten') - - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'Overwritten') - - if os.name != 'nt': - mode = stat.S_IMODE(os.stat(f1).st_mode) - nt.assert_equal(mode, orig_mode) - - if have_symlink: - # Check that writing over a file preserves a symlink - with atomic_writing(f2) as f: - f.write(u'written from symlink') - - with stdlib_io.open(f1, 'r') as f: - nt.assert_equal(f.read(), u'written from symlink') - -def _save_umask(): - global umask - umask = os.umask(0) - os.umask(umask) - -def _restore_umask(): - os.umask(umask) - -@skip_win32 -@nt.with_setup(_save_umask, _restore_umask) -def test_atomic_writing_umask(): - with TemporaryDirectory() as td: - os.umask(0o022) - f1 = os.path.join(td, '1') - with atomic_writing(f1) as f: - f.write(u'1') - mode = stat.S_IMODE(os.stat(f1).st_mode) - nt.assert_equal(mode, 0o644, '{:o} != 644'.format(mode)) - - os.umask(0o057) - f2 = os.path.join(td, '2') - with atomic_writing(f2) as f: - f.write(u'2') - mode = stat.S_IMODE(os.stat(f2).st_mode) - nt.assert_equal(mode, 0o620, '{:o} != 620'.format(mode)) - - -def test_atomic_writing_newlines(): - with TemporaryDirectory() as td: - path = os.path.join(td, 'testfile') - - lf = u'a\nb\nc\n' - plat = lf.replace(u'\n', os.linesep) - crlf = lf.replace(u'\n', u'\r\n') - - # test default - with stdlib_io.open(path, 'w') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, plat) - - # test newline=LF - with stdlib_io.open(path, 'w', newline='\n') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, lf) - - # test newline=CRLF - with atomic_writing(path, newline='\r\n') as f: - f.write(lf) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, crlf) - - # test newline=no convert - text = u'crlf\r\ncr\rlf\n' - with atomic_writing(path, newline='') as f: - f.write(text) - with stdlib_io.open(path, 'r', newline='') as f: - read = f.read() - nt.assert_equal(read, text) diff --git a/notebook/services/contents/tests/test_largefilemanager.py b/notebook/services/contents/tests/test_largefilemanager.py deleted file mode 100644 index 13d294b9b0..0000000000 --- a/notebook/services/contents/tests/test_largefilemanager.py +++ /dev/null @@ -1,113 +0,0 @@ -from unittest import TestCase -from ipython_genutils.tempdir import TemporaryDirectory -from ..largefilemanager import LargeFileManager -import os -from tornado import web - - -def _make_dir(contents_manager, api_path): - """ - Make a directory. - """ - os_path = contents_manager._get_os_path(api_path) - try: - os.makedirs(os_path) - except OSError: - print("Directory already exists: %r" % os_path) - - -class TestLargeFileManager(TestCase): - - def setUp(self): - self._temp_dir = TemporaryDirectory() - self.td = self._temp_dir.name - self.contents_manager = LargeFileManager(root_dir=self.td) - - def make_dir(self, api_path): - """make a subdirectory at api_path - - override in subclasses if contents are not on the filesystem. - """ - _make_dir(self.contents_manager, api_path) - - def test_save(self): - - cm = self.contents_manager - # Create a notebook - model = cm.new_untitled(type='notebook') - name = model['name'] - path = model['path'] - - # Get the model with 'content' - full_model = cm.get(path) - # Save the notebook - model = cm.save(full_model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], name) - self.assertEqual(model['path'], path) - - try: - model = {'name': 'test', 'path': 'test', 'chunk': 1} - cm.save(model, model['path']) - except web.HTTPError as e: - self.assertEqual('HTTP 400: Bad Request (No file type provided)', str(e)) - - try: - model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'notebook'} - cm.save(model, model['path']) - except web.HTTPError as e: - self.assertEqual('HTTP 400: Bad Request (File type "notebook" is not supported for large file transfer)', str(e)) - - try: - model = {'name': 'test', 'path': 'test', 'chunk': 1, 'type': 'file'} - cm.save(model, model['path']) - except web.HTTPError as e: - self.assertEqual('HTTP 400: Bad Request (No file content provided)', str(e)) - - try: - model = {'name': 'test', 'path': 'test', 'chunk': 2, 'type': 'file', - 'content': u'test', 'format': 'json'} - cm.save(model, model['path']) - except web.HTTPError as e: - self.assertEqual("HTTP 400: Bad Request (Must specify format of file contents as 'text' or 'base64')", - str(e)) - - # Save model for different chunks - model = {'name': 'test', 'path': 'test', 'type': 'file', - 'content': u'test==', 'format': 'text'} - name = model['name'] - path = model['path'] - cm.save(model, path) - - for chunk in (1, 2, -1): - for fm in ('text', 'base64'): - full_model = cm.get(path) - full_model['chunk'] = chunk - full_model['format'] = fm - model_res = cm.save(full_model, path) - assert isinstance(model_res, dict) - - self.assertIn('name', model_res) - self.assertIn('path', model_res) - self.assertNotIn('chunk', model_res) - self.assertEqual(model_res['name'], name) - self.assertEqual(model_res['path'], path) - - # Test in sub-directory - # Create a directory and notebook in that directory - sub_dir = '/foo/' - self.make_dir('foo') - model = cm.new_untitled(path=sub_dir, type='notebook') - name = model['name'] - path = model['path'] - model = cm.get(path) - - # Change the name in the model for rename - model = cm.save(model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], 'Untitled.ipynb') - self.assertEqual(model['path'], 'foo/Untitled.ipynb') diff --git a/notebook/services/contents/tests/test_manager.py b/notebook/services/contents/tests/test_manager.py deleted file mode 100644 index 0e6b0fb2b2..0000000000 --- a/notebook/services/contents/tests/test_manager.py +++ /dev/null @@ -1,642 +0,0 @@ -# coding: utf-8 -"""Tests for the notebook manager.""" -from __future__ import print_function - -import os -import sys -import time -from contextlib import contextmanager -from itertools import combinations - -from nose import SkipTest -from tornado.web import HTTPError -from unittest import TestCase -from tempfile import NamedTemporaryFile - -from nbformat import v4 as nbformat - -from ipython_genutils.tempdir import TemporaryDirectory -from traitlets import TraitError -from ipython_genutils.testing import decorators as dec - -from ..filemanager import FileContentsManager - - -def _make_dir(contents_manager, api_path): - """ - Make a directory. - """ - os_path = contents_manager._get_os_path(api_path) - try: - os.makedirs(os_path) - except OSError: - print("Directory already exists: %r" % os_path) - - -class TestFileContentsManager(TestCase): - - @contextmanager - def assertRaisesHTTPError(self, status, msg=None): - msg = msg or "Should have raised HTTPError(%i)" % status - try: - yield - except HTTPError as e: - self.assertEqual(e.status_code, status) - else: - self.fail(msg) - - def symlink(self, contents_manager, src, dst): - """Make a symlink to src from dst - - src and dst are api_paths - """ - src_os_path = contents_manager._get_os_path(src) - dst_os_path = contents_manager._get_os_path(dst) - print(src_os_path, dst_os_path, os.path.isfile(src_os_path)) - os.symlink(src_os_path, dst_os_path) - - def test_root_dir(self): - with TemporaryDirectory() as td: - fm = FileContentsManager(root_dir=td) - self.assertEqual(fm.root_dir, td) - - def test_missing_root_dir(self): - with TemporaryDirectory() as td: - root = os.path.join(td, 'notebook', 'dir', 'is', 'missing') - self.assertRaises(TraitError, FileContentsManager, root_dir=root) - - def test_invalid_root_dir(self): - with NamedTemporaryFile() as tf: - self.assertRaises(TraitError, FileContentsManager, root_dir=tf.name) - - def test_get_os_path(self): - # full filesystem path should be returned with correct operating system - # separators. - with TemporaryDirectory() as td: - root = td - fm = FileContentsManager(root_dir=root) - path = fm._get_os_path('/path/to/notebook/test.ipynb') - rel_path_list = '/path/to/notebook/test.ipynb'.split('/') - fs_path = os.path.join(fm.root_dir, *rel_path_list) - self.assertEqual(path, fs_path) - - fm = FileContentsManager(root_dir=root) - path = fm._get_os_path('test.ipynb') - fs_path = os.path.join(fm.root_dir, 'test.ipynb') - self.assertEqual(path, fs_path) - - fm = FileContentsManager(root_dir=root) - path = fm._get_os_path('////test.ipynb') - fs_path = os.path.join(fm.root_dir, 'test.ipynb') - self.assertEqual(path, fs_path) - - def test_checkpoint_subdir(self): - subd = u'sub ∂ir' - cp_name = 'test-cp.ipynb' - with TemporaryDirectory() as td: - root = td - os.mkdir(os.path.join(td, subd)) - fm = FileContentsManager(root_dir=root) - cpm = fm.checkpoints - cp_dir = cpm.checkpoint_path( - 'cp', 'test.ipynb' - ) - cp_subdir = cpm.checkpoint_path( - 'cp', '/%s/test.ipynb' % subd - ) - self.assertNotEqual(cp_dir, cp_subdir) - self.assertEqual(cp_dir, os.path.join(root, cpm.checkpoint_dir, cp_name)) - self.assertEqual(cp_subdir, os.path.join(root, subd, cpm.checkpoint_dir, cp_name)) - - @dec.skipif(sys.platform == 'win32' and sys.version_info[0] < 3) - def test_bad_symlink(self): - with TemporaryDirectory() as td: - cm = FileContentsManager(root_dir=td) - path = 'test bad symlink' - _make_dir(cm, path) - - file_model = cm.new_untitled(path=path, ext='.txt') - - # create a broken symlink - self.symlink(cm, "target", '%s/%s' % (path, 'bad symlink')) - model = cm.get(path) - - contents = { - content['name']: content for content in model['content'] - } - self.assertTrue('untitled.txt' in contents) - self.assertEqual(contents['untitled.txt'], file_model) - # broken symlinks should still be shown in the contents manager - self.assertTrue('bad symlink' in contents) - - @dec.skipif(sys.platform == 'win32' and sys.version_info[0] < 3) - def test_good_symlink(self): - with TemporaryDirectory() as td: - cm = FileContentsManager(root_dir=td) - parent = 'test good symlink' - name = 'good symlink' - path = '{0}/{1}'.format(parent, name) - _make_dir(cm, parent) - - file_model = cm.new(path=parent + '/zfoo.txt') - - # create a good symlink - self.symlink(cm, file_model['path'], path) - symlink_model = cm.get(path, content=False) - dir_model = cm.get(parent) - self.assertEqual( - sorted(dir_model['content'], key=lambda x: x['name']), - [symlink_model, file_model], - ) - - def test_403(self): - if hasattr(os, 'getuid'): - if os.getuid() == 0: - raise SkipTest("Can't test permissions as root") - if sys.platform.startswith('win'): - raise SkipTest("Can't test permissions on Windows") - - with TemporaryDirectory() as td: - cm = FileContentsManager(root_dir=td) - model = cm.new_untitled(type='file') - os_path = cm._get_os_path(model['path']) - - os.chmod(os_path, 0o400) - try: - with cm.open(os_path, 'w') as f: - f.write(u"don't care") - except HTTPError as e: - self.assertEqual(e.status_code, 403) - else: - self.fail("Should have raised HTTPError(403)") - - def test_escape_root(self): - with TemporaryDirectory() as td: - cm = FileContentsManager(root_dir=td) - # make foo, bar next to root - with open(os.path.join(cm.root_dir, '..', 'foo'), 'w') as f: - f.write('foo') - with open(os.path.join(cm.root_dir, '..', 'bar'), 'w') as f: - f.write('bar') - - with self.assertRaisesHTTPError(404): - cm.get('..') - with self.assertRaisesHTTPError(404): - cm.get('foo/../../../bar') - with self.assertRaisesHTTPError(404): - cm.delete('../foo') - with self.assertRaisesHTTPError(404): - cm.rename('../foo', '../bar') - with self.assertRaisesHTTPError(404): - cm.save(model={ - 'type': 'file', - 'content': u'', - 'format': 'text', - }, path='../foo') - - -class TestContentsManager(TestCase): - @contextmanager - def assertRaisesHTTPError(self, status, msg=None): - msg = msg or "Should have raised HTTPError(%i)" % status - try: - yield - except HTTPError as e: - self.assertEqual(e.status_code, status) - else: - self.fail(msg) - - def make_populated_dir(self, api_path): - cm = self.contents_manager - - self.make_dir(api_path) - - cm.new(path="/".join([api_path, "nb.ipynb"])) - cm.new(path="/".join([api_path, "file.txt"])) - - def check_populated_dir_files(self, api_path): - dir_model = self.contents_manager.get(api_path) - - self.assertEqual(dir_model['path'], api_path) - self.assertEqual(dir_model['type'], "directory") - - for entry in dir_model['content']: - if entry['type'] == "directory": - continue - elif entry['type'] == "file": - self.assertEqual(entry['name'], "file.txt") - complete_path = "/".join([api_path, "file.txt"]) - self.assertEqual(entry["path"], complete_path) - elif entry['type'] == "notebook": - self.assertEqual(entry['name'], "nb.ipynb") - complete_path = "/".join([api_path, "nb.ipynb"]) - self.assertEqual(entry["path"], complete_path) - - def setUp(self): - self._temp_dir = TemporaryDirectory() - self.td = self._temp_dir.name - self.contents_manager = FileContentsManager( - root_dir=self.td, - ) - - def tearDown(self): - self._temp_dir.cleanup() - - def make_dir(self, api_path): - """make a subdirectory at api_path - - override in subclasses if contents are not on the filesystem. - """ - _make_dir(self.contents_manager, api_path) - - def add_code_cell(self, nb): - output = nbformat.new_output("display_data", {'application/javascript': "alert('hi');"}) - cell = nbformat.new_code_cell("print('hi')", outputs=[output]) - nb.cells.append(cell) - - def new_notebook(self): - cm = self.contents_manager - model = cm.new_untitled(type='notebook') - name = model['name'] - path = model['path'] - - full_model = cm.get(path) - nb = full_model['content'] - nb['metadata']['counter'] = int(1e6 * time.time()) - self.add_code_cell(nb) - - cm.save(full_model, path) - return nb, name, path - - def test_new_untitled(self): - cm = self.contents_manager - # Test in root directory - model = cm.new_untitled(type='notebook') - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertIn('type', model) - self.assertEqual(model['type'], 'notebook') - self.assertEqual(model['name'], 'Untitled.ipynb') - self.assertEqual(model['path'], 'Untitled.ipynb') - - # Test in sub-directory - model = cm.new_untitled(type='directory') - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertIn('type', model) - self.assertEqual(model['type'], 'directory') - self.assertEqual(model['name'], 'Untitled Folder') - self.assertEqual(model['path'], 'Untitled Folder') - sub_dir = model['path'] - - model = cm.new_untitled(path=sub_dir) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertIn('type', model) - self.assertEqual(model['type'], 'file') - self.assertEqual(model['name'], 'untitled') - self.assertEqual(model['path'], '%s/untitled' % sub_dir) - - # Test with a compound extension - model = cm.new_untitled(path=sub_dir, ext='.foo.bar') - self.assertEqual(model['name'], 'untitled.foo.bar') - model = cm.new_untitled(path=sub_dir, ext='.foo.bar') - self.assertEqual(model['name'], 'untitled1.foo.bar') - - def test_modified_date(self): - - cm = self.contents_manager - - # Create a new notebook. - nb, name, path = self.new_notebook() - model = cm.get(path) - - # Add a cell and save. - self.add_code_cell(model['content']) - cm.save(model, path) - - # Reload notebook and verify that last_modified incremented. - saved = cm.get(path) - self.assertGreaterEqual(saved['last_modified'], model['last_modified']) - - # Move the notebook and verify that last_modified stayed the same. - # (The frontend fires a warning if last_modified increases on the - # renamed file.) - new_path = 'renamed.ipynb' - cm.rename(path, new_path) - renamed = cm.get(new_path) - self.assertGreaterEqual( - renamed['last_modified'], - saved['last_modified'], - ) - - def test_get(self): - cm = self.contents_manager - # Create a notebook - model = cm.new_untitled(type='notebook') - name = model['name'] - path = model['path'] - - # Check that we 'get' on the notebook we just created - model2 = cm.get(path) - assert isinstance(model2, dict) - self.assertIn('name', model2) - self.assertIn('path', model2) - self.assertEqual(model['name'], name) - self.assertEqual(model['path'], path) - - nb_as_file = cm.get(path, content=True, type='file') - self.assertEqual(nb_as_file['path'], path) - self.assertEqual(nb_as_file['type'], 'file') - self.assertEqual(nb_as_file['format'], 'text') - self.assertNotIsInstance(nb_as_file['content'], dict) - - nb_as_bin_file = cm.get(path, content=True, type='file', format='base64') - self.assertEqual(nb_as_bin_file['format'], 'base64') - - # Test in sub-directory - sub_dir = '/foo/' - self.make_dir('foo') - model = cm.new_untitled(path=sub_dir, ext='.ipynb') - model2 = cm.get(sub_dir + name) - assert isinstance(model2, dict) - self.assertIn('name', model2) - self.assertIn('path', model2) - self.assertIn('content', model2) - self.assertEqual(model2['name'], 'Untitled.ipynb') - self.assertEqual(model2['path'], '{0}/{1}'.format(sub_dir.strip('/'), name)) - - # Test with a regular file. - file_model_path = cm.new_untitled(path=sub_dir, ext='.txt')['path'] - file_model = cm.get(file_model_path) - self.assertDictContainsSubset( - { - 'content': u'', - 'format': u'text', - 'mimetype': u'text/plain', - 'name': u'untitled.txt', - 'path': u'foo/untitled.txt', - 'type': u'file', - 'writable': True, - }, - file_model, - ) - self.assertIn('created', file_model) - self.assertIn('last_modified', file_model) - - # Test getting directory model - - # Create a sub-sub directory to test getting directory contents with a - # subdir. - self.make_dir('foo/bar') - dirmodel = cm.get('foo') - self.assertEqual(dirmodel['type'], 'directory') - self.assertIsInstance(dirmodel['content'], list) - self.assertEqual(len(dirmodel['content']), 3) - self.assertEqual(dirmodel['path'], 'foo') - self.assertEqual(dirmodel['name'], 'foo') - - # Directory contents should match the contents of each individual entry - # when requested with content=False. - model2_no_content = cm.get(sub_dir + name, content=False) - file_model_no_content = cm.get(u'foo/untitled.txt', content=False) - sub_sub_dir_no_content = cm.get('foo/bar', content=False) - self.assertEqual(sub_sub_dir_no_content['path'], 'foo/bar') - self.assertEqual(sub_sub_dir_no_content['name'], 'bar') - - for entry in dirmodel['content']: - # Order isn't guaranteed by the spec, so this is a hacky way of - # verifying that all entries are matched. - if entry['path'] == sub_sub_dir_no_content['path']: - self.assertEqual(entry, sub_sub_dir_no_content) - elif entry['path'] == model2_no_content['path']: - self.assertEqual(entry, model2_no_content) - elif entry['path'] == file_model_no_content['path']: - self.assertEqual(entry, file_model_no_content) - else: - self.fail("Unexpected directory entry: %s" % entry()) - - with self.assertRaises(HTTPError): - cm.get('foo', type='file') - - def test_update(self): - cm = self.contents_manager - # Create a notebook - model = cm.new_untitled(type='notebook') - name = model['name'] - path = model['path'] - - # Change the name in the model for rename - model['path'] = 'test.ipynb' - model = cm.update(model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], 'test.ipynb') - - # Make sure the old name is gone - self.assertRaises(HTTPError, cm.get, path) - - # Test in sub-directory - # Create a directory and notebook in that directory - sub_dir = '/foo/' - self.make_dir('foo') - model = cm.new_untitled(path=sub_dir, type='notebook') - path = model['path'] - - # Change the name in the model for rename - d = path.rsplit('/', 1)[0] - new_path = model['path'] = d + '/test_in_sub.ipynb' - model = cm.update(model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], 'test_in_sub.ipynb') - self.assertEqual(model['path'], new_path) - - # Make sure the old name is gone - self.assertRaises(HTTPError, cm.get, path) - - def test_save(self): - cm = self.contents_manager - # Create a notebook - model = cm.new_untitled(type='notebook') - name = model['name'] - path = model['path'] - - # Get the model with 'content' - full_model = cm.get(path) - - # Save the notebook - model = cm.save(full_model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], name) - self.assertEqual(model['path'], path) - - # Test in sub-directory - # Create a directory and notebook in that directory - sub_dir = '/foo/' - self.make_dir('foo') - model = cm.new_untitled(path=sub_dir, type='notebook') - name = model['name'] - path = model['path'] - model = cm.get(path) - - # Change the name in the model for rename - model = cm.save(model, path) - assert isinstance(model, dict) - self.assertIn('name', model) - self.assertIn('path', model) - self.assertEqual(model['name'], 'Untitled.ipynb') - self.assertEqual(model['path'], 'foo/Untitled.ipynb') - - def test_delete(self): - cm = self.contents_manager - # Create a notebook - nb, name, path = self.new_notebook() - - # Delete the notebook - cm.delete(path) - - # Check that deleting a non-existent path raises an error. - self.assertRaises(HTTPError, cm.delete, path) - - # Check that a 'get' on the deleted notebook raises and error - self.assertRaises(HTTPError, cm.get, path) - - def test_rename(self): - cm = self.contents_manager - # Create a new notebook - nb, name, path = self.new_notebook() - - # Rename the notebook - cm.rename(path, "changed_path") - - # Attempting to get the notebook under the old name raises an error - self.assertRaises(HTTPError, cm.get, path) - # Fetching the notebook under the new name is successful - assert isinstance(cm.get("changed_path"), dict) - - # Ported tests on nested directory renaming from pgcontents - all_dirs = ['foo', 'bar', 'foo/bar', 'foo/bar/foo', 'foo/bar/foo/bar'] - unchanged_dirs = all_dirs[:2] - changed_dirs = all_dirs[2:] - - for _dir in all_dirs: - self.make_populated_dir(_dir) - self.check_populated_dir_files(_dir) - - # Renaming to an existing directory should fail - for src, dest in combinations(all_dirs, 2): - with self.assertRaisesHTTPError(409): - cm.rename(src, dest) - - # Creating a notebook in a non_existant directory should fail - with self.assertRaisesHTTPError(404): - cm.new_untitled("foo/bar_diff", ext=".ipynb") - - cm.rename("foo/bar", "foo/bar_diff") - - # Assert that unchanged directories remain so - for unchanged in unchanged_dirs: - self.check_populated_dir_files(unchanged) - - # Assert changed directories can no longer be accessed under old names - for changed_dirname in changed_dirs: - with self.assertRaisesHTTPError(404): - cm.get(changed_dirname) - - new_dirname = changed_dirname.replace("foo/bar", "foo/bar_diff", 1) - - self.check_populated_dir_files(new_dirname) - - # Created a notebook in the renamed directory should work - cm.new_untitled("foo/bar_diff", ext=".ipynb") - - def test_delete_root(self): - cm = self.contents_manager - with self.assertRaises(HTTPError) as err: - cm.delete('') - self.assertEqual(err.exception.status_code, 400) - - def test_copy(self): - cm = self.contents_manager - parent = u'å b' - name = u'nb √.ipynb' - path = u'{0}/{1}'.format(parent, name) - self.make_dir(parent) - - orig = cm.new(path=path) - # copy with unspecified name - copy = cm.copy(path) - self.assertEqual(copy['name'], orig['name'].replace('.ipynb', '-Copy1.ipynb')) - - # copy with specified name - copy2 = cm.copy(path, u'å b/copy 2.ipynb') - self.assertEqual(copy2['name'], u'copy 2.ipynb') - self.assertEqual(copy2['path'], u'å b/copy 2.ipynb') - # copy with specified path - copy2 = cm.copy(path, u'/') - self.assertEqual(copy2['name'], name) - self.assertEqual(copy2['path'], name) - - def test_trust_notebook(self): - cm = self.contents_manager - nb, name, path = self.new_notebook() - - untrusted = cm.get(path)['content'] - assert not cm.notary.check_cells(untrusted) - - # print(untrusted) - cm.trust_notebook(path) - trusted = cm.get(path)['content'] - # print(trusted) - assert cm.notary.check_cells(trusted) - - def test_mark_trusted_cells(self): - cm = self.contents_manager - nb, name, path = self.new_notebook() - - cm.mark_trusted_cells(nb, path) - for cell in nb.cells: - if cell.cell_type == 'code': - assert not cell.metadata.trusted - - cm.trust_notebook(path) - nb = cm.get(path)['content'] - for cell in nb.cells: - if cell.cell_type == 'code': - assert cell.metadata.trusted - - def test_check_and_sign(self): - cm = self.contents_manager - nb, name, path = self.new_notebook() - - cm.mark_trusted_cells(nb, path) - cm.check_and_sign(nb, path) - assert not cm.notary.check_signature(nb) - - cm.trust_notebook(path) - nb = cm.get(path)['content'] - cm.mark_trusted_cells(nb, path) - cm.check_and_sign(nb, path) - assert cm.notary.check_signature(nb) - - -class TestContentsManagerNoAtomic(TestContentsManager): - """ - Make same test in no atomic case than in atomic case, using inheritance - """ - - def setUp(self): - self._temp_dir = TemporaryDirectory() - self.td = self._temp_dir.name - self.contents_manager = FileContentsManager( - root_dir = self.td, - ) - self.contents_manager.use_atomic_writing = False diff --git a/notebook/services/kernels/__init__.py b/notebook/services/kernels/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/kernels/handlers.py b/notebook/services/kernels/handlers.py deleted file mode 100644 index 8733968ce3..0000000000 --- a/notebook/services/kernels/handlers.py +++ /dev/null @@ -1,497 +0,0 @@ -"""Tornado handlers for kernels. - -Preliminary documentation at https://github.com/ipython/ipython/wiki/IPEP-16%3A-Notebook-multi-directory-dashboard-and-URL-mapping#kernels-api -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json -import logging -from textwrap import dedent - -from tornado import gen, web -from tornado.concurrent import Future -from tornado.ioloop import IOLoop - -from jupyter_client import protocol_version as client_protocol_version -from jupyter_client.jsonutil import date_default -from ipython_genutils.py3compat import cast_unicode -from notebook.utils import maybe_future, url_path_join, url_escape - -from ...base.handlers import APIHandler -from ...base.zmqhandlers import AuthenticatedZMQStreamHandler, deserialize_binary_message - - -class MainKernelHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self): - km = self.kernel_manager - kernels = yield maybe_future(km.list_kernels()) - self.finish(json.dumps(kernels, default=date_default)) - - @web.authenticated - @gen.coroutine - def post(self): - km = self.kernel_manager - model = self.get_json_body() - if model is None: - model = { - 'name': km.default_kernel_name - } - else: - model.setdefault('name', km.default_kernel_name) - - kernel_id = yield maybe_future(km.start_kernel(kernel_name=model['name'])) - model = yield maybe_future(km.kernel_model(kernel_id)) - location = url_path_join(self.base_url, 'api', 'kernels', url_escape(kernel_id)) - self.set_header('Location', location) - self.set_status(201) - self.finish(json.dumps(model, default=date_default)) - - -class KernelHandler(APIHandler): - - @web.authenticated - def get(self, kernel_id): - km = self.kernel_manager - model = km.kernel_model(kernel_id) - self.finish(json.dumps(model, default=date_default)) - - @web.authenticated - @gen.coroutine - def delete(self, kernel_id): - km = self.kernel_manager - yield maybe_future(km.shutdown_kernel(kernel_id)) - self.set_status(204) - self.finish() - - -class KernelActionHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def post(self, kernel_id, action): - km = self.kernel_manager - if action == 'interrupt': - km.interrupt_kernel(kernel_id) - self.set_status(204) - if action == 'restart': - - try: - yield maybe_future(km.restart_kernel(kernel_id)) - except Exception as e: - self.log.error("Exception restarting kernel", exc_info=True) - self.set_status(500) - else: - model = yield maybe_future(km.kernel_model(kernel_id)) - self.write(json.dumps(model, default=date_default)) - self.finish() - - -class ZMQChannelsHandler(AuthenticatedZMQStreamHandler): - '''There is one ZMQChannelsHandler per running kernel and it oversees all - the sessions. - ''' - - # class-level registry of open sessions - # allows checking for conflict on session-id, - # which is used as a zmq identity and must be unique. - _open_sessions = {} - - @property - def kernel_info_timeout(self): - km_default = self.kernel_manager.kernel_info_timeout - return self.settings.get('kernel_info_timeout', km_default) - - @property - def iopub_msg_rate_limit(self): - return self.settings.get('iopub_msg_rate_limit', 0) - - @property - def iopub_data_rate_limit(self): - return self.settings.get('iopub_data_rate_limit', 0) - - @property - def rate_limit_window(self): - return self.settings.get('rate_limit_window', 1.0) - - def __repr__(self): - return "%s(%s)" % (self.__class__.__name__, getattr(self, 'kernel_id', 'uninitialized')) - - def create_stream(self): - km = self.kernel_manager - identity = self.session.bsession - for channel in ('shell', 'iopub', 'stdin'): - meth = getattr(km, 'connect_' + channel) - self.channels[channel] = stream = meth(self.kernel_id, identity=identity) - stream.channel = channel - - def request_kernel_info(self): - """send a request for kernel_info""" - km = self.kernel_manager - kernel = km.get_kernel(self.kernel_id) - try: - # check for previous request - future = kernel._kernel_info_future - except AttributeError: - self.log.debug("Requesting kernel info from %s", self.kernel_id) - # Create a kernel_info channel to query the kernel protocol version. - # This channel will be closed after the kernel_info reply is received. - if self.kernel_info_channel is None: - self.kernel_info_channel = km.connect_shell(self.kernel_id) - self.kernel_info_channel.on_recv(self._handle_kernel_info_reply) - self.session.send(self.kernel_info_channel, "kernel_info_request") - # store the future on the kernel, so only one request is sent - kernel._kernel_info_future = self._kernel_info_future - else: - if not future.done(): - self.log.debug("Waiting for pending kernel_info request") - future.add_done_callback(lambda f: self._finish_kernel_info(f.result())) - return self._kernel_info_future - - def _handle_kernel_info_reply(self, msg): - """process the kernel_info_reply - - enabling msg spec adaptation, if necessary - """ - idents,msg = self.session.feed_identities(msg) - try: - msg = self.session.deserialize(msg) - except: - self.log.error("Bad kernel_info reply", exc_info=True) - self._kernel_info_future.set_result({}) - return - else: - info = msg['content'] - self.log.debug("Received kernel info: %s", info) - if msg['msg_type'] != 'kernel_info_reply' or 'protocol_version' not in info: - self.log.error("Kernel info request failed, assuming current %s", info) - info = {} - self._finish_kernel_info(info) - - # close the kernel_info channel, we don't need it anymore - if self.kernel_info_channel: - self.kernel_info_channel.close() - self.kernel_info_channel = None - - def _finish_kernel_info(self, info): - """Finish handling kernel_info reply - - Set up protocol adaptation, if needed, - and signal that connection can continue. - """ - protocol_version = info.get('protocol_version', client_protocol_version) - if protocol_version != client_protocol_version: - self.session.adapt_version = int(protocol_version.split('.')[0]) - self.log.info("Adapting from protocol version {protocol_version} (kernel {kernel_id}) to {client_protocol_version} (client).".format(protocol_version=protocol_version, kernel_id=self.kernel_id, client_protocol_version=client_protocol_version)) - if not self._kernel_info_future.done(): - self._kernel_info_future.set_result(info) - - def initialize(self): - super(ZMQChannelsHandler, self).initialize() - self.zmq_stream = None - self.channels = {} - self.kernel_id = None - self.kernel_info_channel = None - self._kernel_info_future = Future() - self._close_future = Future() - self.session_key = '' - - # Rate limiting code - self._iopub_window_msg_count = 0 - self._iopub_window_byte_count = 0 - self._iopub_msgs_exceeded = False - self._iopub_data_exceeded = False - # Queue of (time stamp, byte count) - # Allows you to specify that the byte count should be lowered - # by a delta amount at some point in the future. - self._iopub_window_byte_queue = [] - - @gen.coroutine - def pre_get(self): - # authenticate first - super(ZMQChannelsHandler, self).pre_get() - # check session collision: - yield self._register_session() - # then request kernel info, waiting up to a certain time before giving up. - # We don't want to wait forever, because browsers don't take it well when - # servers never respond to websocket connection requests. - kernel = self.kernel_manager.get_kernel(self.kernel_id) - self.session.key = kernel.session.key - future = self.request_kernel_info() - - def give_up(): - """Don't wait forever for the kernel to reply""" - if future.done(): - return - self.log.warning("Timeout waiting for kernel_info reply from %s", self.kernel_id) - future.set_result({}) - loop = IOLoop.current() - loop.add_timeout(loop.time() + self.kernel_info_timeout, give_up) - # actually wait for it - yield future - - @gen.coroutine - def get(self, kernel_id): - self.kernel_id = cast_unicode(kernel_id, 'ascii') - yield super(ZMQChannelsHandler, self).get(kernel_id=kernel_id) - - @gen.coroutine - def _register_session(self): - """Ensure we aren't creating a duplicate session. - - If a previous identical session is still open, close it to avoid collisions. - This is likely due to a client reconnecting from a lost network connection, - where the socket on our side has not been cleaned up yet. - """ - self.session_key = '%s:%s' % (self.kernel_id, self.session.session) - stale_handler = self._open_sessions.get(self.session_key) - if stale_handler: - self.log.warning("Replacing stale connection: %s", self.session_key) - yield stale_handler.close() - self._open_sessions[self.session_key] = self - - def open(self, kernel_id): - super(ZMQChannelsHandler, self).open() - km = self.kernel_manager - km.notify_connect(kernel_id) - - # on new connections, flush the message buffer - buffer_info = km.get_buffer(kernel_id, self.session_key) - if buffer_info and buffer_info['session_key'] == self.session_key: - self.log.info("Restoring connection for %s", self.session_key) - self.channels = buffer_info['channels'] - replay_buffer = buffer_info['buffer'] - if replay_buffer: - self.log.info("Replaying %s buffered messages", len(replay_buffer)) - for channel, msg_list in replay_buffer: - stream = self.channels[channel] - self._on_zmq_reply(stream, msg_list) - else: - try: - self.create_stream() - except web.HTTPError as e: - self.log.error("Error opening stream: %s", e) - # WebSockets don't response to traditional error codes so we - # close the connection. - for channel, stream in self.channels.items(): - if not stream.closed(): - stream.close() - self.close() - return - - km.add_restart_callback(self.kernel_id, self.on_kernel_restarted) - km.add_restart_callback(self.kernel_id, self.on_restart_failed, 'dead') - - for channel, stream in self.channels.items(): - stream.on_recv_stream(self._on_zmq_reply) - - def on_message(self, msg): - if not self.channels: - # already closed, ignore the message - self.log.debug("Received message on closed websocket %r", msg) - return - if isinstance(msg, bytes): - msg = deserialize_binary_message(msg) - else: - msg = json.loads(msg) - channel = msg.pop('channel', None) - if channel is None: - self.log.warning("No channel specified, assuming shell: %s", msg) - channel = 'shell' - if channel not in self.channels: - self.log.warning("No such channel: %r", channel) - return - am = self.kernel_manager.allowed_message_types - mt = msg['header']['msg_type'] - if am and mt not in am: - self.log.warning('Received message of type "%s", which is not allowed. Ignoring.' % mt) - else: - stream = self.channels[channel] - self.session.send(stream, msg) - - def _on_zmq_reply(self, stream, msg_list): - idents, fed_msg_list = self.session.feed_identities(msg_list) - msg = self.session.deserialize(fed_msg_list) - parent = msg['parent_header'] - def write_stderr(error_message): - self.log.warning(error_message) - msg = self.session.msg("stream", - content={"text": error_message + '\n', "name": "stderr"}, - parent=parent - ) - msg['channel'] = 'iopub' - self.write_message(json.dumps(msg, default=date_default)) - channel = getattr(stream, 'channel', None) - msg_type = msg['header']['msg_type'] - - if channel == 'iopub' and msg_type == 'status' and msg['content'].get('execution_state') == 'idle': - # reset rate limit counter on status=idle, - # to avoid 'Run All' hitting limits prematurely. - self._iopub_window_byte_queue = [] - self._iopub_window_msg_count = 0 - self._iopub_window_byte_count = 0 - self._iopub_msgs_exceeded = False - self._iopub_data_exceeded = False - - if channel == 'iopub' and msg_type not in {'status', 'comm_open', 'execute_input'}: - - # Remove the counts queued for removal. - now = IOLoop.current().time() - while len(self._iopub_window_byte_queue) > 0: - queued = self._iopub_window_byte_queue[0] - if (now >= queued[0]): - self._iopub_window_byte_count -= queued[1] - self._iopub_window_msg_count -= 1 - del self._iopub_window_byte_queue[0] - else: - # This part of the queue hasn't be reached yet, so we can - # abort the loop. - break - - # Increment the bytes and message count - self._iopub_window_msg_count += 1 - if msg_type == 'stream': - byte_count = sum([len(x) for x in msg_list]) - else: - byte_count = 0 - self._iopub_window_byte_count += byte_count - - # Queue a removal of the byte and message count for a time in the - # future, when we are no longer interested in it. - self._iopub_window_byte_queue.append((now + self.rate_limit_window, byte_count)) - - # Check the limits, set the limit flags, and reset the - # message and data counts. - msg_rate = float(self._iopub_window_msg_count) / self.rate_limit_window - data_rate = float(self._iopub_window_byte_count) / self.rate_limit_window - - # Check the msg rate - if self.iopub_msg_rate_limit > 0 and msg_rate > self.iopub_msg_rate_limit: - if not self._iopub_msgs_exceeded: - self._iopub_msgs_exceeded = True - write_stderr(dedent("""\ - IOPub message rate exceeded. - The notebook server will temporarily stop sending output - to the client in order to avoid crashing it. - To change this limit, set the config variable - `--NotebookApp.iopub_msg_rate_limit`. - - Current values: - NotebookApp.iopub_msg_rate_limit={} (msgs/sec) - NotebookApp.rate_limit_window={} (secs) - """.format(self.iopub_msg_rate_limit, self.rate_limit_window))) - else: - # resume once we've got some headroom below the limit - if self._iopub_msgs_exceeded and msg_rate < (0.8 * self.iopub_msg_rate_limit): - self._iopub_msgs_exceeded = False - if not self._iopub_data_exceeded: - self.log.warning("iopub messages resumed") - - # Check the data rate - if self.iopub_data_rate_limit > 0 and data_rate > self.iopub_data_rate_limit: - if not self._iopub_data_exceeded: - self._iopub_data_exceeded = True - write_stderr(dedent("""\ - IOPub data rate exceeded. - The notebook server will temporarily stop sending output - to the client in order to avoid crashing it. - To change this limit, set the config variable - `--NotebookApp.iopub_data_rate_limit`. - - Current values: - NotebookApp.iopub_data_rate_limit={} (bytes/sec) - NotebookApp.rate_limit_window={} (secs) - """.format(self.iopub_data_rate_limit, self.rate_limit_window))) - else: - # resume once we've got some headroom below the limit - if self._iopub_data_exceeded and data_rate < (0.8 * self.iopub_data_rate_limit): - self._iopub_data_exceeded = False - if not self._iopub_msgs_exceeded: - self.log.warning("iopub messages resumed") - - # If either of the limit flags are set, do not send the message. - if self._iopub_msgs_exceeded or self._iopub_data_exceeded: - # we didn't send it, remove the current message from the calculus - self._iopub_window_msg_count -= 1 - self._iopub_window_byte_count -= byte_count - self._iopub_window_byte_queue.pop(-1) - return - super(ZMQChannelsHandler, self)._on_zmq_reply(stream, msg) - - def close(self): - super(ZMQChannelsHandler, self).close() - return self._close_future - - def on_close(self): - self.log.debug("Websocket closed %s", self.session_key) - # unregister myself as an open session (only if it's really me) - if self._open_sessions.get(self.session_key) is self: - self._open_sessions.pop(self.session_key) - - km = self.kernel_manager - if self.kernel_id in km: - km.notify_disconnect(self.kernel_id) - km.remove_restart_callback( - self.kernel_id, self.on_kernel_restarted, - ) - km.remove_restart_callback( - self.kernel_id, self.on_restart_failed, 'dead', - ) - - # start buffering instead of closing if this was the last connection - if km._kernel_connections[self.kernel_id] == 0: - km.start_buffering(self.kernel_id, self.session_key, self.channels) - self._close_future.set_result(None) - return - - # This method can be called twice, once by self.kernel_died and once - # from the WebSocket close event. If the WebSocket connection is - # closed before the ZMQ streams are setup, they could be None. - for channel, stream in self.channels.items(): - if stream is not None and not stream.closed(): - stream.on_recv(None) - stream.close() - - self.channels = {} - self._close_future.set_result(None) - - def _send_status_message(self, status): - iopub = self.channels.get('iopub', None) - if iopub and not iopub.closed(): - # flush IOPub before sending a restarting/dead status message - # ensures proper ordering on the IOPub channel - # that all messages from the stopped kernel have been delivered - iopub.flush() - msg = self.session.msg("status", - {'execution_state': status} - ) - msg['channel'] = 'iopub' - self.write_message(json.dumps(msg, default=date_default)) - - def on_kernel_restarted(self): - logging.warn("kernel %s restarted", self.kernel_id) - self._send_status_message('restarting') - - def on_restart_failed(self): - logging.error("kernel %s restarted failed!", self.kernel_id) - self._send_status_message('dead') - - -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - - -_kernel_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" -_kernel_action_regex = r"(?Prestart|interrupt)" - -default_handlers = [ - (r"/api/kernels", MainKernelHandler), - (r"/api/kernels/%s" % _kernel_id_regex, KernelHandler), - (r"/api/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler), - (r"/api/kernels/%s/channels" % _kernel_id_regex, ZMQChannelsHandler), -] diff --git a/notebook/services/kernels/kernelmanager.py b/notebook/services/kernels/kernelmanager.py deleted file mode 100644 index b072e014b9..0000000000 --- a/notebook/services/kernels/kernelmanager.py +++ /dev/null @@ -1,475 +0,0 @@ -"""A MultiKernelManager for use in the notebook webserver - -- raises HTTPErrors -- creates REST API models -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from collections import defaultdict -from datetime import datetime, timedelta -from functools import partial -import os - -from tornado import gen, web -from tornado.concurrent import Future -from tornado.ioloop import IOLoop, PeriodicCallback - -from jupyter_client.session import Session -from jupyter_client.multikernelmanager import MultiKernelManager -from traitlets import (Any, Bool, Dict, List, Unicode, TraitError, Integer, - Float, Instance, default, validate -) - -from notebook.utils import maybe_future, to_os_path, exists -from notebook._tz import utcnow, isoformat -from ipython_genutils.py3compat import getcwd - -from notebook.prometheus.metrics import KERNEL_CURRENTLY_RUNNING_TOTAL - - -class MappingKernelManager(MultiKernelManager): - """A KernelManager that handles notebook mapping and HTTP error handling""" - - @default('kernel_manager_class') - def _default_kernel_manager_class(self): - return "jupyter_client.ioloop.IOLoopKernelManager" - - kernel_argv = List(Unicode()) - - root_dir = Unicode(config=True) - - _kernel_connections = Dict() - - _culler_callback = None - - _initialized_culler = False - - @default('root_dir') - def _default_root_dir(self): - try: - return self.parent.notebook_dir - except AttributeError: - return getcwd() - - @validate('root_dir') - def _update_root_dir(self, proposal): - """Do a bit of validation of the root dir.""" - value = proposal['value'] - if not os.path.isabs(value): - # If we receive a non-absolute path, make it absolute. - value = os.path.abspath(value) - if not exists(value) or not os.path.isdir(value): - raise TraitError("kernel root dir %r is not a directory" % value) - return value - - cull_idle_timeout = Integer(0, config=True, - help="""Timeout (in seconds) after which a kernel is considered idle and ready to be culled. - Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled - for users with poor network connections.""" - ) - - cull_interval_default = 300 # 5 minutes - cull_interval = Integer(cull_interval_default, config=True, - help="""The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value.""" - ) - - cull_connected = Bool(False, config=True, - help="""Whether to consider culling kernels which have one or more connections. - Only effective if cull_idle_timeout > 0.""" - ) - - cull_busy = Bool(False, config=True, - help="""Whether to consider culling kernels which are busy. - Only effective if cull_idle_timeout > 0.""" - ) - - buffer_offline_messages = Bool(True, config=True, - help="""Whether messages from kernels whose frontends have disconnected should be buffered in-memory. - - When True (default), messages are buffered and replayed on reconnect, - avoiding lost messages due to interrupted connectivity. - - Disable if long-running kernels will produce too much output while - no frontends are connected. - """ - ) - - kernel_info_timeout = Float(60, config=True, - help="""Timeout for giving up on a kernel (in seconds). - - On starting and restarting kernels, we check whether the - kernel is running and responsive by sending kernel_info_requests. - This sets the timeout in seconds for how long the kernel can take - before being presumed dead. - This affects the MappingKernelManager (which handles kernel restarts) - and the ZMQChannelsHandler (which handles the startup). - """ - ) - - _kernel_buffers = Any() - @default('_kernel_buffers') - def _default_kernel_buffers(self): - return defaultdict(lambda: {'buffer': [], 'session_key': '', 'channels': {}}) - - last_kernel_activity = Instance(datetime, - help="The last activity on any kernel, including shutting down a kernel") - - def __init__(self, **kwargs): - super(MappingKernelManager, self).__init__(**kwargs) - self.last_kernel_activity = utcnow() - - allowed_message_types = List(trait=Unicode(), config=True, - help="""White list of allowed kernel message types. - When the list is empty, all message types are allowed. - """ - ) - - #------------------------------------------------------------------------- - # Methods for managing kernels and sessions - #------------------------------------------------------------------------- - - def _handle_kernel_died(self, kernel_id): - """notice that a kernel died""" - self.log.warning("Kernel %s died, removing from map.", kernel_id) - self.remove_kernel(kernel_id) - - def cwd_for_path(self, path): - """Turn API path into absolute OS path.""" - os_path = to_os_path(path, self.root_dir) - # in the case of notebooks and kernels not being on the same filesystem, - # walk up to root_dir if the paths don't exist - while not os.path.isdir(os_path) and os_path != self.root_dir: - os_path = os.path.dirname(os_path) - return os_path - - @gen.coroutine - def start_kernel(self, kernel_id=None, path=None, **kwargs): - """Start a kernel for a session and return its kernel_id. - - Parameters - ---------- - kernel_id : uuid - The uuid to associate the new kernel with. If this - is not None, this kernel will be persistent whenever it is - requested. - path : API path - The API path (unicode, '/' delimited) for the cwd. - Will be transformed to an OS path relative to root_dir. - kernel_name : str - The name identifying which kernel spec to launch. This is ignored if - an existing kernel is returned, but it may be checked in the future. - """ - if kernel_id is None: - if path is not None: - kwargs['cwd'] = self.cwd_for_path(path) - kernel_id = yield maybe_future( - super(MappingKernelManager, self).start_kernel(**kwargs) - ) - self._kernel_connections[kernel_id] = 0 - self.start_watching_activity(kernel_id) - self.log.info("Kernel started: %s" % kernel_id) - self.log.debug("Kernel args: %r" % kwargs) - # register callback for failed auto-restart - self.add_restart_callback(kernel_id, - lambda : self._handle_kernel_died(kernel_id), - 'dead', - ) - - # Increase the metric of number of kernels running - # for the relevant kernel type by 1 - KERNEL_CURRENTLY_RUNNING_TOTAL.labels( - type=self._kernels[kernel_id].kernel_name - ).inc() - - else: - self._check_kernel_id(kernel_id) - self.log.info("Using existing kernel: %s" % kernel_id) - - # Initialize culling if not already - if not self._initialized_culler: - self.initialize_culler() - - # py2-compat - raise gen.Return(kernel_id) - - def start_buffering(self, kernel_id, session_key, channels): - """Start buffering messages for a kernel - - Parameters - ---------- - kernel_id : str - The id of the kernel to stop buffering. - session_key: str - The session_key, if any, that should get the buffer. - If the session_key matches the current buffered session_key, - the buffer will be returned. - channels: dict({'channel': ZMQStream}) - The zmq channels whose messages should be buffered. - """ - - if not self.buffer_offline_messages: - for channel, stream in channels.items(): - stream.close() - return - - self.log.info("Starting buffering for %s", session_key) - self._check_kernel_id(kernel_id) - # clear previous buffering state - self.stop_buffering(kernel_id) - buffer_info = self._kernel_buffers[kernel_id] - # record the session key because only one session can buffer - buffer_info['session_key'] = session_key - # TODO: the buffer should likely be a memory bounded queue, we're starting with a list to keep it simple - buffer_info['buffer'] = [] - buffer_info['channels'] = channels - - # forward any future messages to the internal buffer - def buffer_msg(channel, msg_parts): - self.log.debug("Buffering msg on %s:%s", kernel_id, channel) - buffer_info['buffer'].append((channel, msg_parts)) - - for channel, stream in channels.items(): - stream.on_recv(partial(buffer_msg, channel)) - - def get_buffer(self, kernel_id, session_key): - """Get the buffer for a given kernel - - Parameters - ---------- - kernel_id : str - The id of the kernel to stop buffering. - session_key: str, optional - The session_key, if any, that should get the buffer. - If the session_key matches the current buffered session_key, - the buffer will be returned. - """ - self.log.debug("Getting buffer for %s", kernel_id) - if kernel_id not in self._kernel_buffers: - return - - buffer_info = self._kernel_buffers[kernel_id] - if buffer_info['session_key'] == session_key: - # remove buffer - self._kernel_buffers.pop(kernel_id) - # only return buffer_info if it's a match - return buffer_info - else: - self.stop_buffering(kernel_id) - - def stop_buffering(self, kernel_id): - """Stop buffering kernel messages - - Parameters - ---------- - kernel_id : str - The id of the kernel to stop buffering. - """ - self.log.debug("Clearing buffer for %s", kernel_id) - self._check_kernel_id(kernel_id) - - if kernel_id not in self._kernel_buffers: - return - buffer_info = self._kernel_buffers.pop(kernel_id) - # close buffering streams - for stream in buffer_info['channels'].values(): - if not stream.closed(): - stream.on_recv(None) - stream.close() - - msg_buffer = buffer_info['buffer'] - if msg_buffer: - self.log.info("Discarding %s buffered messages for %s", - len(msg_buffer), buffer_info['session_key']) - - def shutdown_kernel(self, kernel_id, now=False): - """Shutdown a kernel by kernel_id""" - self._check_kernel_id(kernel_id) - kernel = self._kernels[kernel_id] - if kernel._activity_stream: - kernel._activity_stream.close() - kernel._activity_stream = None - self.stop_buffering(kernel_id) - self._kernel_connections.pop(kernel_id, None) - self.last_kernel_activity = utcnow() - - # Decrease the metric of number of kernels - # running for the relevant kernel type by 1 - KERNEL_CURRENTLY_RUNNING_TOTAL.labels( - type=self._kernels[kernel_id].kernel_name - ).dec() - - return super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now) - - @gen.coroutine - def restart_kernel(self, kernel_id): - """Restart a kernel by kernel_id""" - self._check_kernel_id(kernel_id) - yield maybe_future(super(MappingKernelManager, self).restart_kernel(kernel_id)) - kernel = self.get_kernel(kernel_id) - # return a Future that will resolve when the kernel has successfully restarted - channel = kernel.connect_shell() - future = Future() - - def finish(): - """Common cleanup when restart finishes/fails for any reason.""" - if not channel.closed(): - channel.close() - loop.remove_timeout(timeout) - kernel.remove_restart_callback(on_restart_failed, 'dead') - - def on_reply(msg): - self.log.debug("Kernel info reply received: %s", kernel_id) - finish() - if not future.done(): - future.set_result(msg) - - def on_timeout(): - self.log.warning("Timeout waiting for kernel_info_reply: %s", kernel_id) - finish() - if not future.done(): - future.set_exception(gen.TimeoutError("Timeout waiting for restart")) - - def on_restart_failed(): - self.log.warning("Restarting kernel failed: %s", kernel_id) - finish() - if not future.done(): - future.set_exception(RuntimeError("Restart failed")) - - kernel.add_restart_callback(on_restart_failed, 'dead') - kernel.session.send(channel, "kernel_info_request") - channel.on_recv(on_reply) - loop = IOLoop.current() - timeout = loop.add_timeout(loop.time() + self.kernel_info_timeout, on_timeout) - # wait for restart to complete - yield future - - def notify_connect(self, kernel_id): - """Notice a new connection to a kernel""" - if kernel_id in self._kernel_connections: - self._kernel_connections[kernel_id] += 1 - - def notify_disconnect(self, kernel_id): - """Notice a disconnection from a kernel""" - if kernel_id in self._kernel_connections: - self._kernel_connections[kernel_id] -= 1 - - def kernel_model(self, kernel_id): - """Return a JSON-safe dict representing a kernel - - For use in representing kernels in the JSON APIs. - """ - self._check_kernel_id(kernel_id) - kernel = self._kernels[kernel_id] - - model = { - "id":kernel_id, - "name": kernel.kernel_name, - "last_activity": isoformat(kernel.last_activity), - "execution_state": kernel.execution_state, - "connections": self._kernel_connections[kernel_id], - } - return model - - def list_kernels(self): - """Returns a list of kernel_id's of kernels running.""" - kernels = [] - kernel_ids = super(MappingKernelManager, self).list_kernel_ids() - for kernel_id in kernel_ids: - model = self.kernel_model(kernel_id) - kernels.append(model) - return kernels - - # override _check_kernel_id to raise 404 instead of KeyError - def _check_kernel_id(self, kernel_id): - """Check a that a kernel_id exists and raise 404 if not.""" - if kernel_id not in self: - raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id) - - # monitoring activity: - - def start_watching_activity(self, kernel_id): - """Start watching IOPub messages on a kernel for activity. - - - update last_activity on every message - - record execution_state from status messages - """ - kernel = self._kernels[kernel_id] - # add busy/activity markers: - kernel.execution_state = 'starting' - kernel.last_activity = utcnow() - kernel._activity_stream = kernel.connect_iopub() - session = Session( - config=kernel.session.config, - key=kernel.session.key, - ) - - def record_activity(msg_list): - """Record an IOPub message arriving from a kernel""" - self.last_kernel_activity = kernel.last_activity = utcnow() - - idents, fed_msg_list = session.feed_identities(msg_list) - msg = session.deserialize(fed_msg_list) - - msg_type = msg['header']['msg_type'] - if msg_type == 'status': - kernel.execution_state = msg['content']['execution_state'] - self.log.debug("activity on %s: %s (%s)", kernel_id, msg_type, kernel.execution_state) - else: - self.log.debug("activity on %s: %s", kernel_id, msg_type) - - kernel._activity_stream.on_recv(record_activity) - - def initialize_culler(self): - """Start idle culler if 'cull_idle_timeout' is greater than zero. - - Regardless of that value, set flag that we've been here. - """ - if not self._initialized_culler and self.cull_idle_timeout > 0: - if self._culler_callback is None: - loop = IOLoop.current() - if self.cull_interval <= 0: #handle case where user set invalid value - self.log.warning("Invalid value for 'cull_interval' detected (%s) - using default value (%s).", - self.cull_interval, self.cull_interval_default) - self.cull_interval = self.cull_interval_default - self._culler_callback = PeriodicCallback( - self.cull_kernels, 1000*self.cull_interval) - self.log.info("Culling kernels with idle durations > %s seconds at %s second intervals ...", - self.cull_idle_timeout, self.cull_interval) - if self.cull_busy: - self.log.info("Culling kernels even if busy") - if self.cull_connected: - self.log.info("Culling kernels even with connected clients") - self._culler_callback.start() - - self._initialized_culler = True - - def cull_kernels(self): - self.log.debug("Polling every %s seconds for kernels idle > %s seconds...", - self.cull_interval, self.cull_idle_timeout) - """Create a separate list of kernels to avoid conflicting updates while iterating""" - for kernel_id in list(self._kernels): - try: - self.cull_kernel_if_idle(kernel_id) - except Exception as e: - self.log.exception("The following exception was encountered while checking the idle duration of kernel %s: %s", - kernel_id, e) - - def cull_kernel_if_idle(self, kernel_id): - kernel = self._kernels[kernel_id] - self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity) - if kernel.last_activity is not None: - dt_now = utcnow() - dt_idle = dt_now - kernel.last_activity - # Compute idle properties - is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout) - is_idle_execute = self.cull_busy or (kernel.execution_state != 'busy') - connections = self._kernel_connections.get(kernel_id, 0) - is_idle_connected = self.cull_connected or not connections - # Cull the kernel if all three criteria are met - if (is_idle_time and is_idle_execute and is_idle_connected): - idle_duration = int(dt_idle.total_seconds()) - self.log.warning("Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.", - kernel.execution_state, kernel.kernel_name, kernel_id, connections, idle_duration) - self.shutdown_kernel(kernel_id) diff --git a/notebook/services/kernels/tests/__init__.py b/notebook/services/kernels/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/kernels/tests/test_kernels_api.py b/notebook/services/kernels/tests/test_kernels_api.py deleted file mode 100644 index 83bfb0c3e0..0000000000 --- a/notebook/services/kernels/tests/test_kernels_api.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Test the kernels service API.""" - -import json -import time - -from traitlets.config import Config - -from tornado.httpclient import HTTPRequest -from tornado.ioloop import IOLoop -from tornado.websocket import websocket_connect - -from jupyter_client.kernelspec import NATIVE_KERNEL_NAME - -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase, assert_http_error - - -class KernelAPI(object): - """Wrapper for kernel REST API requests""" - def __init__(self, request, base_url, headers): - self.request = request - self.base_url = base_url - self.headers = headers - - def _req(self, verb, path, body=None): - response = self.request(verb, - url_path_join('api/kernels', path), data=body) - - if 400 <= response.status_code < 600: - try: - response.reason = response.json()['message'] - except: - pass - response.raise_for_status() - - return response - - def list(self): - return self._req('GET', '') - - def get(self, id): - return self._req('GET', id) - - def start(self, name=NATIVE_KERNEL_NAME): - body = json.dumps({'name': name}) - return self._req('POST', '', body) - - def shutdown(self, id): - return self._req('DELETE', id) - - def interrupt(self, id): - return self._req('POST', url_path_join(id, 'interrupt')) - - def restart(self, id): - return self._req('POST', url_path_join(id, 'restart')) - - def websocket(self, id): - loop = IOLoop() - loop.make_current() - req = HTTPRequest( - url_path_join(self.base_url.replace('http', 'ws', 1), 'api/kernels', id, 'channels'), - headers=self.headers, - ) - f = websocket_connect(req) - return loop.run_sync(lambda : f) - - -class KernelAPITest(NotebookTestBase): - """Test the kernels web service API""" - def setUp(self): - self.kern_api = KernelAPI(self.request, - base_url=self.base_url(), - headers=self.auth_headers(), - ) - - def tearDown(self): - for k in self.kern_api.list().json(): - self.kern_api.shutdown(k['id']) - - def test_no_kernels(self): - """Make sure there are no kernels running at the start""" - kernels = self.kern_api.list().json() - self.assertEqual(kernels, []) - - def test_default_kernel(self): - # POST request - r = self.kern_api._req('POST', '') - kern1 = r.json() - self.assertEqual(r.headers['location'], url_path_join(self.url_prefix, 'api/kernels', kern1['id'])) - self.assertEqual(r.status_code, 201) - self.assertIsInstance(kern1, dict) - - report_uri = url_path_join(self.url_prefix, 'api/security/csp-report') - expected_csp = '; '.join([ - "frame-ancestors 'self'", - 'report-uri ' + report_uri, - "default-src 'none'" - ]) - self.assertEqual(r.headers['Content-Security-Policy'], expected_csp) - - def test_main_kernel_handler(self): - # POST request - r = self.kern_api.start() - kern1 = r.json() - self.assertEqual(r.headers['location'], url_path_join(self.url_prefix, 'api/kernels', kern1['id'])) - self.assertEqual(r.status_code, 201) - self.assertIsInstance(kern1, dict) - - report_uri = url_path_join(self.url_prefix, 'api/security/csp-report') - expected_csp = '; '.join([ - "frame-ancestors 'self'", - 'report-uri ' + report_uri, - "default-src 'none'" - ]) - self.assertEqual(r.headers['Content-Security-Policy'], expected_csp) - - # GET request - r = self.kern_api.list() - self.assertEqual(r.status_code, 200) - assert isinstance(r.json(), list) - self.assertEqual(r.json()[0]['id'], kern1['id']) - self.assertEqual(r.json()[0]['name'], kern1['name']) - - # create another kernel and check that they both are added to the - # list of kernels from a GET request - kern2 = self.kern_api.start().json() - assert isinstance(kern2, dict) - r = self.kern_api.list() - kernels = r.json() - self.assertEqual(r.status_code, 200) - assert isinstance(kernels, list) - self.assertEqual(len(kernels), 2) - - # Interrupt a kernel - r = self.kern_api.interrupt(kern2['id']) - self.assertEqual(r.status_code, 204) - - # Restart a kernel - r = self.kern_api.restart(kern2['id']) - rekern = r.json() - self.assertEqual(rekern['id'], kern2['id']) - self.assertEqual(rekern['name'], kern2['name']) - - def test_kernel_handler(self): - # GET kernel with given id - kid = self.kern_api.start().json()['id'] - r = self.kern_api.get(kid) - kern1 = r.json() - self.assertEqual(r.status_code, 200) - assert isinstance(kern1, dict) - self.assertIn('id', kern1) - self.assertEqual(kern1['id'], kid) - - # Request a bad kernel id and check that a JSON - # message is returned! - bad_id = '111-111-111-111-111' - with assert_http_error(404, 'Kernel does not exist: ' + bad_id): - self.kern_api.get(bad_id) - - # DELETE kernel with id - r = self.kern_api.shutdown(kid) - self.assertEqual(r.status_code, 204) - kernels = self.kern_api.list().json() - self.assertEqual(kernels, []) - - # Request to delete a non-existent kernel id - bad_id = '111-111-111-111-111' - with assert_http_error(404, 'Kernel does not exist: ' + bad_id): - self.kern_api.shutdown(bad_id) - - def test_connections(self): - kid = self.kern_api.start().json()['id'] - model = self.kern_api.get(kid).json() - self.assertEqual(model['connections'], 0) - - ws = self.kern_api.websocket(kid) - model = self.kern_api.get(kid).json() - self.assertEqual(model['connections'], 1) - ws.close() - # give it some time to close on the other side: - for i in range(10): - model = self.kern_api.get(kid).json() - if model['connections'] > 0: - time.sleep(0.1) - else: - break - model = self.kern_api.get(kid).json() - self.assertEqual(model['connections'], 0) - - -class KernelFilterTest(NotebookTestBase): - - # A special install of NotebookTestBase where only `kernel_info_request` - # messages are allowed. - config = Config({ - 'NotebookApp': { - 'MappingKernelManager': { - 'allowed_message_types': ['kernel_info_request'] - } - } - }) - # Sanity check verifying that the configurable was properly set. - def test_config(self): - self.assertEqual(self.notebook.kernel_manager.allowed_message_types, ['kernel_info_request']) diff --git a/notebook/services/kernelspecs/__init__.py b/notebook/services/kernelspecs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/kernelspecs/handlers.py b/notebook/services/kernelspecs/handlers.py deleted file mode 100644 index 6302e96dfa..0000000000 --- a/notebook/services/kernelspecs/handlers.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Tornado handlers for kernel specifications. - -Preliminary documentation at https://github.com/ipython/ipython/wiki/IPEP-25%3A-Registry-of-installed-kernels#rest-api -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import glob -import json -import os -pjoin = os.path.join - -from tornado import web, gen - -from ...base.handlers import APIHandler -from ...utils import maybe_future, url_path_join, url_unescape - - - -def kernelspec_model(handler, name, spec_dict, resource_dir): - """Load a KernelSpec by name and return the REST API model""" - d = { - 'name': name, - 'spec': spec_dict, - 'resources': {} - } - - # Add resource files if they exist - resource_dir = resource_dir - for resource in ['kernel.js', 'kernel.css']: - if os.path.exists(pjoin(resource_dir, resource)): - d['resources'][resource] = url_path_join( - handler.base_url, - 'kernelspecs', - name, - resource - ) - for logo_file in glob.glob(pjoin(resource_dir, 'logo-*')): - fname = os.path.basename(logo_file) - no_ext, _ = os.path.splitext(fname) - d['resources'][no_ext] = url_path_join( - handler.base_url, - 'kernelspecs', - name, - fname - ) - return d - - -def is_kernelspec_model(spec_dict): - """Returns True if spec_dict is already in proper form. This will occur when using a gateway.""" - return isinstance(spec_dict, dict) and 'name' in spec_dict and 'spec' in spec_dict and 'resources' in spec_dict - - -class MainKernelSpecHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self): - ksm = self.kernel_spec_manager - km = self.kernel_manager - model = {} - model['default'] = km.default_kernel_name - model['kernelspecs'] = specs = {} - kspecs = yield maybe_future(ksm.get_all_specs()) - for kernel_name, kernel_info in kspecs.items(): - try: - if is_kernelspec_model(kernel_info): - d = kernel_info - else: - d = kernelspec_model(self, kernel_name, kernel_info['spec'], kernel_info['resource_dir']) - except Exception: - self.log.error("Failed to load kernel spec: '%s'", kernel_name, exc_info=True) - continue - specs[kernel_name] = d - self.set_header("Content-Type", 'application/json') - self.finish(json.dumps(model)) - - -class KernelSpecHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self, kernel_name): - ksm = self.kernel_spec_manager - kernel_name = url_unescape(kernel_name) - try: - spec = yield maybe_future(ksm.get_kernel_spec(kernel_name)) - except KeyError: - raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) - if is_kernelspec_model(spec): - model = spec - else: - model = kernelspec_model(self, kernel_name, spec.to_dict(), spec.resource_dir) - self.set_header("Content-Type", 'application/json') - self.finish(json.dumps(model)) - - -# URL to handler mappings - -kernel_name_regex = r"(?P[\w\.\-%]+)" - -default_handlers = [ - (r"/api/kernelspecs", MainKernelSpecHandler), - (r"/api/kernelspecs/%s" % kernel_name_regex, KernelSpecHandler), -] diff --git a/notebook/services/kernelspecs/tests/__init__.py b/notebook/services/kernelspecs/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/kernelspecs/tests/test_kernelspecs_api.py b/notebook/services/kernelspecs/tests/test_kernelspecs_api.py deleted file mode 100644 index 551f1dd558..0000000000 --- a/notebook/services/kernelspecs/tests/test_kernelspecs_api.py +++ /dev/null @@ -1,137 +0,0 @@ -# coding: utf-8 -"""Test the kernel specs webservice API.""" - -import errno -import io -import json -import os -import shutil - -pjoin = os.path.join - -import requests - -from jupyter_client.kernelspec import NATIVE_KERNEL_NAME -from notebook.utils import url_path_join, url_escape -from notebook.tests.launchnotebook import NotebookTestBase, assert_http_error - -# Copied from jupyter_client.tests.test_kernelspec so updating that doesn't -# break these tests -sample_kernel_json = {'argv':['cat', '{connection_file}'], - 'display_name':'Test kernel', - } - -some_resource = u"The very model of a modern major general" - - -class KernelSpecAPI(object): - """Wrapper for notebook API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, path, body=None): - response = self.request(verb, - path, - data=body, - ) - response.raise_for_status() - return response - - def list(self): - return self._req('GET', 'api/kernelspecs') - - def kernel_spec_info(self, name): - return self._req('GET', url_path_join('api/kernelspecs', name)) - - def kernel_resource(self, name, path): - return self._req('GET', url_path_join('kernelspecs', name, path)) - - -class APITest(NotebookTestBase): - """Test the kernelspec web service API""" - def setUp(self): - self.create_spec('sample') - self.create_spec('sample 2') - self.ks_api = KernelSpecAPI(self.request) - - def create_spec(self, name): - sample_kernel_dir = pjoin(self.data_dir, 'kernels', name) - try: - os.makedirs(sample_kernel_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - with open(pjoin(sample_kernel_dir, 'kernel.json'), 'w') as f: - json.dump(sample_kernel_json, f) - - with io.open(pjoin(sample_kernel_dir, 'resource.txt'), 'w', - encoding='utf-8') as f: - f.write(some_resource) - - def test_list_kernelspecs_bad(self): - """Can list kernelspecs when one is invalid""" - bad_kernel_dir = pjoin(self.data_dir, 'kernels', 'bad') - try: - os.makedirs(bad_kernel_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - with open(pjoin(bad_kernel_dir, 'kernel.json'), 'w') as f: - f.write("garbage") - - model = self.ks_api.list().json() - assert isinstance(model, dict) - self.assertEqual(model['default'], NATIVE_KERNEL_NAME) - specs = model['kernelspecs'] - assert isinstance(specs, dict) - # 2: the sample kernelspec created in setUp, and the native Python kernel - self.assertGreaterEqual(len(specs), 2) - - shutil.rmtree(bad_kernel_dir) - - def test_list_kernelspecs(self): - model = self.ks_api.list().json() - assert isinstance(model, dict) - self.assertEqual(model['default'], NATIVE_KERNEL_NAME) - specs = model['kernelspecs'] - assert isinstance(specs, dict) - - # 2: the sample kernelspec created in setUp, and the native Python kernel - self.assertGreaterEqual(len(specs), 2) - - def is_sample_kernelspec(s): - return s['name'] == 'sample' and s['spec']['display_name'] == 'Test kernel' - - def is_default_kernelspec(s): - return s['name'] == NATIVE_KERNEL_NAME and s['spec']['display_name'].startswith("Python") - - assert any(is_sample_kernelspec(s) for s in specs.values()), specs - assert any(is_default_kernelspec(s) for s in specs.values()), specs - - def test_get_kernelspec(self): - model = self.ks_api.kernel_spec_info('Sample').json() # Case insensitive - self.assertEqual(model['name'].lower(), 'sample') - self.assertIsInstance(model['spec'], dict) - self.assertEqual(model['spec']['display_name'], 'Test kernel') - self.assertIsInstance(model['resources'], dict) - - def test_get_kernelspec_spaces(self): - model = self.ks_api.kernel_spec_info('sample%202').json() - self.assertEqual(model['name'].lower(), 'sample 2') - - def test_get_nonexistant_kernelspec(self): - with assert_http_error(404): - self.ks_api.kernel_spec_info('nonexistant') - - def test_get_kernel_resource_file(self): - res = self.ks_api.kernel_resource('sAmple', 'resource.txt') - self.assertEqual(res.text, some_resource) - - def test_get_nonexistant_resource(self): - with assert_http_error(404): - self.ks_api.kernel_resource('nonexistant', 'resource.txt') - - with assert_http_error(404): - self.ks_api.kernel_resource('sample', 'nonexistant.txt') diff --git a/notebook/services/nbconvert/__init__.py b/notebook/services/nbconvert/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/nbconvert/handlers.py b/notebook/services/nbconvert/handlers.py deleted file mode 100644 index 2a9897fc47..0000000000 --- a/notebook/services/nbconvert/handlers.py +++ /dev/null @@ -1,39 +0,0 @@ -import json - -from tornado import web - -from ...base.handlers import APIHandler - - -class NbconvertRootHandler(APIHandler): - - @web.authenticated - def get(self): - self.check_xsrf_cookie() - try: - from nbconvert.exporters import base - except ImportError as e: - raise web.HTTPError(500, "Could not import nbconvert: %s" % e) - res = {} - exporters = base.get_export_names() - for exporter_name in exporters: - try: - exporter_class = base.get_exporter(exporter_name) - except ValueError: - # I think the only way this will happen is if the entrypoint - # is uninstalled while this method is running - continue - # XXX: According to the docs, it looks like this should be set to None - # if the exporter shouldn't be exposed to the front-end and a friendly - # name if it should. However, none of the built-in exports have it defined. - # if not exporter_class.export_from_notebook: - # continue - res[exporter_name] = { - "output_mimetype": exporter_class.output_mimetype, - } - - self.finish(json.dumps(res)) - -default_handlers = [ - (r"/api/nbconvert", NbconvertRootHandler), -] diff --git a/notebook/services/nbconvert/tests/__init__.py b/notebook/services/nbconvert/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/nbconvert/tests/test_nbconvert_api.py b/notebook/services/nbconvert/tests/test_nbconvert_api.py deleted file mode 100644 index d6ef9d2ca5..0000000000 --- a/notebook/services/nbconvert/tests/test_nbconvert_api.py +++ /dev/null @@ -1,31 +0,0 @@ -import requests - -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase - -class NbconvertAPI(object): - """Wrapper for nbconvert API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, path, body=None, params=None): - response = self.request(verb, - url_path_join('api/nbconvert', path), - data=body, params=params, - ) - response.raise_for_status() - return response - - def list_formats(self): - return self._req('GET', '') - -class APITest(NotebookTestBase): - def setUp(self): - self.nbconvert_api = NbconvertAPI(self.request) - - def test_list_formats(self): - formats = self.nbconvert_api.list_formats().json() - self.assertIsInstance(formats, dict) - self.assertIn('python', formats) - self.assertIn('html', formats) - self.assertEqual(formats['python']['output_mimetype'], 'text/x-python') \ No newline at end of file diff --git a/notebook/services/security/__init__.py b/notebook/services/security/__init__.py deleted file mode 100644 index 9cf0d476b1..0000000000 --- a/notebook/services/security/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# URI for the CSP Report. Included here to prevent a cyclic dependency. -# csp_report_uri is needed both by the BaseHandler (for setting the report-uri) -# and by the CSPReportHandler (which depends on the BaseHandler). -csp_report_uri = r"/api/security/csp-report" diff --git a/notebook/services/security/handlers.py b/notebook/services/security/handlers.py deleted file mode 100644 index 82a00d234b..0000000000 --- a/notebook/services/security/handlers.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Tornado handlers for security logging.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from tornado import web - -from ...base.handlers import APIHandler -from . import csp_report_uri - -class CSPReportHandler(APIHandler): - '''Accepts a content security policy violation report''' - - _track_activity = False - - def skip_check_origin(self): - """Don't check origin when reporting origin-check violations!""" - return True - - def check_xsrf_cookie(self): - # don't check XSRF for CSP reports - return - - @web.authenticated - def post(self): - '''Log a content security policy violation report''' - self.log.warning("Content security violation: %s", - self.request.body.decode('utf8', 'replace')) - -default_handlers = [ - (csp_report_uri, CSPReportHandler) -] diff --git a/notebook/services/sessions/__init__.py b/notebook/services/sessions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/sessions/handlers.py b/notebook/services/sessions/handlers.py deleted file mode 100644 index 49e030df87..0000000000 --- a/notebook/services/sessions/handlers.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Tornado handlers for the sessions web service. - -Preliminary documentation at https://github.com/ipython/ipython/wiki/IPEP-16%3A-Notebook-multi-directory-dashboard-and-URL-mapping#sessions-api -""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import json - -from tornado import gen, web - -from ...base.handlers import APIHandler -from jupyter_client.jsonutil import date_default -from notebook.utils import maybe_future, url_path_join -from jupyter_client.kernelspec import NoSuchKernel - - -class SessionRootHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self): - # Return a list of running sessions - sm = self.session_manager - sessions = yield maybe_future(sm.list_sessions()) - self.finish(json.dumps(sessions, default=date_default)) - - @web.authenticated - @gen.coroutine - def post(self): - # Creates a new session - #(unless a session already exists for the named session) - sm = self.session_manager - - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, "No JSON data provided") - - if 'notebook' in model and 'path' in model['notebook']: - self.log.warning('Sessions API changed, see updated swagger docs') - model['path'] = model['notebook']['path'] - model['type'] = 'notebook' - - try: - path = model['path'] - except KeyError: - raise web.HTTPError(400, "Missing field in JSON data: path") - - try: - mtype = model['type'] - except KeyError: - raise web.HTTPError(400, "Missing field in JSON data: type") - - name = model.get('name', None) - kernel = model.get('kernel', {}) - kernel_name = kernel.get('name', None) - kernel_id = kernel.get('id', None) - - if not kernel_id and not kernel_name: - self.log.debug("No kernel specified, using default kernel") - kernel_name = None - - exists = yield maybe_future(sm.session_exists(path=path)) - if exists: - model = yield maybe_future(sm.get_session(path=path)) - else: - try: - model = yield maybe_future( - sm.create_session(path=path, kernel_name=kernel_name, - kernel_id=kernel_id, name=name, - type=mtype)) - except NoSuchKernel: - msg = ("The '%s' kernel is not available. Please pick another " - "suitable kernel instead, or install that kernel." % kernel_name) - status_msg = '%s not found' % kernel_name - self.log.warning('Kernel not found: %s' % kernel_name) - self.set_status(501) - self.finish(json.dumps(dict(message=msg, short_message=status_msg))) - return - - location = url_path_join(self.base_url, 'api', 'sessions', model['id']) - self.set_header('Location', location) - self.set_status(201) - self.finish(json.dumps(model, default=date_default)) - - -class SessionHandler(APIHandler): - - @web.authenticated - @gen.coroutine - def get(self, session_id): - # Returns the JSON model for a single session - sm = self.session_manager - model = yield maybe_future(sm.get_session(session_id=session_id)) - self.finish(json.dumps(model, default=date_default)) - - @web.authenticated - @gen.coroutine - def patch(self, session_id): - """Patch updates sessions: - - - path updates session to track renamed paths - - kernel.name starts a new kernel with a given kernelspec - """ - sm = self.session_manager - km = self.kernel_manager - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, "No JSON data provided") - - # get the previous session model - before = yield maybe_future(sm.get_session(session_id=session_id)) - - changes = {} - if 'notebook' in model and 'path' in model['notebook']: - self.log.warning('Sessions API changed, see updated swagger docs') - model['path'] = model['notebook']['path'] - model['type'] = 'notebook' - if 'path' in model: - changes['path'] = model['path'] - if 'name' in model: - changes['name'] = model['name'] - if 'type' in model: - changes['type'] = model['type'] - if 'kernel' in model: - # Kernel id takes precedence over name. - if model['kernel'].get('id') is not None: - kernel_id = model['kernel']['id'] - if kernel_id not in km: - raise web.HTTPError(400, "No such kernel: %s" % kernel_id) - changes['kernel_id'] = kernel_id - elif model['kernel'].get('name') is not None: - kernel_name = model['kernel']['name'] - kernel_id = yield sm.start_kernel_for_session( - session_id, kernel_name=kernel_name, name=before['name'], - path=before['path'], type=before['type']) - changes['kernel_id'] = kernel_id - - yield maybe_future(sm.update_session(session_id, **changes)) - model = yield maybe_future(sm.get_session(session_id=session_id)) - - if model['kernel']['id'] != before['kernel']['id']: - # kernel_id changed because we got a new kernel - # shutdown the old one - yield maybe_future( - km.shutdown_kernel(before['kernel']['id']) - ) - self.finish(json.dumps(model, default=date_default)) - - @web.authenticated - @gen.coroutine - def delete(self, session_id): - # Deletes the session with given session_id - sm = self.session_manager - try: - yield maybe_future(sm.delete_session(session_id)) - except KeyError: - # the kernel was deleted but the session wasn't! - raise web.HTTPError(410, "Kernel deleted before session") - self.set_status(204) - self.finish() - - -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - -_session_id_regex = r"(?P\w+-\w+-\w+-\w+-\w+)" - -default_handlers = [ - (r"/api/sessions/%s" % _session_id_regex, SessionHandler), - (r"/api/sessions", SessionRootHandler) -] - diff --git a/notebook/services/sessions/sessionmanager.py b/notebook/services/sessions/sessionmanager.py deleted file mode 100644 index 63e1844829..0000000000 --- a/notebook/services/sessions/sessionmanager.py +++ /dev/null @@ -1,275 +0,0 @@ -"""A base class session manager.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -import uuid - -try: - import sqlite3 -except ImportError: - # fallback on pysqlite2 if Python was build without sqlite - from pysqlite2 import dbapi2 as sqlite3 - -from tornado import gen, web - -from traitlets.config.configurable import LoggingConfigurable -from ipython_genutils.py3compat import unicode_type -from traitlets import Instance - -from notebook.utils import maybe_future - - -class SessionManager(LoggingConfigurable): - - kernel_manager = Instance('notebook.services.kernels.kernelmanager.MappingKernelManager') - contents_manager = Instance('notebook.services.contents.manager.ContentsManager') - - # Session database initialized below - _cursor = None - _connection = None - _columns = {'session_id', 'path', 'name', 'type', 'kernel_id'} - - @property - def cursor(self): - """Start a cursor and create a database called 'session'""" - if self._cursor is None: - self._cursor = self.connection.cursor() - self._cursor.execute("""CREATE TABLE session - (session_id, path, name, type, kernel_id)""") - return self._cursor - - @property - def connection(self): - """Start a database connection""" - if self._connection is None: - self._connection = sqlite3.connect(':memory:') - self._connection.row_factory = sqlite3.Row - return self._connection - - def close(self): - """Close the sqlite connection""" - if self._cursor is not None: - self._cursor.close() - self._cursor = None - - def __del__(self): - """Close connection once SessionManager closes""" - self.close() - - @gen.coroutine - def session_exists(self, path): - """Check to see if the session of a given name exists""" - exists = False - self.cursor.execute("SELECT * FROM session WHERE path=?", (path,)) - row = self.cursor.fetchone() - if row is not None: - # Note, although we found a row for the session, the associated kernel may have - # been culled or died unexpectedly. If that's the case, we should delete the - # row, thereby terminating the session. This can be done via a call to - # row_to_model that tolerates that condition. If row_to_model returns None, - # we'll return false, since, at that point, the session doesn't exist anyway. - model = yield maybe_future(self.row_to_model(row, tolerate_culled=True)) - if model is not None: - exists = True - raise gen.Return(exists) - - def new_session_id(self): - "Create a uuid for a new session" - return unicode_type(uuid.uuid4()) - - @gen.coroutine - def create_session(self, path=None, name=None, type=None, kernel_name=None, kernel_id=None): - """Creates a session and returns its model""" - session_id = self.new_session_id() - if kernel_id is not None and kernel_id in self.kernel_manager: - pass - else: - kernel_id = yield self.start_kernel_for_session(session_id, path, name, type, kernel_name) - result = yield maybe_future( - self.save_session(session_id, path=path, name=name, type=type, kernel_id=kernel_id) - ) - # py2-compat - raise gen.Return(result) - - @gen.coroutine - def start_kernel_for_session(self, session_id, path, name, type, kernel_name): - """Start a new kernel for a given session.""" - # allow contents manager to specify kernels cwd - kernel_path = self.contents_manager.get_kernel_path(path=path) - kernel_id = yield maybe_future( - self.kernel_manager.start_kernel(path=kernel_path, kernel_name=kernel_name) - ) - # py2-compat - raise gen.Return(kernel_id) - - @gen.coroutine - def save_session(self, session_id, path=None, name=None, type=None, kernel_id=None): - """Saves the items for the session with the given session_id - - Given a session_id (and any other of the arguments), this method - creates a row in the sqlite session database that holds the information - for a session. - - Parameters - ---------- - session_id : str - uuid for the session; this method must be given a session_id - path : str - the path for the given session - name: str - the name of the session - type: string - the type of the session - kernel_id : str - a uuid for the kernel associated with this session - - Returns - ------- - model : dict - a dictionary of the session model - """ - self.cursor.execute("INSERT INTO session VALUES (?,?,?,?,?)", - (session_id, path, name, type, kernel_id) - ) - result = yield maybe_future(self.get_session(session_id=session_id)) - raise gen.Return(result) - - @gen.coroutine - def get_session(self, **kwargs): - """Returns the model for a particular session. - - Takes a keyword argument and searches for the value in the session - database, then returns the rest of the session's info. - - Parameters - ---------- - **kwargs : keyword argument - must be given one of the keywords and values from the session database - (i.e. session_id, path, name, type, kernel_id) - - Returns - ------- - model : dict - returns a dictionary that includes all the information from the - session described by the kwarg. - """ - if not kwargs: - raise TypeError("must specify a column to query") - - conditions = [] - for column in kwargs.keys(): - if column not in self._columns: - raise TypeError("No such column: %r", column) - conditions.append("%s=?" % column) - - query = "SELECT * FROM session WHERE %s" % (' AND '.join(conditions)) - - self.cursor.execute(query, list(kwargs.values())) - try: - row = self.cursor.fetchone() - except KeyError: - # The kernel is missing, so the session just got deleted. - row = None - - if row is None: - q = [] - for key, value in kwargs.items(): - q.append("%s=%r" % (key, value)) - - raise web.HTTPError(404, u'Session not found: %s' % (', '.join(q))) - - model = yield maybe_future(self.row_to_model(row)) - raise gen.Return(model) - - @gen.coroutine - def update_session(self, session_id, **kwargs): - """Updates the values in the session database. - - Changes the values of the session with the given session_id - with the values from the keyword arguments. - - Parameters - ---------- - session_id : str - a uuid that identifies a session in the sqlite3 database - **kwargs : str - the key must correspond to a column title in session database, - and the value replaces the current value in the session - with session_id. - """ - yield maybe_future(self.get_session(session_id=session_id)) - - if not kwargs: - # no changes - return - - sets = [] - for column in kwargs.keys(): - if column not in self._columns: - raise TypeError("No such column: %r" % column) - sets.append("%s=?" % column) - query = "UPDATE session SET %s WHERE session_id=?" % (', '.join(sets)) - self.cursor.execute(query, list(kwargs.values()) + [session_id]) - - def kernel_culled(self, kernel_id): - """Checks if the kernel is still considered alive and returns true if its not found. """ - return kernel_id not in self.kernel_manager - - @gen.coroutine - def row_to_model(self, row, tolerate_culled=False): - """Takes sqlite database session row and turns it into a dictionary""" - kernel_culled = yield maybe_future(self.kernel_culled(row['kernel_id'])) - if kernel_culled: - # The kernel was culled or died without deleting the session. - # We can't use delete_session here because that tries to find - # and shut down the kernel - so we'll delete the row directly. - # - # If caller wishes to tolerate culled kernels, log a warning - # and return None. Otherwise, raise KeyError with a similar - # message. - self.cursor.execute("DELETE FROM session WHERE session_id=?", - (row['session_id'],)) - msg = "Kernel '{kernel_id}' appears to have been culled or died unexpectedly, " \ - "invalidating session '{session_id}'. The session has been removed.".\ - format(kernel_id=row['kernel_id'],session_id=row['session_id']) - if tolerate_culled: - self.log.warning(msg + " Continuing...") - raise gen.Return(None) - raise KeyError(msg) - - kernel_model = yield maybe_future(self.kernel_manager.kernel_model(row['kernel_id'])) - model = { - 'id': row['session_id'], - 'path': row['path'], - 'name': row['name'], - 'type': row['type'], - 'kernel': kernel_model - } - if row['type'] == 'notebook': - # Provide the deprecated API. - model['notebook'] = {'path': row['path'], 'name': row['name']} - raise gen.Return(model) - - @gen.coroutine - def list_sessions(self): - """Returns a list of dictionaries containing all the information from - the session database""" - c = self.cursor.execute("SELECT * FROM session") - result = [] - # We need to use fetchall() here, because row_to_model can delete rows, - # which messes up the cursor if we're iterating over rows. - for row in c.fetchall(): - try: - model = yield maybe_future(self.row_to_model(row)) - result.append(model) - except KeyError: - pass - raise gen.Return(result) - - @gen.coroutine - def delete_session(self, session_id): - """Deletes the row in the session database with given session_id""" - session = yield maybe_future(self.get_session(session_id=session_id)) - yield maybe_future(self.kernel_manager.shutdown_kernel(session['kernel']['id'])) - self.cursor.execute("DELETE FROM session WHERE session_id=?", (session_id,)) diff --git a/notebook/services/sessions/tests/__init__.py b/notebook/services/sessions/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/services/sessions/tests/test_sessionmanager.py b/notebook/services/sessions/tests/test_sessionmanager.py deleted file mode 100644 index 97331ebf9b..0000000000 --- a/notebook/services/sessions/tests/test_sessionmanager.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Tests for the session manager.""" - -from functools import partial -from unittest import TestCase - -from tornado import gen, web -from tornado.ioloop import IOLoop - -from ..sessionmanager import SessionManager -from notebook.services.kernels.kernelmanager import MappingKernelManager -from notebook.services.contents.manager import ContentsManager -from notebook._tz import utcnow, isoformat - -class DummyKernel(object): - def __init__(self, kernel_name='python'): - self.kernel_name = kernel_name - -dummy_date = utcnow() -dummy_date_s = isoformat(dummy_date) - -class DummyMKM(MappingKernelManager): - """MappingKernelManager interface that doesn't start kernels, for testing""" - def __init__(self, *args, **kwargs): - super(DummyMKM, self).__init__(*args, **kwargs) - self.id_letters = iter(u'ABCDEFGHIJK') - - def _new_id(self): - return next(self.id_letters) - - def start_kernel(self, kernel_id=None, path=None, kernel_name='python', **kwargs): - kernel_id = kernel_id or self._new_id() - k = self._kernels[kernel_id] = DummyKernel(kernel_name=kernel_name) - self._kernel_connections[kernel_id] = 0 - k.last_activity = dummy_date - k.execution_state = 'idle' - return kernel_id - - def shutdown_kernel(self, kernel_id, now=False): - del self._kernels[kernel_id] - - -class TestSessionManager(TestCase): - - def setUp(self): - self.sm = SessionManager( - kernel_manager=DummyMKM(), - contents_manager=ContentsManager(), - ) - self.loop = IOLoop() - self.addCleanup(partial(self.loop.close, all_fds=True)) - - def create_sessions(self, *kwarg_list): - @gen.coroutine - def co_add(): - sessions = [] - for kwargs in kwarg_list: - kwargs.setdefault('type', 'notebook') - session = yield self.sm.create_session(**kwargs) - sessions.append(session) - raise gen.Return(sessions) - return self.loop.run_sync(co_add) - - def create_session(self, **kwargs): - return self.create_sessions(kwargs)[0] - - def test_get_session(self): - sm = self.sm - session_id = self.create_session(path='/path/to/test.ipynb', kernel_name='bar')['id'] - model = self.loop.run_sync(lambda: sm.get_session(session_id=session_id)) - expected = {'id':session_id, - 'path': u'/path/to/test.ipynb', - 'notebook': {'path': u'/path/to/test.ipynb', 'name': None}, - 'type': 'notebook', - 'name': None, - 'kernel': { - 'id': 'A', - 'name': 'bar', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - }} - self.assertEqual(model, expected) - - def test_bad_get_session(self): - # Should raise error if a bad key is passed to the database. - sm = self.sm - session_id = self.create_session(path='/path/to/test.ipynb', - kernel_name='foo')['id'] - with self.assertRaises(TypeError): - self.loop.run_sync(lambda: sm.get_session(bad_id=session_id)) # Bad keyword - - def test_get_session_dead_kernel(self): - sm = self.sm - session = self.create_session(path='/path/to/1/test1.ipynb', kernel_name='python') - # kill the kernel - sm.kernel_manager.shutdown_kernel(session['kernel']['id']) - with self.assertRaises(KeyError): - self.loop.run_sync(lambda: sm.get_session(session_id=session['id'])) - # no sessions left - listed = self.loop.run_sync(lambda: sm.list_sessions()) - self.assertEqual(listed, []) - - def test_list_sessions(self): - sm = self.sm - sessions = self.create_sessions( - dict(path='/path/to/1/test1.ipynb', kernel_name='python'), - dict(path='/path/to/2/test2.py', type='file', kernel_name='python'), - dict(path='/path/to/3', name='foo', type='console', kernel_name='python'), - ) - - sessions = self.loop.run_sync(lambda: sm.list_sessions()) - expected = [ - { - 'id':sessions[0]['id'], - 'path': u'/path/to/1/test1.ipynb', - 'type': 'notebook', - 'notebook': {'path': u'/path/to/1/test1.ipynb', 'name': None}, - 'name': None, - 'kernel': { - 'id': 'A', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - }, { - 'id':sessions[1]['id'], - 'path': u'/path/to/2/test2.py', - 'type': 'file', - 'name': None, - 'kernel': { - 'id': 'B', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - }, { - 'id':sessions[2]['id'], - 'path': u'/path/to/3', - 'type': 'console', - 'name': 'foo', - 'kernel': { - 'id': 'C', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - } - ] - self.assertEqual(sessions, expected) - - def test_list_sessions_dead_kernel(self): - sm = self.sm - sessions = self.create_sessions( - dict(path='/path/to/1/test1.ipynb', kernel_name='python'), - dict(path='/path/to/2/test2.ipynb', kernel_name='python'), - ) - # kill one of the kernels - sm.kernel_manager.shutdown_kernel(sessions[0]['kernel']['id']) - listed = self.loop.run_sync(lambda: sm.list_sessions()) - expected = [ - { - 'id': sessions[1]['id'], - 'path': u'/path/to/2/test2.ipynb', - 'type': 'notebook', - 'name': None, - 'notebook': {'path': u'/path/to/2/test2.ipynb', 'name': None}, - 'kernel': { - 'id': 'B', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - } - ] - self.assertEqual(listed, expected) - - def test_update_session(self): - sm = self.sm - session_id = self.create_session(path='/path/to/test.ipynb', - kernel_name='julia')['id'] - self.loop.run_sync(lambda: sm.update_session(session_id, path='/path/to/new_name.ipynb')) - model = self.loop.run_sync(lambda: sm.get_session(session_id=session_id)) - expected = {'id':session_id, - 'path': u'/path/to/new_name.ipynb', - 'type': 'notebook', - 'name': None, - 'notebook': {'path': u'/path/to/new_name.ipynb', 'name': None}, - 'kernel': { - 'id': 'A', - 'name':'julia', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - } - self.assertEqual(model, expected) - - def test_bad_update_session(self): - # try to update a session with a bad keyword ~ raise error - sm = self.sm - session_id = self.create_session(path='/path/to/test.ipynb', - kernel_name='ir')['id'] - with self.assertRaises(TypeError): - self.loop.run_sync(lambda: sm.update_session(session_id=session_id, bad_kw='test.ipynb')) # Bad keyword - - def test_delete_session(self): - sm = self.sm - sessions = self.create_sessions( - dict(path='/path/to/1/test1.ipynb', kernel_name='python'), - dict(path='/path/to/2/test2.ipynb', kernel_name='python'), - dict(path='/path/to/3', name='foo', type='console', kernel_name='python'), - ) - self.loop.run_sync(lambda: sm.delete_session(sessions[1]['id'])) - new_sessions = self.loop.run_sync(lambda: sm.list_sessions()) - expected = [{ - 'id': sessions[0]['id'], - 'path': u'/path/to/1/test1.ipynb', - 'type': 'notebook', - 'name': None, - 'notebook': {'path': u'/path/to/1/test1.ipynb', 'name': None}, - 'kernel': { - 'id': 'A', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - }, { - 'id': sessions[2]['id'], - 'type': 'console', - 'path': u'/path/to/3', - 'name': 'foo', - 'kernel': { - 'id': 'C', - 'name':'python', - 'connections': 0, - 'last_activity': dummy_date_s, - 'execution_state': 'idle', - } - } - ] - self.assertEqual(new_sessions, expected) - - def test_bad_delete_session(self): - # try to delete a session that doesn't exist ~ raise error - sm = self.sm - self.create_session(path='/path/to/test.ipynb', kernel_name='python') - with self.assertRaises(TypeError): - self.loop.run_sync(lambda : sm.delete_session(bad_kwarg='23424')) # Bad keyword - with self.assertRaises(web.HTTPError): - self.loop.run_sync(lambda : sm.delete_session(session_id='23424')) # nonexistent - diff --git a/notebook/services/sessions/tests/test_sessions_api.py b/notebook/services/sessions/tests/test_sessions_api.py deleted file mode 100644 index 9c551fc792..0000000000 --- a/notebook/services/sessions/tests/test_sessions_api.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Test the sessions web service API.""" - -import errno -from functools import partial -import io -import os -import json -import requests -import shutil -import time - -pjoin = os.path.join - -from notebook.utils import url_path_join -from notebook.tests.launchnotebook import NotebookTestBase, assert_http_error -from nbformat.v4 import new_notebook -from nbformat import write - -class SessionAPI(object): - """Wrapper for notebook API calls.""" - def __init__(self, request): - self.request = request - - def _req(self, verb, path, body=None): - response = self.request(verb, - url_path_join('api/sessions', path), data=body) - - if 400 <= response.status_code < 600: - try: - response.reason = response.json()['message'] - except: - pass - response.raise_for_status() - - return response - - def list(self): - return self._req('GET', '') - - def get(self, id): - return self._req('GET', id) - - def create(self, path, type='notebook', kernel_name='python', kernel_id=None): - body = json.dumps({'path': path, - 'type': type, - 'kernel': {'name': kernel_name, - 'id': kernel_id}}) - return self._req('POST', '', body) - - def create_deprecated(self, path): - body = json.dumps({'notebook': {'path': path}, - 'kernel': {'name': 'python', - 'id': 'foo'}}) - return self._req('POST', '', body) - - def modify_path(self, id, path): - body = json.dumps({'path': path}) - return self._req('PATCH', id, body) - - def modify_path_deprecated(self, id, path): - body = json.dumps({'notebook': {'path': path}}) - return self._req('PATCH', id, body) - - def modify_type(self, id, type): - body = json.dumps({'type': type}) - return self._req('PATCH', id, body) - - def modify_kernel_name(self, id, kernel_name): - body = json.dumps({'kernel': {'name': kernel_name}}) - return self._req('PATCH', id, body) - - def modify_kernel_id(self, id, kernel_id): - # Also send a dummy name to show that id takes precedence. - body = json.dumps({'kernel': {'id': kernel_id, 'name': 'foo'}}) - return self._req('PATCH', id, body) - - def delete(self, id): - return self._req('DELETE', id) - -class SessionAPITest(NotebookTestBase): - """Test the sessions web service API""" - def setUp(self): - nbdir = self.notebook_dir - subdir = pjoin(nbdir, 'foo') - - try: - os.mkdir(subdir) - except OSError as e: - # Deleting the folder in an earlier test may have failed - if e.errno != errno.EEXIST: - raise - self.addCleanup(partial(shutil.rmtree, subdir, ignore_errors=True)) - - with io.open(pjoin(subdir, 'nb1.ipynb'), 'w', encoding='utf-8') as f: - nb = new_notebook() - write(nb, f, version=4) - - self.sess_api = SessionAPI(self.request) - - @self.addCleanup - def cleanup_sessions(): - for session in self.sess_api.list().json(): - self.sess_api.delete(session['id']) - - # This is necessary in some situations on Windows: without it, it - # fails to delete the directory because something is still using - # it. I think there is a brief period after the kernel terminates - # where Windows still treats its working directory as in use. On my - # Windows VM, 0.01s is not long enough, but 0.1s appears to work - # reliably. -- TK, 15 December 2014 - time.sleep(0.1) - - def test_create(self): - sessions = self.sess_api.list().json() - self.assertEqual(len(sessions), 0) - - resp = self.sess_api.create('foo/nb1.ipynb') - self.assertEqual(resp.status_code, 201) - newsession = resp.json() - self.assertIn('id', newsession) - self.assertEqual(newsession['path'], 'foo/nb1.ipynb') - self.assertEqual(newsession['type'], 'notebook') - self.assertEqual(resp.headers['Location'], self.url_prefix + 'api/sessions/{0}'.format(newsession['id'])) - - sessions = self.sess_api.list().json() - self.assertEqual(sessions, [newsession]) - - # Retrieve it - sid = newsession['id'] - got = self.sess_api.get(sid).json() - self.assertEqual(got, newsession) - - def test_create_file_session(self): - resp = self.sess_api.create('foo/nb1.py', type='file') - self.assertEqual(resp.status_code, 201) - newsession = resp.json() - self.assertEqual(newsession['path'], 'foo/nb1.py') - self.assertEqual(newsession['type'], 'file') - - def test_create_console_session(self): - resp = self.sess_api.create('foo/abc123', type='console') - self.assertEqual(resp.status_code, 201) - newsession = resp.json() - self.assertEqual(newsession['path'], 'foo/abc123') - self.assertEqual(newsession['type'], 'console') - - def test_create_deprecated(self): - resp = self.sess_api.create_deprecated('foo/nb1.ipynb') - self.assertEqual(resp.status_code, 201) - newsession = resp.json() - self.assertEqual(newsession['path'], 'foo/nb1.ipynb') - self.assertEqual(newsession['type'], 'notebook') - self.assertEqual(newsession['notebook']['path'], 'foo/nb1.ipynb') - - def test_create_with_kernel_id(self): - # create a new kernel - r = self.request('POST', 'api/kernels') - r.raise_for_status() - kernel = r.json() - - resp = self.sess_api.create('foo/nb1.ipynb', kernel_id=kernel['id']) - self.assertEqual(resp.status_code, 201) - newsession = resp.json() - self.assertIn('id', newsession) - self.assertEqual(newsession['path'], 'foo/nb1.ipynb') - self.assertEqual(newsession['kernel']['id'], kernel['id']) - self.assertEqual(resp.headers['Location'], self.url_prefix + 'api/sessions/{0}'.format(newsession['id'])) - - sessions = self.sess_api.list().json() - self.assertEqual(sessions, [newsession]) - - # Retrieve it - sid = newsession['id'] - got = self.sess_api.get(sid).json() - self.assertEqual(got, newsession) - - def test_delete(self): - newsession = self.sess_api.create('foo/nb1.ipynb').json() - sid = newsession['id'] - - resp = self.sess_api.delete(sid) - self.assertEqual(resp.status_code, 204) - - sessions = self.sess_api.list().json() - self.assertEqual(sessions, []) - - with assert_http_error(404): - self.sess_api.get(sid) - - def test_modify_path(self): - newsession = self.sess_api.create('foo/nb1.ipynb').json() - sid = newsession['id'] - - changed = self.sess_api.modify_path(sid, 'nb2.ipynb').json() - self.assertEqual(changed['id'], sid) - self.assertEqual(changed['path'], 'nb2.ipynb') - - def test_modify_path_deprecated(self): - newsession = self.sess_api.create('foo/nb1.ipynb').json() - sid = newsession['id'] - - changed = self.sess_api.modify_path_deprecated(sid, 'nb2.ipynb').json() - self.assertEqual(changed['id'], sid) - self.assertEqual(changed['notebook']['path'], 'nb2.ipynb') - - def test_modify_type(self): - newsession = self.sess_api.create('foo/nb1.ipynb').json() - sid = newsession['id'] - - changed = self.sess_api.modify_type(sid, 'console').json() - self.assertEqual(changed['id'], sid) - self.assertEqual(changed['type'], 'console') - - def test_modify_kernel_name(self): - before = self.sess_api.create('foo/nb1.ipynb').json() - sid = before['id'] - - after = self.sess_api.modify_kernel_name(sid, before['kernel']['name']).json() - self.assertEqual(after['id'], sid) - self.assertEqual(after['path'], before['path']) - self.assertEqual(after['type'], before['type']) - self.assertNotEqual(after['kernel']['id'], before['kernel']['id']) - - # check kernel list, to be sure previous kernel was cleaned up - r = self.request('GET', 'api/kernels') - r.raise_for_status() - kernel_list = r.json() - after['kernel'].pop('last_activity') - [ k.pop('last_activity') for k in kernel_list ] - self.assertEqual(kernel_list, [after['kernel']]) - - def test_modify_kernel_id(self): - before = self.sess_api.create('foo/nb1.ipynb').json() - sid = before['id'] - - # create a new kernel - r = self.request('POST', 'api/kernels') - r.raise_for_status() - kernel = r.json() - - # Attach our session to the existing kernel - after = self.sess_api.modify_kernel_id(sid, kernel['id']).json() - self.assertEqual(after['id'], sid) - self.assertEqual(after['path'], before['path']) - self.assertEqual(after['type'], before['type']) - self.assertNotEqual(after['kernel']['id'], before['kernel']['id']) - self.assertEqual(after['kernel']['id'], kernel['id']) - - # check kernel list, to be sure previous kernel was cleaned up - r = self.request('GET', 'api/kernels') - r.raise_for_status() - kernel_list = r.json() - - kernel.pop('last_activity') - [ k.pop('last_activity') for k in kernel_list ] - self.assertEqual(kernel_list, [kernel]) diff --git a/notebook/services/shutdown.py b/notebook/services/shutdown.py deleted file mode 100644 index 78d1f2ad6e..0000000000 --- a/notebook/services/shutdown.py +++ /dev/null @@ -1,15 +0,0 @@ -"""HTTP handler to shut down the notebook server. -""" -from tornado import web, ioloop -from notebook.base.handlers import IPythonHandler - -class ShutdownHandler(IPythonHandler): - @web.authenticated - def post(self): - self.log.info("Shutting down on /api/shutdown request.") - ioloop.IOLoop.current().stop() - - -default_handlers = [ - (r"/api/shutdown", ShutdownHandler), -] diff --git a/notebook/templates/tree.html b/notebook/templates/tree.html index d84a973c1f..cd153e179f 100644 --- a/notebook/templates/tree.html +++ b/notebook/templates/tree.html @@ -204,4 +204,5 @@ + {% endblock %} diff --git a/notebook/terminal/__init__.py b/notebook/terminal/__init__.py deleted file mode 100644 index 37b5a8196f..0000000000 --- a/notebook/terminal/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -import terminado -from ..utils import check_version - -if not check_version(terminado.__version__, '0.8.1'): - raise ImportError("terminado >= 0.8.1 required, found %s" % terminado.__version__) - -from ipython_genutils.py3compat import which -from terminado import NamedTermManager -from tornado.log import app_log -from notebook.utils import url_path_join as ujoin -from .handlers import TerminalHandler, TermSocket -from . import api_handlers - -def initialize(webapp, notebook_dir, connection_url, settings): - if os.name == 'nt': - default_shell = 'powershell.exe' - else: - default_shell = which('sh') - shell = settings.get('shell_command', - [os.environ.get('SHELL') or default_shell] - ) - # Enable login mode - to automatically source the /etc/profile script - if os.name != 'nt': - shell.append('-l') - terminal_manager = webapp.settings['terminal_manager'] = NamedTermManager( - shell_command=shell, - extra_env={'JUPYTER_SERVER_ROOT': notebook_dir, - 'JUPYTER_SERVER_URL': connection_url, - }, - ) - terminal_manager.log = app_log - base_url = webapp.settings['base_url'] - handlers = [ - (ujoin(base_url, r"/terminals/(\w+)"), TerminalHandler), - (ujoin(base_url, r"/terminals/websocket/(\w+)"), TermSocket, - {'term_manager': terminal_manager}), - (ujoin(base_url, r"/api/terminals"), api_handlers.TerminalRootHandler), - (ujoin(base_url, r"/api/terminals/(\w+)"), api_handlers.TerminalHandler), - ] - webapp.add_handlers(".*$", handlers) diff --git a/notebook/terminal/api_handlers.py b/notebook/terminal/api_handlers.py deleted file mode 100644 index c99b37c6ad..0000000000 --- a/notebook/terminal/api_handlers.py +++ /dev/null @@ -1,54 +0,0 @@ -import json -from tornado import web, gen -from ..base.handlers import APIHandler -from ..prometheus.metrics import TERMINAL_CURRENTLY_RUNNING_TOTAL - - -class TerminalRootHandler(APIHandler): - @web.authenticated - def get(self): - tm = self.terminal_manager - terms = [{'name': name} for name in tm.terminals] - self.finish(json.dumps(terms)) - - # Update the metric below to the length of the list 'terms' - TERMINAL_CURRENTLY_RUNNING_TOTAL.set( - len(terms) - ) - - @web.authenticated - def post(self): - """POST /terminals creates a new terminal and redirects to it""" - name, _ = self.terminal_manager.new_named_terminal() - self.finish(json.dumps({'name': name})) - - # Increase the metric by one because a new terminal was created - TERMINAL_CURRENTLY_RUNNING_TOTAL.inc() - - -class TerminalHandler(APIHandler): - SUPPORTED_METHODS = ('GET', 'DELETE') - - @web.authenticated - def get(self, name): - tm = self.terminal_manager - if name in tm.terminals: - self.finish(json.dumps({'name': name})) - else: - raise web.HTTPError(404, "Terminal not found: %r" % name) - - @web.authenticated - @gen.coroutine - def delete(self, name): - tm = self.terminal_manager - if name in tm.terminals: - yield tm.terminate(name, force=True) - self.set_status(204) - self.finish() - - # Decrease the metric below by one - # because a terminal has been shutdown - TERMINAL_CURRENTLY_RUNNING_TOTAL.dec() - - else: - raise web.HTTPError(404, "Terminal not found: %r" % name) diff --git a/notebook/terminal/handlers.py b/notebook/terminal/handlers.py deleted file mode 100644 index 6a66aa2f0f..0000000000 --- a/notebook/terminal/handlers.py +++ /dev/null @@ -1,42 +0,0 @@ -#encoding: utf-8 -"""Tornado handlers for the terminal emulator.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from tornado import web -import terminado -from notebook._tz import utcnow -from ..base.handlers import IPythonHandler -from ..base.zmqhandlers import WebSocketMixin - - -class TerminalHandler(IPythonHandler): - """Render the terminal interface.""" - @web.authenticated - def get(self, term_name): - self.write(self.render_template('terminal.html', - ws_path="terminals/websocket/%s" % term_name)) - - -class TermSocket(WebSocketMixin, IPythonHandler, terminado.TermSocket): - - def origin_check(self): - """Terminado adds redundant origin_check - - Tornado already calls check_origin, so don't do anything here. - """ - return True - - def get(self, *args, **kwargs): - if not self.get_current_user(): - raise web.HTTPError(403) - return super(TermSocket, self).get(*args, **kwargs) - - def on_message(self, message): - super(TermSocket, self).on_message(message) - self.application.settings['terminal_last_activity'] = utcnow() - - def write_message(self, message, binary=False): - super(TermSocket, self).write_message(message, binary=binary) - self.application.settings['terminal_last_activity'] = utcnow() diff --git a/notebook/tree/handlers.py b/notebook/tree/handlers.py index 4cc4c8062f..acbece4635 100644 --- a/notebook/tree/handlers.py +++ b/notebook/tree/handlers.py @@ -5,11 +5,13 @@ from tornado import web import os -from ..base.handlers import IPythonHandler, path_regex -from ..utils import url_path_join, url_escape +from jupyter_server.base.handlers import path_regex +from jupyter_server.utils import url_path_join, url_escape +from ..base.handlers import BaseHandler -class TreeHandler(IPythonHandler): + +class TreeHandler(BaseHandler): """Render the tree view, listing notebooks, etc.""" def generate_breadcrumbs(self, path): diff --git a/notebook/view/__init__.py b/notebook/view/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/notebook/view/handlers.py b/notebook/view/handlers.py deleted file mode 100644 index 3e89faccca..0000000000 --- a/notebook/view/handlers.py +++ /dev/null @@ -1,27 +0,0 @@ -#encoding: utf-8 -"""Tornado handlers for viewing HTML files.""" - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from tornado import web -from ..base.handlers import IPythonHandler, path_regex -from ..utils import url_escape, url_path_join - -class ViewHandler(IPythonHandler): - """Render HTML files within an iframe.""" - @web.authenticated - def get(self, path): - path = path.strip('/') - if not self.contents_manager.file_exists(path): - raise web.HTTPError(404, u'File does not exist: %s' % path) - - basename = path.rsplit('/', 1)[-1] - file_url = url_path_join(self.base_url, 'files', url_escape(path)) - self.write( - self.render_template('view.html', file_url=file_url, page_title=basename) - ) - -default_handlers = [ - (r"/view%s" % path_regex, ViewHandler), -] diff --git a/setup.py b/setup.py index 68ccf606d8..5f2106d460 100755 --- a/setup.py +++ b/setup.py @@ -125,8 +125,6 @@ 'console_scripts': [ 'jupyter-notebook = notebook.notebookapp:main', 'jupyter-nbextension = notebook.nbextensions:main', - 'jupyter-serverextension = notebook.serverextensions:main', - 'jupyter-bundlerextension = notebook.bundler.bundlerextensions:main', ] }, ) diff --git a/setupbase.py b/setupbase.py index 51cb9fac50..9f580076a4 100644 --- a/setupbase.py +++ b/setupbase.py @@ -209,8 +209,8 @@ def find_package_data(): package_data = { 'notebook' : ['templates/*'] + static_data, 'notebook.tests' : js_tests, - 'notebook.bundler.tests': ['resources/*', 'resources/*/*', 'resources/*/*/.*'], - 'notebook.services.api': ['api.yaml'], + # 'notebook.bundler.tests': ['resources/*', 'resources/*/*', 'resources/*/*/.*'], + # 'notebook.services.api': ['api.yaml'], 'notebook.i18n': ['*/LC_MESSAGES/*.*'], }