diff --git a/.gitignore b/.gitignore index 23819d04..e54071d6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ localconfig.py .whoosh docs/code +src/pyff/web +MANIFEST +.vscode diff --git a/npm_modules.txt b/npm_modules.txt new file mode 100644 index 00000000..2c9dbed9 --- /dev/null +++ b/npm_modules.txt @@ -0,0 +1,2 @@ +@sunet/mdq-browser@1.0.2 +@theidentityselector/thiss@1.5.0-dev0 diff --git a/setup.py b/setup.py index 313cacfd..b0384da0 100755 --- a/setup.py +++ b/setup.py @@ -2,14 +2,33 @@ # -*- encoding: utf-8 -*- from distutils.core import setup +from distutils.command.sdist import sdist +from distutils.dir_util import copy_tree from pathlib import PurePath from platform import python_implementation from typing import List - +from tempfile import TemporaryDirectory from setuptools import find_packages __author__ = 'Leif Johansson' -__version__ = '2.0.0' +__version__ = '2.1.0dev0' + + +class NPMSdist(sdist): + def run(self): + import subprocess + + npm_modules = load_requirements(here.with_name('npm_modules.txt')) + with TemporaryDirectory() as tmp: + for npm_module in npm_modules: + subprocess.check_call(['npm', 'install', '--production', '--prefix', tmp, npm_module]) + for npm_module in npm_modules: + (npm_module_path, _, _) = npm_module.rpartition('@') + copy_tree( + "{}/node_modules/{}/dist".format(tmp, npm_module_path), './src/pyff/web/{}'.format(npm_module_path) + ) + + super().run() def load_requirements(path: PurePath) -> List[str]: @@ -37,6 +56,7 @@ def load_requirements(path: PurePath) -> List[str]: setup( name='pyFF', + cmdclass={'sdist': NPMSdist}, version=__version__, description="Federation Feeder", long_description=README + '\n\n' + NEWS, @@ -55,7 +75,7 @@ def load_requirements(path: PurePath) -> List[str]: packages=find_packages('src'), package_dir={'': 'src'}, include_package_data=True, - package_data={'pyff': ['xslt/*.xsl', 'schema/*.xsd']}, + package_data={'pyff': ['xslt/*.xsl', 'schema/*.xsd', 'web/**/*']}, zip_safe=False, install_requires=install_requires, scripts=['scripts/mirror-mdq.sh'], @@ -64,6 +84,11 @@ def load_requirements(path: PurePath) -> List[str]: 'paste.app_factory': ['pyffapp=pyff.wsgi:app_factory'], 'paste.server_runner': ['pyffs=pyff.wsgi:server_runner'], }, - message_extractors={'src': [('**.py', 'python', None), ('**/templates/**.html', 'mako', None),]}, + message_extractors={ + 'src': [ + ('**.py', 'python', None), + ('**/templates/**.html', 'mako', None), + ] + }, python_requires='>=3.7', ) diff --git a/src/pyff/api.py b/src/pyff/api.py index f876c1d3..4803f92f 100644 --- a/src/pyff/api.py +++ b/src/pyff/api.py @@ -2,8 +2,8 @@ import threading from datetime import datetime, timedelta from json import dumps +import os from typing import Any, Dict, Generator, Iterable, List, Mapping, Optional, Tuple - import pkg_resources import pyramid.httpexceptions as exc import pytz @@ -25,7 +25,7 @@ from pyff.repo import MDRepository from pyff.resource import Resource from pyff.samlmd import entity_display_name -from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now +from pyff.utils import b2u, dumptree, hash_id, json_serializer, utc_now, FrontendApp, resource_filename log = get_log(__name__) @@ -58,6 +58,12 @@ def robots_handler(request: Request) -> Response: ) +def json_response(data) -> Response: + response = Response(dumps(data, default=json_serializer)) + response.headers['Content-Type'] = 'application/json' + return response + + def status_handler(request: Request) -> Response: """ Implements the /api/status endpoint @@ -77,9 +83,7 @@ def status_handler(request: Request) -> Response: threads=[t.name for t in threading.enumerate()], store=dict(size=request.registry.md.store.size()), ) - response = Response(dumps(_status, default=json_serializer)) - response.headers['Content-Type'] = 'application/json' - return response + return json_response(_status) class MediaAccept(object): @@ -392,10 +396,7 @@ def _links(url: str, title: Any = None) -> None: for v in request.registry.md.store.attribute(aliases[a]): _links('%s/%s' % (a, quote_plus(v))) - response = Response(dumps(jrd, default=json_serializer)) - response.headers['Content-Type'] = 'application/json' - - return response + return json_response(jrd) def resources_handler(request: Request) -> Response: @@ -420,10 +421,7 @@ def _info(r: Resource) -> Mapping[str, Any]: return nfo - response = Response(dumps(_infos(request.registry.md.rm.children), default=json_serializer)) - response.headers['Content-Type'] = 'application/json' - - return response + return json_response(_infos(request.registry.md.rm.children)) def pipeline_handler(request: Request) -> Response: @@ -433,10 +431,7 @@ def pipeline_handler(request: Request) -> Response: :param request: the HTTP request :return: a JSON representation of the active pipeline """ - response = Response(dumps(request.registry.plumbings, default=json_serializer)) - response.headers['Content-Type'] = 'application/json' - - return response + return json_response(request.registry.plumbings) def search_handler(request: Request) -> Response: @@ -496,6 +491,27 @@ def launch_memory_usage_server(port: int = 9002) -> None: cherrypy.engine.start() +class ExtensionPredicate: + def __init__(self, val, info): + self.segment_name = val[0] + self.extensions = tuple(val[0:]) + + def text(self): + return "extensions = {}".format(self.extensions) + + phash = text + + def __call__(self, info, request): + match = info['match'] + if match[self.segment_name] == '': + return True + + for ext in self.extensions: + if match[self.segment_name].endswith(ext): + return True + return False + + def mkapp(*args: Any, **kwargs: Any) -> Any: md = kwargs.pop('md', None) if md is None: @@ -556,6 +572,23 @@ def mkapp(*args: Any, **kwargs: Any) -> Any: ctx.add_route('call', '/api/call/{entry}', request_method=['POST', 'PUT']) ctx.add_view(process_handler, route_name='call') + if config.mdq_browser is not None or config.thiss is not None: + ctx.add_route_predicate('ext', ExtensionPredicate) + + if config.mdq_browser is not None and len(config.mdq_browser) == 0: + config.mdq_browser = resource_filename('web/@sunet/mdq-browser') + + if config.mdq_browser: + log.debug("serving mdq-browser from {}".format(config.mdq_browser)) + FrontendApp.load('/', 'mdq_browser', config.mdq_browser).add_route(ctx) + + if config.thiss is not None and len(config.thiss) == 0: + config.thiss = resource_filename('web/@theidentityselector/thiss') + + if config.thiss: + log.debug("serving thiss from {}".format(config.thiss)) + FrontendApp.load('/thiss/', 'thiss', config.thiss).add_route(ctx) + ctx.add_route('request', '/*path', request_method='GET') ctx.add_view(request_handler, route_name='request') diff --git a/src/pyff/constants.py b/src/pyff/constants.py index 083ec32f..bbcffcfd 100644 --- a/src/pyff/constants.py +++ b/src/pyff/constants.py @@ -9,7 +9,7 @@ import re import sys from distutils.util import strtobool - +from typing import Tuple, Union, Any import pyconfig import six @@ -253,15 +253,17 @@ class Config(object): version = DummySetting('version', info="Show pyff version information", short='v', typeconv=as_bool) module = DummySetting("module", info="load additional plugins from the specified module", short='m') alias = DummySetting('alias', info="add an alias to the server - argument must be on the form alias=uri", short='A') + env = DummySetting('env', info='add an environment variable to the server', short='E') # deprecated settings google_api_key = S("google_api_key", deprecated=True) caching_delay = S("caching_delay", default=300, typeconv=as_int, short='D', deprecated=True) proxy = S("proxy", default=False, typeconv=as_bool, deprecated=True) - public_url = S("public_url", typeconv=as_string, deprecated=True) allow_shutdown = S("allow_shutdown", default=False, typeconv=as_bool, deprecated=True) ds_template = S("ds_template", default="ds.html", deprecated=True) + public_url = S("public_url", typeconv=as_string, info="the public URL of the service - not often needed") + loglevel = S("loglevel", default=logging.WARN, info="set the loglevel") access_log = S("access_log", cmdline=['pyffd'], info="a log target (file) to use for access logs") @@ -312,6 +314,15 @@ class Config(object): info="a set of aliases to add to the server", ) + environ = S( + "environ", + default=dict(), + typeconv=as_dict_of_string, + cmdline=['pyffd'], + hidden=True, + info="a set of environement variables to add to the server", + ) + base_dir = S("base_dir", info="change to this directory before executing the pipeline") modules = S("modules", default=[], typeconv=as_list_of_string, hidden=True, info="modules providing plugins") @@ -460,6 +471,9 @@ class Config(object): default="/var/run/pyff/backup", ) + mdq_browser = S('mdq_browser', typeconv=as_string, info="the directory where mdq-browser can be found") + thiss = S('thiss', typeconv=as_string, info="the directory where thiss-js can be found") + @property def base_url(self): if self.public_url: @@ -509,6 +523,14 @@ def help(prg): config = Config() +def opt_eq_split(s: str) -> Tuple[Any, Any]: + for sep in [':', '=']: + d = tuple(s.rsplit(sep)) + if len(d) == 2: + return d[0], d[1] + return None, None + + def parse_options(program, docs): (short_args, long_args) = config.args(program) docs += config.help(program) @@ -525,6 +547,9 @@ def parse_options(program, docs): if config.aliases is None or len(config.aliases) == 0: config.aliases = dict(metadata=entities) + if config.environ is None or len(config.environ) == 0: + config.environ = dict() + if config.modules is None: config.modules = [] @@ -541,6 +566,10 @@ def parse_options(program, docs): assert colon == ':' if a and uri: config.aliases[a] = uri + elif o in ('-E', '--env'): + (k, v) = opt_eq_split(a) + if k and v: + config.environ[k] = v elif o in ('-m', '--module'): config.modules.append(a) else: diff --git a/src/pyff/mdq.py b/src/pyff/mdq.py index 0515c2db..278311bc 100644 --- a/src/pyff/mdq.py +++ b/src/pyff/mdq.py @@ -51,6 +51,7 @@ def main(): 'workers': config.worker_pool_size, 'loglevel': config.loglevel, 'preload_app': True, + 'env': config.environ, 'daemon': config.daemonize, 'capture_output': False, 'timeout': config.worker_timeout, diff --git a/src/pyff/resource.py b/src/pyff/resource.py index 196c9683..bcdfc700 100644 --- a/src/pyff/resource.py +++ b/src/pyff/resource.py @@ -264,7 +264,7 @@ def local_copy_fn(self): @property def post( - self, + self, ) -> Iterable[Callable]: # TODO: move classes to make this work -> List[Union['Lambda', 'PipelineCallback']]: return self.opts.via diff --git a/src/pyff/utils.py b/src/pyff/utils.py index 6c87ee7e..039e57e5 100644 --- a/src/pyff/utils.py +++ b/src/pyff/utils.py @@ -6,6 +6,8 @@ This module contains various utilities. """ +from __future__ import annotations + import base64 import cgi import contextlib @@ -26,7 +28,8 @@ from threading import local from time import gmtime, strftime from typing import Any, BinaryIO, Callable, Dict, List, Optional, Sequence, Set, Tuple, Union - +from pyramid.request import Request as PyramidRequest +from pyramid.response import Response as PyramidResponse import pkg_resources import requests import xmlsec @@ -50,6 +53,10 @@ from pyff.exceptions import * from pyff.logs import get_log +from pydantic import BaseModel +from pyramid.static import static_view + + etree.set_default_parser(etree.XMLParser(resolve_entities=False)) __author__ = 'leifj' @@ -980,3 +987,48 @@ def notify(self, *args, **kwargs): def utc_now() -> datetime: """ Return current time with tz=UTC """ return datetime.now(tz=timezone.utc) + + +class FrontendApp(BaseModel): + url_path: str + name: str + directory: str + dirs: List[str] = [] + exts: Set[str] = set() + env: Dict[str, str] = dict() + + @staticmethod + def load(url_path: str, name: str, directory: str, env: Optional[Mapping[str, str]] = None) -> FrontendApp: + if env is None: + env = config.environ + fa = FrontendApp(url_path=url_path, name=name, directory=directory, env=env) + with os.scandir(fa.directory) as it: + for entry in it: + if not entry.name.startswith('.'): + if entry.is_dir(): + fa.dirs.append(entry.name) + else: + fn, ext = os.path.splitext(entry.name) + fa.exts.add(ext) + return fa + + def env_js_handler(self, request: PyramidRequest) -> PyramidResponse: + env_js = "window.env = {" + ",".join([f"{k}: '{v}'" for (k, v) in self.env.items()]) + "};" + response = PyramidResponse(env_js) + response.headers['Content-Type'] = 'text/javascript' + return response + + def add_route(self, ctx): + env_route = '{}_env_js'.format(self.name) + ctx.add_route(env_route, '/env.js', request_method='GET') + ctx.add_view(self.env_js_handler, route_name=env_route) + for uri_part in [self.url_path] + [self.url_path + d for d in self.dirs]: + route = '{}_{}'.format(self.name, uri_part) + path = '{:s}{{sep:/?}}{{path:.*}}'.format(uri_part) + ctx.add_route( + route, + path, + request_method='GET', + ext=['path'] + list(self.exts), + ) + ctx.add_view(static_view(self.directory, use_subpath=False), route_name=route)