diff --git a/.circleci/build_artifacts.py b/.circleci/build_artifacts.py new file mode 100644 index 00000000..edc7e163 --- /dev/null +++ b/.circleci/build_artifacts.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +import logging +import json +import os +import subprocess +import shutil +import sys +import tarfile +import tempfile +import time +import types +import typing + +from datetime import datetime + +from junitparser import TestCase, TestSuite, JUnitXml, Skipped, Error + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +root.addHandler(handler) + +logger = logging.getLogger(__file__) + +IPYDB_REQUIRED_FILES: typing.List[str] = ['requirements.txt'] +ENCODING: str = 'utf-8' +ARTIFACT_DEST_DIR: str = '/tmp/artifacts' +TEST_OUTPUT_DIR: str = '/tmp/test-results' +BUILD_STATE: typing.Dict[str, typing.Any] = {} +if not os.path.exists(TEST_OUTPUT_DIR): + os.makedirs(TEST_OUTPUT_DIR) +TEST_CASES: typing.List[TestCase] = [] + +def run_command(cmd: typing.List[str]) -> types.GeneratorType: + proc = subprocess.Popen(' '.join(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + while proc.poll() is None: + time.sleep(.1) + + if proc.poll() > 0: + yield proc.poll(), proc.stderr.read().decode(ENCODING) + + elif proc.poll() != None: + yield proc.poll(), proc.stdout.read().decode(ENCODING) + + else: + # if proc.poll() is None, its still running the subprocess. + # block until done + pass + + +def find_artifacts(start_dir: str) -> types.GeneratorType: + for root, dirnames, filenames in os.walk(start_dir): + for filename in filenames: + if filename.endswith('.tar.gz'): + yield os.path.join(start_dir, filename) + +for artifact_path in find_artifacts(ARTIFACT_DEST_DIR): + logger.info(f'Found Artifact in path[{artifact_path}]. Building Artifact') + notebook_name: str = os.path.basename(artifact_path).rsplit('.', 1)[0] + extraction_path: str = tempfile.mkdtemp(prefix=notebook_name) + build_script_path: str = None + with tarfile.open(artifact_path, "r:gz") as tar: + for member in tar.getmembers(): + if member.isdir(): + dir_path: str = os.path.join(extraction_path, member.path) + os.makedirs(dir_path) + + elif member.isfile(): + filepath: str = os.path.join(extraction_path, member.path) + with open(filepath, 'wb') as stream: + stream.write(tar.extractfile(member).read()) + + if os.path.basename(member.path) == 'build.sh': + build_script_path = filepath + + else: + raise NotImplementedError + + + owd: str = os.getcwd() + build_dir: str = os.path.dirname(build_script_path) + logger.info(f'Changing to build_dir[{build_dir}]') + os.chdir(build_dir) + BUILD_STATE[notebook_name] = {'stdout': [], 'stderr': []} + start = datetime.utcnow() + for return_code, comm, in run_command(['bash', 'build.sh']): + if return_code > 0: + logger.error(comm) + BUILD_STATE[notebook_name]['exit-code'] = return_code + BUILD_STATE[notebook_name]['stderr'].append(comm) + + else: + BUILD_STATE[notebook_name]['exit-code'] = return_code + BUILD_STATE[notebook_name]['stdout'].append(comm) + logger.info(comm) + + delta = datetime.utcnow() - start + logger.info(f'Changing back to old working dir[{owd}]') + os.chdir(owd) + test_case = TestCase(f'{notebook_name} Test') + if BUILD_STATE[notebook_name]['exit-code'] > 0: + test_case.result = Error('\n'.join(BUILD_STATE[notebook_name]['stderr']), BUILD_STATE[notebook_name]['exit-code']) + TEST_CASES.append(test_case) + + TEST_CASES.append(test_case) + +test_suite = TestSuite(f'Notebooks Test Suite') +[test_suite.add_testcase(case) for case in TEST_CASES] +test_output_path: str = os.path.join(TEST_OUTPUT_DIR, f'results.xml') +xml = JUnitXml() +xml.add_testsuite(test_suite) +xml.write(test_output_path) + +# from nbpages import make_parser, run_parsed, make_html_index +# +# args = make_parser().parse_args() +# +# converted = run_parsed('.', output_type='HTML', args=args) +# +# converted = [item for item in converted if not os.path.basename(item) in ['test-fail.html', 'test-succeed.html']] +# make_html_index(converted, './index.tpl') + diff --git a/.circleci/config.yml b/.circleci/config.yml index 3883101e..33b8cb2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,95 +7,43 @@ executors: working_directory: ~/repo jobs: - build_and_check: + Build_Notebooks: executor: notebook-executor steps: - - checkout - - - restore_cache: - keys: - - v1-env-{{ checksum "environment.yml" }} - - run: name: Setup Environment command: | ./.circleci/setup_env.sh - - save_cache: - key: v1-env-{{ checksum "environment.yml" }} - paths: - - /opt/conda - - run: - name: Run/convert notebooks + name: Create Artifacts command: | - conda info --envs - source activate notebooks_env - conda info --envs - mkdir test-results - python ./convert.py --report test-results/results.xml - no_output_timeout: 20m + python ./.circleci/create_artifacts.py - - store_test_results: - path: test-results + - store_artifacts: + path: /tmp/artifacts - run: - name: Check notebooks + name: Build Artifacts command: | - COMMIT_RANGE=$(echo "${CIRCLE_COMPARE_URL}" | cut -d/ -f7) - if [[ $COMMIT_RANGE != *"..."* ]]; then - COMMIT_RANGE="${COMMIT_RANGE}...${COMMIT_RANGE}" - fi source activate notebooks_env - conda info --envs - python -m "nbpages.check_nbs" --commit-range=${COMMIT_RANGE} + python ./.circleci/build_artifacts.py - - persist_to_workspace: - root: ~/ - paths: - - repo - - - store_artifacts: - path: ~/repo + - store_test_results: + path: /tmp/test-results - run: - name: "Built notebooks are available at:" - command: NB_URL="${CIRCLE_BUILD_URL}/artifacts/${CIRCLE_NODE_INDEX}/${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}/index.html"; echo $NB_URL - - - deploy: - executor: notebook-executor - environment: - - DEPLOY_BRANCH: gh-pages - steps: - - attach_workspace: - at: ~/ + name: Make Pages + command: | + source activate notebooks_env + python ./.circleci/make_pages.py - add_ssh_keys - - # clone gh-pages branch, copy over files from build, commit and push - # export and sources are needed before script because CircleCI does not support - # interpolation when setting environment variables - # i.e. can't set environment vars to other environment vars (for use in script) - # see https://circleci.com/docs/2.0/env-vars/#setting-an-environment-variable-in-a-shell-command - - deploy: - name: Deploy + - run: + name: Deploy Pages command: | - if [[ -z ${CIRCLE_PULL_REQUEST} ]]; then - echo "export WORKSPACE=${CIRCLE_WORKING_DIRECTORY}" >> $BASH_ENV - echo "export REPO_URL=${CIRCLE_REPOSITORY_URL}" >> $BASH_ENV - source $BASH_ENV - git config --global user.email $GH_EMAIL - git config --global user.name $GH_NAME - echo "Host * " >> ~/.ssh/config - echo " StrictHostKeyChecking no" >> ~/.ssh/config - ./gh_pages_deploy.sh - else - echo "PR, not deploying..." - echo "${CIRCLE_PULL_REQUEST}" - echo "${CIRCLE_PULL_NUMBER}" - fi + bash ./.circleci/deploy-notebooks.sh workflows: @@ -103,11 +51,4 @@ workflows: build-deploy: jobs: - - build_and_check - - deploy: - requires: - - build_and_check - filters: - branches: - only: - - master + - Build_Notebooks diff --git a/.circleci/create_artifacts.py b/.circleci/create_artifacts.py new file mode 100644 index 00000000..d74dac3c --- /dev/null +++ b/.circleci/create_artifacts.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +import logging +import os +import shutil +import sys +import tarfile +import tempfile +import types +import typing + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +root.addHandler(handler) + +logger = logging.getLogger(__file__) + +IPYDB_REQUIRED_FILES: typing.List[str] = ['requirements.txt'] +ENCODING: str = 'utf-8' +ARTIFACT_DEST_DIR: str = '/tmp/artifacts' + +def find_ipynb_files(start_path: str) -> types.GeneratorType: + for root, dirnames, filenames in os.walk(start_path): + is_ipydb_directory: bool = False + for filename in filenames: + if filename.endswith('.ipynb'): + is_ipydb_directory = True + break + + if is_ipydb_directory: + has_error: bool = False + for filename in IPYDB_REQUIRED_FILES: + if not filename in filenames: + logger.error(f'Missing file[{filename}] in dir[{os.path.relpath(root)}]') + has_error = True + + if has_error is False: + yield os.path.abspath(root) + +for notebook_path in find_ipynb_files(os.getcwd()): + logger.info(f'Found notebook in path[{os.path.relpath(notebook_path)}]. Building Artifact') + notebook_name: str = os.path.basename(notebook_path) + notebook_name_plain: str = notebook_name.rsplit('.', 1)[0] + build_path = tempfile.mkdtemp(prefix=notebook_name) + shutil.rmtree(build_path) + build_script_path = os.path.join(build_path, 'build.sh') + shutil.copytree(notebook_path, build_path) + setup_script: str = f"""#!/usr/bin/env bash +set -e +cd {build_path} +source activate notebooks_env +virtualenv -p $(which python3) env +conda deactivate +source env/bin/activate +pip install -r requirements.txt +pip install jupyter +jupyter nbconvert --stdout --to html {notebook_name} > {notebook_name_plain}.html +cd - +""" + with open(build_script_path, 'w') as stream: + stream.write(setup_script) + + logger.info(f'Taring Notebook[{notebook_name}]') + artifact_name: str = f'{notebook_name_plain}.tar.gz' + artifact_dir_path: str = os.path.dirname(tempfile.NamedTemporaryFile().name) + artifact_path: str = os.path.join(artifact_dir_path, artifact_name) + with tarfile.open(artifact_path, "w:gz") as tar: + tar.add(build_path, arcname=os.path.basename(build_path)) + + if not os.path.exists(ARTIFACT_DEST_DIR): + os.makedirs(ARTIFACT_DEST_DIR) + + artifact_dest: str = os.path.join(ARTIFACT_DEST_DIR, artifact_name) + logger.info(f'Moving Notebook[{notebook_name_plain}]') + shutil.move(artifact_path, artifact_dest) + diff --git a/.circleci/deploy-notebooks.sh b/.circleci/deploy-notebooks.sh new file mode 100644 index 00000000..4b4d0592 --- /dev/null +++ b/.circleci/deploy-notebooks.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e +# Only runs on master +if [ -z "${CIRCLE_PULL_REQUEST}" ]; then + git config --global user.email devnull@circleci.com + git config --global user.name CircleCI + mkdir -p ~/.ssh + echo 'Host * ' >> ~/.ssh/config + echo ' StrictHostKeyChecking no' >> ~/.ssh/config + # Deploy gh-pages + git clone -b gh-pages --single-branch ${CIRCLE_REPOSITORY_URL} /tmp/out + cd /tmp/out + git add . + git commit -m 'Automated deployment to Github Pages: ${BUILD_TAG}' -a || true + git push origin gh-pages + git clean -dfx +fi +exit 0 diff --git a/.circleci/make_pages.py b/.circleci/make_pages.py new file mode 100644 index 00000000..e1c30bba --- /dev/null +++ b/.circleci/make_pages.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import logging +import os +import sys + +from nbpages import make_parser, run_parsed, make_html_index + +root = logging.getLogger() +root.setLevel(logging.INFO) +handler = logging.StreamHandler(sys.stdout) +handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) +root.addHandler(handler) + +logger = logging.getLogger(__file__) + +args = make_parser().parse_args() +logger.info('Converting notebooks into HTML') +converted = run_parsed('.', output_type='HTML', args=args) +# converted = [item for item in converted if not os.path.basename(item) in ['test-fail.html', 'test-succeed.html']] + +logger.info('Creating HTML Index') +make_html_index(converted, './index.tpl') diff --git a/.circleci/setup_env.sh b/.circleci/setup_env.sh index 3a007a3c..8900a2e5 100755 --- a/.circleci/setup_env.sh +++ b/.circleci/setup_env.sh @@ -1,10 +1,8 @@ #!/bin/bash -if [[ ! -d /opt/conda/envs/notebooks_env ]]; then - conda info --envs - conda env update --file=environment.yml - source activate notebooks_env - conda info --envs -else - echo "Using cached miniconda environment"; -fi +apt-get update +apt-get install build-essential gcc-4.8 -y +conda info --envs +conda env update --file=environment.yml +source activate notebooks_env +conda info --envs diff --git a/.gitignore b/.gitignore index eff340f7..3c49240c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ **/*.ipynb_checkpoints/ *.fits +*.swp +src +index.html +env diff --git a/environment.yml b/environment.yml index a94401ff..ad9a1aad 100644 --- a/environment.yml +++ b/environment.yml @@ -1,30 +1,32 @@ name: notebooks_env - channels: - astropy - http://ssb.stsci.edu/astroconda - defaults dependencies: + - python=3.7 + - ca-certificates=2019.8.28 + - pip + - virtualenv=16.0.0 - numpy=1.15.4 - - astropy - - scipy - - astroquery - - matplotlib - - jupyter - - pandas - - drizzlepac - - crds - - ccdproc - - acstools - - photutils>=0.6 - - regions - - pyregion - - webbpsf>=0.8 - - scikit-learn - + - astropy=3.0.4 - pip: - - k2flix - - junit_xml - - git+https://github.com/eteq/nbpages.git + - future==0.17.1 + - junitparser==1.3.4 + - '-e git+https://github.com/eteq/nbpages.git@b9ec8410803357939210e068af7e14a6f0625fab#egg=nbpages' + # Below is required by nbpages + - jupyter==1.0.0 + - jupyter-client==5.3.3 + - astroquery==0.3.10 + - matplotlib>=2.2.2 + - numpy>=1.15.4 + - photutils>=0.6 + - webbpsf>=0.8 + - scikit-learn>=0.20 + - k2flix>=2.4.0 + - bokeh>=0.12.16 + - pandas>=0.23.4 + - ccdproc>=1.3.0 + - drizzlepac>=2.2.4