From 29c210032e750d6bff1d41910434b8605eebc0f6 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Wed, 13 Jan 2021 17:41:49 -0800 Subject: [PATCH 1/2] ci: backport new CI to 2.4 release branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Generated with `git restore -s origin/master .travis.yml .github/ ci/`, plus the following manual changes: - removed the `build-data-server-pip` CI job, since 2.4 doesn’t have any Rust packages to build; - reinstated `flake8` for Python 3.5; - removed CI check for absence of `"@npm_angular_bazel//:index.bzl"`; - downgraded CI Bazel to 2.1.0 to match WORKSPACE file; - cherry-picked `test_pip_package.sh` changes for `--tf-version notf`. Test Plan: CI test run passed: wchargin-branch: ci-backport-2.4 wchargin-source: a6833768fb4e5c516f426d5c32b4044063fcfefc --- .github/workflows/ci.yml | 142 +++++++++++++------- .travis.yml | 115 ---------------- ci/bazelrc | 26 ++-- tensorboard/pip_package/test_pip_package.sh | 5 +- 4 files changed, 114 insertions(+), 174 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d778456d6..9d6a1df84d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,9 +6,6 @@ # Helpful YAML parser to clarify YAML syntax: # https://yaml-online-parser.appspot.com/ -# For now, we only use GitHub Actions for lint checks, pending better -# support for hermetic-style caching. See: -# https://github.com/actions/cache/issues/109 name: CI on: @@ -18,13 +15,94 @@ on: - '[0-9]+.*' - 'ci-*' pull_request: {} + schedule: + # 13:00 UTC is 05:00 in Pacific standard time (UTC-8), which is well + # after nightly TensorFlow wheels are released (around 2--3 AM) and + # just after nightly TensorBoard wheels are released (around 04:15). + # (cron syntax: minute hour day-of-month month day-of-week) + - cron: '0 13 * * *' env: + # Keep this Bazel version in sync with the `versions.check` directive + # in our WORKSPACE file. + BAZEL_VERSION: '2.1.0' + BAZEL_SHA256SUM: 'e13581d44faad6ac807dd917e682fef20359d26728166ac35dadd8ee653a580d' BUILDTOOLS_VERSION: '3.0.0' BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db' BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d' jobs: + build: + runs-on: ubuntu-16.04 + needs: lint-python-flake8 # fail fast in case of "undefined variable" errors + strategy: + fail-fast: false + matrix: + tf_version_id: ['tf-nightly', 'notf'] + python_version: ['3.7'] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python_version }} + architecture: 'x64' + - name: 'Set up Bazel' + run: | + ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel + sudo mv ~/bazel /usr/local/bin/bazel + sudo chmod +x /usr/local/bin/bazel + cp ./ci/bazelrc ~/.bazelrc + - name: 'Configure build cache write credentials' + env: + CREDS: ${{ secrets.BAZEL_CACHE_SERVICE_ACCOUNT_CREDS }} + EVENT_TYPE: ${{ github.event_name }} + run: | + if [ -z "${CREDS}" ]; then + printf 'Using read-only cache (no credentials)\n' + exit + fi + if [ "${EVENT_TYPE}" = pull_request ]; then + printf 'Using read-only cache (PR build)\n' + exit + fi + printf 'Using writable cache\n' + creds_file=/tmp/service_account_creds.json + printf '%s\n' "${CREDS}" >"${creds_file}" + printf '%s\n' >>~/.bazelrc \ + "common --google_credentials=${creds_file}" \ + "common --remote_upload_local_results=true" \ + ; + - name: 'Install Python dependencies' + run: | + python -m pip install -U pip + pip install \ + -r ./tensorboard/pip_package/requirements.txt \ + -r ./tensorboard/pip_package/requirements_dev.txt \ + ; + - name: 'Install TensorFlow' + run: pip install "${{ matrix.tf_version_id }}" + if: matrix.tf_version_id != 'notf' + - name: 'Check Pip state' + run: pip freeze --all + - name: 'Bazel: fetch' + run: bazel fetch //tensorboard/... + - name: 'Bazel: build' + run: bazel build //tensorboard/... + - name: 'Bazel: test (with TensorFlow support)' + run: bazel test //tensorboard/... + if: matrix.tf_version_id != 'notf' + - name: 'Bazel: test (non-TensorFlow only)' + run: bazel test //tensorboard/... --test_tag_filters="support_notf" + if: matrix.tf_version_id == 'notf' + - name: 'Bazel: run Pip package test' + run: | + bazel run //tensorboard/pip_package:test_pip_package -- \ + --tf-version "${{ matrix.tf_version_id }}" + - name: 'Bazel: run manual tests' + run: | + bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test && + bazel test //tensorboard/summary/writer:event_file_writer_s3_test + lint-python-flake8: runs-on: ubuntu-16.04 strategy: @@ -50,7 +128,7 @@ jobs: # Use the comment '# noqa: ' to suppress. run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics - lint-python: + lint-python-yaml-docs: runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 @@ -58,24 +136,31 @@ jobs: with: python-version: '3.6' architecture: 'x64' - - name: 'Install black' + - name: 'Install black, yamllint, and the TensorFlow docs notebook tools' run: | python -m pip install -U pip - pip install black -c ./tensorboard/pip_package/requirements_dev.txt + nbfmt_version="174c9a5c1cc51a3af1de98d84824c811ecd49029" + pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt + pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version} - run: pip freeze --all - name: 'Lint Python code for style with Black' # You can run `black .` to fix all Black complaints. run: black --check --diff . + - name: 'Lint YAML for gotchas with yamllint' + # Use '# yamllint disable-line rule:foo' to suppress. + run: yamllint -c docs/.yamllint docs docs/.yamllint + - name: 'Lint Colab notebooks for formatting with nbfmt' + run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test lint-rust: runs-on: ubuntu-16.04 strategy: matrix: - rust_version: ['1.47.0'] + rust_version: ['1.48.0'] cargo_raze_version: ['0.7.0'] steps: - uses: actions/checkout@v1 - - name: 'Cache Cargo home directory' + - name: 'Cache Cargo artifacts' uses: actions/cache@v2 with: path: | @@ -88,7 +173,7 @@ jobs: # Needed for installing binaries (`cargo-raze`) with cache ~/.cargo/.crates.toml ~/.cargo/.crates2.json - key: ${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }} + key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }} - name: 'Install Rust toolchain' uses: actions-rs/toolchain@v1 with: @@ -101,37 +186,14 @@ jobs: uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - args: --manifest-path tensorboard/data/server/Cargo.toml + args: --tests --manifest-path tensorboard/data/server/Cargo.toml - name: 'Check cargo-raze freshness' run: | rm -rf third_party/rust/ - (cd tensorboard/data/server/ && cargo generate-lockfile && cargo raze) + (cd tensorboard/data/server/ && cargo fetch && cargo raze) git add . git diff --staged --exit-code - lint-docs: - runs-on: ubuntu-16.04 - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 - with: - python-version: '3.6' - architecture: 'x64' - - name: 'Install yamllint' - run: | - python -m pip install -U pip - pip install yamllint -c ./tensorboard/pip_package/requirements_dev.txt - - run: pip freeze --all - - name: 'Lint YAML for gotchas with yamllint' - # Use '# yamllint disable-line rule:foo' to suppress. - run: yamllint -c docs/.yamllint docs docs/.yamllint - - name: 'Install the TensorFlow docs notebook tools' - run: | - nbfmt_version="174c9a5c1cc51a3af1de98d84824c811ecd49029" - python3 -m pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version} - - name: 'Use nbfmt to check Colab notebooks for formatting' - run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test - lint-frontend: runs-on: ubuntu-16.04 steps: @@ -152,7 +214,7 @@ jobs: - run: | ! git grep -E '"@npm//d3"|"@npm//@types/d3"' 'tensorboard/webapp/**/*BUILD' ':!tensorboard/webapp/third_party/**' - lint-build: + lint-misc: # build, protos, etc. runs-on: ubuntu-16.04 steps: - uses: actions/checkout@v1 @@ -179,11 +241,6 @@ jobs: # https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code run: | buildozer '//tensorboard/...:%licenses' remove_comment && false || test $? = 3 - - lint-proto: - runs-on: ubuntu-16.04 - steps: - - uses: actions/checkout@v1 - name: clang-format lint uses: DoozyX/clang-format-lint-action@v0.5 with: @@ -192,11 +249,6 @@ jobs: exclude: ./tensorboard/compat/proto extensions: 'proto' clangFormatVersion: 9 - - check-misc: - runs-on: ubuntu-16.04 - steps: - - uses: actions/checkout@v1 - run: ./tensorboard/tools/do_not_submit_test.sh - run: ./tensorboard/tools/license_test.sh - run: ./tensorboard/tools/whitespace_hygiene_test.py diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1131967b47..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,115 +0,0 @@ -dist: xenial -language: python -python: - - "3.6" - -branches: - only: - - master - - /^\d+\.\d+(\.\d+)?(-\S*)?$/ - -# Update the `nvm use` stanza below when updating this. -node_js: - - "11" - -env: - # Keep this Bazel version in sync with the `versions.check` directive - # near the top of our WORKSPACE file. - # - # Grab the BAZEL_SHA256SUM from the Bazel releases page; e.g.: - # bazel-0.20.0-linux-x86_64.sha256 - global: - - BAZEL=2.1.0 - - BAZEL_SHA256SUM=e13581d44faad6ac807dd917e682fef20359d26728166ac35dadd8ee653a580d - matrix: - # HACK: do not try the 2.4.0rc1 because Travis CI cannot reuse any caches. - # - TF_VERSION_ID=tensorflow==2.4.0rc1 - - TF_VERSION_ID= # Do not install TensorFlow in this case - -cache: - # Don't cache the Pip directory. We pull in a new `tf-nightly` wheel - # every day, and Pip caches are never evicted, so this quickly bloats - # to many gigabytes and adds minutes to the CI time. - pip: false - # Cache directories for Bazel. See ci/bazelrc for details. - directories: - - $HOME/.cache/tb-bazel-repo - - $HOME/.cache/tb-bazel-disk - -# Each bullet point is displayed in the Travis log as one collapsed line, which -# indicates how long it took. Travis will check the return code at the end. We -# can't use `set -e` in the YAML file since it might impact Travis internals. -# If inline scripts get too long, Travis surprisingly prints them twice. - -before_install: - - elapsed() { TZ=UTC printf "Time %(%T)T %s\n" "$SECONDS" "$1"; } - - elapsed "before_install" - - ci/download_bazel.sh "${BAZEL}" "${BAZEL_SHA256SUM}" ~/bazel - - sudo mv ~/bazel /usr/local/bin/bazel - - cp ci/bazelrc ~/.bazelrc - - elapsed "before_install (done)" - -install: - - elapsed "install" - - "PY3=\"$(python -c 'if __import__(\"sys\").version_info[0] > 2: print(1)')\"" - # Older versions of Pip sometimes resolve specifiers like `tf-nightly` - # to versions other than the most recent(!). - - pip install -U pip - # Uninstall older Travis numpy to avoid upgrade-in-place issues. - - pip uninstall -y numpy - - | - pip install \ - -r tensorboard/pip_package/requirements.txt \ - -r tensorboard/pip_package/requirements_dev.txt \ - ; - # Keep the node version in sync with node_js key above. - - nvm use v11 - # HACK: intentionally test just 2.4.0rc1, since the notf version was tested - # and historically passed in: - # https://travis-ci.org/github/tensorflow/tensorboard/builds/742878781 - - export TF_VERSION_ID=tensorflow==2.4.0rc1 - - | - # Install TensorFlow if requested - if [ -n "${TF_VERSION_ID}" ]; then - pip install -I "${TF_VERSION_ID}" - fi - # Workaround for https://github.com/travis-ci/travis-ci/issues/7940 - - sudo rm -f /etc/boto.cfg - - pip freeze # print installed distributions, for debugging purposes - - elapsed "install (done)" - -before_script: - # Note: Lint checks happen on GitHub Actions; see .github/workflows/ci.yml. - - elapsed "before_script" - - | - # Specify subset of tests to run depending on TF installation config. - # We condition the value of --test_tag_filters so that we can run the - # bazel test command unconditionally which produces nicer log output. - if [ -z "${TF_VERSION_ID}" ]; then - test_tag_filters=support_notf - else - test_tag_filters= - fi - - elapsed "before_script (done)" - -# Commands in this section should only fail if it's our fault. Travis will -# categorize them as 'failed', rather than 'error' for other sections. -script: - - elapsed "script" - # Note: bazel test implies fetch+build, but this gives us timing. - - elapsed && bazel fetch //tensorboard/... - - elapsed && bazel build //tensorboard/... - - elapsed && bazel test //tensorboard/... --test_tag_filters="${test_tag_filters}" - - elapsed && bazel run //tensorboard/pip_package:test_pip_package -- --tf-version "${TF_VERSION_ID}" - # Run manual S3 test - - elapsed && bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test - - elapsed && bazel test //tensorboard/summary/writer:event_file_writer_s3_test - - elapsed "script (done)" - -after_script: - # Bazel launches daemons unless --batch is used. - - elapsed "after_script" - - bazel shutdown - -notifications: - email: false diff --git a/ci/bazelrc b/ci/bazelrc index 10435bae8a..4bda9653d6 100644 --- a/ci/bazelrc +++ b/ci/bazelrc @@ -1,5 +1,6 @@ -# Limit resources since Travis Trusty GCE VMs have 2 cores and 7.5 GB RAM. -build --local_resources=4000,2,1.0 +# Limit resources since GitHub Actions VMs have 2 cores and 7 GB RAM. +build --local_cpu_resources=2 +build --local_ram_resources=4000 build --worker_max_instances=2 # Ensure sandboxing is on to increase hermeticity. @@ -17,18 +18,17 @@ build --worker_sandboxing # https://github.com/bazelbuild/bazel/issues/7026 (future of action_env) build --action_env=PATH -# Set up caching on local disk so incremental builds are faster. -# See https://bazel.build/designs/2016/09/30/repository-cache.html -build --repository_cache=~/.cache/tb-bazel-repo -fetch --repository_cache=~/.cache/tb-bazel-repo -query --repository_cache=~/.cache/tb-bazel-repo -# See https://docs.bazel.build/versions/master/remote-caching.html#disk-cache -build --disk_cache=~/.cache/tb-bazel-disk - -# Log more information to help with debugging, and disable curses output which -# just adds more clutter to the log. (Travis spoofs an interactive terminal.) -common --curses=no +# Log more information to help with debugging. build --verbose_failures build --worker_verbose test --test_output=errors test --test_verbose_timeout_warnings + +# Enable remote caching. This cache is publicly readable, but only writable on +# trusted commits (e.g., commits to master). We don't write the actual test +# statuses (or read them if they happen to be there) because not all tests are +# actually hermetic, and they run pretty quickly, anyway. +common --remote_cache=https://storage.googleapis.com/tensorboard-build-cache +common --remote_upload_local_results=false +test --remote_upload_local_results=false +test --cache_test_results=false diff --git a/tensorboard/pip_package/test_pip_package.sh b/tensorboard/pip_package/test_pip_package.sh index 738d5cd4dd..1595bcd860 100755 --- a/tensorboard/pip_package/test_pip_package.sh +++ b/tensorboard/pip_package/test_pip_package.sh @@ -24,7 +24,7 @@ Test pre-built TensorBoard Pip packages. Options: --tf-version VERSION: Test against the provided version of TensorFlow, given as a Pip package specifier like "tensorflow==2.0.0a0" or - "tf-nightly". If empty, will test without installing TensorFlow. + "tf-nightly". If empty or "notf", will test without installing TensorFlow. Defaults to "tf-nightly". EOF } @@ -104,6 +104,9 @@ smoke() ( smoke_venv="${virtualenvs_root}/venv-${smoke_python}/" set +x printf '\n\n%70s\n' '' | tr ' ' '=' + if [ "${tf_version}" = notf ]; then + tf_version= + fi if [ -z "${tf_version}" ]; then echo "Smoke testing with ${smoke_python} and no tensorflow..." else From 877829cfe1808d6bffe4ff2546c8c4ab06836c03 Mon Sep 17 00:00:00 2001 From: Joyce Er Date: Mon, 16 Nov 2020 18:32:06 -0800 Subject: [PATCH 2/2] Remove `frame-ancestors *` CSP directive (#4332) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR essentially reverses https://github.com/tensorflow/tensorboard/pull/2797. Currently this doesn't work because the `frame-ancestors *` directive prevents VS Code from framing TensorBoard. This is because VS Code is an Electron application, and Electron appears to be unable to frame websites which set `frame-ancestors *` in its response headers: https://github.com/electron/electron/issues/26369 If I'm reading the CSP specification correctly, omitting the frame-ancestors directive altogether is equivalent to setting `frame-ancestors *`, so to my knowledge this PR should not result in a behavior change for environments which correctly implement the CSP spec. From https://w3c.github.io/webappsec-csp/2/#directive-frame-ancestors: > The term allowed frame ancestors refers to the result of parsing the frame-ancestors directive’s value as a source list. If a frame-ancestors directive is not explicitly included in the policy, then allowed frame ancestors is "*". --- tensorboard/backend/http_util.py | 1 - tensorboard/backend/http_util_test.py | 12 ++++++------ tensorboard/backend/security_validator.py | 2 -- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/tensorboard/backend/http_util.py b/tensorboard/backend/http_util.py index cc32e08362..28838cd2a7 100644 --- a/tensorboard/backend/http_util.py +++ b/tensorboard/backend/http_util.py @@ -222,7 +222,6 @@ def Respond( "default-src 'self'", "font-src %s" % _create_csp_string("'self'", *_CSP_FONT_DOMAINS_WHITELIST), - "frame-ancestors *", # Dynamic plugins are rendered inside an iframe. "frame-src %s" % _create_csp_string("'self'", *_CSP_FRAME_DOMAINS_WHITELIST), diff --git a/tensorboard/backend/http_util_test.py b/tensorboard/backend/http_util_test.py index 5423a96c5d..a1e2c1a88e 100644 --- a/tensorboard/backend/http_util_test.py +++ b/tensorboard/backend/http_util_test.py @@ -239,7 +239,7 @@ def testCsp(self): q, "hello", "text/html", csp_scripts_sha256s=["abcdefghi"] ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self';img-src 'self' data: blob:;object-src 'none';" "style-src 'self' https://www.gstatic.com data: 'unsafe-inline';" "connect-src 'self';script-src 'self' 'unsafe-eval' 'sha256-abcdefghi'" @@ -253,7 +253,7 @@ def testCsp_noHash(self): q, "hello", "text/html", csp_scripts_sha256s=None ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self';img-src 'self' data: blob:;object-src 'none';" "style-src 'self' https://www.gstatic.com data: 'unsafe-inline';" "connect-src 'self';script-src 'unsafe-eval'" @@ -268,7 +268,7 @@ def testCsp_noHash_noUnsafeEval(self): q, "hello", "text/html", csp_scripts_sha256s=None ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self';img-src 'self' data: blob:;object-src 'none';" "style-src 'self' https://www.gstatic.com data: 'unsafe-inline';" "connect-src 'self';script-src 'none'" @@ -283,7 +283,7 @@ def testCsp_onlySelf(self): q, "hello", "text/html", csp_scripts_sha256s=None ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self';img-src 'self' data: blob:;object-src 'none';" "style-src 'self' https://www.gstatic.com data: 'unsafe-inline';" "connect-src 'self';script-src 'self'" @@ -297,7 +297,7 @@ def testCsp_disableUnsafeEval(self): q, "hello", "text/html", csp_scripts_sha256s=["abcdefghi"] ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self';img-src 'self' data: blob:;object-src 'none';" "style-src 'self' https://www.gstatic.com data: 'unsafe-inline';" "connect-src 'self';script-src 'self' 'sha256-abcdefghi'" @@ -324,7 +324,7 @@ def testCsp_globalDomainWhiteList(self): q, "hello", "text/html", csp_scripts_sha256s=["abcd"] ) expected_csp = ( - "default-src 'self';font-src 'self' data:;frame-ancestors *;" + "default-src 'self';font-src 'self' data:;" "frame-src 'self' https://myframe.com;" "img-src 'self' data: blob: https://example.com;" "object-src 'none';style-src 'self' https://www.gstatic.com data: " diff --git a/tensorboard/backend/security_validator.py b/tensorboard/backend/security_validator.py index a7bc1cf5d2..34926d5821 100644 --- a/tensorboard/backend/security_validator.py +++ b/tensorboard/backend/security_validator.py @@ -34,8 +34,6 @@ _CSP_DEFAULT_SRC = "default-src" # Whitelist of allowed CSP violations. _CSP_IGNORE = { - # Allow TensorBoard to be iframed. - "frame-ancestors": ["*"], # Polymer-based code uses unsafe-inline. "style-src": ["'unsafe-inline'", "data:"], # Used in canvas