-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs: add docs build kokoro config for django spanner (#644)
* docs: added trampoline v2 file and docs config for kokoro * fix: change library_type to INTEGRATION * docs: add kokoro docker build for docs * fix:repo metadata * fix: updated PROJECT_ROOT for django spanner docs build using kokoro * fix: remove sudo apt update because sudo is not allowed * fix: add build_nox.sh as original build.sh file is used by kokoro workers to run parallelized tests * refactor: renamed build_nox.sh to build-nox.sh * refactor: change build-nox to nox.sh * refactor: change build-nox to nox.sh * refactor: change kokoro build to run different test for docs and different test for workers * refactor: change kokoro build to run different test for docs and different test for workers * refactor: add log msg for debugging * refactor: remove log msg added for debugging * fix: correct docs build command * refactor: remove unused nox file * feat: add .trampolinerc file * fix: correct repo_metadata release_level to beta
- Loading branch information
Showing
10 changed files
with
890 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
# Copyright 2020 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
from ubuntu:20.04 | ||
|
||
ENV DEBIAN_FRONTEND noninteractive | ||
|
||
# Ensure local Python is preferred over distribution Python. | ||
ENV PATH /usr/local/bin:$PATH | ||
|
||
# Install dependencies. | ||
RUN apt-get update \ | ||
&& apt-get install -y --no-install-recommends \ | ||
apt-transport-https \ | ||
build-essential \ | ||
ca-certificates \ | ||
curl \ | ||
dirmngr \ | ||
git \ | ||
gpg-agent \ | ||
graphviz \ | ||
libbz2-dev \ | ||
libdb5.3-dev \ | ||
libexpat1-dev \ | ||
libffi-dev \ | ||
liblzma-dev \ | ||
libreadline-dev \ | ||
libsnappy-dev \ | ||
libssl-dev \ | ||
libsqlite3-dev \ | ||
portaudio19-dev \ | ||
redis-server \ | ||
software-properties-common \ | ||
ssh \ | ||
sudo \ | ||
tcl \ | ||
tcl-dev \ | ||
tk \ | ||
tk-dev \ | ||
uuid-dev \ | ||
wget \ | ||
zlib1g-dev \ | ||
&& add-apt-repository universe \ | ||
&& apt-get update \ | ||
&& apt-get -y install jq \ | ||
&& apt-get clean autoclean \ | ||
&& apt-get autoremove -y \ | ||
&& rm -rf /var/lib/apt/lists/* \ | ||
&& rm -f /var/cache/apt/archives/*.deb | ||
|
||
|
||
COPY fetch_gpg_keys.sh /tmp | ||
# Install the desired versions of Python. | ||
RUN set -ex \ | ||
&& export GNUPGHOME="$(mktemp -d)" \ | ||
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \ | ||
&& /tmp/fetch_gpg_keys.sh \ | ||
&& for PYTHON_VERSION in 3.7.8 3.8.5; do \ | ||
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \ | ||
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \ | ||
&& gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \ | ||
&& rm -r python-${PYTHON_VERSION}.tar.xz.asc \ | ||
&& mkdir -p /usr/src/python-${PYTHON_VERSION} \ | ||
&& tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \ | ||
&& rm python-${PYTHON_VERSION}.tar.xz \ | ||
&& cd /usr/src/python-${PYTHON_VERSION} \ | ||
&& ./configure \ | ||
--enable-shared \ | ||
# This works only on Python 2.7 and throws a warning on every other | ||
# version, but seems otherwise harmless. | ||
--enable-unicode=ucs4 \ | ||
--with-system-ffi \ | ||
--without-ensurepip \ | ||
&& make -j$(nproc) \ | ||
&& make install \ | ||
&& ldconfig \ | ||
; done \ | ||
&& rm -rf "${GNUPGHOME}" \ | ||
&& rm -rf /usr/src/python* \ | ||
&& rm -rf ~/.cache/ | ||
|
||
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ | ||
&& python3.7 /tmp/get-pip.py \ | ||
&& python3.8 /tmp/get-pip.py \ | ||
&& rm /tmp/get-pip.py | ||
|
||
CMD ["python3.7"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
#!/bin/bash | ||
# Copyright 2020 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
# A script to fetch gpg keys with retry. | ||
# Avoid jinja parsing the file. | ||
# | ||
|
||
function retry { | ||
if [[ "${#}" -le 1 ]]; then | ||
echo "Usage: ${0} retry_count commands.." | ||
exit 1 | ||
fi | ||
local retries=${1} | ||
local command="${@:2}" | ||
until [[ "${retries}" -le 0 ]]; do | ||
$command && return 0 | ||
if [[ $? -ne 0 ]]; then | ||
echo "command failed, retrying" | ||
((retries--)) | ||
fi | ||
done | ||
return 1 | ||
} | ||
|
||
# 3.6.9, 3.7.5 (Ned Deily) | ||
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ | ||
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D | ||
|
||
# 3.8.0 (Łukasz Langa) | ||
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \ | ||
E3FF2839C048B25C084DEBE9B26995E310250568 | ||
|
||
# |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
# Format: //devtools/kokoro/config/proto/build.proto | ||
|
||
# Build logs will be here | ||
action { | ||
define_artifacts { | ||
regex: "**/*sponge_log.xml" | ||
} | ||
} | ||
|
||
# Download trampoline resources. | ||
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" | ||
|
||
# Use the trampoline script to run in docker. | ||
build_file: "python-spanner-django/.kokoro/trampoline_v2.sh" | ||
|
||
# Configure the docker image for kokoro-trampoline. | ||
env_vars: { | ||
key: "TRAMPOLINE_IMAGE" | ||
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs" | ||
} | ||
env_vars: { | ||
key: "TRAMPOLINE_BUILD_FILE" | ||
value: "github/python-spanner-django/.kokoro/publish-docs.sh" | ||
} | ||
|
||
env_vars: { | ||
key: "STAGING_BUCKET" | ||
value: "docs-staging" | ||
} | ||
|
||
env_vars: { | ||
key: "V2_STAGING_BUCKET" | ||
value: "docs-staging-v2" | ||
} | ||
|
||
# It will upload the docker image after successful builds. | ||
env_vars: { | ||
key: "TRAMPOLINE_IMAGE_UPLOAD" | ||
value: "true" | ||
} | ||
|
||
# It will always build the docker image. | ||
env_vars: { | ||
key: "TRAMPOLINE_DOCKERFILE" | ||
value: ".kokoro/docker/docs/Dockerfile" | ||
} | ||
|
||
# Fetch the token needed for reporting release status to GitHub | ||
before_action { | ||
fetch_keystore { | ||
keystore_resource { | ||
keystore_config_id: 73713 | ||
keyname: "yoshi-automation-github-key" | ||
} | ||
} | ||
} | ||
|
||
before_action { | ||
fetch_keystore { | ||
keystore_resource { | ||
keystore_config_id: 73713 | ||
keyname: "docuploader_service_account" | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
# Format: //devtools/kokoro/config/proto/build.proto | ||
|
||
env_vars: { | ||
key: "STAGING_BUCKET" | ||
value: "gcloud-python-test" | ||
} | ||
|
||
env_vars: { | ||
key: "V2_STAGING_BUCKET" | ||
value: "gcloud-python-test" | ||
} | ||
|
||
# We only upload the image in the main `docs` build. | ||
env_vars: { | ||
key: "TRAMPOLINE_IMAGE_UPLOAD" | ||
value: "false" | ||
} | ||
|
||
env_vars: { | ||
key: "TRAMPOLINE_BUILD_FILE" | ||
value: "github/python-spanner-django/.kokoro/build.sh" | ||
} | ||
|
||
# Only run this nox session. | ||
env_vars: { | ||
key: "NOX_SESSION" | ||
value: "docs docfx" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
# Format: //devtools/kokoro/config/proto/build.proto |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
#!/bin/bash | ||
# Copyright 2020 Google LLC | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# https://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
set -eo pipefail | ||
|
||
# Disable buffering, so that the logs stream through. | ||
export PYTHONUNBUFFERED=1 | ||
|
||
export PATH="${HOME}/.local/bin:${PATH}" | ||
|
||
# Install nox | ||
python3 -m pip install --user --upgrade --quiet nox | ||
python3 -m nox --version | ||
|
||
# build docs | ||
nox -s docs | ||
|
||
python3 -m pip install --user gcp-docuploader | ||
|
||
# create metadata | ||
python3 -m docuploader create-metadata \ | ||
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ | ||
--version=$(python3 setup.py --version) \ | ||
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ | ||
--distribution-name=$(python3 setup.py --name) \ | ||
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ | ||
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ | ||
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) | ||
|
||
cat docs.metadata | ||
|
||
# upload docs | ||
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}" | ||
|
||
|
||
# docfx yaml files | ||
nox -s docfx | ||
|
||
# create metadata. | ||
python3 -m docuploader create-metadata \ | ||
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \ | ||
--version=$(python3 setup.py --version) \ | ||
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \ | ||
--distribution-name=$(python3 setup.py --name) \ | ||
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \ | ||
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \ | ||
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json) | ||
|
||
cat docs.metadata | ||
|
||
# upload docs | ||
python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}" |
Oops, something went wrong.