diff --git a/.gitignore b/.gitignore index ded6eaad..13bdf8d8 100644 --- a/.gitignore +++ b/.gitignore @@ -171,3 +171,5 @@ target/ #Ipython Notebook .ipynb_checkpoints +# Local Netlify folder +.netlify diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 00000000..2fed5128 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,17 @@ +[build] + base = "site" + publish = "public" + command = "npm install && hugo --gc --minify" + +[context.deploy-preview.environment] + HUGO_VERSION = "0.92.0" + NODE_VERSION = "16" + +[context.production.environment] + HUGO_VERSION = "0.92.0" + HUGO_ENV = "production" + NODE_VERSION = "16" + +[context.branch-deploy.environment] + HUGO_VERSION = "0.92.0" + NODE_VERSION = "16" diff --git a/site/.gitignore b/site/.gitignore new file mode 100644 index 00000000..cc87aecd --- /dev/null +++ b/site/.gitignore @@ -0,0 +1,12 @@ +public/ + +# Modules generated by node.js for CSS editing +node_modules/ + +# Resources directory generated by Hugo local build +resources/ + +# Notebooks +.ipynb_checkpoints +# +.hugo_build.lock diff --git a/site/OWNERS b/site/OWNERS new file mode 100644 index 00000000..dfa91ceb --- /dev/null +++ b/site/OWNERS @@ -0,0 +1,4 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- moficodes diff --git a/site/README.md b/site/README.md deleted file mode 100644 index b3a42524..00000000 --- a/site/README.md +++ /dev/null @@ -1 +0,0 @@ -placeholder \ No newline at end of file diff --git a/site/archetypes/default.md b/site/archetypes/default.md new file mode 100644 index 00000000..26f317f3 --- /dev/null +++ b/site/archetypes/default.md @@ -0,0 +1,5 @@ +--- +title: "{{ replace .Name "-" " " | title }}" +date: {{ .Date }} +draft: true +--- diff --git a/site/assets/icons/logo.svg b/site/assets/icons/logo.svg new file mode 100644 index 00000000..f9bd6b6c --- /dev/null +++ b/site/assets/icons/logo.svg @@ -0,0 +1,84 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + + diff --git a/site/assets/scss/_styles_project.scss b/site/assets/scss/_styles_project.scss new file mode 100644 index 00000000..fccde58c --- /dev/null +++ b/site/assets/scss/_styles_project.scss @@ -0,0 +1,174 @@ +// -------------------------------------------------- +// remove whitespace after footer +// -------------------------------------------------- +footer { + min-height: auto; +} + +// -------------------------------------------------- +// prevent overflow of long names in sidebar +// -------------------------------------------------- +.td-sidebar-nav__section > ul { + overflow-wrap: break-word; +} + +.td-overlay--dark::after { + background-color: rgba(33, 61, 122, 0.6); +} + +#main_navbar { + justify-content: end; + padding-right: 16px; +} + +.row > * { + flex-shrink: 1; +} + +.row > .container { + max-width: 1200px; +} + +// -------------------------------------------------- +// custom navbar with larger logo, dropdown on mobile +// -------------------------------------------------- +.td-navbar { + min-height: auto; + + .navbar-brand { + margin: 0; + padding: 0; + + .text-uppercase { + display: none; + } + + .navbar-logo { + svg { + display: inline-block; + position: absolute; + top: 0; + z-index: 33; + padding: 10px; + height: 95px; + background: white; + border: 2px solid #4279f4; + border-top: none; + + @include media-breakpoint-down(md) { + width: 80px; + height: auto; + padding: 6px; + } + } + } + } + + .navbar-nav { + @include media-breakpoint-down(md) { + font-size: .875rem; + .dropdown { + min-width: inherit; + } + } + } +} + +.td-sidebar { + padding-bottom: 0.5rem; + + .td-sidebar__inner { + padding-top: 30px; + + @include media-breakpoint-down(md) { + padding-top: 10px; + } + } +} + +.td-sidebar-toc { + @supports (position: sticky) { + position: sticky; + top: 60px; + height: calc(100vh - 120px); + overflow-y: auto; + } +} + + +// -------------------------------------------------- +// 404 page +// -------------------------------------------------- +.error-page { + margin-top: 120px; + + ul { + margin-bottom: 50px; + list-style-type: none; + display: flex; + align-items: center; + justify-content: center; + flex-wrap: wrap; + padding-left: 0; + } + + li { + margin-left: 10px; + margin-right: 10px; + } + + h1 { + text-align: center; + margin-bottom: 20px; + } + + @media (min-width: 768px) and (max-width: 991px) { + margin-top: 100px; + } + + @media (max-width: 767px) { + margin-top: 50px; + } +} + +figcaption { + font-size: 0.8rem; + text-align: center; + font-style: italic; + color: #6c757d; +} + +// -------------------------------------------------- +// for tabbed code blocks +// -------------------------------------------------- +.nav-tabs { + border-bottom: none !important; +} + +.td-content > ul li, +.td-content > ol li.nav-item { + margin-bottom: 0px; +} + +.td-content .tab-content .highlight { + margin: 0; +} + +.tab-pane { + border-radius: 0.25rem; + padding: 0 16px 16px; + + border: 1px solid #dee2e6; + + &:first-of-type.active { + border-top-left-radius: 0; + } +} + +nav.foldable-nav .ul-1 .with-child>label:hover:before { + transform: none; +} + +nav.foldable-nav .ul-1 .with-child>input:checked~label:hover:before { + transform: rotate(90deg) !important; +} \ No newline at end of file diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss new file mode 100644 index 00000000..f9614eb5 --- /dev/null +++ b/site/assets/scss/_variables_project.scss @@ -0,0 +1,151 @@ +/* +Add styles or override the theme's variables here. +*/ + +html.smooth-scroll { + scroll-behavior: smooth; +} + +// Theme colors +$primary: #4279f4; +$secondary: #fff; +$dark: #213d7a; +$info: #adb5bd; +$light: #dee2e6; + +// Nav bar colors +$white: #fff; +$navbar-dark-color: rgba($white, 1); +$navbar-dark-hover-color: rgba($white, 0.75); +$navbar-dark-active-color: $navbar-dark-color; + +// Fonts +$google_font_family: "Open+Sans:300,300i,400,400i,600,600i,700,700i&display=swap" !default; + +// Front page styling +.card-img-top { + object-fit: scale-down; +} + +.text-white { + font-weight: 400; +} + +.bg-primary-dark { + background-color: $dark; + + a { + border-bottom: 1px dotted paleturquoise; + + color: paleturquoise !important; + font-weight: 600; + padding: 0 2px 1px 2px; + text-decoration: none; + + &:hover { + border-bottom: 1px dotted #fff; + + color: #fff !important; + } + } +} + +.border-primary-dark { + border-color: $info !important; +} + +.section-head { + font-size: 1.25em; + color: $primary; + font-weight: bold; + padding: 0 0 1em 0; +} + +.contain { + margin: 0 auto; + max-width: 1200px; } + +.image { + display: block; } + .image img { + display: block; + width: 100%; + height: auto; } + .image.left, .image.right { + max-width: 45%; } + .image.left::after, .image.right::after { + clear: both; + content: ""; + display: block; } + .image.left { + float: left; + margin: 0 1.5em 1.5em 0; } + .image.right { + float: right; + margin: 0 0 1.5em 1.5em; } + +#overview, #community { + padding: 6em 1.5em 3em 1.5em; + text-align: center; + margin: 0; } + #overview p, #community p { + font-size: 1.125em;} + +#overview { + border-bottom: 2px solid #b6d0ff; +} + +#community { + border-top: 2px solid #b6d0ff; +} + +#pageContent .lead { + margin: 0 1.5em 3em 1.5em;} + #pageContent .lead > .image { + padding: 0 1.5em; + max-width: 60%; + margin-bottom: 1.5em; } + #pageContent .lead > .text p { + font-size: 1em; } + +@media screen and (min-width: 480px) { + #overview, #community { + font-size: 1.125em; } + #pageContent .lead > .image { + max-width: 35%; + margin-bottom: 3em; } + #pageContent .lead > .text p { + font-size: 1em; } } + +@media screen and (min-width: 769px) { + #pageContent .lead { + margin-top: 1em; + display: -ms-flexbox; + display: flex; + -ms-flex-direction: row; + flex-direction: row; + -ms-flex-wrap: nowrap; + flex-wrap: nowrap; + -ms-flex-pack: justify; + justify-content: space-between; + -ms-flex-align: stretch; + align-items: stretch; } + #pageContent .lead > * { + -ms-flex: 0 1 auto; + flex: 0 1 auto; } + #pageContent .lead > .image { + display: block; + margin: 0 auto; + max-width: 100%; } + #pageContent .lead > .image > img { + max-width: 80%; + margin: 0 auto; } + #pageContent .lead > .text { + -ms-flex-preferred-size: 70%; + flex-basis: 70%; } + #pageContent .lead:nth-child(2n+0) > .image { + -ms-flex-order: 2; + order: 2; } + #pageContent .lead:nth-child(2n+0) > .text { + -ms-flex-order: 1; + order: 1; } } diff --git a/site/config.toml b/site/config.toml new file mode 100644 index 00000000..53519ed6 --- /dev/null +++ b/site/config.toml @@ -0,0 +1,161 @@ +baseURL = "/" +title = "Jobset" + +enableRobotsTXT = true + +theme = "docsy" +themesDir = "node_modules" + +############################################################################### +# Docsy +############################################################################### +enableGitInfo = true + +# language settings +contentDir = "content/en" +defaultContentLanguage = "en" +# tell Hugo not to include the /en/ element in the URL path for English docs +defaultContentLanguageInSubdir = false +# useful when translating +enableMissingTranslationPlaceholders = true +# disable taxonomies +disableKinds = ["taxonomy", "taxonomyTerm"] +# deprecated directories +ignoreFiles = [] +############################################################################### +# Hugo - Top-level navigation (horizontal) +############################################################################### +[menu] + + [[menu.main]] + name = "GitHub" + weight = 99 + pre = "" + url = "https://github.com/kubernetes-sigs/jobset" + +############################################################################### +# Docsy - Goldmark markdown parser +############################################################################### +[markup] + [markup.goldmark] + [markup.goldmark.renderer] + unsafe = true + [markup.highlight] + # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html + style = "tango" + guessSyntax = "true" + +############################################################################### +# Docsy - Image processing configuration +############################################################################### +[imaging] + resampleFilter = "CatmullRom" + quality = 75 + anchor = "smart" + +############################################################################### +# Docsy - Services configuration +############################################################################### +# [services] +# [services.googleAnalytics] +# id = "UA-XXXXXXXXX-X" + +############################################################################### +# Docsy - Language configuration +############################################################################### +[languages] + [languages.en] + languageName ="English" + # weight used for sorting in the language menu + weight = 1 + +############################################################################### +# Docsy - Site Parameters +############################################################################### +[params] + github_repo = "https://github.com/kubernetes-sigs/jobset" + github_project_repo = "https://github.com/kubernetes-sigs/jobset" + github_subdir = "site" + RSSLink = "/index.xml" + author = "kubernetes" # add your company name + github = "kubernetes" # add your github profile name + twitter = "kubernetesio" # add your twitter profile + + copyright = "The Kubernetes Authors" + + # Google Custom Search Engine ID. + # gcs_engine_id = "007239566369470735695:624rglujm-w" + prism_syntax_highlighting = true + # Text label for the version menu in the top bar of the website. + version_menu = "Version" + + # The major.minor version tag for the version of the docs represented in this + # branch of the repository. Used in the "version-banner" partial to display a + # version number for this doc set. + version = "v0.5.3" + + # Flag used in the "version-banner" partial to decide whether to display a + # banner on every page indicating that this is an archived version of the docs. + archived_version = false + + # A link to latest version of the docs. Used in the "version-banner" partial to + # point people to the main doc site. + url_latest_version = "https://sigs.k8s.io/jobset/docs" + + # A variable used in various docs to determine URLs for config files etc. + # To find occurrences, search the repo for 'params "githubbranch"'. + githubbranch = "main" + + # These entries appear in the drop-down menu at the top of the website. + # [[params.versions]] + # version = "main" + # githubbranch = "main" + # url = "https://jobset.sigs.k8s.io" + + # User interface configuration + [params.ui] + # Enable the logo + navbar_logo = true + # Enable to show the side bar menu in its compact state. + sidebar_menu_compact = true + # Enable the search box in the side bar. + sidebar_search_disable = false + # Set to true to disable breadcrumb navigation. + breadcrumb_disable = false + # Show expand/collapse icon for sidebar sections + sidebar_menu_foldable = true + # Disable about button in footer + footer_about_disable = true + + # Adds a H2 section titled "Feedback" to the bottom of each doc. The responses are sent to Google Analytics as events. + # This feature depends on [services.googleAnalytics] and will be disabled if "services.googleAnalytics.id" is not set. + # If you want this feature, but occasionally need to remove the "Feedback" section from a single page, + # add "hide_feedback: true" to the page's front matter. + [params.ui.feedback] + enable = true + # The responses that the user sees after clicking "yes" (the page was helpful) or "no" (the page was not helpful). + yes = 'Glad to hear it! Please tell us how we can improve.' + no = 'Sorry to hear that. Please tell us how we can improve.' + +[params.links] +# End user relevant links. These will show up on left side of footer and in the community page if you have one. + [[params.links.user]] + name = "User mailing list" + url = "https://groups.google.com/a/kubernetes.io/g/wg-batch" + icon = "fa fa-envelope" + desc = "Discussion and help from your fellow users" + [[params.links.user]] + name ="Twitter" + url = "https://twitter.com/kubernetesio" + icon = "fab fa-twitter" + desc = "Follow us on Twitter to get the latest news!" + [[params.links.user]] + name = "Stack Overflow" + url = "https://stackoverflow.com/questions/tagged/kubernetes" + icon = "fab fa-stack-overflow" + desc = "Practical questions and curated answers" + [[params.links.user]] + name = "Slack" + url = "https://kubernetes.slack.com/" + icon = "fab fa-slack" + desc = "Chat with other project developers" diff --git a/site/content/en/_index.html b/site/content/en/_index.html new file mode 100644 index 00000000..7cdd155b --- /dev/null +++ b/site/content/en/_index.html @@ -0,0 +1,50 @@ ++++ +title = "Jobset" +linkTitle = "Jobset" +description = "TBD" ++++ + +{{< blocks/cover title="Welcome to Jobset" image_anchor="top" height="full" color="orange" >}} +
+ + Read the docs + + + Github + +

Kubernetes-native Job Queueing

+ {{< blocks/link-down color="info" id="description" >}} +
+{{< /blocks/cover >}} + +
+ +{{% blocks/lead color="primary"%}} +

TBD

+{{% /blocks/lead %}} + +{{% blocks/lead color="secondary" %}} +

TBD +

+{{% /blocks/lead %}} + +{{% blocks/lead color="dark" %}} +

TBD

+{{% /blocks/lead %}} + +{{< blocks/section color="light" >}} + +{{% blocks/feature icon="fab fa-github" title="Contributions welcome!" url="https://github.com/kubernetes-sigs/jobset" %}} +We do a [Pull Request](https://github.com/kubernetes-sigs/jobset/pulls) contributions workflow on **GitHub**. New users are always welcome! +{{% /blocks/feature %}} + +{{% blocks/feature icon="fab fa-slack" title="Connect with us" url="https://kubernetes.slack.com" %}} +Talk to contributors on [#wg-batch](https://kubernetes.slack.com/messages/wg-batch) channel +{{% /blocks/feature %}} + +{{% blocks/feature icon="fa fa-envelope" title="Join the mailing group" url="https://groups.google.com/a/kubernetes.io/g/wg-batch" %}} +Join the conversation on the [mailing group](https://groups.google.com/a/kubernetes.io/g/wg-batch) +{{% /blocks/feature %}} + + +{{< /blocks/section >}} diff --git a/site/content/en/docs/_index.md b/site/content/en/docs/_index.md new file mode 100755 index 00000000..f72f7329 --- /dev/null +++ b/site/content/en/docs/_index.md @@ -0,0 +1,8 @@ +--- +title: "Documentation" +linkTitle: "Documentation" +weight: 20 +menu: + main: + weight: 20 +--- diff --git a/site/content/en/docs/concepts/_index.md b/site/content/en/docs/concepts/_index.md new file mode 100644 index 00000000..0427edb4 --- /dev/null +++ b/site/content/en/docs/concepts/_index.md @@ -0,0 +1,152 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 4 +description: > + Core Jobset Concepts +no_list: true +--- + +## Jobset + +A JobSet creates one or more Jobs. It allows you to create sets of jobs of different or identical templates, and control their lifecycle together. + +## Running an Example JobSet + +Here is an example JobSet. It runs a distributed PyTorch training workload. + +```yaml +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: pytorch +spec: + replicatedJobs: + - name: workers + template: + spec: + parallelism: 4 + completions: 4 + backoffLimit: 0 + template: + spec: + containers: + - name: pytorch + image: gcr.io/k8s-staging-jobset/pytorch-resnet:latest + ports: + - containerPort: 3389 + env: + - name: MASTER_ADDR + value: "pytorch-workers-0-0.pytorch-workers" + - name: MASTER_PORT + value: "3389" + command: + - bash + - -xc + - | + torchrun --nproc_per_node=1 --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT resnet.py --backend=gloo +``` + +To list all the jobs that belong to a JobSet, you can use a command like this: + +```shell +kubectl get jobs --selector=jobset.sigs.k8s.io/jobset-name=pytorch +``` + +The output is similar to + +``` +NAME COMPLETIONS DURATION AGE +pytorch-workers-0 0/4 6m12s 6m12s +``` + +To list all the pods that belong to a JobSet, you can use a command like this: + +```shell +kubectl get pods --selector=jobset.sigs.k8s.io/jobset-name=pytorch +``` + +The output is similar to + +``` +NAME READY STATUS RESTARTS AGE +pytorch-workers-0-0-tcngx 1/1 Running 0 13m +pytorch-workers-0-1-sbhxs 1/1 Running 0 13m +pytorch-workers-0-2-5m6d6 1/1 Running 0 13m +pytorch-workers-0-3-mn8c8 1/1 Running 0 13m +``` + +## JobSet defaults for Jobs and Pods + +- Job [`completionMode`](https://kubernetes.io/docs/concepts/workloads/controllers/job/#completion-mode) is defaulted to `Indexed` +- Pod [`restartPolicy`](https://kubernetes.io/docs/concepts/workloads/controllers/job/#pod-template) is defaulted to `OnFailure` + + +## JobSet labels + +JobSet labels will have `jobset.x-k8s.io/` prefix. JobSet sets the following labels on both the jobs and pods: +- `jobset.sigs.k8s.io/jobset-name`: `.metadata.name` +- `jobset.sigs.k8s.io/replicatedjob-name`: `.spec.replicatedJobs[*].name` +- `jobset.sigs.k8s.io/replicatedjob-replicas`: `.spec.replicatedJobs[*].replicas` +- `jobset.sigs.k8s.io/job-index`: ordinal index of a job within a `spec.replicatedJobs[*]` + + +## ReplicatedJob + +The list `.spec.replicatedJobs` allows the user to define groups of Jobs of different templates. + +Each entry of `.spec.replicatedJobs` defines a Job template in `spec.replicatedJobs[*].template`, +and the number replicas that should be created in `spec.replicatedJobs[*].replicas`. When +unset, it is defaulted to 1. + +Each Job in each `spec.replicatedJobs` gets a different job-index in the range 0 to `.spec.replicatedJob[*].replicas-1`. +The Job name will have the following format: `--`. + + +### DNS hostnames for Pods + +By default, JobSet configures DNS for Pods by creating a headless service for each `spec.replicatedJobs`. +The headless service name, which determines the subdomain, is `.metadata.name-.spec.replicatedJobs[*].name` + +To list all the headless services that belong to a JobSet, you can use a command like this: + +```shell +kubectl get services --selector=jobset.sigs.k8s.io/jobset-name=pytorch +``` + +The output is similar to + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +pytorch-workers ClusterIP None 25m +``` + +### Exclusive Job to topology placement + +The JobSet annotation `alpha.jobset.sigs.k8s.io/exclusive-topology` defines 1:1 job to topology placement. +For example, consider the case where the nodes are assigned a rack label. To optimize for network +performance, we want to assign each job exclusively to one rack. This can be done as follows: + +```yaml +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: pytorch + annotations: + alpha.jobset.sigs.k8s.io/exclusive-topology: rack +spec: + replicatedJobs: + - name: workers + template: + spec: + ... +``` + +## JobSet termination + +A JobSet is marked as successful when ALL the Jobs it created completes successfully. + +A JobSet failure is counted when ANY of its child Jobs fail. `spec.failurePolicy.maxRestarts` defines how many times +to automatically restart the JobSet. A restart is done by recreating all child jobs. + +A JobSet is terminally failed when the number of failures reaches `spec.failurePolicy.maxRestarts` diff --git a/site/content/en/docs/contribution guidelines/_index.md b/site/content/en/docs/contribution guidelines/_index.md new file mode 100644 index 00000000..839fd044 --- /dev/null +++ b/site/content/en/docs/contribution guidelines/_index.md @@ -0,0 +1,32 @@ +--- +title: "Contribution Guidelines" +linkTitle: "Contribution Guidelines" +weight: 9 +description: > + How to contribute to Jobset +--- + +Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: + +_As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ + +## Getting Started + +We have full documentation on how to get started contributing here: + + + +- [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests +- [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) +- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet) - Common resources for existing developers + +## Mentorship + +- [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! + +## Contact Information + +- [Slack](https://kubernetes.slack.com/messages/sig-scheduling) +- [Mailing List](https://groups.google.com/forum/#!forum/kubernetes-sig-scheduling) diff --git a/site/content/en/docs/faq/_index.md b/site/content/en/docs/faq/_index.md new file mode 100755 index 00000000..17a54030 --- /dev/null +++ b/site/content/en/docs/faq/_index.md @@ -0,0 +1,61 @@ +--- + +title: "Faqs" +linkTitle: "Faqs" +weight: 10 +date: 2022-02-14 +description: > + Frequently asked questions about JobSet. +no_list: true +--- + +## Troubleshooting Common Issues + +## 1. "Webhook not available" error when attempting to create a JobSet + +Example error: `failed calling webhook "mjobset.kb.io": failed to call webhook: Post "https://jobset-webhook-service.jobset-system.svc:443/mutate-jobset-x-k8s-io-v1alpha1-jobset?timeout=10s": no endpoints available for service "jobset-webhook-service"` + +**Cause**: Usually this means the JobSet controller manager Deployment pods are unschedulable for some reason. + +**Solution**: Check if jobset-controller-manager deployment pods are running (`kubectl get pods -n jobset-system`). +If they are in a `Pending` state, describe the pod to see why (`kubectl describe pod -n jobset-system`), you +should see a message in the pod Events indicating why they are unschedulable. The solution will depend on why the pods +are unschedulable. For example, if they unschedulable due to insufficient CPU/memory, the solution is to scale up your CPU node pools or turn on autoscaling. + +## 2. JobSet is created but child jobs and/or pods are not being created + +Check the jobset controller logs to see why the jobs are not being created: + +- `kubectl get pods -n jobset-system` +- `kubectl logs -n jobset-system` + +Inspect the logs to look for one of the following issues: + +1. Error message indicating an index does not exist (example: ` "error": "Index with name field:.metadata.controller does not exist"`) + +**Cause**: In older versions of JobSet (older than v0.2.1) if the indexes could not be built for some reason, the JobSet controller would log the error and launch anyway. This resulted in confusing behavior later when trying to create JobSets, where the controller would encounter this "index not found" error and not be able to create any jobs. This bug was fixed +in v0.2.1 so the JobSet controller now fails fast and exits with an error if indexes cannot be built. + +**Solution**: Uninstall JobSet and re-install using the latest release (or at minimum, JobSet v0.2.1). See [installation guide](/docs/setup/install.md) for the commands to do this. + +2. Validation error creating Jobs and/or Services, indicating the Job/Service name is invalid. + +**Cause**: Generated child job names or headless services names (which are derived from the JobSet name and ReplicatedJob names) are not valid. + +**Solution**: Validation has been added to fail the JobSet creation if the generated job/service names will be invalid, but the fix is not included in a release yet. For now, to resolve this simply delete/recreate the JobSet with a name such that: + +* The generated Job names (format: `---.`) will be valid DNS labels as defined in RFC 1035. +* The subdomain name (manually specified in `js.Spec.Network.Subdomain` or defaulted to the JobSet name if unspecified) is both [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123) compliant and [RFC 1035](https://datatracker.ietf.org/doc/html/rfc1035) compliant. + + +## 3. Using JobSet + Kueue, preempted workloads never resume + +Look at the JobSet controller logs and you'll probably see an error like this: + +``` + ERROR resuming jobset {"controller": "jobset", "controllerGroup": "jobset.x-k8s.io", "controllerKind": "JobSet", "JobSet": {"name":"js","namespace":"default"}, "namespace": "default", "name": "js", "reconcileID": "e1ab5e21-586c-496e-96b7-8629cd702f3b", "jobset": {"name":"js","namespace":"default"}, "error": "jobs.batch \"js-slice-job-1\" is forbidden: User \"system:serviceaccount:jobset-system:jobset-controller-manager\" cannot update resource \"jobs/status\" in API group \"batch\" in the namespace \"default\""} + ``` + +**Cause**: This could be due to a known bug in an older version of JobSet, or a known bug in an older version of Kueue. JobSet and Kueue integration requires JobSet v0.2.3+ and Kueue v0.4.1+. + +**Solution**: If you're using JobSet version less than v0.2.3, uninstall and re-install using a versoin >= v0.2.3 (see the JobSet [installation guide](/docs/setup/install.md) for the commands to do this). If you're using a Kueue version less than v0.4.1, uninstall and re-install using a v0.4.1 (see the Kueue [installation guide](https://kueue.sigs.k8s.io/docs/installation/) for the commands to do this). diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md new file mode 100644 index 00000000..f147b2b6 --- /dev/null +++ b/site/content/en/docs/installation/_index.md @@ -0,0 +1,203 @@ +--- +title: "Installation" +linkTitle: "Installation" +weight: 2 +description: > + Installing Jobset to a Kubernetes Cluster +--- + + +- [Before you begin](#before-you-begin) +- [Install a released version](#install-a-released-version) + - [Uninstall](#uninstall) +- [Install the latest development version](#install-the-latest-development-version) + - [Uninstall](#uninstall-1) +- [Build and install from source](#build-and-install-from-source) + - [Uninstall](#uninstall-2) +- [Use Cert Manager instead of internal cert](#optional-use-cert-manager-instead-of-internal-cert) + + + +## Before you begin + +Make sure the following conditions are met: + +- A Kubernetes cluster with version 1.21 or newer is running. Learn how to [install the Kubernetes tools](https://kubernetes.io/docs/tasks/tools/). +- Your cluster has at least 1 node with 2+ CPUs and 512+ MB of memory available for the JobSet controller manager Deployment to run on. **NOTE: On some cloud providers, the default node machine type will not have sufficient resources to run the JobSet controller manager and all the required kube-system pods, so you'll need to use a larger +machine type for your nodes.** +- The `SuspendJob` [feature gate][feature_gate] is enabled. In Kubernetes 1.22 or newer, the feature gate is enabled by default and reached stable in Kubernetes 1.24. +- The kubectl command-line tool has communication with your cluster. + + + + + + + + + + + + + +[feature_gate]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + + +# Install a released version + +To install a released version of Jobset in your cluster, run the following command: + +```shell +VERSION=v0.3.1 +kubectl apply --server-side -f https://github.com/kubernetes-sigs/jobset/releases/download/$VERSION/manifests.yaml +``` + +### Optional: Add metrics scraping for prometheus-operator + +If you are using [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) +to scrape metrics from jobset components, run the following command: + +```shell +VERSION=v0.3.1 +kubectl apply -f https://github.com/kubernetes-sigs/jobset/releases/download/$VERSION/prometheus.yaml +``` + +If you are using [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus), metrics +can be scraped without performing this step. + + +## Uninstall + +To uninstall a released version of JobSet from your cluster, run the following command: + +```shell +VERSION=v0.3.1 +kubectl delete -f https://github.com/kubernetes-sigs/jobset/releases/download/$VERSION/manifests.yaml +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# Install the latest development version + +To install the latest development version of Jobset in your cluster, run the +following command: + +```shell +kubectl apply --server-side -k github.com/kubernetes-sigs/jobset/config/default?ref=main +``` + +The controller runs in the `jobset-system` namespace. + +### Optional: Add metrics scraping for prometheus-operator + +If you are using [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) +to scrape metrics from jobset components, you need to run the following command so it can access +the metrics: + +```shell +kubectl apply --server-side -k github.com/kubernetes-sigs/jobset/config/prometheus?ref=main +``` +If you are using [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus), metrics +can be scraped without performing this step. + +## Uninstall + +To uninstall JobSet, run the following command: + +```shell +kubectl delete -k github.com/kubernetes-sigs/jobset/config/default +``` + +# Build and install from source + +To build Jobset from source and install Jobset in your cluster, run the following +commands: + +```sh +git clone https://github.com/kubernetes-sigs/jobset.git +cd jobset +IMAGE_REGISTRY=/ make image-push deploy +``` + +### Optional: Add metrics scraping for prometheus-operator + +If you are using [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) +to scrape metrics from jobset components, you need to run the following command so it has access +to the metrics: + +```shell +make prometheus +``` + +If you are using [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus), metrics +can be scraped without performing this step. + +## Uninstall + +To uninstall JobSet, run the following command: + +```sh +make undeploy +``` + +# Optional: Use cert manager instead of internal cert +JobSet webhooks use an internal certificate by default. However, if you wish to use cert-manager (which +supports cert rotation), instead of internal cert, you can by performing the following steps. + +First, install cert-manager on your cluster by running the following command: + +```shell +VERSION=v1.11.0 +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$VERSION/cert-manager.yaml +``` + +Next, in the file ``jobset/config/default/kustomization.yaml`` replace ``../components/internalcert`` with +``../components/certmanager`` then uncomment all the lines beginning with ``[CERTMANAGER]``. + +Finally, apply these configurations to your cluster with ``kubectl apply --server-side -k config/default``. \ No newline at end of file diff --git a/site/content/en/docs/reference/_index.md b/site/content/en/docs/reference/_index.md new file mode 100644 index 00000000..3abcdcfa --- /dev/null +++ b/site/content/en/docs/reference/_index.md @@ -0,0 +1,7 @@ +--- +title: "Reference" +linkTitle: "Reference" +weight: 9 +description: > + This section contains the Jobset reference information. +--- diff --git a/site/content/en/docs/reference/metrics.md b/site/content/en/docs/reference/metrics.md new file mode 100644 index 00000000..9213c49c --- /dev/null +++ b/site/content/en/docs/reference/metrics.md @@ -0,0 +1,21 @@ +--- +title: "Prometheus Metrics" +linkTitle: "Prometheus Metrics" +date: 2022-02-14 +description: > + Prometheus metrics exported by Jobset +--- + +## Prometheus Metrics + +JobSet exposes [prometheus](https://prometheus.io) metrics to monitor the health +of the controller. + +## JobSet controller health + +Use the following metrics to monitor the health of the jobset controller: + +| Metric name | Type | Description | Labels | +| ----------- | ---- | ----------- | ------ | +| `controller_runtime_reconcile_errors_total` | Counter | The total number of reconciliation errors encountered by each controller. | `controller`: name of controller (i.e. use value `jobset` to obtain metrics for jobset controller) | +| `controller_runtime_reconcile_time_seconds` | Histogram | The latency of a reconciliation attempt in seconds. | `controller`: name of controller (i.e. use value `jobset` to obtain metrics for jobset controller) | diff --git a/site/content/en/docs/tasks/_index.md b/site/content/en/docs/tasks/_index.md new file mode 100755 index 00000000..47d9c330 --- /dev/null +++ b/site/content/en/docs/tasks/_index.md @@ -0,0 +1,61 @@ +--- + +title: "Tasks" +linkTitle: "Tasks" +weight: 6 +date: 2022-02-14 +description: > + Doing common Jobset tasks +no_list: true +--- + +## PyTorch Examples + +In [pytorch](examples/pytorch), there are two examples using pytorch + +- [mnist](examples/pytorch/mnist.yaml) +- [resnet](examples/pytorch/resnet.yaml) + +Each of these examples demonstrate how you use the JobSet API to run pytorch jobs. + +Machine Learning images can be quite large so it may take some time to pull the images. + +## Simple Examples + +In [simple](examples/simple), we have some examples demonstrating features for the JobSet. + +- [success-policy](examples/simple/driver-worker-success-policy.yaml) +- [max-restarts](examples/simple/max-restarts.yaml) +- [paralleljobs](examples/simple/paralleljobs.yaml) + +[Success Policy](examples/simple/driver-worker-success-policy.yaml) demonstrates an example of utilizing `successPolicy`. +Success Policy allows one to specify when to mark a JobSet as success. +This example showcases an example of using the success policy to mark the JobSet as successful if the worker replicated job completes. + +[Max Restarts](examples/simple/max-restarts.yaml) demonstrates an example of utilizing `failurePolicy`. +Failure Policy allows one to control how many restarts a JobSet can do before declaring the JobSet as failed. + +[Parallel Jobs](examples/simple/paralleljobs.yaml) demonstates how we can submit multiple replicated jobs in a jobset. + +## Tensorflow Examples + +In [tensorflow](examples/tensorflow), we have some examples demonstrating how to use Tensorflow with a JobSet. + +- [mnist](examples/tensorflow/mnist.yaml) + +[mnist](examples/tensorflow/mnist.yaml) runs an example job for a single epoch. +You can view the progress of your jobs via `kubectl logs jobs/tensorflow-tensorflow-0`. + +``` +Train Epoch: 1 [0/60000 (0%)] loss=2.3130, accuracy=12.5000 +Train Epoch: 1 [6400/60000 (11%)] loss=0.4624, accuracy=86.4171 +Train Epoch: 1 [12800/60000 (21%)] loss=0.3381, accuracy=90.0109 +Train Epoch: 1 [19200/60000 (32%)] loss=0.2724, accuracy=91.8916 +Train Epoch: 1 [25600/60000 (43%)] loss=0.2367, accuracy=92.9941 +Train Epoch: 1 [32000/60000 (53%)] loss=0.2111, accuracy=93.7063 +Train Epoch: 1 [38400/60000 (64%)] loss=0.1925, accuracy=94.2882 +Train Epoch: 1 [44800/60000 (75%)] loss=0.1796, accuracy=94.6416 +Train Epoch: 1 [51200/60000 (85%)] loss=0.1677, accuracy=94.9945 +Train Epoch: 1 [57600/60000 (96%)] loss=0.1565, accuracy=95.3229 +Test Loss: 0.0635, Test Accuracy: 97.8400 +``` diff --git a/site/content/en/featured-background.png b/site/content/en/featured-background.png new file mode 100644 index 00000000..07436c39 Binary files /dev/null and b/site/content/en/featured-background.png differ diff --git a/site/content/en/search.md b/site/content/en/search.md new file mode 100644 index 00000000..e3690fd5 --- /dev/null +++ b/site/content/en/search.md @@ -0,0 +1,6 @@ +--- +title: Search Results +layout: search + +--- + diff --git a/site/layouts/404.html b/site/layouts/404.html new file mode 100644 index 00000000..4429eab8 --- /dev/null +++ b/site/layouts/404.html @@ -0,0 +1,19 @@ +{{ define "main"}} +
+

404 Page not found

+
    + {{ range .Site.Menus.main }} +
  • + {{ $url := urls.Parse .URL }} + {{ $baseurl := urls.Parse $.Site.Params.Baseurl }} + + {{ .Name }} + +
  • + {{ end }} +
+
+{{ end }} \ No newline at end of file diff --git a/site/layouts/partials/feedback.html b/site/layouts/partials/feedback.html new file mode 100644 index 00000000..78e65fec --- /dev/null +++ b/site/layouts/partials/feedback.html @@ -0,0 +1,61 @@ + +
+
+

Feedback

+

Was this page helpful?

+ + + + +
+
+ diff --git a/site/layouts/partials/footer.html b/site/layouts/partials/footer.html new file mode 100644 index 00000000..56ec4720 --- /dev/null +++ b/site/layouts/partials/footer.html @@ -0,0 +1,39 @@ +{{ $links := .Site.Params.links }} +
+
+
+
+ {{ with $links }} + {{ with index . "user"}} + {{ template "footer-links-block" . }} + {{ end }} + {{ end }} +
+
+ {{ with $links }} + {{ with index . "developer"}} + {{ template "footer-links-block" . }} + {{ end }} + {{ end }} +
+
+ {{ with .Site.Params.copyright }}© {{ now.Year}} {{ .}} {{ "| Documentation Distributed under CC BY 4.0" }}{{ end }} + {{ with .Site.Params.privacy_policy }}

{{ T "footer_privacy_policy" }}

{{ end }} + {{ if not .Site.Params.ui.footer_about_disable }} + {{ with .Site.GetPage "about" }}

{{ .Title }}

{{ end }} + {{ end }} +
+
+
+
+{{ define "footer-links-block" }} +
    + {{ range . }} +
  • + + + +
  • + {{ end }} +
+{{ end }} diff --git a/site/layouts/partials/head.html b/site/layouts/partials/head.html new file mode 100644 index 00000000..7e474dd0 --- /dev/null +++ b/site/layouts/partials/head.html @@ -0,0 +1,56 @@ + + +{{ hugo.Generator }} +{{ range .AlternativeOutputFormats -}} + +{{ end -}} + +{{ $outputFormat := partial "outputformat.html" . -}} +{{ if and hugo.IsProduction (ne $outputFormat "print") -}} + +{{ else -}} + +{{ end -}} + +{{ partial "seo_schema" . }} + +{{ partialCached "favicons.html" . }} + + {{- if .IsHome -}} + {{ .Site.Title -}} + {{ else -}} + {{ with .Title }}{{ . }} | {{ end -}} + {{ .Site.Title -}} + {{ end -}} + +{{ $desc := .Page.Description | default (.Page.Content | safeHTML | truncate 150) -}} + +{{ template "_internal/opengraph.html" . -}} +{{ template "_internal/schema.html" . -}} +{{ template "_internal/twitter_cards.html" . -}} +{{ partialCached "head-css.html" . "asdf" -}} + +{{ if .Site.Params.offlineSearch -}} + +{{ end -}} + +{{ if .Site.Params.prism_syntax_highlighting -}} + +{{ end -}} + +{{ partial "hooks/head-end.html" . -}} + +{{/* To comply with GDPR, cookie consent scripts places in head-end must execute before Google Analytics is enabled */ -}} +{{ if hugo.IsProduction -}} + {{ if hasPrefix .Site.GoogleAnalytics "G-" -}} + {{ template "_internal/google_analytics.html" . -}} + {{ else -}} + {{ template "_internal/google_analytics_async.html" . -}} + {{ end -}} +{{ end -}} diff --git a/site/layouts/partials/navbar-version-selector.html b/site/layouts/partials/navbar-version-selector.html new file mode 100644 index 00000000..04aebe32 --- /dev/null +++ b/site/layouts/partials/navbar-version-selector.html @@ -0,0 +1,8 @@ + + diff --git a/site/layouts/partials/navbar.html b/site/layouts/partials/navbar.html new file mode 100644 index 00000000..d65dc0b5 --- /dev/null +++ b/site/layouts/partials/navbar.html @@ -0,0 +1,37 @@ +{{ $cover := and (.HasShortcode "blocks/cover") (not .Site.Params.ui.navbar_translucent_over_cover_disable) }} + diff --git a/site/layouts/partials/seo_schema.html b/site/layouts/partials/seo_schema.html new file mode 100644 index 00000000..d7e1e675 --- /dev/null +++ b/site/layouts/partials/seo_schema.html @@ -0,0 +1,26 @@ + \ No newline at end of file diff --git a/site/layouts/shortcodes/blocks/content-item.html b/site/layouts/shortcodes/blocks/content-item.html new file mode 100644 index 00000000..c5dbd4da --- /dev/null +++ b/site/layouts/shortcodes/blocks/content-item.html @@ -0,0 +1,9 @@ +{{ $url_text := .Get "url_text" }} +
+ {{ with .Get "title" }}

{{ . }}

{{ end }} + +

{{ .Get "date" }}

+ +

{{ .Inner }}

+ {{ with .Get "url" }}

{{ with $url_text }}{{ $url_text }}{{ else }}{{ "Read" }}{{ end }}

{{ end }} +
diff --git a/site/layouts/shortcodes/blocks/content-section.html b/site/layouts/shortcodes/blocks/content-section.html new file mode 100644 index 00000000..df2d996f --- /dev/null +++ b/site/layouts/shortcodes/blocks/content-section.html @@ -0,0 +1,12 @@ +{{ $col_id := .Get "color" | default .Ordinal }} +{{ $height := .Get "height" | default "auto" }} + + +
+
+

{{ .Get "title" | markdownify }}

+
+ {{ .Inner }} +
+
+
\ No newline at end of file diff --git a/site/layouts/shortcodes/blocks/link-down.html b/site/layouts/shortcodes/blocks/link-down.html new file mode 100644 index 00000000..365d8d11 --- /dev/null +++ b/site/layouts/shortcodes/blocks/link-down.html @@ -0,0 +1,7 @@ +{{ with .Parent }} +{{ $id := $.Get "id" | default "overview" }} +{{ $color := $.Get "color" | default "blue" }} + +{{ else }} +{{ errorf "The link-down shortcode is supposed to be nested inside a shortcode"}} +{{ end }} \ No newline at end of file diff --git a/site/layouts/shortcodes/blocks/sample-section.html b/site/layouts/shortcodes/blocks/sample-section.html new file mode 100644 index 00000000..0c679891 --- /dev/null +++ b/site/layouts/shortcodes/blocks/sample-section.html @@ -0,0 +1,10 @@ +{{ $api := .Get "api" }} +{{ $last_updated := "" }} +{{ $version := .Get "kfctl" }} +
+ {{ with .Get "title" }}

{{ . }}

{{ end }} + {{ with getJSON $api }} {{ $last_updated = (index (index (index (index . 0) "commit") "committer") "date") | dateFormat "2006/01/02" }} +

{{ "Last update " }} {{ $last_updated }} {{ with $version }}{{ "Jobset " }}{{ . }}{{ end }}

{{ end }} +

{{ .Inner }}

+ {{ with .Get "url" }}

{{ "Go to sample" }}

{{ end }} +
diff --git a/site/layouts/shortcodes/blocks/tab.html b/site/layouts/shortcodes/blocks/tab.html new file mode 100644 index 00000000..4db7465a --- /dev/null +++ b/site/layouts/shortcodes/blocks/tab.html @@ -0,0 +1,19 @@ +{{ if .Parent }} + {{ $name := trim (.Get "name") " " }} + {{ $include := trim (.Get "include") " "}} + {{ $codelang := .Get "codelang" }} + {{ if not (.Parent.Scratch.Get "tabs") }} + {{ .Parent.Scratch.Set "tabs" slice }} + {{ end }} + {{ with .Inner }} + {{ if $codelang }} + {{ $.Parent.Scratch.Add "tabs" (dict "name" $name "content" (highlight . $codelang "") ) }} + {{ else }} + {{ $.Parent.Scratch.Add "tabs" (dict "name" $name "content" . ) }} + {{ end }} + {{ else }} + {{ $.Parent.Scratch.Add "tabs" (dict "name" $name "include" $include "codelang" $codelang) }} + {{ end }} +{{ else }} + {{- errorf "[%s] %q: tab shortcode missing its parent" site.Language.Lang .Page.Path -}} +{{ end}} diff --git a/site/layouts/shortcodes/blocks/tabs.html b/site/layouts/shortcodes/blocks/tabs.html new file mode 100644 index 00000000..aeb9582b --- /dev/null +++ b/site/layouts/shortcodes/blocks/tabs.html @@ -0,0 +1,50 @@ +{{- .Page.Scratch.Add "tabset-counter" 1 -}} +{{- $tab_set_id := .Get "name" | default (printf "tabset-%s-%d" (.Page.RelPermalink) (.Page.Scratch.Get "tabset-counter") ) | anchorize -}} +{{- $tabs := .Scratch.Get "tabs" -}} +{{- if .Inner -}}{{- /* We don't use the inner content, but Hugo will complain if we don't reference it. */ -}}{{- end -}} + +
+{{- range $i, $e := $tabs -}} +{{- $id := printf "%s-%d" $tab_set_id $i -}} +{{- if (eq $i 0) -}} +
+{{ else }} +
+{{ end }} +

+ {{- with .content -}} + {{- . -}} + {{- else -}} + {{- if eq $.Page.BundleType "leaf" -}} + {{- /* find the file somewhere inside the bundle. Note the use of double asterisk */ -}} + {{- with $.Page.Resources.GetMatch (printf "**%s*" .include) -}} + {{- if ne .ResourceType "page" -}} + {{- /* Assume it is a file that needs code highlighting. */ -}} + {{- $codelang := $e.codelang | default ( path.Ext .Name | strings.TrimPrefix ".") -}} + {{- highlight .Content $codelang "" -}} + {{- else -}} + {{- .Content -}} + {{- end -}} + {{- end -}} + {{- else -}} + {{- $path := path.Join $.Page.File.Dir .include -}} + {{- $page := site.GetPage "page" $path -}} + {{- with $page -}} + {{- .Content -}} + {{- else -}} + {{- errorf "[%s] tabs include not found for path %q" site.Language.Lang $path -}} + {{- end -}} + {{- end -}} + {{- end -}} +

+{{- end -}} +
diff --git a/site/layouts/shortcodes/include.html b/site/layouts/shortcodes/include.html new file mode 100644 index 00000000..0c71b0bb --- /dev/null +++ b/site/layouts/shortcodes/include.html @@ -0,0 +1,43 @@ + +{{/* Handle the "file" and "lang" named parameter or two unnamed parameter as the filepath and coding language*/}} +{{/* The first parameter should be a relative path to the "static" directory. Ex: "examples/jobs/sample-job.yaml" */}} +{{ if .IsNamedParams }} + {{ $.Scratch.Set "fparameter" ( .Get "file" ) }} + {{ $.Scratch.Set "lang" ( .Get "lang") }} +{{ else }} + {{ $.Scratch.Set "fparameter" ( .Get 0 ) }} + {{ $.Scratch.Set "lang" ( .Get 1 ) }} +{{ end }} + +{{/* If coding language are specified, then the static file is a code file; If not, then the static are non-code file. */}} +{{ if eq ($.Scratch.Get "lang") "" }} +{{ $.Scratch.Set "iscode" "false"}} +{{ else }} +{{ $.Scratch.Set "iscode" "true" }} +{{ end }} + +{{/* `fparameter is a relative path of the static file from the `static` diretory. For exampl, `/example/simple/exclusive-placement.yaml` */}} +{{ $.Scratch.Set "filepath" "/" }} +{{ $.Scratch.Add "filepath" "static/" }} +{{ $.Scratch.Add "filepath" ($.Scratch.Get "fparameter") }} + +{{/* If the file exists, read it and highlight it if it's code. +Throw a compile error or print an error on the page if the file is not found */}} + +{{ if fileExists ($.Scratch.Get "filepath") }} + {{ $filename := (path.Split ($.Scratch.Get "filepath")).File }} + {{ $link := printf "/%s" ($.Scratch.Get "fparameter") | safeURL }} +
+ + {{ if eq ($.Scratch.Get "iscode") "true" }} + {{- highlight ($.Scratch.Get "filepath" | readFile | htmlUnescape | + safeHTML ) ($.Scratch.Get "lang") "" -}} + {{ else }} + {{- $.Scratch.Get "filepath" | os.ReadFile | .Page.RenderString | safeHTML -}} + {{ end }} +
+{{ else if eq (.Get "draft") "true" }} + +

The file {{ $.Scratch.Get "filepath" }} was not found.

+ +{{ else }}{{- errorf "Shortcode %q: file %q not found at %s" .Name ($.Scratch.Get "filepath") .Position -}}{{ end }} \ No newline at end of file diff --git a/site/layouts/shortcodes/needs-update.html b/site/layouts/shortcodes/needs-update.html new file mode 100644 index 00000000..e0a875b3 --- /dev/null +++ b/site/layouts/shortcodes/needs-update.html @@ -0,0 +1,4 @@ + diff --git a/site/layouts/shortcodes/params.html b/site/layouts/shortcodes/params.html new file mode 100644 index 00000000..499251b9 --- /dev/null +++ b/site/layouts/shortcodes/params.html @@ -0,0 +1 @@ +{{- .Page.Param (.Get 0) -}} \ No newline at end of file diff --git a/site/layouts/sitemap.xml b/site/layouts/sitemap.xml new file mode 100644 index 00000000..60b935d8 --- /dev/null +++ b/site/layouts/sitemap.xml @@ -0,0 +1,21 @@ + + {{ range .Data.Pages }} + + https://jobset.sigs.k8s.io{{ .Permalink }}{{ if not .Lastmod.IsZero }} + {{ safeHTML ( .Lastmod.Format "2006-01-02T15:04:05-07:00" ) }}{{ end }}{{ with .Sitemap.ChangeFreq }} + {{ . }}{{ end }}{{ if ge .Sitemap.Priority 0.0 }} + {{ .Sitemap.Priority }}{{ end }}{{ if .IsTranslated }}{{ range .Translations }} + {{ end }} + {{ end }} + + {{ end }} + \ No newline at end of file diff --git a/site/package-lock.json b/site/package-lock.json new file mode 100644 index 00000000..d1031bc5 --- /dev/null +++ b/site/package-lock.json @@ -0,0 +1,1020 @@ +{ + "name": "site", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "devDependencies": { + "autoprefixer": "^10.4.13", + "docsy": "github:google/docsy", + "postcss": "^8.4.21", + "postcss-cli": "^10.1.0" + } + }, + "node_modules/@fortawesome/fontawesome-free": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.2.1.tgz", + "integrity": "sha512-viouXhegu/TjkvYQoiRZK3aax69dGXxgEjpvZW81wIJdxm5Fnvp3VVIP4VHKqX4SvFw6qpmkILkD4RJWAdrt7A==", + "dev": true, + "hasInstallScript": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.6", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz", + "integrity": "sha512-50/17A98tWUfQ176raKiOGXuYpLyyVMkxxG6oylzL3BPOlA6ADGdK7EYunSa4I064xerltq9TGXs8HmOk5E+vw==", + "dev": true, + "peer": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/autoprefixer": { + "version": "10.4.13", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.13.tgz", + "integrity": "sha512-49vKpMqcZYsJjwotvt4+h/BCjJVnhGwcLpDt5xkcaOG3eLrG/HUYLagrihYsQ+qrIBgIzX1Rw7a6L8I/ZA1Atg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/autoprefixer" + } + ], + "dependencies": { + "browserslist": "^4.21.4", + "caniuse-lite": "^1.0.30001426", + "fraction.js": "^4.2.0", + "normalize-range": "^0.1.2", + "picocolors": "^1.0.0", + "postcss-value-parser": "^4.2.0" + }, + "bin": { + "autoprefixer": "bin/autoprefixer" + }, + "engines": { + "node": "^10 || ^12 || >=14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/bootstrap": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-5.2.3.tgz", + "integrity": "sha512-cEKPM+fwb3cT8NzQZYEu4HilJ3anCrWqh3CHAok1p9jXqMPsPTBhU25fBckEJHJ/p+tTxTFTsFQGM+gaHpi3QQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/twbs" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/bootstrap" + } + ], + "peerDependencies": { + "@popperjs/core": "^2.11.6" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.21.5", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", + "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "caniuse-lite": "^1.0.30001449", + "electron-to-chromium": "^1.4.284", + "node-releases": "^2.0.8", + "update-browserslist-db": "^1.0.10" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001449", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001449.tgz", + "integrity": "sha512-CPB+UL9XMT/Av+pJxCKGhdx+yg1hzplvFJQlJ2n68PyQGMz9L/E2zCyLdOL8uasbouTUgnPl+y0tccI/se+BEw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + } + ] + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/dependency-graph": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", + "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/docsy": { + "version": "0.6.0", + "resolved": "git+ssh://git@github.com/google/docsy.git#dc94be3e13e2090ff6cd1c50edd650ab7ebd968b", + "dev": true, + "dependencies": { + "@fortawesome/fontawesome-free": "6.2.1", + "bootstrap": "5.2.3" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.4.284", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz", + "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", + "dev": true + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fraction.js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", + "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", + "dev": true, + "engines": { + "node": "*" + }, + "funding": { + "type": "patreon", + "url": "https://www.patreon.com/infusion" + } + }, + "node_modules/fs-extra": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.0.tgz", + "integrity": "sha512-0rcTq621PD5jM/e0a3EJoGC/1TC5ZBCERW82LQuwfGnCa1V8w7dpYH1yNu+SLb6E5dkeCBzKEyLGlFrnr+dUyw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stdin": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", + "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.3.tgz", + "integrity": "sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==", + "dev": true, + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/globby/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "node_modules/ignore": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lilconfig": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", + "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/nanoid": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.4.tgz", + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dev": true, + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.9.tgz", + "integrity": "sha512-2xfmOrRkGogbTK9R6Leda0DGiXeY3p2NJpy4+gNCffdUvV6mdEJnaDEic1i3Ec2djAo8jWYoJMR5PB0MSMpxUA==", + "dev": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-range": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "integrity": "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss": { + "version": "8.4.21", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz", + "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + ], + "dependencies": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-cli": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.1.0.tgz", + "integrity": "sha512-Zu7PLORkE9YwNdvOeOVKPmWghprOtjFQU3srMUGbdz3pHJiFh7yZ4geiZFMkjMfB0mtTFR3h8RemR62rPkbOPA==", + "dev": true, + "dependencies": { + "chokidar": "^3.3.0", + "dependency-graph": "^0.11.0", + "fs-extra": "^11.0.0", + "get-stdin": "^9.0.0", + "globby": "^13.0.0", + "picocolors": "^1.0.0", + "postcss-load-config": "^4.0.0", + "postcss-reporter": "^7.0.0", + "pretty-hrtime": "^1.0.3", + "read-cache": "^1.0.0", + "slash": "^5.0.0", + "yargs": "^17.0.0" + }, + "bin": { + "postcss": "index.js" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-load-config": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", + "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", + "dev": true, + "dependencies": { + "lilconfig": "^2.0.5", + "yaml": "^2.1.1" + }, + "engines": { + "node": ">= 14" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": ">=8.0.9", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "postcss": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/postcss-reporter": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", + "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", + "dev": true, + "dependencies": { + "picocolors": "^1.0.0", + "thenby": "^1.3.4" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "node_modules/pretty-hrtime": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", + "integrity": "sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/slash": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.0.0.tgz", + "integrity": "sha512-n6KkmvKS0623igEVj3FF0OZs1gYYJ0o0Hj939yc1fyxl2xt+xYpLnzJB6xBSqOfV9ZFLEWodBBN/heZJahuIJQ==", + "dev": true, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/thenby": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", + "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", + "dev": true + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz", + "integrity": "sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + } + ], + "dependencies": { + "escalade": "^3.1.1", + "picocolors": "^1.0.0" + }, + "bin": { + "browserslist-lint": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", + "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/yargs": { + "version": "17.6.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", + "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + } + } +} diff --git a/site/package.json b/site/package.json new file mode 100644 index 00000000..5000fb60 --- /dev/null +++ b/site/package.json @@ -0,0 +1,8 @@ +{ + "devDependencies": { + "autoprefixer": "^10.4.13", + "docsy": "github:google/docsy", + "postcss": "^8.4.21", + "postcss-cli": "^10.1.0" + } +} diff --git a/site/static/examples/pytorch/cnn-mnist/Dockerfile b/site/static/examples/pytorch/cnn-mnist/Dockerfile new file mode 100644 index 00000000..d9e198ab --- /dev/null +++ b/site/static/examples/pytorch/cnn-mnist/Dockerfile @@ -0,0 +1,4 @@ +FROM pytorch/pytorch:latest +RUN pip install tqdm +COPY mnist.py mnist.py +CMD printenv diff --git a/site/static/examples/pytorch/cnn-mnist/mnist.py b/site/static/examples/pytorch/cnn-mnist/mnist.py new file mode 100644 index 00000000..512cdcc5 --- /dev/null +++ b/site/static/examples/pytorch/cnn-mnist/mnist.py @@ -0,0 +1,155 @@ +import argparse +from tqdm import tqdm +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.optim.lr_scheduler import StepLR +from torch.utils.data.distributed import DistributedSampler + + +class CNN(nn.Module): + ''' + Convolutional neural network. + ''' + def __init__(self): + super(CNN, self).__init__() + self.conv1 = nn.Conv2d(1, 32, 3, 1) + self.conv2 = nn.Conv2d(32, 64, 3, 1) + self.dropout1 = nn.Dropout(0.25) + self.dropout2 = nn.Dropout(0.5) + self.fc1 = nn.Linear(9216, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = self.conv1(x) + x = F.relu(x) + x = self.conv2(x) + x = F.relu(x) + x = F.max_pool2d(x, 2) + x = self.dropout1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + +def train(args, model, device, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in tqdm(enumerate(train_loader)): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + if batch_idx % args.log_interval == 0: + print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + if args.dry_run: + break + + +def test(model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + + test_loss /= len(test_loader.dataset) + + print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( + test_loss, correct, len(test_loader.dataset), + 100. * correct / len(test_loader.dataset))) + + +def main(): + # Training settings + parser = argparse.ArgumentParser(description='PyTorch MNIST Example') + parser.add_argument('--batch-size', type=int, default=64, metavar='N', + help='input batch size for training (default: 64)') + parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', + help='input batch size for testing (default: 1000)') + parser.add_argument('--epochs', type=int, default=14, metavar='N', + help='number of epochs to train (default: 14)') + parser.add_argument('--lr', type=float, default=1.0, metavar='LR', + help='learning rate (default: 1.0)') + parser.add_argument('--gamma', type=float, default=0.7, metavar='M', + help='Learning rate step gamma (default: 0.7)') + parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + parser.add_argument('--no-mps', action='store_true', default=False, + help='disables macOS GPU training') + parser.add_argument('--dry-run', action='store_true', default=False, + help='quickly check a single pass') + parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') + parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='how many batches to wait before logging training status') + parser.add_argument('--save-model', action='store_true', default=False, + help='For Saving the current Model') + args = parser.parse_args() + use_cuda = not args.no_cuda and torch.cuda.is_available() + use_mps = not args.no_mps and torch.backends.mps.is_available() + + torch.manual_seed(args.seed) + + # Initialize distributed training coordation. + torch.distributed.init_process_group(backend="gloo") + + device = "cpu" if not torch.cuda.is_available() else "cuda" + + train_kwargs = {'batch_size': args.batch_size} + test_kwargs = {'batch_size': args.test_batch_size} + if use_cuda: + cuda_kwargs = {'num_workers': 1, + 'pin_memory': True} + train_kwargs.update(cuda_kwargs) + test_kwargs.update(cuda_kwargs) + + # Set up distributed training to use DDP. + model = torch.nn.parallel.DistributedDataParallel(CNN().to(device)) + + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ]) + + train_set = datasets.MNIST('../data', train=True, download=True, + transform=transform) + test_set = datasets.MNIST('../data', train=False, + transform=transform) + + # Set up distributed data sampling so each worker in our DDP set up will + # process a specific subset/partition of the training data. + train_sampler = DistributedSampler(dataset=train_set) + + train_loader = torch.utils.data.DataLoader(train_set,**train_kwargs, sampler=train_sampler) + test_loader = torch.utils.data.DataLoader(test_set, **test_kwargs) + + + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + train(args, model, device, train_loader, optimizer, epoch) + test(model, device, test_loader) + scheduler.step() + + if args.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/site/static/examples/pytorch/cnn-mnist/mnist.yaml b/site/static/examples/pytorch/cnn-mnist/mnist.yaml new file mode 100644 index 00000000..7062b457 --- /dev/null +++ b/site/static/examples/pytorch/cnn-mnist/mnist.yaml @@ -0,0 +1,38 @@ +# Distributed training of a traditional CNN model to do image classification +# using the MNIST dataset and PyTorch. +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: pytorch +spec: + replicatedJobs: + - name: workers + template: + spec: + parallelism: 4 + completions: 4 + backoffLimit: 0 + template: + spec: + containers: + - name: pytorch + image: gcr.io/k8s-staging-jobset/pytorch-mnist:latest + ports: + - containerPort: 3389 + env: + - name: MASTER_ADDR + value: "pytorch-workers-0-0.pytorch" + - name: MASTER_PORT + value: "3389" + - name: RANK + valueFrom: + fieldRef: + fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index'] + # Force python to not buffer output and write directly to stdout, so we can view training logs via `kubectl logs`. + - name: PYTHONUNBUFFERED + value: "0" + command: + - bash + - -xc + - | + torchrun --rdzv_id=123 --nnodes=4 --nproc_per_node=1 --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT --node_rank=$RANK mnist.py --epochs=1 --log-interval=1 \ No newline at end of file diff --git a/site/static/examples/pytorch/resnet-cifar10/resnet.yaml b/site/static/examples/pytorch/resnet-cifar10/resnet.yaml new file mode 100644 index 00000000..c08efe38 --- /dev/null +++ b/site/static/examples/pytorch/resnet-cifar10/resnet.yaml @@ -0,0 +1,35 @@ +# Distributed training of a ResNet18 model to do image classification +# using the CIFAR-10 dataset and PyTorch. +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: pytorch +spec: + replicatedJobs: + - name: workers + template: + spec: + parallelism: 4 + completions: 4 + backoffLimit: 0 + template: + spec: + containers: + - name: pytorch + image: gcr.io/k8s-staging-jobset/pytorch-resnet:latest + ports: + - containerPort: 3389 + env: + - name: MASTER_ADDR + value: "pytorch-workers-0-0.pytorch" + - name: MASTER_PORT + value: "3389" + # Force python to not buffer output and write directly to stdout, so we can view training logs via `kubectl logs`. + - name: PYTHONUNBUFFERED + value: "0" + command: + - bash + - -xc + - | + torchrun --nproc_per_node=1 --master_addr=$MASTER_ADDR --master_port=$MASTER_PORT resnet.py --backend=gloo --num_epochs=2 + \ No newline at end of file diff --git a/site/static/examples/simple/exclusive-placement.yaml b/site/static/examples/simple/exclusive-placement.yaml new file mode 100644 index 00000000..ecbf3c9f --- /dev/null +++ b/site/static/examples/simple/exclusive-placement.yaml @@ -0,0 +1,26 @@ +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: exclusive-placement + annotations: + alpha.jobset.sigs.k8s.io/exclusive-topology: cloud.google.com/gke-nodepool # 1:1 job replica to node pool assignment +spec: + failurePolicy: + maxRestarts: 3 + replicatedJobs: + - name: workers + replicas: 3 # set to number of node pools + template: + spec: + parallelism: 3 + completions: 3 + backoffLimit: 10 + template: + spec: + containers: + - name: sleep + image: busybox + command: + - sleep + args: + - 1000s diff --git a/site/static/examples/simple/max-restarts.yaml b/site/static/examples/simple/max-restarts.yaml new file mode 100644 index 00000000..bffde898 --- /dev/null +++ b/site/static/examples/simple/max-restarts.yaml @@ -0,0 +1,61 @@ +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: max-restarts +spec: + # On failure, restart all jobs up to 3 times. + failurePolicy: + maxRestarts: 3 + replicatedJobs: + - name: leader + replicas: 1 + template: + spec: + # Set backoff limit to 0 so job will immediately fail if any pod fails. + backoffLimit: 0 + completions: 2 + parallelism: 2 + template: + spec: + containers: + - name: leader + image: bash:latest + # Default failure policy is to recreate all jobs if any jobs fails. + # The bash script provides a simple demonstration of it by failing + # the pod with completion index 0, which will trigger job failure + # and the jobset controller wil recreate all jobs. + command: + - bash + - -xc + - | + echo "JOB_COMPLETION_INDEX=$JOB_COMPLETION_INDEX" + if [[ "$JOB_COMPLETION_INDEX" == "0" ]]; then + for i in $(seq 10 -1 1) + do + echo "Sleeping in $i" + sleep 1 + done + exit 1 + fi + for i in $(seq 1 1000) + do + echo "$i" + sleep 1 + done + - name: workers + replicas: 1 + template: + spec: + backoffLimit: 0 + completions: 2 + parallelism: 2 + template: + spec: + containers: + - name: worker + image: bash:latest + command: + - bash + - -xc + - | + sleep 1000 \ No newline at end of file diff --git a/site/static/examples/simple/paralleljobs.yaml b/site/static/examples/simple/paralleljobs.yaml new file mode 100644 index 00000000..665712bb --- /dev/null +++ b/site/static/examples/simple/paralleljobs.yaml @@ -0,0 +1,38 @@ +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: paralleljobs +spec: + replicatedJobs: + - name: workers + template: + spec: + parallelism: 4 + completions: 4 + backoffLimit: 0 + template: + spec: + containers: + - name: sleep + image: busybox + command: + - sleep + args: + - 100s + - name: driver + template: + spec: + parallelism: 1 + completions: 1 + backoffLimit: 0 + template: + spec: + containers: + - name: sleep + image: busybox + command: + - sleep + args: + - 100s + + \ No newline at end of file diff --git a/site/static/examples/simple/success-policy.yaml b/site/static/examples/simple/success-policy.yaml new file mode 100644 index 00000000..a259db22 --- /dev/null +++ b/site/static/examples/simple/success-policy.yaml @@ -0,0 +1,47 @@ +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: success-policy +spec: +# We want to declare our JobSet successful if workers finish. +# If workers finish we should clean up the remaining replicatedJobs. + successPolicy: + operator: All + targetReplicatedJobs: + - workers + replicatedJobs: + - name: leader + replicas: 1 + template: + spec: + # Set backoff limit to 0 so job will immediately fail if any pod fails. + backoffLimit: 0 + completions: 1 + parallelism: 1 + template: + spec: + containers: + - name: leader + image: bash:latest + command: + - bash + - -xc + - | + sleep 10000 + - name: workers + replicas: 1 + template: + spec: + backoffLimit: 0 + completions: 2 + parallelism: 2 + template: + spec: + containers: + - name: worker + image: bash:latest + command: + - bash + - -xc + - | + sleep 10 \ No newline at end of file diff --git a/site/static/examples/tensorflow/mnist.yaml b/site/static/examples/tensorflow/mnist.yaml new file mode 100644 index 00000000..36458f7e --- /dev/null +++ b/site/static/examples/tensorflow/mnist.yaml @@ -0,0 +1,22 @@ +apiVersion: jobset.x-k8s.io/v1alpha2 +kind: JobSet +metadata: + name: tensorflow +spec: + replicatedJobs: + - name: tensorflow + template: + spec: + parallelism: 2 + completions: 2 + backoffLimit: 5 + template: + spec: + containers: + - name: tensorflow + image: docker.io/kubeflowkatib/tf-mnist-with-summaries:latest + command: + - "python" + - "/opt/tf-mnist-with-summaries/mnist.py" + - "--epochs=1" + - "--log-path=/mnist-with-summaries-logs" \ No newline at end of file diff --git a/site/static/favicons/android-chrome-192x192.png b/site/static/favicons/android-chrome-192x192.png new file mode 100644 index 00000000..58c03756 Binary files /dev/null and b/site/static/favicons/android-chrome-192x192.png differ diff --git a/site/static/favicons/android-chrome-512x512.png b/site/static/favicons/android-chrome-512x512.png new file mode 100644 index 00000000..ab4601df Binary files /dev/null and b/site/static/favicons/android-chrome-512x512.png differ diff --git a/site/static/favicons/apple-touch-icon.png b/site/static/favicons/apple-touch-icon.png new file mode 100644 index 00000000..136c618a Binary files /dev/null and b/site/static/favicons/apple-touch-icon.png differ diff --git a/site/static/favicons/favicon-16x16.png b/site/static/favicons/favicon-16x16.png new file mode 100644 index 00000000..0b298566 Binary files /dev/null and b/site/static/favicons/favicon-16x16.png differ diff --git a/site/static/favicons/favicon-32x32.png b/site/static/favicons/favicon-32x32.png new file mode 100644 index 00000000..cdd9e372 Binary files /dev/null and b/site/static/favicons/favicon-32x32.png differ diff --git a/site/static/favicons/favicon.ico b/site/static/favicons/favicon.ico new file mode 100644 index 00000000..612e9c15 Binary files /dev/null and b/site/static/favicons/favicon.ico differ diff --git a/site/static/favicons/site.webmanifest b/site/static/favicons/site.webmanifest new file mode 100644 index 00000000..45dc8a20 --- /dev/null +++ b/site/static/favicons/site.webmanifest @@ -0,0 +1 @@ +{"name":"","short_name":"","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ffffff","background_color":"#ffffff","display":"standalone"} \ No newline at end of file diff --git a/site/static/images/logo.svg b/site/static/images/logo.svg new file mode 100644 index 00000000..f9bd6b6c --- /dev/null +++ b/site/static/images/logo.svg @@ -0,0 +1,84 @@ + + + + + + + + + + image/svg+xml + + + + + + + + + + + +