diff --git a/_config.yml b/_config.yml index 3471320708cc3..87698f6193e58 100644 --- a/_config.yml +++ b/_config.yml @@ -88,3 +88,4 @@ tocs: - reference - samples - search + - imported diff --git a/_data/globals.yml b/_data/globals.yml index 797470b0c8cab..664d7ed66ae3b 100644 --- a/_data/globals.yml +++ b/_data/globals.yml @@ -8,3 +8,4 @@ tocs: - reference - samples - support +- imported diff --git a/_data/imported.yml b/_data/imported.yml new file mode 100644 index 0000000000000..dee00131d6a02 --- /dev/null +++ b/_data/imported.yml @@ -0,0 +1,11 @@ +bigheader: "Imported Docs" +abstract: "Docs imported from other Kubernetes SIGs and repos." +toc: +- docs/imported/index.md + +- title: Community + section: + - docs/imported/community/guide.md + - docs/imported/community/devel.md + - docs/imported/community/mentoring.md + - docs/imported/community/keps.md diff --git a/docs/imported/community/devel.md b/docs/imported/community/devel.md new file mode 100755 index 0000000000000..62fd70223f736 --- /dev/null +++ b/docs/imported/community/devel.md @@ -0,0 +1,81 @@ +--- +title: Kubernetes Developer Guide +--- + +The developer guide is for anyone wanting to either write code which directly accesses the +Kubernetes API, or to contribute directly to the Kubernetes project. +It assumes some familiarity with concepts in the [User Guide](http://kubernetes.io/docs/user-guide/) and the [Cluster Admin +Guide](http://kubernetes.io/docs/admin/). + + +## The process of developing and contributing code to the Kubernetes project + +* **Contributor Guide** + ([Please start here](https://github.com/kubernetes/community/tree/master/contributors/guide/README.md)) to learn about how to contribute to Kubernetes + +* **GitHub Issues** ([issues.md](https://github.com/kubernetes/community/tree/master/contributors/devel/issues.md)): How incoming issues are triaged. + +* **Pull Request Process** ([pull-requests.md](https://github.com/kubernetes/community/tree/master/contributors/devel/pull-requests.md)): When and why pull requests are closed. + +* **Getting Recent Builds** ([getting-builds.md](https://github.com/kubernetes/community/tree/master/contributors/devel/getting-builds.md)): How to get recent builds including the latest builds that pass CI. + +* **Automated Tools** ([automation.md](https://github.com/kubernetes/community/tree/master/contributors/devel/automation.md)): Descriptions of the automation that is running on our github repository. + + +## Setting up your dev environment, coding, and debugging + +* **Development Guide** ([development.md](https://github.com/kubernetes/community/tree/master/contributors/devel/development.md)): Setting up your development environment. + +* **Testing** ([testing.md](https://github.com/kubernetes/community/tree/master/contributors/devel/testing.md)): How to run unit, integration, and end-to-end tests in your development sandbox. + +* **Hunting flaky tests** ([flaky-tests.md](https://github.com/kubernetes/community/tree/master/contributors/devel/flaky-tests.md)): We have a goal of 99.9% flake free tests. + Here's how to run your tests many times. + +* **Logging Conventions** ([logging.md](https://github.com/kubernetes/community/tree/master/contributors/devel/logging.md)): Glog levels. + +* **Profiling Kubernetes** ([profiling.md](https://github.com/kubernetes/community/tree/master/contributors/devel/profiling.md)): How to plug in go pprof profiler to Kubernetes. + +* **Instrumenting Kubernetes with a new metric** + ([instrumentation.md](https://github.com/kubernetes/community/tree/master/contributors/devel/instrumentation.md)): How to add a new metrics to the + Kubernetes code base. + +* **Coding Conventions** ([coding-conventions.md](https://github.com/kubernetes/community/tree/master/contributors/devel/coding-conventions.md)): + Coding style advice for contributors. + +* **Document Conventions** ([how-to-doc.md](https://github.com/kubernetes/community/tree/master/contributors/devel/how-to-doc.md)) + Document style advice for contributors. + +* **Running a cluster locally** ([running-locally.md](https://github.com/kubernetes/community/tree/master/contributors/devel/running-locally.md)): + A fast and lightweight local cluster deployment for development. + +## Developing against the Kubernetes API + +* The [REST API documentation](http://kubernetes.io/docs/reference/) explains the REST + API exposed by apiserver. + +* **Annotations** ([Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)): are for attaching arbitrary non-identifying metadata to objects. + Programs that automate Kubernetes objects may use annotations to store small amounts of their state. + +* **API Conventions** ([api-conventions.md](https://github.com/kubernetes/community/tree/master/contributors/devel/api-conventions.md)): + Defining the verbs and resources used in the Kubernetes API. + +* **API Client Libraries** ([client-libraries.md](https://github.com/kubernetes/community/tree/master/contributors/devel/client-libraries.md)): + A list of existing client libraries, both supported and user-contributed. + + +## Writing plugins + +* **Authentication** ([Authentication](http://kubernetes.io/docs/admin/authentication/)): + The current and planned states of authentication tokens. + +* **Authorization Plugins** ([Authorization](http://kubernetes.io/docs/admin/authorization/)): + Authorization applies to all HTTP requests on the main apiserver port. + This doc explains the available authorization implementations. + +* **Admission Control Plugins** ([admission_control](https://github.com/kubernetes/community/tree/master/contributors/design-proposals/api-machinery/admission_control.md)) + + +## Building releases + +See the [kubernetes/release](https://github.com/kubernetes/release) repository for details on creating releases and related tools and helper scripts. +ed tools and helper scripts. diff --git a/docs/imported/community/guide.md b/docs/imported/community/guide.md new file mode 100755 index 0000000000000..37e5a5e684c2a --- /dev/null +++ b/docs/imported/community/guide.md @@ -0,0 +1,251 @@ +--- +title: Kubernetes Contributor Guide +owner: sig-contributor-experience +--- +**OWNER:** + +sig-contributor-experience + +## Disclaimer +Hello! This is the starting point for our brand new contributor guide, currently underway as per [issue#6102](https://github.com/kubernetes/website/issues/6102) and in need of help. Please be patient, or fix a section below that needs improvement, and submit a pull request! + +Many of the links below should lead to relevant documents scattered across the community repository. Often, the linked instructions need to be updated or cleaned up. + +* If you do so, please move the relevant file from its previous location to the community/contributors/guide folder, and delete its previous location. +* Our goal is that all contributor guide specific files live in this folder. + +Please find _Improvements needed_ sections below and help us out. + +For example: + +_Improvements needed_ +* kubernetes/community/CONTRIBUTING.md -> Needs a rewrite + +* kubernetes/community/README.md -> Needs a rewrite + +* Individual SIG contributing documents -> add a link to this guide + +# Welcome + +Welcome to Kubernetes! This document is the single source of truth for how to contribute to the code base. Please leave comments / suggestions if you find something is missing or incorrect. + +- [Before you get started](#before-you-get-started) + - [Sign the CLA](#sign-the-cla) + - [Setting up your development + environment](#setting-up-your-development-environment) + - [Community Expectations](#community-expectations) + - [Code of Conduct](#code-of-conduct) + - [Thanks](#thanks) +- [Your First Contribution](#your-first-contribution) + - [Find something to work on](#find-something-to-work-on) + - [Find a good first topic](#find-a-good-first-topic) + - [Learn about SIGs](#learn-about-sigs) + - [File an Issue](#file-an-issue) +- [Contributing](#contributing) + - [Communication](#communication) + - [GitHub workflow](#github-workflow) + - [Open a Pull Request](#open-a-pull-request) + - [Code Review](#code-review) + - [Testing](#testing) + - [Security](#security) + - [Documentation](#documentation) + - [Issues Management or Triage](#issues-management-or-triage) +- [Community](#community) + - [Events](#events) + - [Meetups](#meetups) + - [KubeCon](#kubecon) + - [Mentorship](#mentorship) + +# Before you get started + +## Sign the CLA + +Before you can contribute, you will need to sign the [Contributor License Agreement](https://github.com/kubernetes/community/tree/master/CLA.md). + +## Setting up your development environment + +If you haven’t set up your environment, please find resources [here](https://github.com/kubernetes/community/tree/master/contributors/devel). These resources are not well organized currently; please have patience as we are working on it. + +_Improvements needed_ +* A new developer guide will be created and linked to in this section. + + * RyanJ from Red Hat is working on this + +## Community Expectations + +Kubernetes is a community project. Consequently, it is wholly dependent on its community to provide a productive, friendly and collaborative environment. + +The first and foremost goal of the Kubernetes community to develop orchestration technology that radically simplifies the process of creating reliable distributed systems. However a second, equally important goal is the creation of a community that fosters easy, agile development of such orchestration systems. + +We therefore describe the expectations for members of the Kubernetes community. This document is intended to be a living one that evolves as the community evolves via the same pull request and code review process that shapes the rest of the project. It currently covers the expectations of conduct that govern all members of the community as well as the expectations around code review that govern all active contributors to Kubernetes. + +### Code of Conduct + +Please make sure to read and observe our [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) + +### Thanks + +Many thanks in advance to everyone who contributes their time and effort to making Kubernetes both a successful system as well as a successful community. The strength of our software shines in the strengths of each individual community member. Thanks! + +# Your First Contribution + +Have you ever wanted to contribute to the coolest cloud technology? We will help you understand the organization of the Kubernetes project and direct you to the best places to get started. You'll be able to pick up issues, write code to fix them, and get your work reviewed and merged. + +Please be aware that due to the large number of issues our triage team deals with, we cannot offer technical support in GitHub issues. If you have questions about the development process, feel free to jump into our [Slack Channel](http://slack.k8s.io/) or join our [mailing list](https://groups.google.com/forum/#!forum/kubernetes-dev). You can also ask questions on [ServerFault](https://serverfault.com/questions/tagged/kubernetes) or [Stack Overflow](https://stackoverflow.com/questions/tagged/kubernetes). The Kubernetes team scans Stack Overflow on a regular basis, and will try to ensure your questions don't go unanswered. + +## Find something to work on + +Help is always welcome! For example, documentation (like the text you are reading now) can always use improvement. There's always code that can be clarified and variables or functions that can be renamed or commented. There's always a need for more test coverage. +You get the idea - if you ever see something you think should be fixed, you should own it. Here is how you get started. + +### Find a good first topic + +There are multiple repositories within the Kubernetes community and a full list of repositories can be found [here](https://github.com/kubernetes/). +Each repository in the Kubernetes organization has beginner-friendly issues that provide a good first issue. For example, [kubernetes/kubernetes](https://git.k8s.io/kubernetes) has [help wanted issues](https://go.k8s.io/help-wanted) that should not need deep knowledge of the system. +Another good strategy is to find a documentation improvement, such as a missing/broken link, which will give you exposure to the code submission/review process without the added complication of technical depth. Please see [Contributing](#contributing) below for the workflow. + +### Learn about SIGs + +#### Sig structure + +You may have noticed that some repositories in the Kubernetes Organization are owned by Special Interest Groups, or SIGs. We organize the Kubernetes community into SIGs in order to improve our workflow and more easily manage what is a very large community project. The developers within each SIG have autonomy and ownership over that SIG's part of Kubernetes. +SIGs also have their own CONTRIBUTING.md files, which may contain extra information or guidelines in addition to these general ones. These are located in the SIG specific community documentation directories, for example: sig-docs' is in the kubernetes/community repo's [/sig-docs/CONTRIBUTING.md](https://github.com/kubernetes/community/tree/master/sig-docs/CONTRIBUTING.md) file and similarly for other SIGs. + +Like everything else in Kubernetes, a SIG is an open, community, effort. Anybody is welcome to jump into a SIG and begin fixing issues, critiquing design proposals and reviewing code. SIGs have regular [video meetings](https://kubernetes.io/community/) which everyone is welcome to. Each SIG has a kubernetes slack channel that you can join as well. + +There is an entire SIG ([sig-contributor-experience](https://github.com/kubernetes/community/tree/master/sig-contributor-experience/README.md)) devoted to improving your experience as a contributor. +Contributing to Kubernetes should be easy. If you find a rough edge, let us know! Better yet, help us fix it by joining the SIG; just +show up to one of the [bi-weekly meetings](https://docs.google.com/document/d/1qf-02B7EOrItQgwXFxgqZ5qjW0mtfu5qkYIF1Hl4ZLI/edit). + +#### Find a SIG that is related to your contribution + +Finding the appropriate SIG for your contribution will help you ask questions in the correct place and give your contribution higher visibility and a faster community response. + +For Pull Requests, the automatically assigned reviewer will add a SIG label if you haven't done so. See [Open A Pull Request](#open-a-pull-request) below. + +For Issues we are still working on a more automated workflow. Since SIGs do not directly map onto Kubernetes subrepositories, it may be difficult to find which SIG your contribution belongs in. Here is the [list of SIGs](https://github.com/kubernetes/community/tree/master/sig-list.md). Determine which is most likely related to your contribution. + +*Example:* if you are filing a cni issue, you should choose SIG-networking. + +Follow the link in the SIG name column to reach each SIGs README. Most SIGs will have a set of GitHub Teams with tags that can be mentioned in a comment on issues and pull requests for higher visibility. If you are not sure about the correct SIG for an issue, you can try SIG-contributor-experience [here](https://github.com/kubernetes/community/tree/master/sig-contributor-experience#github-teams), or [ask in Slack](http://slack.k8s.io/). + +_Improvements needed_ + +* Open pull requests with all applicable SIGs to not have duplicate information in their CONTRIBUTING.md and instead link here. Keep it light, keep it clean, have only one source of truth. + +### File an Issue + +Not ready to contribute code, but see something that needs work? While the community encourages everyone to contribute code, it is also appreciated when someone reports an issue (aka problem). Issues should be filed under the appropriate Kubernetes subrepository. + +*Example:* a documentation issue should be opened to [kubernetes/website](https://github.com/kubernetes/website/issues). + +Make sure to adhere to the prompted submission guidelines while opening an issue. + +# Contributing + +(From:[here](https://github.com/kubernetes/community/tree/master/contributors/devel/collab.md)) + +Kubernetes is open source, but many of the people working on it do so as their day job. In order to avoid forcing people to be "at work" effectively 24/7, we want to establish some semi-formal protocols around development. Hopefully these rules make things go more smoothly. If you find that this is not the case, please complain loudly. + +As a potential contributor, your changes and ideas are welcome at any hour of the day or night, weekdays, weekends, and holidays. Please do not ever hesitate to ask a question or send a pull request. + +Our community guiding principles on how to create great code as a big group are found [here](https://github.com/kubernetes/community/tree/master/contributors/devel/collab.md). Beginner focused information can be found below in [Open a Pull Request](#open-a-pull-request) and [Code Review](#code-review). + +### Communication + +It is best to contact your [SIG](#learn-about-sigs) for issues related to the SIG's topic. Your SIG will be able to help you much more quickly than a general question would. + +For questions and troubleshooting, please feel free to use any of the methods of communication listed [here](https://github.com/kubernetes/community/tree/master/communication.md). The [kubernetes website](https://kubernetes.io/docs/tasks/debug-application-cluster/troubleshooting/) also lists this information. + +## GitHub workflow + +To check out code to work on, please refer to [this guide](https://github.com/kubernetes/community/tree/master/contributors/guide/./github-workflow.md). + +## Open a Pull Request + +Pull requests are often called simply "PR". Kubernetes generally follows the standard [github pull request](https://help.github.com/articles/about-pull-requests/) process, but there is a layer of additional kubernetes specific (and sometimes SIG specific) differences. + +The first difference you'll see is that a bot will begin applying structured labels to your PR. + +The bot may also make some helpful suggestions for commands to run in your PR to facilitate review. These `/command` options can be entered in comments to trigger auto-labeling and notifications. The command reference is [here](https://go.k8s.io/bot-commands). + +Common new contributor PR issues are: + +* not having correctly signed the CLA ahead of your first PR (see [Sign the CLA](#sign-the-cla) section) +* finding the right SIG or reviewer(s) for the PR (see [Code Review](#code-review) section) and following any SIG specific contributing guidelines +* dealing with test cases which fail on your PR, unrelated to the changes you introduce (see [Test Flakes](http://velodrome.k8s.io/dashboard/db/bigquery-metrics?orgId=1)) + +The pull request workflow is described in detail [here](https://github.com/kubernetes/community/tree/master/contributors/devel/pull-requests.md#the-testing-and-merge-workflow). + +## Code Review + +For a brief description of the importance of code review, please read [On Code Review](https://github.com/kubernetes/community/tree/master/contributors/devel/community-expectations.md#code-review). There are two aspects of code review: giving and receiving. + +To make it easier for your PR to receive reviews, consider the reviewers will need you to: + +* write [good commit messages](https://chris.beams.io/posts/git-commit/) +* break large changes into a logical series of smaller patches which individually make easily understandable changes, and in aggregate solve a broader issue +* label PRs with appropriate SIGs and reviewers: to do this read the messages the bot sends you to guide you through the PR process + +Reviewers, the people giving review, are highly encouraged to revisit the [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) and must go above and beyond to promote a collaborative, respectful Kubernetes community. When reviewing PRs from others [The Gentle Art of Patch Review](http://sage.thesharps.us/2014/09/01/the-gentle-art-of-patch-review/) suggests an iterative series of focuses which is designed to lead new contributors to positive collaboration without inundating them initially with nuances: + +* Is the idea behind the contribution sound? +* Is the contribution architected correctly? +* Is the contribution polished? + +## Testing + +Testing is the responsibility of all contributors and is in part owned by all sigs, but is also coordinated by [sig-testing](https://github.com/kubernetes/community/tree/master/sig-testing). + +The main testing overview document is [here](https://github.com/kubernetes/community/tree/master/contributors/devel/testing.md). + +There are three types of test in kubernetes. The location of the test code varies with type, as does the specifics of the environment needed to successfully run the test: + +* Unit: These confirm that a particular function behaves as intended. Golang includes native ability for unit testing via the [testing](https://golang.org/pkg/testing/) package. Unit test source code can be found adjacent to the corresponding source code within a given package. For example: functions defined in [kubernetes/cmd/kubeadm/app/util/version.go](https://git.k8s.io/kubernetes/cmd/kubeadm/app/util/version.go) will have unit tests in [kubernetes/cmd/kubeadm/app/util/version_test.go](https://git.k8s.io/kubernetes/cmd/kubeadm/app/util/version_test.go). These are easily run locally be any developer on any OS. +* Integration: These tests cover interactions of package components or interactions between kubernetes components and some other non-kubernetes system resource (eg: etcd). An example would be testing whether a piece of code can correctly store data to or retrieve data from etcd. Integration tests are stored in [kubernetes/test/integration/](https://git.k8s.io/kubernetes/test/integration). Running these can require the developer set up additional functionality on their development system. +* End-to-end ("e2e"): These are broad tests of overall kubernetes system behavior and coherence. These are more complicated as they require a functional kubernetes cluster built from the sources to be tested. A separate document [here](https://github.com/kubernetes/community/tree/master/contributors/devel/e2e-tests.md) details e2e testing and test cases themselves can be found in [kubernetes/test/e2e/](https://git.k8s.io/kubernetes/test/e2e). + +Continuous integration will run these tests either as pre-submits on PRs, post-submits against master/release branches, or both. The results appear on [testgrid](https://testgrid.k8s.io). + +sig-testing is responsible for that official infrastructure and CI. The associated automation is tracked in the [test-infra repo](https://git.k8s.io/test-infra). If you're looking to run e2e tests on your own infrastructure, [kubetest](https://git.k8s.io/test-infra/kubetest) is the mechanism. + +## Security + +_Improvements needed_ +* Please help write this section. + +## Documentation + +_Improvements needed_ +* Please help write this section. + +## Issues Management or Triage + +Have you ever noticed the total number of [open issues](https://issues.k8s.io)? This number at any given time is typically high. Helping to manage or triage these open issues can be a great contribution to the Kubernetes project. This is also a great opportunity to learn about the various areas of the project. Refer to the [Kubernetes Issue Triage Guidelines](https://github.com/kubernetes/community/tree/master/contributors/devel/issues.md) for more information. + +# Community + +If you haven't noticed by now, we have a large, lively, and friendly open-source community. We depend on new people becoming members and regular code contributors, so we would like you to come join us. To find out more about our community structure, different levels of membership and code contributors, please [explore here](https://github.com/kubernetes/community/tree/master/community-membership.md). + +_Improvements needed_ + +* The top level k/community/README.md should be a good starting point for what the community is and does. (see above instructions on rewriting this file) + +## Events +Kubernetes is the main focus of CloudNativeCon/KubeCon, held twice per year in EMEA and in North America. Information about these and other community events is available on the CNCF [events](https://www.cncf.io/events/) pages. + +### Meetups + +_Improvements needed_ +* include link to meetups +* information on CNCF support for founding a Meetup + +### KubeCon + +_Improvements needed_ +* write friendly blurb about KubeCon, and include links + +## Mentorship + +_Improvements needed_ +* Link and mini description for Kubernetes Pilots should go here. diff --git a/docs/imported/community/keps.md b/docs/imported/community/keps.md new file mode 100755 index 0000000000000..ee2f8b66df6c2 --- /dev/null +++ b/docs/imported/community/keps.md @@ -0,0 +1,460 @@ +--- +title: Kubernetes Enhancement Proposal Process +--- + +## Metadata +``` +--- +kep-number: 1 +title: Kubernetes Enhancement Proposal Process +authors: + - name: Caleb Miles + github: calebamiles + slack: calebamiles + - name: Joe Beda + github: jbeda + email: joe@heptio.com + slack: jbeda +owning-sig: sig-architecture +participating-sigs: + - `kubernetes-wide` +reviewers: + - name: TBD +approvers: + - name: TBD +editor: + name: TBD +creation-date: 2017-08-22 +status: draft +``` + +## Table of Contents + +* [Kubernetes Enhancement Proposal Process](#kubernetes-enhancement-proposal-process) + * [Metadata](#metadata) + * [Table of Contents](#table-of-contents) + * [Summary](#summary) + * [Motivation](#motivation) + * [Reference-level explanation](#reference-level-explanation) + * [What type of work should be tracked by a KEP](#what-type-of-work-should-be-tracked-by-a-kep) + * [KEP Template](#kep-template) + * [KEP Metadata](#kep-metadata) + * [KEP Workflow](#kep-workflow) + * [Git and GitHub Implementation](#git-and-github-implementation) + * [KEP Editor Role](#kep-editor-role) + * [Important Metrics](#important-metrics) + * [Prior Art](#prior-art) + * [Graduation Criteria](#graduation-criteria) + * [Drawbacks](#drawbacks) + * [Alternatives](#alternatives) + * [Unresolved Questions](#unresolved-questions) + * [Mentors](#mentors) + +## Summary + +A standardized development process for Kubernetes is proposed in order to + +- provide a common structure for proposing changes to Kubernetes +- ensure that the motivation for a change is clear +- allow for the enumeration stability milestones and stability graduation + criteria +- persist project information in a Version Control System (VCS) for future + Kubernauts +- support the creation of _high value user facing_ information such as: + - an overall project development roadmap + - motivation for impactful user facing changes +- support development across multiple repositories beyond `kubernetes/kubernetes` +- reserve GitHub issues for tracking work in flight rather than creating "umbrella" + issues +- ensure community participants are successfully able to drive changes to + completion across one or more releases while stakeholders are adequately + represented throughout the process + +This process is supported by a unit of work called a Kubernetes Enhancement +Proposal or KEP. A KEP attempts to combine aspects of a + +- feature, and effort tracking document +- a product requirements document +- design document + +into one file which is created incrementally in collaboration with one or more +Special Interest Groups (SIGs). + +## Motivation + +For cross project SIGs such as SIG PM and SIG Release an abstraction beyond a +single GitHub Issue or Pull request seems to be required in order to understand +and communicate upcoming changes to Kubernetes. In a blog post describing the +[road to Go 2][], Russ Cox explains + +> that it is difficult but essential to describe the significance of a problem +> in a way that someone working in a different environment can understand + +as a project it is vital to be able to track the chain of custody for a proposed +enhancement from conception through implementation. This proposal does not +attempt to mandate how SIGs track their work internally, however, it is +suggested that SIGs which do not adhere to a process which allows for their hard +work to be explained to others in the wider Kubernetes community will see their +work wallow in the shadows of obscurity. At the very least [survey data][] +suggest that high quality documentation is crucial to project adoption. +Documentation can take many forms and it is imperative to ensure that it is easy +to produce high quality user or developer focused documentation for a complex +project like Kubernetes. + +Without a standardized mechanism for describing important enhancements our +talented technical writers and product managers struggle to weave a coherent +narrative explaining why a particular release is important. Additionally for +critical infrastructure such as Kubernetes adopters need a forward looking road +map in order to plan their adoption strategy. + +The purpose of the KEP process is to reduce the amount of "tribal knowledge" in +our community. By moving decisions from a smattering of mailing lists, video +calls and hallway conversations into a well tracked artifact this process aims +to enhance communication and discoverability. + +A KEP is broken into sections which can be merged into source control +incrementally in order to support an iterative development process. An important +goal of the KEP process is ensuring that the process for submitting the content +contained in [design proposals][] is both clear and efficient. The KEP process +is intended to create high quality uniform design and implementation documents +for SIGs to deliberate. + +[tell a story]: https://blog.rust-lang.org/2017/08/31/Rust-1.20.html +[road to Go 2]: https://blog.golang.org/toward-go2 +[survey data]: http://opensourcesurvey.org/2017/ +[design proposals]: /contributors/design-proposals + + +## Reference-level explanation + +### What type of work should be tracked by a KEP + +The definition of what constitutes an "enhancement" is a foundational concern +for the Kubernetes project. Roughly any Kubernetes user or operator facing +enhancement should follow the KEP process: if an enhancement would be described +in either written or verbal communication to anyone besides the KEP author or +developer then consider creating a KEP. One concrete example is an enhancement +which should be communicated to SIG Release or SIG PM. + +Similarly, any technical effort (refactoring, major architectural change) that +will impact a large section of the development community should also be +communicated widely. The KEP process is suited for this even if it will have +zero impact on the typical user or operator. + +As the local bodies of governance, SIGs should have broad latitude in describing +what constitutes an enhancement which should be tracked through the KEP process. +SIGs may find that helpful to enumerate what _does not_ require a KEP rather +than what does. SIGs also have the freedom to customize the KEP template +according to their SIG specific concerns. For example the KEP template used to +track API changes will likely have different subsections than the template for +proposing governance changes. However, as changes start impacting other SIGs or +the larger developer community outside of a SIG, the KEP process should be used +to coordinate and communicate. + +### KEP Template + +The template for a KEP is precisely defined in the [template proposal][] + +[template proposal]: https://github.com/kubernetes/community/pull/1124 + +### KEP Metadata + +There is a place in each KEP for a YAML document that has standard metadata. +This will be used to support tooling around filtering and display. It is also +critical to clearly communicate the status of a KEP. + +Metadata items: +* **kep-number** Required + * Each proposal has a number. This is to make all references to proposals as + clear as possible. This is especially important as we create a network + cross references between proposals. + * Before having the `Approved` status, the number for the KEP will be in the + form of `draft-YYYYMMDD`. The `YYYYMMDD` is replaced with the current date + when first creating the KEP. The goal is to enable fast parallel merges of + pre-acceptance KEPs. + * On acceptance a sequential dense number will be assigned. This will be done + by the editor and will be done in such a way as to minimize the chances of + conflicts. The final number for a KEP will have no prefix. +* **title** Required + * The title of the KEP in plain language. The title will also be used in the + KEP filename. See the template for instructions and details. +* **status** Required + * The current state of the KEP. + * Must be one of `Draft`, `Deferred`, `Approved`, `Rejected`, `Withdrawn`, + `Final`, `Replaced`. +* **authors** Required + * A list of authors for the KEP. We require a name (which can be a psuedonym) + along with a github ID. Other ways to contact the author is strongly + encouraged. This is a list of maps. Subkeys of each item: `name`, + `github`, `email` (optional), `slack` (optional). +* **owning-sig** Required + * The SIG that is most closely associated with this KEP. If there is code or + other artifacts that will result from this KEP, then it is expected that + this SIG will take responsiblity for the bulk of those artifacts. + * Sigs are listed as `sig-abc-def` where the name matches up with the + directory in the `kubernetes/community` repo. +* **participating-sigs** Optional + * A list of SIGs that are involved or impacted by this KEP. + * A special value of `kubernetes-wide` will indicate that this KEP has impact + across the entire project. +* **reviewers** Required + * Reviewer(s) chosen after triage according to proposal process + * If not yet chosen replace with `TBD` + * Same name/contact scheme as `authors` +* **approvers** Required + * Approver(s) chosen after triage according to proposal process + * If not yet chosen replace with `TBD` + * Same name/contact scheme as `authors` +* **editor** Required + * Someone to keep things moving forward. + * If not yet chosen replace with `TBD` + * Same name/contact scheme as `authors` +* **creation-date** Required + * The date that the KEP was first submitted in a PR. + * In the form `yyyy-mm-dd` + * While this info will also be in source control, it is helpful to have the set of KEP files stand on their own. +* **last-updated** Optional + * The date that the KEP was last changed significantly. + * In the form `yyyy-mm-dd` +* **see-also** Optional + * A list of other KEPs that are relevant to this KEP. + * In the form `KEP-123` +* **replaces** Optional + * A list of KEPs that this KEP replaces. Those KEPs should list this KEP in + their `superseded-by`. + * In the form `KEP-123` +* **superseded-by** + * A list of KEPs that supersede this KEP. Use of this should be paired with + this KEP moving into the `Replaced` status. + * In the form `KEP-123` + + +### KEP Workflow + +TODO(jbeda) Rationalize this with status entires in the Metadata above. + +A KEP is proposed to have the following states + +- **opened**: a new KEP has been filed but not triaged by the responsible SIG or + working group +- **accepted**: the motivation has been accepted by the SIG or working group as in + road map +- **scoped**: the design has been approved by the SIG or working group +- **started**: the implementation of the KEP has begun +- **implemented**: the implementation of the KEP is complete +- **deferred**: the KEP has been postponed by the SIG or working group despite + agreement on the motivation +- **superseded**: the KEP has been superseded by another KEP +- **retired**: the implementation of the KEP has been removed +- **rejected**: the KEP has been rejected by the SIG or working group +- **orphaned**: the author or developer of the KEP is no longer willing or able + to complete implementation + +with possible paths through the state space + +- opened -> deferred (a) +- opened -> rejected (b) +- opened -> orphaned (c) +- opened -> accepted -> orphaned (d) +- opened -> accepted -> scoped -> superseded (e) +- opened -> accepted -> scoped -> orphaned (f) +- opened -> accepted -> scoped -> started -> retired (g) +- opened -> accepted -> scoped -> started -> orphaned (h) +- opened -> accepted -> scoped -> started -> superseded (i) +- opened -> accepted -> scoped -> started -> implemented (j) +- opened -> accepted -> scoped -> started -> implemented -> retired (k) + +the happy path is denoted by (j) where an KEP is opened; accepted by a SIG as in +their roadmap; fleshed out with a design; started; and finally implemented. As +Kubernetes continues to mature, hopefully metrics on the utilization of features +will drive decisions on what features to maintain and which to deprecate and so +it is possible that a KEP would be retired if its functionality no longer provides +sufficient value to the community. + +### Git and GitHub Implementation + +Practically an KEP would be implemented as a pull request to a central repository +with the following example structure + +``` +├── 0000-kep-template.md +├── CODEOWNERS +├── index.md +├── sig-architecture +│   ├── deferred +│   ├── orphaned +│   └── retired +├── sig-network +│   ├── deferred +│   ├── kube-dns +│   ├── orphaned +│   └── retired +├── sig-node +│   ├── deferred +│   ├── kubelet +│   ├── orphaned +│   └── retired +├── sig-release +│   ├── deferred +│   ├── orphaned +│   └── retired +├── sig-storage +│   ├── deferred +│   ├── orphaned +│   └── retired +├── unsorted-to-be-used-by-newcomers-only +└── wg-resource-management + ├── deferred + ├── orphaned + └── retired +``` + +where each SIG or working group is given a top level directory with subprojects +maintained by the SIG listed in sub directories. For newcomers to the community +an `unsorted-to-be-used-by-newcomers-only` directory may be used before an KEP +can be properly routed to a SIG although hopefully if discussion for a potential +KEP begins on the mailing lists proper routing information will be provided to +the KEP author. Additionally a top level index of KEPs may be helpful for people +looking for a complete list of KEPs. There should be basic CI to ensure that an +`index.md` remains up to date. + +Ideally no work would begin within the repositories of the Kubernetes organization +before a KEP has been approved by the responsible SIG or working group. While the +details of how SIGs organize their work is beyond the scope of this proposal one +possibility would be for each charter SIG to create a top level repository within +the Kubernetes org where implementation issues managed by that SIG would be filed. + +### KEP Editor Role + +Taking a cue from the [Python PEP process][], I believe that a group of KEP editors +will be required to make this process successful; the job of an KEP editor is +likely very similar to the [PEP editor responsibilities][] and will hopefully +provide another opportunity for people who do not write code daily to contribute +to Kubernetes. + +In keeping with the PEP editors which + +> Read the PEP to check if it is ready: sound and complete. The ideas must make +> technical sense, even if they don't seem likely to be accepted. +> The title should accurately describe the content. +> Edit the PEP for language (spelling, grammar, sentence structure, etc.), markup +> (for reST PEPs), code style (examples should match PEP 8 & 7). + +KEP editors should generally not pass judgement on a KEP beyond editorial +corrections. + +[Python PEP process]: https://www.python.org/dev/peps/pep-0001/ +[PEP editor responsibilities]: https://www.python.org/dev/peps/pep-0001/#pep-editor-responsibilities-workflow + +### Important Metrics + +It is proposed that the primary metrics which would signal the success or +failure of the KEP process are + +- how many "features" are tracked with a KEP +- distribution of time a KEP spends in each state +- KEP rejection rate +- PRs referencing a KEP merged per week +- number of issued open which reference a KEP +- number of contributors who authored a KEP +- number of contributors who authored a KEP for the first time +- number of orphaned KEPs +- number of retired KEPs +- number of superseded KEPs + +### Prior Art + +The KEP process as proposed was essentially stolen from the [Rust RFC process] which +itself seems to be very similar to the [Python PEP process][] + +[Rust RFC process]: https://github.com/rust-lang/rfcs + +## Graduation Criteria + +should hit at least the following milestones + +- a release note draft can be generated by referring primarily to KEP content +- a yearly road map is expressed as a KEP + +## Drawbacks + +Any additional process has the potential to engender resentment within the +community. There is also a risk that the KEP process as designed will not +sufficiently address the scaling challenges we face today. PR review bandwidth is +already at a premium and we may find that the KEP process introduces an unreasonable +bottleneck on our development velocity. + +It certainly can be argued that the lack of a dedicated issue/defect tracker +beyond GitHub issues contributes to our challenges in managing a project as large +as Kubernetes, however, given that other large organizations, including GitHub +itself, make effective use of GitHub issues perhaps the argument is overblown. + +The centrality of Git and GitHub within the KEP process also may place too high +a barrier to potential contributors, however, given that both Git and GitHub are +required to contribute code changes to Kubernetes today perhaps it would be reasonable +to invest in providing support to those unfamiliar with this tooling. + +Expanding the proposal template beyond the single sentence description currently +required in the [features issue template][] may be a heavy burden for non native +English speakers and here the role of the KEP editor combined with kindness and +empathy will be crucial to making the process successful. + +[features issue template]: https://git.k8s.io/features/ISSUE_TEMPLATE.md + +## Alternatives + +This KEP process is related to +- the generation of a [architectural roadmap][] +- the fact that the [what constitutes a feature][] is still undefined +- [issue management][] +- the difference between an [accepted design and a proposal][] +- [the organization of design proposals][] + +this proposal attempts to place these concerns within a general framework. + +[architectural roadmap]: https://github.com/kubernetes/community/issues/952 +[what constitutes a feature]: https://github.com/kubernetes/community/issues/531 +[issue management]: https://github.com/kubernetes/community/issues/580 +[accepted design and a proposal]: https://github.com/kubernetes/community/issues/914 +[the organization of design proposals]: https://github.com/kubernetes/community/issues/918 + +### Github issues vs. KEPs + +The use of GitHub issues when proposing changes does not provide SIGs good +facilities for signaling approval or rejection of a proposed change to Kubernetes +since anyone can open a GitHub issue at any time. Additionally managing a proposed +change across multiple releases is somewhat cumbersome as labels and milestones +need to be updated for every release that a change spans. These long lived GitHub +issues lead to an ever increasing number of issues open against +`kubernetes/features` which itself has become a management problem. + +In addition to the challenge of managing issues over time, searching for text +within an issue can be challenging. The flat hierarchy of issues can also make +navigation and categorization tricky. While not all community members might +not be comfortable using Git directly, it is imperative that as a community we +work to educate people on a standard set of tools so they can take their +experience to other projects they may decide to work on in the future. While +git is a fantastic version control system (VCS), it is not a project management +tool nor a cogent way of managing an architectural catalog or backlog; this +proposal is limited to motivating the creation of a standardized definition of +work in order to facilitate project management. This primitive for describing +a unit of work may also allow contributors to create their own personalized +view of the state of the project while relying on Git and GitHub for consistency +and durable storage. + +## Unresolved Questions + +- How reviewers and approvers are assigned to a KEP +- Approval decision process for a KEP +- Example schedule, deadline, and time frame for each stage of a KEP +- Communication/notification mechanisms +- Review meetings and escalation procedure +- Decision on where development should occur + +## Mentors + +- caleb miles + - github: [calebamiles](https://github.com/calebamiles/) + - slack: [calebamiles](https://coreos.slack.com/team/caleb.miles) + - email: [caleb.miles@coreos.com](mailto:caleb.miles@coreos.com) + - pronoun: "he" diff --git a/docs/imported/community/mentoring.md b/docs/imported/community/mentoring.md new file mode 100755 index 0000000000000..081d441db8608 --- /dev/null +++ b/docs/imported/community/mentoring.md @@ -0,0 +1,41 @@ +--- +title: Kubernetes Mentoring Initiatives +--- + +This folder will be used for all mentoring initiatives for Kubernetes. + +--- +## Kubernetes Pilots + +We understand that everyone has different learning styles and we want to support as many of those as possible. Mentoring is vital to the growth of an individual and organization of every kind. For Kubernetes, the larger the project becomes, it's necessary to keep a continuous pipeline of quality contributors. + +*What's a Pilot?* +A pilot is a Kubernetes mentor helping new and current members navigate the seas of our repos. + +## Current mentoring activities: +All are currently in an incubation phase. Please reach out to Paris Pittman (parispittman@google.com or Paris on Kubernetes slack channel) for more information on how to get involved. The preliminary deck for mentoring strategies is [here](https://docs.google.com/presentation/d/1bRjDEPEn3autWzaEFirbLfHagbZV04Q9kTCalYmnnXw/edit?usp=sharing0). + +[Contributor Office Hours](https://github.com/kubernetes/community/tree/master/community/office-hours.md) +[Group Mentoring Cohorts](https://github.com/kubernetes/community/tree/master/mentoring/group-mentoring.md) +[Outreachy](https://github.com/kubernetes/community/tree/master/sig-cli/outreachy.md) + +#### Inspiration and Thanks +This is not an out of the box program but was largely inspired by the following: +* [Ada Developer Academy](https://adadevelopersacademy.org/) +* [Google Summer of Code](https://developers.google.com/open-source/gsoc/) +* [exercism.io](https://github.com/OperationCode/exercism-io-mentoring) +* [OpenStack Mentoring](https://wiki.openstack.org/wiki/Mentoring) +* [Apache Mentoring Programme](https://community.apache.org/mentoringprogramme.html) + +Thanks to: +* the many contributors who reviewed and participated in brainstorming, +* founding mentees for their willingness to try this out, +* founding Pilots (@chrislovecnm, @luxas, @kow3ns) + +We welcome PRs, suggestions, and help! +welcome PRs, suggestions, and help! + the many contributors who reviewed and participated in brainstorming, +* founding mentees for their willingness to try this out, +* founding Pilots (@chrislovecnm, @luxas, @kow3ns) + +We welcome PRs, suggestions, and help! diff --git a/docs/imported/index.md b/docs/imported/index.md new file mode 100644 index 0000000000000..2c9888d98bac7 --- /dev/null +++ b/docs/imported/index.md @@ -0,0 +1,9 @@ +--- +title: Imported Docs +--- + +This section of the Kubernetes documentation surfaces key topics imported from various repos in the broader Kubernetes community for better findability. + +## Community + +[kubernetes/community](https://github.com/kubernetes/community) is the starting point for becoming a contributor -- improving docs, improving code, giving talks etc. diff --git a/update-imported-docs/README b/update-imported-docs/README deleted file mode 100644 index 0314070896771..0000000000000 --- a/update-imported-docs/README +++ /dev/null @@ -1,5 +0,0 @@ -update-imported-docs/update-imported-docs.go will update the target files generated from other repos. -You should modify update-imported-docs/config.yaml to reflect the desired src and dst path. -``` -go run update-imported-docs/update-imported-docs.go -``` diff --git a/update-imported-docs/README.md b/update-imported-docs/README.md new file mode 100644 index 0000000000000..f00a44b0f530c --- /dev/null +++ b/update-imported-docs/README.md @@ -0,0 +1,75 @@ +# Update imported docs + +This script updates the target files generated from other repos listed in the file, which is specified as the command line argument. + +## Requirements + +Imported docs must follow these guidelines: + +1. Be listed somewhere in the `/_data/imported.yml` table of contents file. +1. Have `title` defined in the front matter. For example: + + ``` + --- + title: Title Displayed in Table of Contents + --- + + Rest of the .md file... + ``` + +1. Adhere to the [Documentation Style Guide](/docs/home/contribute/style-guide/). + +## Usage + +From within this directory, run the following command: + +``` +./update-imported-docs +``` + +The output should look similar to the following: + +``` +Website root directory: /Users/someuser/git/kubernetes-website + + * * * + +Cloning repo "community"... + + * * * + +Docs imported! Run 'git add .' 'git commit -m ' and 'git push' to upload them. +``` + +## Config file format + +Each config file may contain multiple repos, which will be imported together. You should modify the corresponding `update-imported-docs/` file to reflect the desired `src` and `dst` paths. + +You may also create new config files for different groups of documents to import. The following is an example of the YAML file format: + +``` +repos: +- name: kubernetes #tmp directory name + remote: https://github.com/kubernetes/kubernetes.git + branch: release-1.9 + generate-command: hack/generate-docs.sh #optional command to run + files: + - src: docs/admin/cloud-controller-manager.md + dst: docs/reference/generated/cloud-controller-manager.md + - src: docs/admin/kube-apiserver.md + dst: docs/reference/generated/kube-apiserver.md +- name: community #tmp directory name + remote: https://github.com/kubernetes/community.git + branch: master + files: + - src: contributors/devel/README.md + dst: docs/imported/community/devel.md + - src: contributors/guide/README.md + dst: docs/imported/community/guide.md +``` + +Note: `generate-command` is an optional entry, which can be used to run a given command to auto-generate the docs from within that repo. + +## Fixing Links + +To fix relative links within your imported files, set the repo config's `gen-absolute-links` value to `true`. You can see an example of this in [`community.yml`](community.yml). diff --git a/update-imported-docs/community.yml b/update-imported-docs/community.yml new file mode 100644 index 0000000000000..6a57c71630ea6 --- /dev/null +++ b/update-imported-docs/community.yml @@ -0,0 +1,14 @@ +repos: +- name: community + remote: https://github.com/kubernetes/community.git + branch: master + gen-absolute-links: true + files: + - src: contributors/devel/README.md + dst: docs/imported/community/devel.md + - src: contributors/guide/README.md + dst: docs/imported/community/guide.md + - src: mentoring/README.md + dst: docs/imported/community/mentoring.md + - src: keps/1-kubernetes-enhancement-proposal-process.md + dst: docs/imported/community/keps.md diff --git a/update-imported-docs/config.yaml b/update-imported-docs/reference.yml similarity index 95% rename from update-imported-docs/config.yaml rename to update-imported-docs/reference.yml index 340bd25fd2a19..3569e71b49d26 100644 --- a/update-imported-docs/config.yaml +++ b/update-imported-docs/reference.yml @@ -2,6 +2,7 @@ repos: - name: kubernetes remote: https://github.com/kubernetes/kubernetes.git branch: release-1.9 + generate-command: hack/generate-docs.sh files: - src: docs/admin/cloud-controller-manager.md dst: docs/reference/generated/cloud-controller-manager.md @@ -21,6 +22,7 @@ repos: remote: https://github.com/kubernetes/federation.git # # Change this to a release branch when federation has release branches. branch: master + generate-command: hack/generate-docs.sh files: - src: docs/admin/federation-apiserver.md dst: docs/reference/generated/federation-apiserver.md diff --git a/update-imported-docs/update-imported-docs b/update-imported-docs/update-imported-docs new file mode 100755 index 0000000000000..3bf65a5a0b263 Binary files /dev/null and b/update-imported-docs/update-imported-docs differ diff --git a/update-imported-docs/update-imported-docs.go b/update-imported-docs/update-imported-docs.go index cb8de3aabbcbb..46912150d468b 100644 --- a/update-imported-docs/update-imported-docs.go +++ b/update-imported-docs/update-imported-docs.go @@ -1,35 +1,66 @@ package main import ( + "bufio" "fmt" - "io" "io/ioutil" "os" "os/exec" "path" "path/filepath" "regexp" - + "strings" "github.com/ghodss/yaml" ) func main() { - websiteRepo, err := os.Getwd() + + //get command line arguments without executable + clArgs := os.Args[1:] + + //check that an argument has been passed in + if len(clArgs) == 0 { + fmt.Fprintf(os.Stderr, "Please specify a config file as a command line argument.\n") + os.Exit(1) + } + configFile := clArgs[0] + + //get directory of executable + ex, err := os.Executable() checkError(err) + exPath := filepath.Dir(ex) //file path of updated-imported-docs executable + suffix := filepath.Base(exPath) //should be "updated-imported-docs" - content, err := ioutil.ReadFile("update-imported-docs/config.yaml") + //check if suffix is "updated-imported-docs" + if suffix != "update-imported-docs" { + fmt.Fprintf(os.Stderr, "Instead of `go run update-imported-docs.go `, use the compiled binary `./update-imported-docs `\n") + os.Exit(1) + } + + //set root directory of website + websiteRepo := filepath.Clean(strings.TrimSuffix(exPath,suffix)) //path of parent directory + fmt.Fprintf(os.Stdout, "Website root directory: %s\n", websiteRepo) + + //read config.yaml file specified by first command line argument + content, err := ioutil.ReadFile(configFile) if err != nil { - fmt.Fprintf(os.Stderr, "error when reading file: %v\n", err) + fmt.Fprintf(os.Stderr, "Error when reading file: %v\n", err) os.Exit(1) } + //convert contents of config.yml file into a map var config map[string]interface{} err = yaml.Unmarshal(content, &config) if err != nil { - fmt.Fprintf(os.Stderr, "error when unmarshal the config file: %v\n", err) + fmt.Fprintf(os.Stderr, "Error when unmarshal the config file: %v\n", err) os.Exit(1) } + //change working directory to website root + err = os.Chdir(websiteRepo) + checkError(err) + + //clean out temp directory tmpDir := "/tmp/update_docs" os.RemoveAll(tmpDir) os.Mkdir(tmpDir, 0750) @@ -40,32 +71,74 @@ func main() { // title: *** // notile: *** // --- - titleRegex := regexp.MustCompile("^---\n(.*\n)*---\n") + titleRegex := regexp.MustCompile("^---\ntitle:(.*\n)*?---\n") + // To extract repo path prefix from `remote` + remoteGitRegex := regexp.MustCompile("(https://.*)\\.git$") + + //execute for each repo repos := config["repos"].([]interface{}) for _, repo := range repos { err = os.Chdir(tmpDir) checkError(err) + //get config info for repo, clone repo locally r := repo.(map[string]interface{}) repoName := r["name"].(string) + remotePathMatch := remoteGitRegex.FindAllStringSubmatch(r["remote"].(string), -1) + if (len(remotePathMatch) == 0) { + fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nInvalid remote path %q. Schema should look like: https://.git\n", r["remote"].(string)) + os.Exit(1) + } + remotePrefix := fmt.Sprintf("%s/tree/master", remotePathMatch[0][1]) + cmd := "git" args := []string{"clone", "--depth=1", "-b", r["branch"].(string), r["remote"].(string), repoName} - fmt.Fprintf(os.Stdout, "Cloning repo %q\n", repoName) + fmt.Fprintf(os.Stdout, "\n\t\t\t*\t*\t*\n\nCloning repo %q...\n", repoName) if err := exec.Command(cmd, args...).Run(); err != nil { - fmt.Fprintf(os.Stderr, "error when cloning repo %q: %v\n", repoName, err) + fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nError when cloning repo %q: %v\n", repoName, err) os.Exit(1) } err = os.Chdir(repoName) checkError(err) - fmt.Fprintf(os.Stdout, "Generating docs for repo %q\n", repoName) - if err := exec.Command("hack/generate-docs.sh").Run(); err != nil { - fmt.Fprintf(os.Stderr, "error when generating docs for repo %q: %v\n", repoName, err) - os.Exit(1) + //if generate-command is specified in the repo config, + //run the command for that repo, e.g. "hack/generate-docs.sh" + if r["generate-command"] != nil { + genCmd := r["generate-command"].(string) + + fmt.Fprintf(os.Stdout, "Generating docs for repo %q with %q...\n\n", repoName, genCmd) + cmd := exec.Command(genCmd) + cmdReader, err := cmd.StdoutPipe() + if err != nil { + fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nError when generating docs for repo %q: %v\n", repoName, err) + os.Exit(1) + } + + //display running output of generate command + scanner := bufio.NewScanner(cmdReader) + go func() { + for scanner.Scan() { + fmt.Printf("generator output | %s\n", scanner.Text()) + } + }() + + err = cmd.Start() + if err != nil { + fmt.Fprintln(os.Stderr, "Error starting %q command\n", genCmd, err) + os.Exit(1) + } + + err = cmd.Wait() + if err != nil { + fmt.Fprintln(os.Stderr, "Error waiting for %q command\n", genCmd, err) + os.Exit(1) + } + } + //copy and rename files from src -> dst specified in config err = os.Chdir(websiteRepo) checkError(err) files := r["files"].([]interface{}) @@ -73,6 +146,7 @@ func main() { f := file.(map[string]interface{}) src := f["src"].(string) dst := f["dst"].(string) + srcDir := filepath.Dir(src) absSrc, err := filepath.Abs(path.Join(tmpDir, repoName, src)) checkError(err) absDst, err := filepath.Abs(dst) @@ -82,38 +156,53 @@ func main() { titleBlock := titleRegex.Find(content) content, err = ioutil.ReadFile(absSrc) checkError(err) + + // Write to new output file dstFile, err := os.OpenFile(absDst, os.O_RDWR|os.O_CREATE, 0755) checkError(err) defer dstFile.Close() _, err = dstFile.Write(titleBlock) checkError(err) + + // Process content if necessary + if r["gen-absolute-links"] != nil { + content = processLinks(content, remotePrefix, srcDir) + } _, err = dstFile.Write(content) checkError(err) dstFile.Sync() } } - fmt.Fprintf(os.Stdout, "Docs imported! Run 'git add .' 'git commit -m ' and 'git push' to upload them\n") + fmt.Fprintf(os.Stdout, "\n\t\t\t*\t*\t*\n\nDocs imported! Run 'git add .' 'git commit -m ' and 'git push' to upload them.\n") } -func copyFile(src, dst string) error { - sf, err := os.Open(src) - if err != nil { - return err - } - defer sf.Close() - - df, err := os.Create(dst) - if err != nil { - return err - } - defer df.Close() +// +func processLinks(content []byte, remotePrefix string, subPath string) []byte { + // To catch anything of the form [text](url) + linkRegex := regexp.MustCompile("(\\[.+?\\])\\(([^\\s\\)]+)\\)") + // Regexes to skip + absUrlRegex := regexp.MustCompile("https*://") + mailRegex := regexp.MustCompile("mailto:") + + processedContent := linkRegex.ReplaceAllFunc(content, func(b []byte) []byte { + if (absUrlRegex.Match(b) || mailRegex.Match(b)) { + return b // no processing needed + } + match := linkRegex.FindAllStringSubmatch(string(b), -1) + url := match[0][2] + if url[0] == '#' { // link on current page + return b + } else if url[0] == '/' { // link at root of repo + return []byte(fmt.Sprintf("%s(%s/%s)", match[0][1], remotePrefix, url[1:])) + } else { // link relative to current page + return []byte(fmt.Sprintf("%s(%s/%s/%s)", match[0][1], remotePrefix, subPath, url)) + } + }) - _, err = io.Copy(df, sf) - if err != nil { - return err - } + h1Regex := regexp.MustCompile("^(# .*)?\n") + processedContent = h1Regex.ReplaceAll(processedContent, []byte("")) - return df.Sync() + return processedContent } func checkError(err error) {