diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f3800a04330..0a33d117b35b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2088,8 +2088,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 - id: merge-coverage-artifacts with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 21dbfbe3e707..2a4d7fd73fa1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2153,8 +2153,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 - id: merge-coverage-artifacts with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* @@ -2770,7 +2770,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-windows path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ff943fe4a17a..acaff8835a92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -363,7 +363,7 @@ jobs: branch: ${{ github.ref }} - name: Create Github Release - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1 with: artifactErrorsFailBuild: true artifacts: ${{ steps.prepare-release.outputs.release-artifacts }} diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 2ccef7efda89..6641cdb1f614 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2135,8 +2135,8 @@ jobs: - name: Merge All Code Coverage Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 - id: merge-coverage-artifacts with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 509dc6fcea76..ec32bd547de0 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -2595,7 +2595,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-windows path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/build-windows-repo.yml.jinja b/.github/workflows/templates/build-windows-repo.yml.jinja index dfa281b18ff6..67a2d82e17f6 100644 --- a/.github/workflows/templates/build-windows-repo.yml.jinja +++ b/.github/workflows/templates/build-windows-repo.yml.jinja @@ -85,7 +85,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-windows path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 93d534c5d90f..400b3211cd45 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -348,8 +348,8 @@ #} - name: Merge All Code Coverage Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 - id: merge-coverage-artifacts with: name: all-testrun-coverage-artifacts pattern: all-testrun-coverage-artifacts-* diff --git a/.github/workflows/templates/release.yml.jinja b/.github/workflows/templates/release.yml.jinja index efeef41f8a45..cb6d251b966c 100644 --- a/.github/workflows/templates/release.yml.jinja +++ b/.github/workflows/templates/release.yml.jinja @@ -372,7 +372,7 @@ permissions: branch: ${{ github.ref }} - name: Create Github Release - uses: ncipollo/release-action@v1.12.0 + uses: ncipollo/release-action@v1 with: artifactErrorsFailBuild: true artifacts: ${{ steps.prepare-release.outputs.release-artifacts }} diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index bbf77fdc29f2..95e5d3d4964f 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -127,6 +127,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -275,7 +280,7 @@ jobs: if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ @@ -283,7 +288,7 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/xml-unittests-output/ @@ -291,13 +296,13 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/logs report: name: Test Reports - if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test @@ -313,6 +318,8 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge JUnit XML Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} @@ -321,6 +328,8 @@ jobs: delete-merged: true - name: Merge Log Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} @@ -329,9 +338,9 @@ jobs: delete-merged: true - name: Merge Code Coverage Test Run Artifacts - uses: actions/upload-artifact/merge@v4 if: ${{ inputs.skip-code-coverage == false }} - id: merge-coverage-artifacts + continue-on-error: true + uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }} pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-* @@ -347,6 +356,7 @@ jobs: path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts + if: ${{ inputs.skip-code-coverage == false }} run: | tree -a artifacts diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 5105682d3735..a9fc7fc31533 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -120,6 +120,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -303,7 +308,7 @@ jobs: if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ @@ -311,7 +316,7 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} path: | artifacts/xml-unittests-output/ @@ -319,13 +324,13 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} path: | artifacts/logs report: name: Test Reports - if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test @@ -341,28 +346,32 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge JUnit XML Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Log Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Code Coverage Test Run Artifacts - uses: actions/upload-artifact/merge@v4 if: ${{ inputs.skip-code-coverage == false }} - id: merge-coverage-artifacts + continue-on-error: true + uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true @@ -375,6 +384,7 @@ jobs: path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts + if: ${{ inputs.skip-code-coverage == false }} run: | tree -a artifacts diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index ff4ab30758c3..d8e57dbad0fe 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -127,6 +127,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -275,7 +280,7 @@ jobs: if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/coverage/ @@ -283,7 +288,7 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/xml-unittests-output/ @@ -291,14 +296,14 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }} path: | artifacts/logs report: name: Test Reports - if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test @@ -314,28 +319,32 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge JUnit XML Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Log Test Run Artifacts + if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true - name: Merge Code Coverage Test Run Artifacts - uses: actions/upload-artifact/merge@v4 if: ${{ inputs.skip-code-coverage == false }} - id: merge-coverage-artifacts + continue-on-error: true + uses: actions/upload-artifact/merge@v4 with: name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} - pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}* separate-directories: false delete-merged: true @@ -348,6 +357,7 @@ jobs: path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts + if: ${{ inputs.skip-code-coverage == false }} run: | tree -a artifacts diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index a319f9f262eb..a5c00db88eac 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -83,6 +83,7 @@ jobs: - x86_64 outputs: pkg-matrix-include: ${{ steps.generate-pkg-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-pkg-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -126,6 +127,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -221,7 +227,7 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}}-${{ env.TIMESTAMP }} path: | artifacts !artifacts/pkg/* @@ -231,8 +237,9 @@ jobs: report: name: Report runs-on: ubuntu-latest - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' needs: + - generate-matrix - test steps: @@ -245,6 +252,7 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.pkg-type }} diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index df107359f03f..2e27f1e9849c 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -79,6 +79,7 @@ jobs: - x86_64 outputs: pkg-matrix-include: ${{ steps.generate-pkg-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-pkg-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -118,6 +119,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -204,7 +210,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}}-${{ env.TIMESTAMP }} path: | artifacts !artifacts/pkg/* @@ -214,8 +220,9 @@ jobs: report: name: Report runs-on: ubuntu-latest - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' needs: + - generate-matrix - test steps: @@ -228,6 +235,7 @@ jobs: t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - name: Merge Test Run Artifacts + continue-on-error: true uses: actions/upload-artifact/merge@v4 with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index a3198ac6ffb6..967482ac2040 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -82,6 +82,7 @@ jobs: - x86_64 outputs: pkg-matrix-include: ${{ steps.generate-pkg-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-pkg-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -125,6 +126,11 @@ jobs: run: | t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + - name: Checkout Source Code uses: actions/checkout@v4 @@ -220,7 +226,7 @@ jobs: if: always() && steps.download-artifacts-from-vm.outcome == 'success' uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}}-${{ env.TIMESTAMP }} path: | artifacts !artifacts/pkg/* @@ -230,8 +236,9 @@ jobs: report: name: Report runs-on: ubuntu-latest - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped' needs: + - generate-matrix - test steps: @@ -245,6 +252,7 @@ jobs: - name: Merge Test Run Artifacts uses: actions/upload-artifact/merge@v4 + continue-on-error: true with: name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }} pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-* diff --git a/CHANGELOG.md b/CHANGELOG.md index a0f77032b3fd..a5a5912826dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,50 @@ Versions are `MAJOR.PATCH`. # Changelog +## 3007.1 (2024-05-19) + + +### Removed + +- The ``salt.utils.psutil_compat`` was deprecated and now removed in Salt 3008. Please use the ``psutil`` module directly. [#66160](https://github.com/saltstack/salt/issues/66160) + + +### Fixed + +- Fixes multiple issues with the cmd module on Windows. Scripts are called using + the ``-File`` parameter to the ``powershell.exe`` binary. ``CLIXML`` data in + stderr is now removed (only applies to encoded commands). Commands can now be + sent to ``cmd.powershell`` as a list. Makes sure JSON data returned is valid. + Strips whitespace from the return when using ``runas``. [#61166](https://github.com/saltstack/salt/issues/61166) +- Fixed the win_lgpo_netsh salt util to handle non-English systems. This was a + rewrite to use PowerShell instead of netsh to make the changes on the system [#61534](https://github.com/saltstack/salt/issues/61534) +- Fix typo in nftables module to ensure unique nft family values [#65295](https://github.com/saltstack/salt/issues/65295) +- Corrected x509_v2 CRL creation `last_update` and `next_update` values when system timezone is not UTC [#65837](https://github.com/saltstack/salt/issues/65837) +- Fix for NoneType can't be used in 'await' expression error. [#66177](https://github.com/saltstack/salt/issues/66177) +- Log "Publish server binding pub to" messages to debug instead of error level. [#66179](https://github.com/saltstack/salt/issues/66179) +- Fix syndic startup by making payload handler a coroutine [#66237](https://github.com/saltstack/salt/issues/66237) +- Fixed `aptpkg.remove` "unable to locate package" error for non-existent package [#66260](https://github.com/saltstack/salt/issues/66260) +- Fixed pillar.ls doesn't accept kwargs [#66262](https://github.com/saltstack/salt/issues/66262) +- Fix cache directory setting in Master Cluster tutorial [#66264](https://github.com/saltstack/salt/issues/66264) +- Change log level of successful master cluster key exchange from error to info. [#66266](https://github.com/saltstack/salt/issues/66266) +- Made `file.managed` skip download of a remote source if the managed file already exists with the correct hash [#66342](https://github.com/saltstack/salt/issues/66342) +- Fixed nftables.build_rule breaks ipv6 rules by using the wrong syntax for source and destination addresses [#66382](https://github.com/saltstack/salt/issues/66382) + + +### Added + +- Added the ability to pass a version of chocolatey to install to the + chocolatey.bootstrap function. Also added states to bootstrap and + unbootstrap chocolatey. [#64722](https://github.com/saltstack/salt/issues/64722) +- Add Ubuntu 24.04 support [#66180](https://github.com/saltstack/salt/issues/66180) +- Add Fedora 40 support, replacing Fedora 39 [#66300](https://github.com/saltstack/salt/issues/66300) + + +### Security + +- Bump to `pydantic==2.6.4` due to https://github.com/advisories/GHSA-mr82-8j83-vxmv [#66433](https://github.com/saltstack/salt/issues/66433) +- Bump to ``jinja2==3.1.4`` due to https://github.com/advisories/GHSA-h75v-3vvj-5mfj [#66488](https://github.com/saltstack/salt/issues/66488) + ## 3006.8 (2024-04-29) diff --git a/changelog/61166.fixed.md b/changelog/61166.fixed.md deleted file mode 100644 index f197c324c9e9..000000000000 --- a/changelog/61166.fixed.md +++ /dev/null @@ -1,5 +0,0 @@ -Fixes multiple issues with the cmd module on Windows. Scripts are called using -the ``-File`` parameter to the ``powershell.exe`` binary. ``CLIXML`` data in -stderr is now removed (only applies to encoded commands). Commands can now be -sent to ``cmd.powershell`` as a list. Makes sure JSON data returned is valid. -Strips whitespace from the return when using ``runas``. diff --git a/changelog/61534.fixed.md b/changelog/61534.fixed.md deleted file mode 100644 index ed6c44011409..000000000000 --- a/changelog/61534.fixed.md +++ /dev/null @@ -1,2 +0,0 @@ -Fixed the win_lgpo_netsh salt util to handle non-English systems. This was a -rewrite to use PowerShell instead of netsh to make the changes on the system diff --git a/changelog/64722.added.md b/changelog/64722.added.md deleted file mode 100644 index 6a05d2f59d6b..000000000000 --- a/changelog/64722.added.md +++ /dev/null @@ -1,3 +0,0 @@ -Added the ability to pass a version of chocolatey to install to the -chocolatey.bootstrap function. Also added states to bootstrap and -unbootstrap chocolatey. diff --git a/changelog/65295.fixed.md b/changelog/65295.fixed.md deleted file mode 100644 index c672de05b756..000000000000 --- a/changelog/65295.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix typo in nftables module to ensure unique nft family values diff --git a/changelog/65837.fixed.md b/changelog/65837.fixed.md deleted file mode 100644 index 72f4a30fbda6..000000000000 --- a/changelog/65837.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Corrected x509_v2 CRL creation `last_update` and `next_update` values when system timezone is not UTC diff --git a/changelog/66160.removed.md b/changelog/66160.removed.md deleted file mode 100644 index 319112ac9d9c..000000000000 --- a/changelog/66160.removed.md +++ /dev/null @@ -1 +0,0 @@ -The ``salt.utils.psutil_compat`` was deprecated and now removed in Salt 3008. Please use the ``psutil`` module directly. diff --git a/changelog/66177.fixed.md b/changelog/66177.fixed.md deleted file mode 100644 index eff19eb10ec1..000000000000 --- a/changelog/66177.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix for NoneType can't be used in 'await' expression error. diff --git a/changelog/66179.fixed.md b/changelog/66179.fixed.md deleted file mode 100644 index 5dba9a55262a..000000000000 --- a/changelog/66179.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Log "Publish server binding pub to" messages to debug instead of error level. diff --git a/changelog/66180.added.md b/changelog/66180.added.md deleted file mode 100644 index 92925b9f9070..000000000000 --- a/changelog/66180.added.md +++ /dev/null @@ -1 +0,0 @@ -Add Ubuntu 24.04 support diff --git a/changelog/66237.fixed.md b/changelog/66237.fixed.md deleted file mode 100644 index 48f292a265c8..000000000000 --- a/changelog/66237.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix syndic startup by making payload handler a coroutine diff --git a/changelog/66260.fixed.md b/changelog/66260.fixed.md deleted file mode 100644 index 73d07f178576..000000000000 --- a/changelog/66260.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed `aptpkg.remove` "unable to locate package" error for non-existent package diff --git a/changelog/66262.fixed.md b/changelog/66262.fixed.md deleted file mode 100644 index dea7787f21c7..000000000000 --- a/changelog/66262.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed pillar.ls doesn't accept kwargs diff --git a/changelog/66264.fixed.md b/changelog/66264.fixed.md deleted file mode 100644 index 84a86f546d76..000000000000 --- a/changelog/66264.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix cache directory setting in Master Cluster tutorial diff --git a/changelog/66266.fixed.md b/changelog/66266.fixed.md deleted file mode 100644 index 8632ef1191e7..000000000000 --- a/changelog/66266.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Change log level of successful master cluster key exchange from error to info. diff --git a/changelog/66300.added.md b/changelog/66300.added.md deleted file mode 100644 index 18b4964110f6..000000000000 --- a/changelog/66300.added.md +++ /dev/null @@ -1 +0,0 @@ -Add Fedora 40 support, replacing Fedora 39 diff --git a/changelog/66342.fixed.md b/changelog/66342.fixed.md deleted file mode 100644 index da57b2926d05..000000000000 --- a/changelog/66342.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Made `file.managed` skip download of a remote source if the managed file already exists with the correct hash diff --git a/changelog/66382.fixed.md b/changelog/66382.fixed.md deleted file mode 100644 index 15875838cff0..000000000000 --- a/changelog/66382.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed nftables.build_rule breaks ipv6 rules by using the wrong syntax for source and destination addresses diff --git a/changelog/66433.security.md b/changelog/66433.security.md deleted file mode 100644 index 816b14516e20..000000000000 --- a/changelog/66433.security.md +++ /dev/null @@ -1 +0,0 @@ -Bump to `pydantic==2.6.4` due to https://github.com/advisories/GHSA-mr82-8j83-vxmv diff --git a/changelog/66488.security.md b/changelog/66488.security.md deleted file mode 100644 index 7871bb678db7..000000000000 --- a/changelog/66488.security.md +++ /dev/null @@ -1 +0,0 @@ -Bump to ``jinja2==3.1.4`` due to https://github.com/advisories/GHSA-h75v-3vvj-5mfj diff --git a/doc/man/salt-api.1 b/doc/man/salt-api.1 index 96f6e137793c..2d12576615cf 100644 --- a/doc/man/salt-api.1 +++ b/doc/man/salt-api.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-API" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-API" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-api \- salt-api Command .sp diff --git a/doc/man/salt-call.1 b/doc/man/salt-call.1 index 2ed60593bb7e..ad3a79277967 100644 --- a/doc/man/salt-call.1 +++ b/doc/man/salt-call.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-CALL" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-CALL" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-call \- salt-call Documentation .SH SYNOPSIS diff --git a/doc/man/salt-cloud.1 b/doc/man/salt-cloud.1 index cc2139878bc0..8c043719848d 100644 --- a/doc/man/salt-cloud.1 +++ b/doc/man/salt-cloud.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-CLOUD" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-CLOUD" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-cloud \- Salt Cloud Command .sp diff --git a/doc/man/salt-cp.1 b/doc/man/salt-cp.1 index 952c6008c9d2..b45cc58eb15a 100644 --- a/doc/man/salt-cp.1 +++ b/doc/man/salt-cp.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-CP" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-CP" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-cp \- salt-cp Documentation .sp diff --git a/doc/man/salt-key.1 b/doc/man/salt-key.1 index f6ac175d835b..8066d9bf73b7 100644 --- a/doc/man/salt-key.1 +++ b/doc/man/salt-key.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-KEY" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-KEY" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-key \- salt-key Documentation .SH SYNOPSIS diff --git a/doc/man/salt-master.1 b/doc/man/salt-master.1 index e7004f2fcafc..20147774055c 100644 --- a/doc/man/salt-master.1 +++ b/doc/man/salt-master.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-MASTER" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-MASTER" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-master \- salt-master Documentation .sp diff --git a/doc/man/salt-minion.1 b/doc/man/salt-minion.1 index 1231e4d55511..f302c3ccbe4b 100644 --- a/doc/man/salt-minion.1 +++ b/doc/man/salt-minion.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-MINION" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-MINION" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-minion \- salt-minion Documentation .sp diff --git a/doc/man/salt-proxy.1 b/doc/man/salt-proxy.1 index 18346acad6f0..47f27cdcb4c6 100644 --- a/doc/man/salt-proxy.1 +++ b/doc/man/salt-proxy.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-PROXY" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-PROXY" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-proxy \- salt-proxy Documentation .sp diff --git a/doc/man/salt-run.1 b/doc/man/salt-run.1 index 5b456b4f1bc0..1bf2f1115fa4 100644 --- a/doc/man/salt-run.1 +++ b/doc/man/salt-run.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-RUN" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-RUN" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-run \- salt-run Documentation .sp diff --git a/doc/man/salt-ssh.1 b/doc/man/salt-ssh.1 index 186d13f2fea2..0b1840ed9289 100644 --- a/doc/man/salt-ssh.1 +++ b/doc/man/salt-ssh.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-SSH" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-SSH" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-ssh \- salt-ssh Documentation .SH SYNOPSIS diff --git a/doc/man/salt-syndic.1 b/doc/man/salt-syndic.1 index 8207be769d1f..7e06e30540d2 100644 --- a/doc/man/salt-syndic.1 +++ b/doc/man/salt-syndic.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT-SYNDIC" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT-SYNDIC" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt-syndic \- salt-syndic Documentation .sp diff --git a/doc/man/salt.1 b/doc/man/salt.1 index 21511c9e0355..be57335582a5 100644 --- a/doc/man/salt.1 +++ b/doc/man/salt.1 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT" "1" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT" "1" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt \- salt .SH SYNOPSIS diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 25cf54d9fbb0..8e0c6bff484d 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -27,7 +27,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "SALT" "7" "Generated on April 29, 2024 at 03:20:12 AM UTC." "3006.8" "Salt" +.TH "SALT" "7" "Generated on May 19, 2024 at 12:51:08 PM UTC." "3007.1" "Salt" .SH NAME salt \- Salt Documentation .SH SALT PROJECT @@ -1693,6 +1693,293 @@ for the current installation instructions. .sp This section explains how to configure user access, view and store job results, secure and troubleshoot, and how to perform many other administrative tasks. +.SS Configuring Salt +.sp +Salt configuration is very simple. The default configuration for the +\fI\%master\fP will work for most installations and the only requirement for +setting up a \fI\%minion\fP is to set the location of the master in the minion +configuration file. +.sp +The configuration files will be installed to \fB/etc/salt\fP and are named +after the respective components, \fB/etc/salt/master\fP, and +\fB/etc/salt/minion\fP\&. +.SS Master Configuration +.sp +By default the Salt master listens on ports 4505 and 4506 on all +interfaces (0.0.0.0). To bind Salt to a specific IP, redefine the +\(dqinterface\(dq directive in the master configuration file, typically +\fB/etc/salt/master\fP, as follows: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\- #interface: 0.0.0.0 ++ interface: 10.0.0.1 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +After updating the configuration file, restart the Salt master. +See the \fI\%master configuration reference\fP +for more details about other configurable options. +.SS Minion Configuration +.sp +Although there are many Salt Minion configuration options, configuring +a Salt Minion is very simple. By default a Salt Minion will +try to connect to the DNS name \(dqsalt\(dq; if the Minion is able to +resolve that name correctly, no configuration is needed. +.sp +If the DNS name \(dqsalt\(dq does not resolve to point to the correct +location of the Master, redefine the \(dqmaster\(dq directive in the minion +configuration file, typically \fB/etc/salt/minion\fP, as follows: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +\- #master: salt ++ master: 10.0.0.1 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +After updating the configuration file, restart the Salt minion. +See the \fI\%minion configuration reference\fP +for more details about other configurable options. +.SS Proxy Minion Configuration +.sp +A proxy minion emulates the behaviour of a regular minion +and inherits their options. +.sp +Similarly, the configuration file is \fB/etc/salt/proxy\fP and the proxy +tries to connect to the DNS name \(dqsalt\(dq. +.sp +In addition to the regular minion options, +there are several proxy\-specific \- see the +\fI\%proxy minion configuration reference\fP\&. +.SS Running Salt +.INDENT 0.0 +.IP 1. 3 +Start the master in the foreground (to daemonize the process, pass the +\fI\%\-d flag\fP): +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-master +.ft P +.fi +.UNINDENT +.UNINDENT +.IP 2. 3 +Start the minion in the foreground (to daemonize the process, pass the +\fI\%\-d flag\fP): +.INDENT 3.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-minion +.ft P +.fi +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.IP "Having trouble?" +.sp +The simplest way to troubleshoot Salt is to run the master and minion in +the foreground with \fI\%log level\fP set to \fBdebug\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-master \-\-log\-level=debug +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For information on salt\(aqs logging system please see the \fI\%logging +document\fP\&. +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.IP "Run as an unprivileged (non\-root) user" +.sp +To run Salt as another user, set the \fI\%user\fP parameter in the +master config file. +.sp +Additionally, ownership, and permissions need to be set such that the +desired user can read from and write to the following directories (and +their subdirectories, where applicable): +.INDENT 0.0 +.IP \(bu 2 +/etc/salt +.IP \(bu 2 +/var/cache/salt +.IP \(bu 2 +/var/log/salt +.IP \(bu 2 +/var/run/salt +.UNINDENT +.sp +More information about running salt as a non\-privileged user can be found +\fI\%here\fP\&. +.UNINDENT +.UNINDENT +.sp +There is also a full \fI\%troubleshooting guide\fP +available. +.SS Key Identity +.sp +Salt provides commands to validate the identity of your Salt master +and Salt minions before the initial key exchange. Validating key identity helps +avoid inadvertently connecting to the wrong Salt master, and helps prevent +a potential MiTM attack when establishing the initial connection. +.SS Master Key Fingerprint +.sp +Print the master key fingerprint by running the following command on the Salt master: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-key \-F master +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Copy the \fBmaster.pub\fP fingerprint from the \fILocal Keys\fP section, and then set this value +as the \fI\%master_finger\fP in the minion configuration file. Save the configuration +file and then restart the Salt minion. +.SS Minion Key Fingerprint +.sp +Run the following command on each Salt minion to view the minion key fingerprint: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-call \-\-local key.finger +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Compare this value to the value that is displayed when you run the +\fBsalt\-key \-\-finger \fP command on the Salt master. +.SS Key Management +.sp +Salt uses AES encryption for all communication between the Master and +the Minion. This ensures that the commands sent to the Minions cannot +be tampered with, and that communication between Master and Minion is +authenticated through trusted, accepted keys. +.sp +Before commands can be sent to a Minion, its key must be accepted on +the Master. Run the \fBsalt\-key\fP command to list the keys known to +the Salt Master: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[root@master ~]# salt\-key \-L +Unaccepted Keys: +alpha +bravo +charlie +delta +Accepted Keys: +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This example shows that the Salt Master is aware of four Minions, but none of +the keys has been accepted. To accept the keys and allow the Minions to be +controlled by the Master, again use the \fBsalt\-key\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[root@master ~]# salt\-key \-A +[root@master ~]# salt\-key \-L +Unaccepted Keys: +Accepted Keys: +alpha +bravo +charlie +delta +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +The \fBsalt\-key\fP command allows for signing keys individually or in bulk. The +example above, using \fB\-A\fP bulk\-accepts all pending keys. To accept keys +individually use the lowercase of the same option, \fB\-a keyname\fP\&. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%salt\-key manpage\fP +.UNINDENT +.UNINDENT +.SS Sending Commands +.sp +Communication between the Master and a Minion may be verified by running +the \fBtest.version\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[root@master ~]# salt alpha test.version +alpha: + 2018.3.4 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Communication between the Master and all Minions may be tested in a +similar way: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[root@master ~]# salt \(aq*\(aq test.version +alpha: + 2018.3.4 +bravo: + 2018.3.4 +charlie: + 2018.3.4 +delta: + 2018.3.4 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Each of the Minions should send a \fB2018.3.4\fP response as shown above, +or any other salt version installed. +.SS What\(aqs Next? +.sp +Understanding \fI\%targeting\fP is important. From there, depending +on the way you wish to use Salt, you should also proceed to learn about +\fI\%Remote Execution\fP and \fI\%Configuration Management\fP\&. .SS Configuring the Salt Master .sp The Salt system is amazingly simple and easy to configure, the two components @@ -1816,8 +2103,13 @@ enable_ssh_minions: True \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Cross\-minion communication is still not possible. The Salt mine and -publish.publish do not work between minion types. +Enabling this does not influence the limitations on cross\-minion communication. +The Salt mine and \fBpublish.publish\fP do not work from regular minions +to SSH minions, the other way around is partly possible since 3007.0 +(during state rendering on the master). +This means you can use the mentioned functions to call out to regular minions +in \fBsls\fP templates and wrapper modules, but state modules +(which are executed on the remote) relying on them still do not work. .UNINDENT .UNINDENT .SS \fBret_port\fP @@ -1913,6 +2205,62 @@ pki_dir: /etc/salt/pki/master .fi .UNINDENT .UNINDENT +.SS \fBcluster_id\fP +.sp +New in version 3007. + +.sp +When defined, the master will operate in cluster mode. The master will send the +cluster key and id to minions instead of its own key and id. The master will +also forward its local event bus to other masters defined by \fBcluster_peers\fP +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cluster_id: master +.ft P +.fi +.UNINDENT +.UNINDENT +.SS \fBcluster_peers\fP +.sp +New in version 3007. + +.sp +When \fBcluster_id\fP is defined, this setting is a list of other master +(hostnames or ips) that will be in the cluster. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cluster_peers: + \- master2 + \- master3 +.ft P +.fi +.UNINDENT +.UNINDENT +.SS \fBcluster_pki_dir\fP +.sp +New in version 3007. + +.sp +When \fBcluster_id\fP is defined, this sets the location of where this cluster +will store its cluster public and private key as well as any minion keys. This +setting will default to the value of \fBpki_dir\fP, but should be changed +to the filesystem location shared between peers in the cluster. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +cluster_pki: /my/gluster/share/pki +.ft P +.fi +.UNINDENT +.UNINDENT .SS \fBextension_modules\fP .sp Changed in version 2016.3.0: The default location for this directory has been moved. Prior to this @@ -6352,31 +6700,6 @@ minionfs_update_interval: 120 .fi .UNINDENT .UNINDENT -.SS azurefs: Azure File Server Backend -.sp -New in version 2015.8.0. - -.sp -See the \fI\%azurefs documentation\fP for usage -examples. -.SS \fBazurefs_update_interval\fP -.sp -New in version 2018.3.0. - -.sp -Default: \fB60\fP -.sp -This option defines the update interval (in seconds) for azurefs. -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -azurefs_update_interval: 120 -.ft P -.fi -.UNINDENT -.UNINDENT .SS s3fs: S3 File Server Backend .sp New in version 0.16.0. @@ -7893,9 +8216,9 @@ and pkg modules. .nf .ft C peer: - foo.example.com: - \- test.* - \- pkg.* + foo\e.example\e.com: + \- test\e..* + \- pkg\e..* .ft P .fi .UNINDENT @@ -7918,22 +8241,34 @@ peer: This is not recommended, since it would allow anyone who gets root on any single minion to instantly have root on all of the minions! .sp -By adding an additional layer you can limit the target hosts in addition to the -accessible commands: +It is also possible to limit target hosts with the \fI\%Compound Matcher\fP\&. +You can achieve this by adding another layer in between the source and the +allowed functions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C peer: - foo.example.com: - \(aqdb*\(aq: - \- test.* - \- pkg.* + \(aq.*\e.example\e.com\(aq: + \- \(aqG@role:db\(aq: + \- test\e..* + \- pkg\e..* .ft P .fi .UNINDENT .UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Notice that the source hosts are matched by a regular expression +on their minion ID, while target hosts can be matched by any of +the \fI\%available matchers\fP\&. +.sp +Note that globbing and regex matching on pillar values is not supported. You can only match exact values. +.UNINDENT +.UNINDENT .SS \fBpeer_run\fP .sp Default: \fB{}\fP @@ -11356,8 +11691,6 @@ pillar .UNINDENT .UNINDENT .SS Top File Settings -.sp -These parameters only have an effect if running a masterless minion. .SS \fBstate_top\fP .sp Default: \fBtop.sls\fP @@ -17916,7 +18249,7 @@ peer: .UNINDENT .UNINDENT .sp -This configuration will allow minions with IDs ending in example.com access +This configuration allows minions with IDs ending in \fB\&.example.com\fP access to the test, ps, and pkg module functions. .INDENT 0.0 .INDENT 3.5 @@ -17924,10 +18257,10 @@ to the test, ps, and pkg module functions. .nf .ft C peer: - .*example.com: - \- test.* - \- ps.* - \- pkg.* + .*\e.example.com: + \- test\e..* + \- ps\e..* + \- pkg\e..* .ft P .fi .UNINDENT @@ -17943,14 +18276,14 @@ allow minions ending with foo.org access to the publisher. .nf .ft C peer: - .*example.com: - \- test.* - \- ps.* - \- pkg.* - .*foo.org: - \- test.* - \- ps.* - \- pkg.* + .*\e.example.com: + \- test\e..* + \- ps\e..* + \- pkg\e..* + .*\e.foo.org: + \- test\e..* + \- ps\e..* + \- pkg\e..* .ft P .fi .UNINDENT @@ -17959,7 +18292,36 @@ peer: \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Functions are matched using regular expressions. +Functions are matched using regular expressions as well. +.UNINDENT +.UNINDENT +.sp +It is also possible to limit target hosts with the \fI\%Compound Matcher\fP\&. +You can achieve this by adding another layer in between the source and the +allowed functions: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +peer: + \(aq.*\e.example\e.com\(aq: + \- \(aqG@role:db\(aq: + \- test\e..* + \- pkg\e..* +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Notice that the source hosts are matched by a regular expression +on their minion ID, while target hosts can be matched by any of +the \fI\%available matchers\fP\&. +.sp +Note that globbing and regex matching on pillar values is not supported. You can only match exact values. .UNINDENT .UNINDENT .SS Peer Runner Communication @@ -20722,10 +21084,12 @@ first.git: edit/vim.sls edit/vimrc nginx/init.sls + shell/init.sls second.git: edit/dev_vimrc haproxy/init.sls + shell.sls third: haproxy/haproxy.conf @@ -20749,6 +21113,14 @@ A request for the file \fBsalt://haproxy/haproxy.conf\fP will be served from the \fBfile:///root/third\fP repo. .UNINDENT .sp +Also a requested state file overrules a directory with an \fIinit.sls\fP\-file. +For example: +.INDENT 0.0 +.IP \(bu 2 +A request for \fBstate.apply shell\fP will be served from the +\fBhttps://github.com/example/second.git\fP git repo. +.UNINDENT +.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 @@ -23365,6 +23737,46 @@ salt\-call sdb.get sdb://kevinopenstack/password .UNINDENT .UNINDENT .sp +For SDB sub\-keys, ie users[\(aquser1\(aq][\(aqid\(aq] +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +users: + user1: + id: 12345 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To get SDB sub\-keys from the CLI, use a colon to separate sub key values. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +salt\-call sdb.get sdb://users:user1:id +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To get SDB sub\-keys in a state file, use this syntax: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +users: + user1: + id: sdb.get sdb://users:user1:id +.ft P +.fi +.UNINDENT +.UNINDENT +.sp \fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 @@ -24154,6 +24566,10 @@ use the same transport. We\(aqre investigating a report of an error when using mixed transport types at very heavy loads. .UNINDENT .UNINDENT +.SS TLS Support +.sp +The TLS transport supports full encryption and verification using both server +and client certificates. See \fI\%Transport TLS Support\fP for more details. .SS Wire Protocol .sp This implementation over TCP focuses on flexibility over absolute efficiency. @@ -24177,19 +24593,76 @@ Head contains header information (such as \(dqmessage id\(dq). The Body contains actual message that we are sending. With this flexible wire protocol we can implement any message semantics that we\(aqd like\-\- including multiplexed message passing on a single socket. +.SS Crypto +.sp +The current implementation uses the same crypto as the \fBzeromq\fP transport. +.SS Publish Server and Client +.sp +For the publish server and client we send messages without \(dqmessage ids\(dq which +the remote end interprets as a one\-way send. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +As of Salt \fI\%2016.3.0\fP, publishes using \fBlist\fP targeting are sent only to relevant minions and not broadcasted. +.sp +As of Salt \fI\%3005\fP, publishes using \fBpcre\fP and \fBglob\fP targeting are also sent only to relevant minions and not broadcasted. Other targeting types are always sent to all minions and rely on minion\-side filtering. +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Salt CLI defaults to \fBglob\fP targeting type, so in order to target specific minions without broadcast, you need to use \fI\-L\fP option, such as \fBsalt \-L my.minion test.ping\fP, for masters before 3005. +.UNINDENT +.UNINDENT +.SS Request Server and Client +.sp +For the request server and client we send messages with a \(dqmessage id\(dq. This +\(dqmessage id\(dq allows us to multiplex messages across the socket. +.SS Websocket Transport +.sp +The Websocket transport is an implementation of Salt\(aqs transport using the websocket protocol. +The Websocket transport is enabled by changing the \fI\%transport\fP setting +to \fBws\fP on each Salt minion and Salt master. .SS TLS Support .sp +The Websocket transport supports full encryption and verification using both server +and client certificates. See \fI\%Transport TLS Support\fP for more details. +.SS Publish Server and Client +.sp +The publish server and client are implemented using aiohttp. +.SS Request Server and Client +.sp +The request server and client are implemented using aiohttp. +.SS Transport TLS Support +.sp +Whenever possible transports should provide TLS Support. Currently the \fI\%TCP Transport\fP and +\fI\%Websocket Transport\fP transports support encryption and verification using TLS. +.sp New in version 2016.11.1. .sp The TCP transport allows for the master/minion communication to be optionally wrapped in a TLS connection. Enabling this is simple, the master and minion need -to be using the tcp connection, then the \fIssl\fP option is enabled. The \fIssl\fP -option is passed as a dict and corresponds to the options passed to the +to be using the tcp connection, then the \fBssl\fP option is enabled. The \fBssl\fP +option is passed as a dict and roughly corresponds to the options passed to the Python \fI\%ssl.wrap_socket\fP -function. +function for backwards compatability. +.sp +New in version 3007.0. + .sp -A simple setup looks like this, on the Salt Master add the \fIssl\fP option to the +The \fBssl\fP option accepts \fBverify_locations\fP and \fBverify_flags\fP\&. The +\fBverify_locations\fP option is a list of strings or dictionaries. Strings are +passed as a single argument to the SSL context\(aqs \fBload_verify_locations\fP +method. Dictionary keys are expected to be one of \fBcafile\fP, \fBcapath\fP, +\fBcadata\fP\&. For each corresponding key, the key and value will be passed as a +keyword argument to \fBload_verify_locations\fP\&. The \fBverify_flags\fP option is +a list of string names of verification flags which will be set on the SSL +context. All paths are assumed to be the full path to the file or directory. +.sp +A simple setup looks like this, on the Salt Master add the \fBssl\fP option to the master configuration file: .INDENT 0.0 .INDENT 3.5 @@ -24199,67 +24672,66 @@ master configuration file: ssl: keyfile: certfile: - ssl_version: PROTOCOL_TLSv1_2 - ciphers: ECDHE\-ECDSA\-CHACHA20\-POLY1305:ECDHE\-ECDSA\-AES256\-GCM\-SHA384:ECDHE\-RSA\-AES256\-GCM\-SHA384 .ft P .fi .UNINDENT .UNINDENT .sp -The minimal \fIssl\fP option in the minion configuration file looks like this: +A more complex setup looks like this, on the Salt Master add the \fBssl\fP +option to the master\(aqs configuration file. In this example the Salt Master will +require valid client side certificates from Minions by setting \fBcert_reqs\fP to +\fBCERT_REQUIRED\fP\&. The Salt Master will also check a certificate revocation list +if one is provided in \fBverify_locations\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -ssl: True -# Versions below 2016.11.4: -ssl: {} +ssl: + keyfile: + certfile: + cert_reqs: CERT_REQUIRED + verify_locations: + \- + \- capath: + \- cafile: + verify_flags: + \- VERIFY_CRL_CHECK_CHAIN .ft P .fi .UNINDENT .UNINDENT .sp -Specific options can be sent to the minion also, as defined in the Python -\fIssl.wrap_socket\fP function. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -While setting the ssl_version is not required, we recommend it. Some older -versions of python do not support the latest TLS protocol and if this is -the case for your version of python we strongly recommend upgrading your -version of Python. Ciphers specification might be omitted, but strongly -recommended as otherwise all available ciphers will be enabled. -.UNINDENT -.UNINDENT -.SS Crypto -.sp -The current implementation uses the same crypto as the \fBzeromq\fP transport. -.SS Publish Server and Client -.sp -For the publish server and client we send messages without \(dqmessage ids\(dq which -the remote end interprets as a one\-way send. -.sp -\fBNOTE:\fP +The minimal \fIssl\fP option in the minion configuration file looks like this: .INDENT 0.0 .INDENT 3.5 -As of Salt \fI\%2016.3.0\fP, publishes using \fBlist\fP targeting are sent only to relevant minions and not broadcasted. .sp -As of Salt \fI\%3005\fP, publishes using \fBpcre\fP and \fBglob\fP targeting are also sent only to relevant minions and not broadcasted. Other targeting types are always sent to all minions and rely on minion\-side filtering. +.nf +.ft C +ssl: True +# Versions below 2016.11.4: +ssl: {} +.ft P +.fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP +A Minion can be configured to present a client certificate to the master like this: .INDENT 0.0 .INDENT 3.5 -Salt CLI defaults to \fBglob\fP targeting type, so in order to target specific minions without broadcast, you need to use \fI\-L\fP option, such as \fBsalt \-L my.minion test.ping\fP, for masters before 3005. +.sp +.nf +.ft C +ssl: + keyfile: + certfile: +.ft P +.fi .UNINDENT .UNINDENT -.SS Request Server and Client .sp -For the request server and client we send messages with a \(dqmessage id\(dq. This -\(dqmessage id\(dq allows us to multiplex messages across the socket. +Specific options can be sent to the minion also, as defined in the Python +\fIssl.wrap_socket\fP function. .SS Master Tops System .sp In 0.10.4 the \fIexternal_nodes\fP system was upgraded to allow for modular @@ -24268,6 +24740,9 @@ subsystems to be used to generate the top file data for a \fI\%highstate\fP run The old \fIexternal_nodes\fP option has been removed. The master tops system provides a pluggable and extendable replacement for it, allowing for multiple different subsystems to provide top file data. +.sp +Changed in version 3007.0: Masterless minions now support master top modules as well. + .sp Using the new \fImaster_tops\fP option is simple: .INDENT 0.0 @@ -24390,6347 +24865,6025 @@ functionality allowing a minion to treat master_tops as the single source of truth, irrespective of the top file. .UNINDENT .UNINDENT -.SS Returners +.SS Renderers .sp -By default the return values of the commands sent to the Salt minions are -returned to the Salt master, however anything at all can be done with the results -data. +The Salt state system operates by gathering information from common data types +such as lists, dictionaries, and strings that would be familiar to any +developer. .sp -By using a Salt returner, results data can be redirected to external data\-stores -for analysis and archival. +Salt Renderers translate input from the format in which it is written into +Python data structures. .sp -Returners pull their configuration values from the Salt minions. Returners are only -configured once, which is generally at load time. +The default renderer is set in the master/minion configuration file using the +\fI\%renderer\fP config option, which defaults to \fBjinja|yaml\fP\&. +.SS Two Kinds of Renderers .sp -The returner interface allows the return data to be sent to any system that -can receive data. This means that return data can be sent to a Redis server, -a MongoDB server, a MySQL server, or any system. +Renderers fall into one of two categories, based on what they output: text or +data. Some exceptions to this would be the \fI\%pure python\fP and \fI\%gpg\fP renderers which could be used in either capacity. +.SS Text Renderers .sp -\fBSEE ALSO:\fP +\fBIMPORTANT:\fP .INDENT 0.0 .INDENT 3.5 -\fI\%Full list of builtin returners\fP +\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt +takes advantage of. Other text \fI\%Renderers\fP do not support this +functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. .UNINDENT .UNINDENT -.SS Using Returners -.sp -All Salt commands will return the command data back to the master. Specifying -returners will ensure that the data is _also_ sent to the specified returner -interfaces. .sp -Specifying what returners to use is done when the command is invoked: +A text renderer returns text. These include templating engines such as +\fI\%jinja\fP, \fI\%mako\fP, and +\fI\%genshi\fP, as well as the \fI\%gpg\fP renderer. The following are all text renderers: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -salt \(aq*\(aq test.version \-\-return redis_return -.ft P -.fi -.UNINDENT +.IP \(bu 2 +\fI\%aws_kms\fP +.IP \(bu 2 +\fI\%cheetah\fP +.IP \(bu 2 +\fI\%genshi\fP +.IP \(bu 2 +\fI\%gpg\fP +.IP \(bu 2 +\fI\%jinja\fP +.IP \(bu 2 +\fI\%mako\fP +.IP \(bu 2 +\fI\%nacl\fP +.IP \(bu 2 +\fI\%pass\fP +.IP \(bu 2 +\fI\%py\fP +.IP \(bu 2 +\fI\%wempy\fP .UNINDENT +.SS Data Renderers .sp -This command will ensure that the redis_return returner is used. -.sp -It is also possible to specify multiple returners: +A data renderer returns a Python data structure (typically a dictionary). The +following are all data renderers: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -salt \(aq*\(aq test.version \-\-return mongo_return,redis_return,cassandra_return -.ft P -.fi -.UNINDENT +.IP \(bu 2 +\fI\%dson\fP +.IP \(bu 2 +\fI\%hjson\fP +.IP \(bu 2 +\fI\%json5\fP +.IP \(bu 2 +\fI\%json\fP +.IP \(bu 2 +\fI\%pydsl\fP +.IP \(bu 2 +\fI\%pyobjects\fP +.IP \(bu 2 +\fI\%py\fP +.IP \(bu 2 +\fI\%stateconf\fP +.IP \(bu 2 +\fI\%yamlex\fP +.IP \(bu 2 +\fI\%yaml\fP +.IP \(bu 2 +\fI\%gpg\fP .UNINDENT +.SS Overriding the Default Renderer .sp -In this scenario all three returners will be called and the data from the -test.version command will be sent out to the three named returners. -.SS Writing a Returner -.sp -Returners are Salt modules that allow the redirection of results data to targets other than the Salt Master. -.SS Returners Are Easy To Write! -.sp -Writing a Salt returner is straightforward. -.sp -A returner is a Python module containing at minimum a \fBreturner\fP function. -Other optional functions can be included to add support for -\fI\%master_job_cache\fP, \fI\%Storing Job Results in an External System\fP, and \fI\%Event Returners\fP\&. -.INDENT 0.0 -.TP -.B \fBreturner\fP -The \fBreturner\fP function must accept a single argument. The argument -contains return data from the called minion function. If the minion -function \fBtest.version\fP is called, the value of the argument will be a -dictionary. Run the following command from a Salt master to get a sample -of the dictionary: -.UNINDENT -.INDENT 0.0 -.INDENT 3.5 +It can sometimes be beneficial to write an SLS file using a renderer other than +the default one. This can be done by using a \(dqshebang\(dq\-like syntax on the first +line of the SLS file: .sp -.nf -.ft C -salt\-call \-\-local \-\-metadata test.version \-\-out=pprint -.ft P -.fi -.UNINDENT -.UNINDENT +Here is an example of using the \fI\%pure python\fP renderer +to install a package: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import redis -import salt.utils.json +#!py -def returner(ret): +def run(): \(dq\(dq\(dq - Return information to a redis server + Install version 1.5\-1.el7 of package \(dqpython\-foo\(dq \(dq\(dq\(dq - # Get a redis connection - serv = redis.Redis(host=\(dqredis\-serv.example.com\(dq, port=6379, db=\(dq0\(dq) - serv.sadd(\(dq%(id)s:jobs\(dq % ret, ret[\(dqjid\(dq]) - serv.set(\(dq%(jid)s:%(id)s\(dq % ret, salt.utils.json.dumps(ret[\(dqreturn\(dq])) - serv.sadd(\(dqjobs\(dq, ret[\(dqjid\(dq]) - serv.sadd(ret[\(dqjid\(dq], ret[\(dqid\(dq]) + return { + \(dqinclude\(dq: [\(dqpython\(dq], + \(dqpython\-foo\(dq: {\(dqpkg.installed\(dq: [{\(dqversion\(dq: \(dq1.5\-1.el7\(dq}]}, + } .ft P .fi .UNINDENT .UNINDENT .sp -The above example of a returner set to send the data to a Redis server -serializes the data as JSON and sets it in redis. -.SS Using Custom Returner Modules -.sp -Place custom returners in a \fB_returners/\fP directory within the -\fI\%file_roots\fP specified by the master config file. -.sp -Custom returners are distributed when any of the following are called: -.INDENT 0.0 -.IP \(bu 2 -\fI\%state.apply\fP -.IP \(bu 2 -\fI\%saltutil.sync_returners\fP -.IP \(bu 2 -\fI\%saltutil.sync_all\fP -.UNINDENT -.sp -Any custom returners which have been synced to a minion that are named the -same as one of Salt\(aqs default set of returners will take the place of the -default returner with the same name. -.SS Naming the Returner -.sp -Note that a returner\(aqs default name is its filename (i.e. \fBfoo.py\fP becomes -returner \fBfoo\fP), but that its name can be overridden by using a -\fI\%__virtual__ function\fP\&. A good example of this can be -found in the \fI\%redis\fP returner, which is named \fBredis_return.py\fP but is -loaded as simply \fBredis\fP: +This would be equivalent to the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -try: - import redis - - HAS_REDIS = True -except ImportError: - HAS_REDIS = False - -__virtualname__ = \(dqredis\(dq - +include: + \- python -def __virtual__(): - if not HAS_REDIS: - return False - return __virtualname__ +python\-foo: + pkg.installed: + \- version: \(aq1.5\-1.el7\(aq .ft P .fi .UNINDENT .UNINDENT -.SS Master Job Cache Support +.SS Composing Renderers (a.k.a. The \(dqRender Pipeline\(dq) .sp -\fI\%master_job_cache\fP, \fI\%Storing Job Results in an External System\fP, and \fI\%Event Returners\fP\&. -Salt\(aqs \fI\%master_job_cache\fP allows returners to be used as a pluggable -replacement for the \fI\%Default Job Cache\fP\&. In order to do so, a returner -must implement the following functions: +A render pipeline can be composed from other renderers by connecting them in a +series of \(dqpipes\(dq (i.e. \fB|\fP). The renderers will be evaluated from left to +right, with each renderer receiving the result of the previous renderer\(aqs +execution. .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -The code samples contained in this section were taken from the cassandra_cql -returner. -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B \fBprep_jid\fP -Ensures that job ids (jid) don\(aqt collide, unless passed_jid is provided. +Take for example the default renderer (\fBjinja|yaml\fP). The file is evaluated +first a jinja template, and the result of that template is evaluated as a YAML +document. .sp -\fBnocache\fP is an optional boolean that indicates if return data -should be cached. \fBpassed_jid\fP is a caller provided jid which should be -returned unconditionally. -.UNINDENT +Other render pipeline combinations include: .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -def prep_jid(nocache, passed_jid=None): # pylint: disable=unused\-argument - \(dq\(dq\(dq - Do any work necessary to prepare a JID, including sending a custom id - \(dq\(dq\(dq - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() -.ft P -.fi -.UNINDENT -.UNINDENT .INDENT 0.0 .TP -.B \fBsave_load\fP -Save job information. The \fBjid\fP is generated by \fBprep_jid\fP and should -be considered a unique identifier for the job. The jid, for example, could -be used as the primary/unique key in a database. The \fBload\fP is what is -returned to a Salt master by a minion. \fBminions\fP is a list of minions -that the job was run against. The following code example stores the load as -a JSON string in the salt.jids table. -.UNINDENT -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -import salt.utils.json - - -def save_load(jid, load, minions=None): - \(dq\(dq\(dq - Save the load to the specified jid id - \(dq\(dq\(dq - query = \(dq\(dq\(dqINSERT INTO salt.jids ( - jid, load - ) VALUES ( - \(aq{0}\(aq, \(aq{1}\(aq - );\(dq\(dq\(dq.format( - jid, salt.utils.json.dumps(load) - ) - - # cassandra_cql.cql_query may raise a CommandExecutionError - try: - __salt__[\(dqcassandra_cql.cql_query\(dq](query) - except CommandExecutionError: - log.critical(\(dqCould not save load in jids table.\(dq) - raise - except Exception as e: - log.critical(\(dqUnexpected error while inserting into jids: {0}\(dq.format(e)) - raise -.ft P -.fi +.B \fByaml\fP +Just YAML, no templating. +.TP +.B \fBmako|yaml\fP +This passes the input to the \fBmako\fP renderer, with its output fed into +the \fByaml\fP renderer. +.TP +.B \fBjinja|mako|yaml\fP +This one allows you to use both jinja and mako templating syntax in the +input and then parse the final rendered output as YAML. .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B \fBget_load\fP -must accept a job id (jid) and return the job load stored by \fBsave_load\fP, -or an empty dictionary when not found. .UNINDENT +.sp +The following is a contrived example SLS file using the \fBjinja|mako|yaml\fP +render pipeline: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -def get_load(jid): - \(dq\(dq\(dq - Return the load data that marks a specified jid - \(dq\(dq\(dq - query = \(dq\(dq\(dqSELECT load FROM salt.jids WHERE jid = \(aq{0}\(aq;\(dq\(dq\(dq.format(jid) - - ret = {} +#!jinja|mako|yaml - # cassandra_cql.cql_query may raise a CommandExecutionError - try: - data = __salt__[\(dqcassandra_cql.cql_query\(dq](query) - if data: - load = data[0].get(\(dqload\(dq) - if load: - ret = json.loads(load) - except CommandExecutionError: - log.critical(\(dqCould not get load from jids table.\(dq) - raise - except Exception as e: - log.critical( - \(dq\(dq\(dqUnexpected error while getting load from - jids: {0}\(dq\(dq\(dq.format( - str(e) - ) - ) - raise +An_Example: + cmd.run: + \- name: | + echo \(dqUsing Salt ${grains[\(aqsaltversion\(aq]}\(dq \e + \(dqfrom path {{grains[\(aqsaltpath\(aq]}}.\(dq + \- cwd: / - return ret +<%doc> ${...} is Mako\(aqs notation, and so is this comment. +{# Similarly, {{...}} is Jinja\(aqs notation, and so is this comment. #} .ft P .fi .UNINDENT .UNINDENT -.SS External Job Cache Support -.sp -Salt\(aqs \fI\%Storing Job Results in an External System\fP extends the \fI\%master_job_cache\fP\&. External -Job Cache support requires the following functions in addition to what is -required for Master Job Cache support: -.INDENT 0.0 -.TP -.B \fBget_jid\fP -Return a dictionary containing the information (load) returned by each -minion when the specified job id was executed. -.UNINDENT .sp -Sample: +\fBIMPORTANT:\fP .INDENT 0.0 .INDENT 3.5 +Keep in mind that not all renderers can be used alone or with any other +renderers. For example, text renderers shouldn\(aqt be used alone as their +outputs are just strings, which still need to be parsed by another renderer +to turn them into Python data structures. .sp -.nf -.ft C -{ - \(dqlocal\(dq: { - \(dqmaster_minion\(dq: { - \(dqfun_args\(dq: [], - \(dqjid\(dq: \(dq20150330121011408195\(dq, - \(dqreturn\(dq: \(dq2018.3.4\(dq, - \(dqretcode\(dq: 0, - \(dqsuccess\(dq: true, - \(dqcmd\(dq: \(dq_return\(dq, - \(dq_stamp\(dq: \(dq2015\-03\-30T12:10:12.708663\(dq, - \(dqfun\(dq: \(dqtest.version\(dq, - \(dqid\(dq: \(dqmaster_minion\(dq - } - } -} -.ft P -.fi -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B \fBget_fun\fP -Return a dictionary of minions that called a given Salt function as their -last function call. -.UNINDENT -.sp -Sample: -.INDENT 0.0 -.INDENT 3.5 +For example, it would not make sense to use \fByaml|jinja\fP because the +output of the \fI\%yaml\fP renderer is a Python data +structure, and the \fI\%jinja\fP renderer only +accepts text as input. .sp -.nf -.ft C -{ - \(dqlocal\(dq: { - \(dqminion1\(dq: \(dqtest.version\(dq, - \(dqminion3\(dq: \(dqtest.version\(dq, - \(dqminion2\(dq: \(dqtest.version\(dq - } -} -.ft P -.fi +Therefore, when combining renderers, you should know what each renderer +accepts as input and what it returns as output. One way of thinking about +it is that you can chain together multiple text renderers, but the pipeline +\fImust\fP end in a data renderer. Similarly, since the text renderers in Salt +don\(aqt accept data structures as input, a text renderer should usually not +come after a data renderer. It\(aqs technically \fIpossible\fP to write a renderer +that takes a data structure as input and returns a string, but no such +renderer is distributed with Salt. .UNINDENT .UNINDENT +.SS Writing Renderers +.sp +A custom renderer must be a Python module which implements a \fBrender\fP +function. This function must implement three positional arguments: .INDENT 0.0 -.TP -.B \fBget_jids\fP -Return a list of all job ids. +.IP 1. 3 +\fBdata\fP \- Can be called whatever you like. This is the input to be +rendered. +.IP 2. 3 +\fBsaltenv\fP +.IP 3. 3 +\fBsls\fP .UNINDENT .sp -Sample: +The first is the important one, and the 2nd and 3rd must be included since Salt +needs to pass this info to each render, even though it is only used by template +renderers. +.sp +Renderers should be written so that the \fBdata\fP argument can accept either +strings or file\-like objects as input. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{ - \(dqlocal\(dq: [ - \(dq20150330121011408195\(dq, - \(dq20150330195922139916\(dq - ] -} +import mycoolmodule +from salt.ext import six + + +def render(data, saltenv=\(dqbase\(dq, sls=\(dq\(dq, **kwargs): + if not isinstance(data, six.string_types): + # Read from file\-like object + data = data.read() + + return mycoolmodule.do_something(data) .ft P .fi .UNINDENT .UNINDENT +.sp +Custom renderers should be placed within \fBsalt://_renderers/\fP, so that they +can be synced to minions. They are synced when any of the following are run: .INDENT 0.0 -.TP -.B \fBget_minions\fP -Returns a list of minions +.IP \(bu 2 +\fI\%state.apply\fP +.IP \(bu 2 +\fI\%saltutil.sync_renderers\fP +.IP \(bu 2 +\fI\%saltutil.sync_all\fP .UNINDENT .sp -Sample: +Any custom renderers which have been synced to a minion, that are named the +same as one of Salt\(aqs default set of renderers, will take the place of the +default renderer with the same name. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{ - \(dqlocal\(dq: [ - \(dqminion3\(dq, - \(dqminion2\(dq, - \(dqminion1\(dq, - \(dqmaster_minion\(dq - ] -} -.ft P -.fi +Renderers can also be synced from \fBsalt://_renderers/\fP to the Master +using either the \fI\%saltutil.sync_renderers\fP or \fI\%saltutil.sync_all\fP runner function. .UNINDENT .UNINDENT +.SS Examples .sp -Please refer to one or more of the existing returners (i.e. mysql, -cassandra_cql) if you need further clarification. -.SS Event Support +The best place to find examples of renderers is in the Salt source code. .sp -An \fBevent_return\fP function must be added to the returner module to allow -events to be logged from a master via the returner. A list of events are passed -to the function by the master. +Documentation for renderers included with Salt can be found here: .sp -The following example was taken from the MySQL returner. In this example, each -event is inserted into the salt_events table keyed on the event tag. The tag -contains the jid and therefore is guaranteed to be unique. +\fI\%salt/renderers\fP +.sp +Here is a simple YAML renderer example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import salt.utils.json - +import salt.utils.yaml +from salt.utils.yamlloader import SaltYamlSafeLoader +from salt.ext import six -def event_return(events): - \(dq\(dq\(dq - Return event to mysql server - Requires that configuration be enabled via \(aqevent_return\(aq - option in master config. - \(dq\(dq\(dq - with _get_serv(events, commit=True) as cur: - for event in events: - tag = event.get(\(dqtag\(dq, \(dq\(dq) - data = event.get(\(dqdata\(dq, \(dq\(dq) - sql = \(dq\(dq\(dqINSERT INTO \(gasalt_events\(ga (\(gatag\(ga, \(gadata\(ga, \(gamaster_id\(ga ) - VALUES (%s, %s, %s)\(dq\(dq\(dq - cur.execute(sql, (tag, salt.utils.json.dumps(data), __opts__[\(dqid\(dq])) -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Testing the Returner -.sp -The \fBreturner\fP, \fBprep_jid\fP, \fBsave_load\fP, \fBget_load\fP, and -\fBevent_return\fP functions can be tested by configuring the -\fI\%master_job_cache\fP and \fI\%Event Returners\fP in the master config -file and submitting a job to \fBtest.version\fP each minion from the master. -.sp -Once you have successfully exercised the Master Job Cache functions, test the -External Job Cache functions using the \fBret\fP execution module. -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -salt\-call ret.get_jids cassandra_cql \-\-output=json -salt\-call ret.get_fun cassandra_cql test.version \-\-output=json -salt\-call ret.get_minions cassandra_cql \-\-output=json -salt\-call ret.get_jid cassandra_cql 20150330121011408195 \-\-output=json +def render(yaml_data, saltenv=\(dq\(dq, sls=\(dq\(dq, **kws): + if not isinstance(yaml_data, six.string_types): + yaml_data = yaml_data.read() + data = salt.utils.yaml.safe_load(yaml_data) + return data if data else {} .ft P .fi .UNINDENT .UNINDENT -.SS Event Returners -.sp -For maximum visibility into the history of events across a Salt -infrastructure, all events seen by a salt master may be logged to one or -more returners. -.sp -To enable event logging, set the \fBevent_return\fP configuration option in the -master config to the returner(s) which should be designated as the handler -for event returns. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Not all returners support event returns. Verify a returner has an -\fBevent_return()\fP function before using. -.UNINDENT -.UNINDENT +.SS Full List of Renderers +.SS renderer modules .sp -\fBNOTE:\fP +\fBIMPORTANT:\fP .INDENT 0.0 .INDENT 3.5 -On larger installations, many hundreds of events may be generated on a -busy master every second. Be certain to closely monitor the storage of -a given returner as Salt can easily overwhelm an underpowered server -with thousands of returns. +\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt +takes advantage of. Other text \fI\%Renderers\fP do not support this +functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. .UNINDENT .UNINDENT -.SS Full List of Returners -.SS returner modules .TS center; |l|l|. _ T{ -\fI\%appoptics_return\fP -T} T{ -Salt returner to return highstate stats to AppOptics Metrics -T} -_ -T{ -\fI\%carbon_return\fP -T} T{ -Take data from salt and \(dqreturn\(dq it into a carbon receiver -T} -_ -T{ -\fI\%cassandra_cql_return\fP -T} T{ -Return data to a cassandra server -T} -_ -T{ -\fI\%couchbase_return\fP -T} T{ -Simple returner for Couchbase. -T} -_ -T{ -\fI\%couchdb_return\fP -T} T{ -Simple returner for CouchDB. -T} -_ -T{ -\fI\%elasticsearch_return\fP -T} T{ -Return data to an elasticsearch server for indexing. -T} -_ -T{ -\fI\%etcd_return\fP -T} T{ -Return data to an etcd server or cluster -T} -_ -T{ -\fI\%highstate_return\fP -T} T{ -Return the results of a highstate (or any other state function that returns data in a compatible format) via an HTML email or HTML file. -T} -_ -T{ -\fI\%influxdb_return\fP -T} T{ -Return data to an influxdb server. -T} -_ -T{ -\fI\%kafka_return\fP -T} T{ -Return data to a Kafka topic -T} -_ -T{ -\fI\%librato_return\fP -T} T{ -Salt returner to return highstate stats to Librato -T} -_ -T{ -\fI\%local\fP -T} T{ -The local returner is used to test the returner interface, it just prints the return data to the console to verify that it is being passed properly -T} -_ -T{ -\fI\%local_cache\fP -T} T{ -Return data to local job cache -T} -_ -T{ -\fI\%mattermost_returner\fP -T} T{ -Return salt data via mattermost -T} -_ -T{ -\fI\%memcache_return\fP -T} T{ -Return data to a memcache server -T} -_ -T{ -\fI\%mongo_future_return\fP -T} T{ -Return data to a mongodb server -T} -_ -T{ -\fI\%mongo_return\fP -T} T{ -Return data to a mongodb server -T} -_ -T{ -\fI\%multi_returner\fP +\fI\%aws_kms\fP T} T{ -Read/Write multiple returners T} _ T{ -\fI\%mysql\fP +\fI\%cheetah\fP T} T{ -Return data to a mysql server +Cheetah Renderer for Salt T} _ T{ -\fI\%nagios_nrdp_return\fP +\fI\%dson\fP T} T{ -Return salt data to Nagios +DSON Renderer for Salt T} _ T{ -\fI\%odbc\fP +\fI\%genshi\fP T} T{ -Return data to an ODBC compliant server. +Genshi Renderer for Salt T} _ T{ -\fI\%pgjsonb\fP +\fI\%gpg\fP T} T{ -Return data to a PostgreSQL server with json data stored in Pg\(aqs jsonb data type +Renderer that will decrypt GPG ciphers T} _ T{ -\fI\%postgres\fP +\fI\%hjson\fP T} T{ -Return data to a postgresql server +hjson renderer for Salt T} _ T{ -\fI\%postgres_local_cache\fP +\fI\%jinja\fP T} T{ -Use a postgresql server for the master job cache. +Jinja loading utils to enable a more powerful backend for jinja templates T} _ T{ -\fI\%pushover_returner\fP +\fI\%json\fP T} T{ -Return salt data via pushover (\fI\%http://www.pushover.net\fP) +JSON Renderer for Salt T} _ T{ -\fI\%rawfile_json\fP +\fI\%json5\fP T} T{ -Take data from salt and \(dqreturn\(dq it into a raw file containing the json, with one line per event. +JSON5 Renderer for Salt T} _ T{ -\fI\%redis_return\fP +\fI\%mako\fP T} T{ -Return data to a redis server +Mako Renderer for Salt T} _ T{ -\fI\%sentry_return\fP +\fI\%msgpack\fP T} T{ -Salt returner that reports execution results back to sentry. T} _ T{ -\fI\%slack_returner\fP +\fI\%nacl\fP T} T{ -Return salt data via slack +Renderer that will decrypt NACL ciphers T} _ T{ -\fI\%slack_webhook_return\fP +\fI\%pass\fP T} T{ -Return salt data via Slack using Incoming Webhooks +Pass Renderer for Salt T} _ T{ -\fI\%sms_return\fP +\fI\%py\fP T} T{ -Return data by SMS. +Pure python state renderer T} _ T{ -\fI\%smtp_return\fP +\fI\%pydsl\fP T} T{ -Return salt data via email +A Python\-based DSL T} _ T{ -\fI\%splunk\fP +\fI\%pyobjects\fP T} T{ -Send json response data to Splunk via the HTTP Event Collector Requires the following config values to be specified in config or pillar: +Python renderer that includes a Pythonic Object based interface T} _ T{ -\fI\%sqlite3_return\fP +\fI\%stateconf\fP T} T{ -Insert minion return data into a sqlite3 database +A flexible renderer that takes a templating engine and a data format T} _ T{ -\fI\%syslog_return\fP +\fI\%tomlmod\fP T} T{ -Return data to the host operating system\(aqs syslog facility T} _ T{ -\fI\%telegram_return\fP +\fI\%wempy\fP T} T{ -Return salt data via Telegram. T} _ T{ -\fI\%xmpp_return\fP +\fI\%yaml\fP T} T{ -Return salt data via xmpp +YAML Renderer for Salt T} _ T{ -\fI\%zabbix_return\fP +\fI\%yamlex\fP T} T{ -Return salt data to Zabbix T} _ .TE -.SS salt.returners.appoptics_return +.SS salt.renderers.aws_kms .sp -Salt returner to return highstate stats to AppOptics Metrics +Renderer that will decrypt ciphers encrypted using \fI\%AWS KMS Envelope Encryption\fP\&. .sp -To enable this returner the minion will need the AppOptics Metrics -client importable on the Python path and the following -values configured in the minion or master config. +Any key in the data to be rendered can be a urlsafe_b64encoded string, and this renderer will attempt +to decrypt it before passing it off to Salt. This allows you to safely store secrets in +source control, in such a way that only your Salt master can decrypt them and +distribute them only to the minions that need them. .sp -The AppOptics python client can be found at: +The typical use\-case would be to use ciphers in your pillar data, and keep the encrypted +data key on your master. This way developers with appropriate AWS IAM privileges can add new secrets +quickly and easily. .sp -\fI\%https://github.com/appoptics/python\-appoptics\-metrics\fP +This renderer requires the \fI\%boto3\fP Python library. +.SS Setup +.sp +First, set up your AWS client. For complete instructions on configuration the AWS client, +please read the \fI\%boto3 configuration documentation\fP\&. By default, this renderer will use +the default AWS profile. You can override the profile name in salt configuration. +For example, if you have a profile in your aws client configuration named \(dqsalt\(dq, +you can add the following salt configuration: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -appoptics.api_token: abc12345def +aws_kms: + profile_name: salt .ft P .fi .UNINDENT .UNINDENT .sp -An example configuration that returns the total number of successes -and failures for your salt highstate runs (the default) would look -like this: +The rest of these instructions assume that you will use the default profile for key generation +and setup. If not, export AWS_PROFILE and set it to the desired value. +.sp +Once the aws client is configured, generate a KMS customer master key and use that to generate +a local data key. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -return: appoptics -appoptics.api_token: +# data_key=$(aws kms generate\-data\-key \-\-key\-id your\-key\-id \-\-key\-spec AES_256 + \-\-query \(aqCiphertextBlob\(aq \-\-output text) +# echo \(aqaws_kms:\(aq +# echo \(aq data_key: !!binary \(dq%s\(dq\en\(aq \(dq$data_key\(dq >> config/master .ft P .fi .UNINDENT .UNINDENT .sp -The returner publishes the following metrics to AppOptics: -.INDENT 0.0 -.IP \(bu 2 -saltstack.failed -.IP \(bu 2 -saltstack.passed -.IP \(bu 2 -saltstack.retcode -.IP \(bu 2 -saltstack.runtime -.IP \(bu 2 -saltstack.total -.UNINDENT -.sp -You can add a tags section to specify which tags should be attached to -all metrics created by the returner. +To apply the renderer on a file\-by\-file basis add the following line to the +top of any pillar with gpg data in it: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -appoptics.tags: - host_hostname_alias: - tier: - cluster: +#!yaml|aws_kms .ft P .fi .UNINDENT .UNINDENT .sp -If no tags are explicitly configured, then the tag key \fBhost_hostname_alias\fP -will be set, with the minion\(aqs \fBid\fP grain being the value. -.sp -In addition to the requested tags, for a highstate run each of these -will be tagged with the \fBkey:value\fP of \fBstate_type: highstate\fP\&. -.sp -In order to return metrics for \fBstate.sls\fP runs (distinct from highstates), you can -specify a list of state names to the key \fBappoptics.sls_states\fP like so: +Now with your renderer configured, you can include your ciphers in your pillar +data like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -appoptics.sls_states: - \- role_salt_master.netapi - \- role_redis.config - \- role_smarty.dummy +#!yaml|aws_kms + +a\-secret: gAAAAABaj5uzShPI3PEz6nL5Vhk2eEHxGXSZj8g71B84CZsVjAAtDFY1mfjNRl\-1Su9YVvkUzNjI4lHCJJfXqdcTvwczBYtKy0Pa7Ri02s10Wn1tF0tbRwk= .ft P .fi .UNINDENT .UNINDENT -.sp -This will report success and failure counts on runs of the -\fBrole_salt_master.netapi\fP, \fBrole_redis.config\fP, and -\fBrole_smarty.dummy\fP states in addition to highstates. -.sp -This will report the same metrics as above, but for these runs the -metrics will be tagged with \fBstate_type: sls\fP and \fBstate_name\fP set to -the name of the state that was invoked, e.g. \fBrole_salt_master.netapi\fP\&. .INDENT 0.0 .TP -.B salt.returners.appoptics_return.returner(ret) -Parse the return data and return metrics to AppOptics. +.B salt.renderers.aws_kms.render(data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) +Decrypt the data to be rendered that was encrypted using AWS KMS envelope encryption. +.UNINDENT +.SS salt.renderers.cheetah .sp -For each state that\(aqs provided in the configuration, return tagged metrics for -the result of that state if it\(aqs present. +Cheetah Renderer for Salt +.INDENT 0.0 +.TP +.B salt.renderers.cheetah.render(cheetah_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, method=\(aqxml\(aq, **kws) +Render a Cheetah template. +.INDENT 7.0 +.TP +.B Return type +A Python data structure .UNINDENT -.SS salt.returners.carbon_return +.UNINDENT +.SS salt.renderers.dson .sp -Take data from salt and \(dqreturn\(dq it into a carbon receiver +DSON Renderer for Salt .sp -Add the following configuration to the minion configuration file: +This renderer is intended for demonstration purposes. Information on the DSON +spec can be found \fI\%here\fP\&. +.sp +This renderer requires \fI\%Dogeon\fP (installable via pip) .INDENT 0.0 -.INDENT 3.5 +.TP +.B salt.renderers.dson.render(dson_input, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kwargs) +Accepts DSON data as a string or as a file object and runs it through the +JSON parser. +.INDENT 7.0 +.TP +.B Return type +A Python data structure +.UNINDENT +.UNINDENT +.SS salt.renderers.genshi .sp -.nf -.ft C -carbon.host: -carbon.port: 2003 -.ft P -.fi +Genshi Renderer for Salt +.INDENT 0.0 +.TP +.B salt.renderers.genshi.render(genshi_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, method=\(aqxml\(aq, **kws) +Render a Genshi template. A method should be passed in as part of the +kwargs. If no method is passed in, xml is assumed. Valid methods are: +.sp +Note that the \fBtext\fP method will call \fBNewTextTemplate\fP\&. If \fBoldtext\fP +is desired, it must be called explicitly +.INDENT 7.0 +.TP +.B Return type +A Python data structure .UNINDENT .UNINDENT +.SS salt.renderers.gpg .sp -Errors when trying to convert data to numbers may be ignored by setting -\fBcarbon.skip_on_error\fP to \fITrue\fP: +Renderer that will decrypt GPG ciphers +.sp +Any value in the SLS file can be a GPG cipher, and this renderer will decrypt it +before passing it off to Salt. This allows you to safely store secrets in +source control, in such a way that only your Salt master can decrypt them and +distribute them only to the minions that need them. +.sp +The typical use\-case would be to use ciphers in your pillar data, and keep a +secret key on your master. You can put the public key in source control so that +developers can add new secrets quickly and easily. +.sp +This renderer requires the \fI\%gpg\fP binary. No python libraries are required as of +the 2015.8.0 release. +.SS GPG Homedir +.sp +The default \fIGPG Homedir \fP is \fB~/.gnupg\fP and needs to be set using +\fBgpg \-\-homedir\fP\&. Be very careful to not forget this option. It is also important +to run \fBgpg\fP commands as the user that owns the keys directory. If the salt\-master +runs as user \fBsalt\fP, then use \fBsu \- salt\fP before running any gpg commands. +.sp +In some cases, it\(aqs preferable to have gpg keys stored on removable media or +other non\-standard locations. This can be done using the \fBgpg_keydir\fP option +on the salt master. This will also require using a different path to \fB\-\-homedir\fP\&. +.sp +The \fB\-\-homedir\fP argument can be configured for the current user using +\fBecho \(aqhomedir /etc/salt/gpgkeys\(aq >> ~/.gnupg\fP, but this should be used with +caution to avoid potential confusion. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -carbon.skip_on_error: True +gpg_keydir: .ft P .fi .UNINDENT .UNINDENT +.SS GPG Keys .sp -By default, data will be sent to carbon using the plaintext protocol. To use -the pickle protocol, set \fBcarbon.mode\fP to \fBpickle\fP: +GPG key pairs include both a public and private key. The private key is akin to +a password and should be kept secure by the owner. A public key is used to +encrypt data being sent to the owner of the private key. +.sp +This means that the public key will be freely distributed so that others can +encrypt pillar data without access to the secret key. +.SS New Key Pair +.sp +To create a new GPG key pair for encrypting data, log in to the master as root +and run the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -carbon.mode: pickle +# mkdir \-p /etc/salt/gpgkeys +# chmod 0700 /etc/salt/gpgkeys +# gpg \-\-homedir /etc/salt/gpgkeys \-\-gen\-key .ft P .fi .UNINDENT .UNINDENT +.sp +Do not supply a password for the keypair and use a name that makes sense for +your application. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B You can also specify the pattern used for the metric base path (except for virt modules metrics): -carbon.metric_base_pattern: carbon.[minion_id].[module].[function] -.TP -.B These tokens can used : -[module]: salt module -[function]: salt function -[minion_id]: minion id -.TP -.B Default is : -carbon.metric_base_pattern: [module].[function].[minion_id] +.INDENT 3.5 +In some situations, gpg may be starved of entropy and will take an incredibly +long time to finish. Two common tools to generate (less secure) pseudo\-random +data are \fBrng\-tools\fP and \fBhaveged\fP\&. +.UNINDENT .UNINDENT .sp -Carbon settings may also be configured as: +The new keys can be seen and verified using \fB\-\-list\-secret\-keys\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -carbon: - host: - port: - skip_on_error: True - mode: (pickle|text) - metric_base_pattern: | [module].[function].[minion_id] +# gpg \-\-homedir /etc/salt/gpgkeys \-\-list\-secret\-keys +/etc/salt/gpgkeys/pubring.kbx +\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- +sec rsa4096 2002\-05\-12 [SC] [expires: 2012\-05\-10] + 2DC47B416EE8C3484450B450A4D44406274AF44E +uid [ultimate] salt\-master (gpg key for salt) +ssb rsa4096 2002\-05\-12 [E] [expires: 2012\-05\-10] .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +In the example above, our KEY\-ID is \fB2DC47B416EE8C3484450B450A4D44406274AF44E\fP\&. +.SS Export Public Key +.sp +To export a public key suitable for public distribution: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.carbon: - host: - port: - skip_on_error: True - mode: (pickle|text) +# gpg \-\-homedir /etc/salt/gpgkeys \-\-armor \-\-export > exported_pubkey.asc .ft P .fi .UNINDENT .UNINDENT +.SS Import Public Key .sp -To use the carbon returner, append \(aq\-\-return carbon\(aq to the salt command. +Users wishing to import the public key into their local keychain may run: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return carbon +$ gpg \-\-import exported_pubkey.asc .ft P .fi .UNINDENT .UNINDENT +.SS Export (Save) Private Key .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. -.sp -New in version 2015.5.0. - +This key protects all gpg\-encrypted pillar data and should be backed up to a +safe and secure location. This command will generate a backup of secret keys +in the \fB/etc/salt/gpgkeys\fP directory to the \fBgpgkeys.secret\fP file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return carbon \-\-return_config alternative +# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-keys \-\-export\-options export\-backup \-o gpgkeys.secret .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +Salt does not support password\-protected private keys, which means this file +is essentially a clear\-text password (just add \fB\-\-armor\fP). Fortunately, it +is trivial to pass this export back to gpg to be encrypted with symmetric key: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return carbon \-\-return_kwargs \(aq{\(dqskip_on_error\(dq: False}\(aq +# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-keys \-\-export\-options export\-backup | gpg \-\-symmetric \-o gpgkeys.gpg .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.carbon_return.event_return(events) -Return event data to remote carbon server .sp -Provide a list of events to be stored in carbon -.UNINDENT +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.carbon_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.INDENT 3.5 +In some cases, particularly when using su/sudo, gpg gets confused and needs +to be told which TTY to use; this can be done with: \fBexport GPG_TTY=$(tty)\fP\&. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.carbon_return.returner(ret) -Return data to a remote carbon server using the text metric protocol +.UNINDENT +.SS Import (Restore) Private Key .sp -Each metric will look like: -.INDENT 7.0 +To import/restore a private key, create a directory with the correct permissions +and import using gpg. +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[module].[function].[minion_id].[metric path [...]].[metric name] +# mkdir \-p /etc/salt/gpgkeys +# chmod 0700 /etc/salt/gpgkeys +# gpg \-\-homedir /etc/salt/gpgkeys \-\-import gpgkeys.secret .ft P .fi .UNINDENT .UNINDENT -.UNINDENT -.SS salt.returners.cassandra_cql_return -.sp -Return data to a cassandra server .sp -New in version 2015.5.0. - +If the export was encrypted using a symmetric key, then decrypt first with: .INDENT 0.0 -.TP -.B maintainer -Corin Kochenower<\fI\%ckochenower@saltstack.com\fP> -.TP -.B maturity -new as of 2015.2 -.TP -.B depends -salt.modules.cassandra_cql -.TP -.B depends -DataStax Python Driver for Apache Cassandra -\fI\%https://github.com/datastax/python\-driver\fP -pip install cassandra\-driver -.TP -.B platform -all -.TP -.B configuration -To enable this returner, the minion will need the DataStax Python Driver -for Apache Cassandra ( \fI\%https://github.com/datastax/python\-driver\fP ) -installed and the following values configured in the minion or master -config. The list of cluster IPs must include at least one cassandra node -IP address. No assumption or default will be used for the cluster IPs. -The cluster IPs will be tried in the order listed. The port, username, -and password values shown below will be the assumed defaults if you do -not provide values.: -.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -cassandra: - cluster: - \- 192.168.50.11 - \- 192.168.50.12 - \- 192.168.50.13 - port: 9042 - username: salt - password: salt +# gpg \-\-decrypt gpgkeys.gpg | gpg \-\-homedir /etc/salt/gpgkeys \-\-import .ft P .fi .UNINDENT .UNINDENT +.SS Adjust trust level of imported keys .sp -Use the following cassandra database schema: -.INDENT 7.0 +In some cases, importing existing keys may not be enough and the trust level of +the key needs to be adjusted. This can be done by editing the key. The \fBKEY\-ID\fP +and the actual trust level of the key can be seen by listing the already imported +keys. +.sp +If the trust\-level is not \fBultimate\fP it needs to be changed by running +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -CREATE KEYSPACE IF NOT EXISTS salt - WITH replication = {\(aqclass\(aq: \(aqSimpleStrategy\(aq, \(aqreplication_factor\(aq : 1}; - -CREATE USER IF NOT EXISTS salt WITH PASSWORD \(aqsalt\(aq NOSUPERUSER; - -GRANT ALL ON KEYSPACE salt TO salt; - -USE salt; - -CREATE TABLE IF NOT EXISTS salt.salt_returns ( - jid text, - minion_id text, - fun text, - alter_time timestamp, - full_ret text, - return text, - success boolean, - PRIMARY KEY (jid, minion_id, fun) -) WITH CLUSTERING ORDER BY (minion_id ASC, fun ASC); -CREATE INDEX IF NOT EXISTS salt_returns_minion_id ON salt.salt_returns (minion_id); -CREATE INDEX IF NOT EXISTS salt_returns_fun ON salt.salt_returns (fun); - -CREATE TABLE IF NOT EXISTS salt.jids ( - jid text PRIMARY KEY, - load text -); - -CREATE TABLE IF NOT EXISTS salt.minions ( - minion_id text PRIMARY KEY, - last_fun text -); -CREATE INDEX IF NOT EXISTS minions_last_fun ON salt.minions (last_fun); - -CREATE TABLE IF NOT EXISTS salt.salt_events ( - id timeuuid, - tag text, - alter_time timestamp, - data text, - master_id text, - PRIMARY KEY (id, tag) -) WITH CLUSTERING ORDER BY (tag ASC); -CREATE INDEX tag ON salt.salt_events (tag); +gpg \-\-homedir /etc/salt/gpgkeys \-\-edit\-key .ft P .fi .UNINDENT .UNINDENT -.UNINDENT .sp -Required python modules: cassandra\-driver +This will open an interactive shell for the management of the GPG encryption key. +Type \fBtrust\fP to be able to set the trust level for the key and then select \fB5 +(I trust ultimately)\fP\&. Then quit the shell by typing \fBsave\fP\&. +.SS Encrypting Data .sp -To use the cassandra returner, append \(aq\-\-return cassandra_cql\(aq to the salt command. ex: +In order to encrypt data to a recipient (salt), the public key must be imported +into the local keyring. Importing the public key is described above in the +\fIImport Public Key \fP section. +.sp +To generate a cipher from a secret: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return_cql cassandra +$ echo \-n \(aqsupersecret\(aq | gpg \-\-trust\-model always \-ear .ft P .fi .UNINDENT .UNINDENT .sp -Note: if your Cassandra instance has not been tuned much you may benefit from -altering some timeouts in \fIcassandra.yaml\fP like so: +To apply the renderer on a file\-by\-file basis add the following line to the +top of any pillar with gpg data in it: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 -# How long the coordinator should wait for seq or index scans to complete -range_request_timeout_in_ms: 20000 -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 20000 -# How long the coordinator should wait for counter writes to complete -counter_write_request_timeout_in_ms: 10000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 5000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -request_timeout_in_ms: 20000 +#!yaml|gpg .ft P .fi .UNINDENT .UNINDENT .sp -As always, your mileage may vary and your Cassandra cluster may have different -needs. SaltStack has seen situations where these timeouts can resolve -some stacktraces that appear to come from the Datastax Python driver. -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.event_return(events) -Return event to one of potentially many clustered cassandra nodes -.sp -Requires that configuration be enabled via \(aqevent_return\(aq -option in master config. -.sp -Cassandra does not support an auto\-increment feature due to the -highly inefficient nature of creating a monotonically increasing -number across all nodes in a distributed database. Each event -will be assigned a uuid by the connecting client. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.prep_jid(nocache, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.returner(ret) -Return data to one of potentially many clustered cassandra nodes -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.save_load(jid, load, minions=None) -Save the load to the specified jid id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.cassandra_cql_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.couchbase_return -.sp -Simple returner for Couchbase. Optional configuration -settings are listed below, along with sane defaults. +Now with your renderer configured, you can include your ciphers in your pillar +data like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -couchbase.host: \(aqsalt\(aq -couchbase.port: 8091 -couchbase.bucket: \(aqsalt\(aq -couchbase.ttl: 86400 -couchbase.password: \(aqpassword\(aq -couchbase.skip_verify_views: False +#!yaml|gpg + +a\-secret: | + \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- + Version: GnuPG v1 + + hQEMAweRHKaPCfNeAQf9GLTN16hCfXAbPwU6BbBK0unOc7i9/etGuVc5CyU9Q6um + QuetdvQVLFO/HkrC4lgeNQdM6D9E8PKonMlgJPyUvC8ggxhj0/IPFEKmrsnv2k6+ + cnEfmVexS7o/U1VOVjoyUeliMCJlAz/30RXaME49Cpi6No2+vKD8a4q4nZN1UZcG + RhkhC0S22zNxOXQ38TBkmtJcqxnqT6YWKTUsjVubW3bVC+u2HGqJHu79wmwuN8tz + m4wBkfCAd8Eyo2jEnWQcM4TcXiF01XPL4z4g1/9AAxh+Q4d8RIRP4fbw7ct4nCJv + Gr9v2DTF7HNigIMl4ivMIn9fp+EZurJNiQskLgNbktJGAeEKYkqX5iCuB1b693hJ + FKlwHiJt5yA8X2dDtfk8/Ph1Jx2TwGS+lGjlZaNqp3R1xuAZzXzZMLyZDe5+i3RJ + skqmFTbOiA===Eqsm + \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- .ft P .fi .UNINDENT .UNINDENT +.SS Encrypted CLI Pillar Data .sp -To use the couchbase returner, append \(aq\-\-return couchbase\(aq to the salt command. ex: +New in version 2016.3.0. + +.sp +Functions like \fI\%state.highstate\fP and +\fI\%state.sls\fP allow for pillar data to be +passed on the CLI. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchbase +salt myminion state.highstate pillar=\(dq{\(aqmypillar\(aq: \(aqfoo\(aq}\(dq .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Starting with the 2016.3.0 release of Salt, it is now possible for this pillar +data to be GPG\-encrypted, and to use the GPG renderer to decrypt it. +.SS Replacing Newlines .sp -New in version 2015.5.0. - +To pass encrypted pillar data on the CLI, the ciphertext must have its newlines +replaced with a literal backslash\-n (\fB\en\fP), as newlines are not supported +within Salt CLI arguments. There are a number of ways to do this: +.sp +With awk or Perl: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchbase \-\-return_config alternative +# awk +ciphertext=\(gaecho \-n \(dqsupersecret\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | awk \(aq{printf \(dq%s\e\en\(dq,$0} END {print \(dq\(dq}\(aq\(ga +# Perl +ciphertext=\(gaecho \-n \(dqsupersecret\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | perl \-pe \(aqs/\en/\e\en/g\(aq\(ga .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +With Python: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchbase \-\-return_kwargs \(aq{\(dqbucket\(dq: \(dqanother\-salt\(dq}\(aq +import subprocess + +secret, stderr = subprocess.Popen( + [\(aqgpg\(aq, \(aq\-\-armor\(aq, \(aq\-\-batch\(aq, \(aq\-\-trust\-model\(aq, \(aqalways\(aq, \(aq\-\-encrypt\(aq, + \(aq\-r\(aq, \(aquser@domain.com\(aq], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate(input=\(aqsupersecret\(aq) + +if secret: + print(secret.replace(\(aq\en\(aq, r\(aq\en\(aq)) +else: + raise ValueError(\(aqNo ciphertext found: {0}\(aq.format(stderr)) .ft P .fi .UNINDENT .UNINDENT -.sp -All of the return data will be stored in documents as follows: -.SS JID -.sp -load: load obj -tgt_minions: list of minions targeted -nocache: should we not cache the return data -.SS JID/MINION_ID -.sp -return: return_data -full_ret: full load of job return -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.prep_jid(nocache=False, passed_jid=None) -Return a job id and prepare the job id directory -This is the function responsible for making sure jids don\(aqt collide (unless -its passed a jid) -So do what you have to do to make sure that stays the case -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.returner(load) -Return data to couchbase bucket -.UNINDENT .INDENT 0.0 -.TP -.B salt.returners.couchbase_return.save_load(jid, clear_load, minion=None) -Save the load to the specified jid +.INDENT 3.5 +.sp +.nf +.ft C +ciphertext=\(gapython /path/to/script.py\(ga +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchbase_return.save_minions(jid, minions, syndic_id=None) -Save/update the minion list for a given jid. The syndic_id argument is -included for API compatibility only. .UNINDENT -.SS salt.returners.couchdb_return .sp -Simple returner for CouchDB. Optional configuration -settings are listed below, along with sane defaults: +The ciphertext can be included in the CLI pillar data like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -couchdb.db: \(aqsalt\(aq -couchdb.url: \(aqhttp://salt:5984/\(aq +salt myminion state.sls secretstuff pillar_enc=gpg pillar=\(dq{secret_pillar: \(aq$ciphertext\(aq}\(dq .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +The \fBpillar_enc=gpg\fP argument tells Salt that there is GPG\-encrypted pillar +data, so that the CLI pillar data is passed through the GPG renderer, which +will iterate recursively though the CLI pillar dictionary to decrypt any +encrypted values. +.SS Encrypting the Entire CLI Pillar Dictionary +.sp +If several values need to be encrypted, it may be more convenient to encrypt +the entire CLI pillar dictionary. Again, this can be done in several ways: +.sp +With awk or Perl: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.couchdb.db: \(aqsalt\(aq -alternative.couchdb.url: \(aqhttp://salt:5984/\(aq +# awk +ciphertext=\(gaecho \-n \(dq{\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, \(aqsecret_b\(aq: \(aqGPG is fun!\(aq}\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | awk \(aq{printf \(dq%s\e\en\(dq,$0} END {print \(dq\(dq}\(aq\(ga +# Perl +ciphertext=\(gaecho \-n \(dq{\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, \(aqsecret_b\(aq: \(aqGPG is fun!\(aq}\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | perl \-pe \(aqs/\en/\e\en/g\(aq\(ga .ft P .fi .UNINDENT .UNINDENT .sp -To use the couchdb returner, append \fB\-\-return couchdb\fP to the salt command. Example: +With Python: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchdb +import subprocess + +pillar_data = {\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, + \(aqsecret_b\(aq: \(aqGPG is fun!\(aq} + +secret, stderr = subprocess.Popen( + [\(aqgpg\(aq, \(aq\-\-armor\(aq, \(aq\-\-batch\(aq, \(aq\-\-trust\-model\(aq, \(aqalways\(aq, \(aq\-\-encrypt\(aq, + \(aq\-r\(aq, \(aquser@domain.com\(aq], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate(input=repr(pillar_data)) + +if secret: + print(secret.replace(\(aq\en\(aq, r\(aq\en\(aq)) +else: + raise ValueError(\(aqNo ciphertext found: {0}\(aq.format(stderr)) .ft P .fi .UNINDENT .UNINDENT -.sp -To use the alternative configuration, append \fB\-\-return_config alternative\fP to the salt command. -.sp -New in version 2015.5.0. - .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchdb \-\-return_config alternative +ciphertext=\(gapython /path/to/script.py\(ga .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +With the entire pillar dictionary now encrypted, it can be included in the CLI +pillar data like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return couchdb \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +salt myminion state.sls secretstuff pillar_enc=gpg pillar=\(dq$ciphertext\(dq .ft P .fi .UNINDENT .UNINDENT -.SS On concurrent database access +.SS Configuration .sp -As this returner creates a couchdb document with the salt job id as document id -and as only one document with a given id can exist in a given couchdb database, -it is advised for most setups that every minion be configured to write to it own -database (the value of \fBcouchdb.db\fP may be suffixed with the minion id), -otherwise multi\-minion targeting can lead to losing output: -.INDENT 0.0 -.IP \(bu 2 -the first returning minion is able to create a document in the database -.IP \(bu 2 -other minions fail with \fB{\(aqerror\(aq: \(aqHTTP Error 409: Conflict\(aq}\fP -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchdb_return.ensure_views() -This function makes sure that all the views that should -exist in the design document do exist. -.UNINDENT +The default behaviour of this renderer is to log a warning if a block could not +be decrypted; in other words, it just returns the ciphertext rather than the +encrypted secret. +.sp +This behaviour can be changed via the \fIgpg_decrypt_must_succeed\fP configuration +option. If set to \fITrue\fP, any gpg block that cannot be decrypted raises a +\fISaltRenderError\fP exception, which registers an error in \fB_errors\fP during +rendering. +.sp +In the Chlorine release, the default behavior will be reversed and an error +message will be added to \fB_errors\fP by default. .INDENT 0.0 .TP -.B salt.returners.couchdb_return.get_fun(fun) -Return a dict with key being minion and value -being the job details of the last run of function \(aqfun\(aq. +.B salt.renderers.gpg.render(gpg_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) +Create a gpg object given a gpg_keydir, and then use it to try to decrypt +the data to be rendered. .UNINDENT +.SS salt.renderers.hjson +.sp +hjson renderer for Salt +.sp +See the \fI\%hjson\fP documentation for more information .INDENT 0.0 .TP -.B salt.returners.couchdb_return.get_jid(jid) -Get the document with a given JID. -.UNINDENT -.INDENT 0.0 +.B salt.renderers.hjson.render(hjson_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts HJSON as a string or as a file object and runs it through the HJSON +parser. +.INDENT 7.0 .TP -.B salt.returners.couchdb_return.get_jids() -List all the jobs that we have.. +.B Return type +A Python data structure .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchdb_return.get_minions() -Return a list of minion identifiers from a request of the view. .UNINDENT +.SS salt.renderers.jinja +.sp +Jinja loading utils to enable a more powerful backend for jinja templates +.sp +\fBIMPORTANT:\fP .INDENT 0.0 -.TP -.B salt.returners.couchdb_return.get_valid_salt_views() -Returns a dict object of views that should be -part of the salt design document. +.INDENT 3.5 +\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt +takes advantage of. Other text \fI\%Renderers\fP do not support this +functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchdb_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id .UNINDENT .INDENT 0.0 .TP -.B salt.returners.couchdb_return.returner(ret) -Take in the return and shove it into the couchdb database. -.UNINDENT -.INDENT 0.0 +.B salt.renderers.jinja.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, context=None, tmplpath=None, **kws) +Render the template_file, passing the functions and grains into the +Jinja rendering system. +.INDENT 7.0 .TP -.B salt.returners.couchdb_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency +.B Return type +string .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.couchdb_return.set_salt_view() -Helper function that sets the salt design -document. Uses get_valid_salt_views and some hardcoded values. .UNINDENT -.SS salt.returners.elasticsearch_return -.sp -Return data to an elasticsearch server for indexing. .INDENT 0.0 .TP -.B maintainer -Jurnell Cockhren <\fI\%jurnell.cockhren@sophicware.com\fP>, Arnold Bechtoldt <\fI\%mail@arnoldbechtoldt.com\fP> -.TP -.B maturity -New -.TP -.B depends -\fI\%elasticsearch\-py\fP -.TP -.B platform -all -.UNINDENT -.sp -To enable this returner the elasticsearch python client must be installed -on the desired minions (all or some subset). +.B class salt.utils.jinja.SerializerExtension(environment) +Yaml and Json manipulation. .sp -Please see documentation of \fI\%elasticsearch execution module\fP -for a valid connection configuration. +\fBFormat filters\fP .sp -\fBWARNING:\fP -.INDENT 0.0 +Allows jsonifying or yamlifying any data structure. For example, this dataset: +.INDENT 7.0 .INDENT 3.5 -The index that you wish to store documents will be created by Elasticsearch automatically if -doesn\(aqt exist yet. It is highly recommended to create predefined index templates with appropriate mapping(s) -that will be used by Elasticsearch upon index creation. Otherwise you will have problems as described in #20826. +.sp +.nf +.ft C +data = { + \(aqfoo\(aq: True, + \(aqbar\(aq: 42, + \(aqbaz\(aq: [1, 2, 3], + \(aqqux\(aq: 2.0 +} +.ft P +.fi .UNINDENT .UNINDENT -.sp -To use the returner per salt call: -.INDENT 0.0 +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return elasticsearch +yaml = {{ data|yaml }} +json = {{ data|json }} +python = {{ data|python }} +xml = {{ {\(aqroot_node\(aq: data}|xml }} .ft P .fi .UNINDENT .UNINDENT .sp -In order to have the returner apply to all minions: -.INDENT 0.0 +will be rendered as: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -ext_job_cache: elasticsearch +yaml = {bar: 42, baz: [1, 2, 3], foo: true, qux: 2.0} +json = {\(dqbaz\(dq: [1, 2, 3], \(dqfoo\(dq: true, \(dqbar\(dq: 42, \(dqqux\(dq: 2.0} +python = {\(aqbar\(aq: 42, \(aqbaz\(aq: [1, 2, 3], \(aqfoo\(aq: True, \(aqqux\(aq: 2.0} +xml = \(dq\(dq\(dq< + + 1 + 2 + 3 + \(dq\(dq\(dq .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B Minion configuration: -.INDENT 7.0 -.TP -.B debug_returner_payload\(aq: False -Output the payload being posted to the log file in debug mode -.TP -.B doc_type: \(aqdefault\(aq -Document type to use for normal return messages -.TP -.B functions_blacklist -Optional list of functions that should not be returned to elasticsearch -.TP -.B index_date: False -Use a dated index (e.g. \-2016.11.29) -.TP -.B master_event_index: \(aqsalt\-master\-event\-cache\(aq -Index to use when returning master events -.TP -.B master_event_doc_type: \(aqefault\(aq -Document type to use got master events -.TP -.B master_job_cache_index: \(aqsalt\-master\-job\-cache\(aq -Index to use for master job cache -.TP -.B master_job_cache_doc_type: \(aqdefault\(aq -Document type to use for master job cache -.TP -.B number_of_shards: 1 -Number of shards to use for the indexes -.TP -.B number_of_replicas: 0 -Number of replicas to use for the indexes -.UNINDENT .sp -NOTE: The following options are valid for \(aqstate.apply\(aq, \(aqstate.sls\(aq and \(aqstate.highstate\(aq functions only. +The yaml filter takes an optional flow_style parameter to control the +default\-flow\-style parameter of the YAML dumper. .INDENT 7.0 -.TP -.B states_count: False -Count the number of states which succeeded or failed and return it in top\-level item called \(aqcounts\(aq. -States reporting None (i.e. changes would be made but it ran in test mode) are counted as successes. -.TP -.B states_order_output: False -Prefix the state UID (e.g. file_|\-yum_configured_|\-/etc/yum.conf_|\-managed) with a zero\-padded version -of the \(aq__run_num__\(aq value to allow for easier sorting. Also store the state function (i.e. file.managed) -into a new key \(aq_func\(aq. Change the index to be \(aq\-ordered\(aq (e.g. salt\-state_apply\-ordered). -.TP -.B states_single_index: False -Store results for state.apply, state.sls and state.highstate in the salt\-state_apply index -(or \-ordered/\-) indexes if enabled -.UNINDENT -.UNINDENT -.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -elasticsearch: - hosts: - \- \(dq10.10.10.10:9200\(dq - \- \(dq10.10.10.11:9200\(dq - \- \(dq10.10.10.12:9200\(dq - index_date: True - number_of_shards: 5 - number_of_replicas: 1 - debug_returner_payload: True - states_count: True - states_order_output: True - states_single_index: True - functions_blacklist: - \- test.ping - \- saltutil.find_job +{{ data|yaml(False) }} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.elasticsearch_return.event_return(events) -Return events to Elasticsearch .sp -Requires that the \fIevent_return\fP configuration be set in master config. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.elasticsearch_return.get_load(jid) -Return the load data that marks a specified jid +will be rendered as: +.INDENT 7.0 +.INDENT 3.5 .sp -New in version 2015.8.1. - -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.elasticsearch_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.nf +.ft C +bar: 42 +baz: + \- 1 + \- 2 + \- 3 +foo: true +qux: 2.0 +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.elasticsearch_return.returner(ret) -Process the return from Salt .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.elasticsearch_return.save_load(jid, load, minions=None) -Save the load to the specified jid id .sp -New in version 2015.8.1. - -.UNINDENT -.SS salt.returners.etcd_return +\fBLoad filters\fP .sp -Return data to an etcd server or cluster -.INDENT 0.0 -.TP -.B depends +Strings and variables can be deserialized with \fBload_yaml\fP and +\fBload_json\fP tags and filters. It allows one to manipulate data directly +in templates, easily: .INDENT 7.0 -.IP \(bu 2 -python\-etcd or etcd3\-py -.UNINDENT -.UNINDENT -.sp -In order to return to an etcd server, a profile should be created in the master -configuration file: -.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_etcd_config: - etcd.host: 127.0.0.1 - etcd.port: 2379 +{%\- set yaml_src = \(dq{foo: it works}\(dq|load_yaml %} +{%\- set json_src = \(aq{\(dqbar\(dq: \(dqfor real\(dq}\(aq|load_json %} +Dude, {{ yaml_src.foo }} {{ json_src.bar }}! .ft P .fi .UNINDENT .UNINDENT .sp -It is technically possible to configure etcd without using a profile, but this -is not considered to be a best practice, especially when multiple etcd servers -or clusters are available. -.INDENT 0.0 +will be rendered as: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.host: 127.0.0.1 -etcd.port: 2379 +Dude, it works for real! .ft P .fi .UNINDENT .UNINDENT .sp -In order to choose whether to use etcd API v2 or v3, you can put the following -configuration option in the same place as your etcd configuration. This option -defaults to true, meaning you will use v2 unless you specify otherwise. -.INDENT 0.0 +\fBLoad tags\fP +.sp +Salt implements \fBload_yaml\fP and \fBload_json\fP tags. They work like +the \fI\%import tag\fP, except that the document is also deserialized. +.sp +Syntaxes are \fB{% load_yaml as [VARIABLE] %}[YOUR DATA]{% endload %}\fP +and \fB{% load_json as [VARIABLE] %}[YOUR DATA]{% endload %}\fP +.sp +For example: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.require_v2: True +{% load_yaml as yaml_src %} + foo: it works +{% endload %} +{% load_json as json_src %} + { + \(dqbar\(dq: \(dqfor real\(dq + } +{% endload %} +Dude, {{ yaml_src.foo }} {{ json_src.bar }}! .ft P .fi .UNINDENT .UNINDENT .sp -When using API v3, there are some specific options available to be configured -within your etcd profile. They are defaulted to the following... -.INDENT 0.0 +will be rendered as: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.encode_keys: False -etcd.encode_values: True -etcd.raw_keys: False -etcd.raw_values: False -etcd.unicode_errors: \(dqsurrogateescape\(dq +Dude, it works for real! .ft P .fi .UNINDENT .UNINDENT .sp -\fBetcd.encode_keys\fP indicates whether you want to pre\-encode keys using msgpack before -adding them to etcd. +\fBImport tags\fP .sp -\fBNOTE:\fP -.INDENT 0.0 +External files can be imported and made available as a Jinja variable. +.INDENT 7.0 .INDENT 3.5 -If you set \fBetcd.encode_keys\fP to \fBTrue\fP, all recursive functionality will no longer work. -This includes \fBtree\fP and \fBls\fP and all other methods if you set \fBrecurse\fP/\fBrecursive\fP to \fBTrue\fP\&. -This is due to the fact that when encoding with msgpack, keys like \fB/salt\fP and \fB/salt/stack\fP will have -differing byte prefixes, and etcd v3 searches recursively using prefixes. +.sp +.nf +.ft C +{% import_yaml \(dqmyfile.yml\(dq as myfile %} +{% import_json \(dqdefaults.json\(dq as defaults %} +{% import_text \(dqcompleteworksofshakespeare.txt\(dq as poems %} +.ft P +.fi .UNINDENT .UNINDENT .sp -\fBetcd.encode_values\fP indicates whether you want to pre\-encode values using msgpack before -adding them to etcd. This defaults to \fBTrue\fP to avoid data loss on non\-string values wherever possible. -.sp -\fBetcd.raw_keys\fP determines whether you want the raw key or a string returned. +\fBCatalog\fP .sp -\fBetcd.raw_values\fP determines whether you want the raw value or a string returned. +\fBimport_*\fP and \fBload_*\fP tags will automatically expose their +target variable to import. This feature makes catalog of data to +handle. .sp -\fBetcd.unicode_errors\fP determines what you policy to follow when there are encoding/decoding errors. +for example: +.INDENT 7.0 +.INDENT 3.5 .sp -Additionally, two more options must be specified in the top\-level configuration -in order to use the etcd returner: -.INDENT 0.0 +.nf +.ft C +# doc1.sls +{% load_yaml as var1 %} + foo: it works +{% endload %} +{% load_yaml as var2 %} + bar: for real +{% endload %} +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.returner: my_etcd_config -etcd.returner_root: /salt/return +# doc2.sls +{% from \(dqdoc1.sls\(dq import var1, var2 as local2 %} +{{ var1.foo }} {{ local2.bar }} .ft P .fi .UNINDENT .UNINDENT .sp -The \fBetcd.returner\fP option specifies which configuration profile to use. The -\fBetcd.returner_root\fP option specifies the path inside etcd to use as the root -of the returner system. +** Escape Filters ** .sp -Once the etcd options are configured, the returner may be used: +New in version 2017.7.0. + .sp -CLI Example: -.INDENT 0.0 -.INDENT 3.5 -salt \(aq*\(aq test.ping \-\-return etcd -.UNINDENT -.UNINDENT +Allows escaping of strings so they can be interpreted literally by another +function. .sp -A username and password can be set: -.INDENT 0.0 +For example: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.username: larry # Optional; requires etcd.password to be set -etcd.password: 123pass # Optional; requires etcd.username to be set +regex_escape = {{ \(aqhttps://example.com?foo=bar%20baz\(aq | regex_escape }} .ft P .fi .UNINDENT .UNINDENT .sp -You can also set a TTL (time to live) value for the returner: -.INDENT 0.0 +will be rendered as: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.ttl: 5 +regex_escape = https\e:\e/\e/example\e.com\e?foo\e=bar\e%20baz .ft P .fi .UNINDENT .UNINDENT .sp -Authentication with username and password, and ttl, currently requires the -\fBmaster\fP branch of \fBpython\-etcd\fP\&. +** Set Theory Filters ** .sp -You may also specify different roles for read and write operations. First, -create the profiles as specified above. Then add: -.INDENT 0.0 +New in version 2017.7.0. + +.sp +Performs set math using Jinja filters. +.sp +For example: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -etcd.returner_read_profile: my_etcd_read -etcd.returner_write_profile: my_etcd_write +unique = {{ [\(aqfoo\(aq, \(aqfoo\(aq, \(aqbar\(aq] | unique }} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.etcd_return.clean_old_jobs() -Included for API consistency -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.etcd_return.get_fun(fun) -Return a dict of the last function called for all minions +.sp +will be rendered as: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +unique = [\(aqfoo\(aq, \(aqbar\(aq] +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.etcd_return.get_jid(jid) -Return the information returned when the specified job id was executed .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.etcd_return.get_jids() -Return a list of all job ids +.sp +** Salt State Parameter Format Filters ** +.sp +New in version 3005. + +.sp +Renders a formatted multi\-line YAML string from a Python dictionary. Each +key/value pair in the dictionary will be added as a single\-key dictionary +to a list that will then be sent to the YAML formatter. +.sp +For example: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +{% set thing_params = { + \(dqname\(dq: \(dqthing\(dq, + \(dqchanges\(dq: True, + \(dqwarnings\(dq: \(dqOMG! Stuff is happening!\(dq + } +%} + +thing: + test.configurable_test_state: + {{ thing_params | dict_to_sls_yaml_params | indent }} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +will be rendered as: +.INDENT 7.0 +.INDENT 3.5 +.sp +.nf +.ft C +\&.. code\-block:: yaml +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.etcd_return.get_load(jid) -Return the load data that marks a specified jid .UNINDENT +.INDENT 7.0 +.INDENT 3.5 .INDENT 0.0 .TP -.B salt.returners.etcd_return.get_minions() -Return a list of minions +.B thing: +.INDENT 7.0 +.TP +.B test.configurable_test_state: +.INDENT 7.0 +.IP \(bu 2 +name: thing +.IP \(bu 2 +changes: true +.IP \(bu 2 +warnings: OMG! Stuff is happening! +.UNINDENT .UNINDENT +.UNINDENT +.UNINDENT +.UNINDENT +.UNINDENT +.SS salt.renderers.json +.sp +JSON Renderer for Salt .INDENT 0.0 .TP -.B salt.returners.etcd_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.B salt.renderers.json.render(json_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts JSON as a string or as a file object and runs it through the JSON +parser. +.INDENT 7.0 +.TP +.B Return type +A Python data structure .UNINDENT +.UNINDENT +.SS salt.renderers.json5 +.sp +JSON5 Renderer for Salt +.sp +New in version 2016.3.0. + +.sp +JSON5 is an unofficial extension to JSON. See \fI\%http://json5.org/\fP for more +information. +.sp +This renderer requires the \fI\%json5 python bindings\fP, installable via pip. .INDENT 0.0 .TP -.B salt.returners.etcd_return.returner(ret) -Return data to an etcd server or cluster +.B salt.renderers.json5.render(json_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts JSON as a string or as a file object and runs it through the JSON +parser. +.INDENT 7.0 +.TP +.B Return type +A Python data structure +.UNINDENT .UNINDENT +.SS salt.renderers.mako +.sp +Mako Renderer for Salt +.sp +This renderer requires the Mako library. +.sp +To install Mako, do the following: .INDENT 0.0 .TP -.B salt.returners.etcd_return.save_load(jid, load, minions=None) -Save the load to the specified jid +.B salt.renderers.mako.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, context=None, tmplpath=None, **kws) +Render the template_file, passing the functions and grains into the +Mako rendering system. +.INDENT 7.0 +.TP +.B Return type +string .UNINDENT +.UNINDENT +.SS salt.renderers.msgpack .INDENT 0.0 .TP -.B salt.returners.etcd_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency +.B salt.renderers.msgpack.render(msgpack_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts a message pack string or a file object, renders said data back to +a python dict. +.INDENT 7.0 +.TP +.B Return type +A Python data structure .UNINDENT -.SS salt.returners.highstate_return -.sp -Return the results of a highstate (or any other state function that returns -data in a compatible format) via an HTML email or HTML file. +.UNINDENT +.SS salt.renderers.nacl .sp -New in version 2017.7.0. - +Renderer that will decrypt NACL ciphers .sp -Similar results can be achieved by using the smtp returner with a custom template, -except an attempt at writing such a template for the complex data structure -returned by highstate function had proven to be a challenge, not to mention -that the smtp module doesn\(aqt support sending HTML mail at the moment. +Any key in the SLS file can be an NACL cipher, and this renderer will decrypt it +before passing it off to Salt. This allows you to safely store secrets in +source control, in such a way that only your Salt master can decrypt them and +distribute them only to the minions that need them. .sp -The main goal of this returner was to produce an easy to read email similar -to the output of highstate outputter used by the CLI. +The typical use\-case would be to use ciphers in your pillar data, and keep a +secret key on your master. You can put the public key in source control so that +developers can add new secrets quickly and easily. .sp -This returner could be very useful during scheduled executions, -but could also be useful for communicating the results of a manual execution. +This renderer requires the libsodium library binary and PyNacl >= 1.0 +.SS Setup .sp -Returner configuration is controlled in a standard fashion either via -highstate group or an alternatively named group. +To set things up, first generate a keypair. On the master, run the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.highstate \-\-return highstate +# salt\-call \-\-local nacl.keygen sk_file=/root/.nacl .ft P .fi .UNINDENT .UNINDENT +.SS Using encrypted pillar .sp -To use the alternative configuration, append \(aq\-\-return_config config\-name\(aq +To encrypt secrets, copy the public key to your local machine and run: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.highstate \-\-return highstate \-\-return_config simple +$ salt\-call \-\-local nacl.enc datatoenc pk_file=/root/.nacl.pub .ft P .fi .UNINDENT .UNINDENT .sp -Here is an example of what the configuration might look like: +To apply the renderer on a file\-by\-file basis add the following line to the +top of any pillar with nacl encrypted data in it: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -simple.highstate: - report_failures: True - report_changes: True - report_everything: False - failure_function: pillar.items - success_function: pillar.items - report_format: html - report_delivery: smtp - smtp_success_subject: \(aqsuccess minion {id} on host {host}\(aq - smtp_failure_subject: \(aqfailure minion {id} on host {host}\(aq - smtp_server: smtp.example.com - smtp_recipients: saltusers@example.com, devops@example.com - smtp_sender: salt@example.com +#!yaml|nacl .ft P .fi .UNINDENT .UNINDENT .sp -The \fIreport_failures\fP, \fIreport_changes\fP, and \fIreport_everything\fP flags provide -filtering of the results. If you want an email to be sent every time, then -\fIreport_everything\fP is your choice. If you want to be notified only when -changes were successfully made use \fIreport_changes\fP\&. And \fIreport_failures\fP will -generate an email if there were failures. -.sp -The configuration allows you to run a salt module function in case of -success (\fIsuccess_function\fP) or failure (\fIfailure_function\fP). -.sp -Any salt function, including ones defined in the _module folder of your salt -repo, could be used here and its output will be displayed under the \(aqextra\(aq -heading of the email. -.sp -Supported values for \fIreport_format\fP are html, json, and yaml. The latter two -are typically used for debugging purposes, but could be used for applying -a template at some later stage. -.sp -The values for \fIreport_delivery\fP are smtp or file. In case of file delivery -the only other applicable option is \fIfile_output\fP\&. -.sp -In case of smtp delivery, smtp_* options demonstrated by the example above -could be used to customize the email. +Now with your renderer configured, you can include your ciphers in your pillar +data like so: +.INDENT 0.0 +.INDENT 3.5 .sp -As you might have noticed, the success and failure subjects contain {id} and {host} -values. Any other grain name could be used. As opposed to using -{{grains[\(aqid\(aq]}}, which will be rendered by the master and contain master\(aqs -values at the time of pillar generation, these will contain minion values at -the time of execution. +.nf +.ft C +#!yaml|nacl + +a\-secret: \(dqNACL[MRN3cc+fmdxyQbz6WMF+jq1hKdU5X5BBI7OjK+atvHo1ll+w1gZ7XyWtZVfq9gK9rQaMfkDxmidJKwE0Mw==]\(dq +.ft P +.fi +.UNINDENT +.UNINDENT .INDENT 0.0 .TP -.B salt.returners.highstate_return.returner(ret) -Check highstate return information and possibly fire off an email -or save a file. +.B salt.renderers.nacl.render(nacl_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) +Decrypt the data to be rendered using the given nacl key or the one given +in config .UNINDENT -.SS salt.returners.influxdb_return +.SS salt.renderers.pass +.SS Pass Renderer for Salt .sp -Return data to an influxdb server. +\fI\%pass\fP is an encrypted on\-disk password store. .sp -New in version 2015.8.0. +New in version 2017.7.0. +.SS Setup .sp -To enable this returner the minion will need the python client for influxdb -installed and the following values configured in the minion or master -config, these are the defaults: +\fINote\fP: \fB\fP needs to be replaced with the user salt\-master will be +running as. +.sp +Have private gpg loaded into \fBuser\fP\(aqs gpg keyring .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -influxdb.db: \(aqsalt\(aq -influxdb.user: \(aqsalt\(aq -influxdb.password: \(aqsalt\(aq -influxdb.host: \(aqlocalhost\(aq -influxdb.port: 8086 +load_private_gpg_key: + cmd.run: + \- name: gpg \-\-import + \- unless: gpg \-\-list\-keys \(aq\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Said private key\(aqs public key should have been used when encrypting pass entries +that are of interest for pillar data. +.sp +Fetch and keep local pass git repo up\-to\-date .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.influxdb.db: \(aqsalt\(aq -alternative.influxdb.user: \(aqsalt\(aq -alternative.influxdb.password: \(aqsalt\(aq -alternative.influxdb.host: \(aqlocalhost\(aq -alternative.influxdb.port: 6379 +update_pass: + git.latest: + \- force_reset: True + \- name: + \- target: //.password\-store + \- identity: + \- require: + \- cmd: load_private_gpg_key .ft P .fi .UNINDENT .UNINDENT .sp -To use the influxdb returner, append \(aq\-\-return influxdb\(aq to the salt command. +Install pass binary .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return influxdb +pass: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Salt master configuration options .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return influxdb \-\-return_config alternative +# If the prefix is *not* set (default behavior), all template variables are +# considered for fetching secrets from Pass. Those that cannot be resolved +# to a secret are passed through. +# +# If the prefix is set, only the template variables with matching prefix are +# considered for fetching the secrets, other variables are passed through. +# +# For ease of use it is recommended to set the following options as well: +# renderer: \(aqjinja|yaml|pass\(aq +# pass_strict_fetch: true +# +pass_variable_prefix: \(aqpass:\(aq + +# If set to \(aqtrue\(aq, error out when unable to fetch a secret for a template variable. +pass_strict_fetch: true + +# Set GNUPGHOME env for Pass. +# Defaults to: ~/.gnupg +pass_gnupghome: + +# Set PASSWORD_STORE_DIR env for Pass. +# Defaults to: ~/.password\-store +pass_dir: .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B salt.renderers.pass.render(pass_info, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) +Fetch secret from pass based on pass_path +.UNINDENT +.SS salt.renderers.py +.SS Pure python state renderer .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +To use this renderer, the SLS file should contain a function called \fBrun\fP +which returns highstate data. .sp -New in version 2016.3.0. - +The highstate data is a dictionary containing identifiers as keys, and execution +dictionaries as values. For example the following state declaration in YAML: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return influxdb \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +common_packages: + pkg.installed: + \- pkgs: + \- curl + \- vim .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.returner(ret) -Return data to a influxdb data store -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.save_load(jid, load, minions=None) -Save the load to the specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.influxdb_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.kafka_return .sp -Return data to a Kafka topic +translates to: .INDENT 0.0 -.TP -.B maintainer -Justin Desilets (\fI\%justin.desilets@gmail.com\fP) -.TP -.B maturity -20181119 -.TP -.B depends -confluent\-kafka -.TP -.B platform -all +.INDENT 3.5 +.sp +.nf +.ft C +{\(aqcommon_packages\(aq: {\(aqpkg.installed\(aq: [{\(aqpkgs\(aq: [\(aqcurl\(aq, \(aqvim\(aq]}]}} +.ft P +.fi +.UNINDENT .UNINDENT .sp -To enable this returner install confluent\-kafka and enable the following -settings in the minion config: -.INDENT 0.0 -.INDENT 3.5 +In this module, a few objects are defined for you, giving access to Salt\(aqs +execution functions, grains, pillar, etc. They are: .INDENT 0.0 -.TP -.B returner.kafka.bootstrap: -.INDENT 7.0 .IP \(bu 2 -\(dqserver1:9092\(dq +\fB__salt__\fP \- \fI\%Execution functions\fP (i.e. +\fB__salt__[\(aqtest.echo\(aq](\(aqfoo\(aq)\fP) .IP \(bu 2 -\(dqserver2:9092\(dq +\fB__grains__\fP \- \fI\%Grains\fP (i.e. \fB__grains__[\(aqos\(aq]\fP) .IP \(bu 2 -\(dqserver3:9092\(dq -.UNINDENT -.UNINDENT -.sp -returner.kafka.topic: \(aqtopic\(aq +\fB__pillar__\fP \- \fI\%Pillar data\fP (i.e. \fB__pillar__[\(aqfoo\(aq]\fP) +.IP \(bu 2 +\fB__opts__\fP \- Minion configuration options +.IP \(bu 2 +\fB__env__\fP \- The effective salt fileserver environment (i.e. \fBbase\fP). Also +referred to as a \(dqsaltenv\(dq. \fB__env__\fP should not be modified in a pure +python SLS file. To use a different environment, the environment should be +set when executing the state. This can be done in a couple different ways: +.INDENT 2.0 +.IP \(bu 2 +Using the \fBsaltenv\fP argument on the salt CLI (i.e. \fBsalt \(aq*\(aq state.sls +foo.bar.baz saltenv=env_name\fP). +.IP \(bu 2 +By adding a \fBsaltenv\fP argument to an individual state within the SLS +file. In other words, adding a line like this to the state\(aqs data +structure: \fB{\(aqsaltenv\(aq: \(aqenv_name\(aq}\fP .UNINDENT +.IP \(bu 2 +\fB__sls__\fP \- The SLS path of the file. For example, if the root of the base +environment is \fB/srv/salt\fP, and the SLS file is +\fB/srv/salt/foo/bar/baz.sls\fP, then \fB__sls__\fP in that file will be +\fBfoo.bar.baz\fP\&. .UNINDENT .sp -To use the kafka returner, append \fI\-\-return kafka\fP to the Salt command, eg; +When used in a scenario where additional user\-provided context data is supplied +(such as with \fI\%file.managed\fP), the additional +data will typically be injected into the script as one or more global +variables: .INDENT 0.0 .INDENT 3.5 -salt \(aq*\(aq test.ping \-\-return kafka -.UNINDENT +.sp +.nf +.ft C +/etc/http/conf/http.conf: + file.managed: + \- source: salt://apache/generate_http_conf.py + \- template: py + \- context: + # Will be injected as the global variable \(dqsite_name\(dq. + site_name: {{ site_name }} +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.kafka_return.returner(ret) -Return information to a Kafka server .UNINDENT -.SS salt.returners.librato_return -.sp -Salt returner to return highstate stats to Librato -.sp -To enable this returner the minion will need the Librato -client importable on the Python path and the following -values configured in the minion or master config. .sp -The Librato python client can be found at: -\fI\%https://github.com/librato/python\-librato\fP +When writing a reactor SLS file the global context \fBdata\fP (same as context +\fB{{ data }}\fP for states written with Jinja + YAML) is available. The +following YAML + Jinja state declaration: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -librato.email: example@librato.com -librato.api_token: abc12345def +{% if data[\(aqid\(aq] == \(aqmysql1\(aq %} +highstate_run: + local.state.apply: + \- tgt: mysql1 +{% endif %} .ft P .fi .UNINDENT .UNINDENT .sp -This return supports multi\-dimension metrics for Librato. To enable -support for more metrics, the tags JSON object can be modified to include -other tags. -.sp -Adding EC2 Tags example: -If ec2_tags:region were desired within the tags for multi\-dimension. The tags -could be modified to include the ec2 tags. Multiple dimensions are added simply -by adding more tags to the submission. +translates to: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_data = __salt__[\(aqpillar.raw\(aq]() -q.add(metric.name, value, tags={\(aqName\(aq: ret[\(aqid\(aq],\(aqRegion\(aq: pillar_data[\(aqec2_tags\(aq][\(aqName\(aq]}) +if data[\(aqid\(aq] == \(aqmysql1\(aq: + return {\(aqhighstate_run\(aq: {\(aqlocal.state.apply\(aq: [{\(aqtgt\(aq: \(aqmysql1\(aq}]}} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.librato_return.returner(ret) -Parse the return data and return metrics to Librato. -.UNINDENT -.SS salt.returners.local -.sp -The local returner is used to test the returner interface, it just prints the -return data to the console to verify that it is being passed properly -.sp -To use the local returner, append \(aq\-\-return local\(aq to the salt command. ex: +.SS Full Example .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return local + #!py + + def run(): + config = {} + + if __grains__[\(aqos\(aq] == \(aqUbuntu\(aq: + user = \(aqubuntu\(aq + group = \(aqubuntu\(aq + home = \(aq/home/{0}\(aq.format(user) + else: + user = \(aqroot\(aq + group = \(aqroot\(aq + home = \(aq/root/\(aq + + config[\(aqs3cmd\(aq] = { + \(aqpkg\(aq: [ + \(aqinstalled\(aq, + {\(aqname\(aq: \(aqs3cmd\(aq}, + ], + } + + config[home + \(aq/.s3cfg\(aq] = { + \(aqfile.managed\(aq: [ + {\(aqsource\(aq: \(aqsalt://s3cfg/templates/s3cfg\(aq}, + {\(aqtemplate\(aq: \(aqjinja\(aq}, + {\(aquser\(aq: user}, + {\(aqgroup\(aq: group}, + {\(aqmode\(aq: 600}, + {\(aqcontext\(aq: { + \(aqaws_key\(aq: __pillar__[\(aqAWS_ACCESS_KEY_ID\(aq], + \(aqaws_secret_key\(aq: __pillar__[\(aqAWS_SECRET_ACCESS_KEY\(aq], + }, + }, + ], + } + + return config .ft P .fi .UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.returners.local.event_return(event) -Print event return data to the terminal to verify functionality -.UNINDENT -.INDENT 0.0 +.B salt.renderers.py.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, tmplpath=None, **kws) +Render the python module\(aqs components +.INDENT 7.0 .TP -.B salt.returners.local.returner(ret) -Print the return data to the terminal to verify functionality +.B Return type +string .UNINDENT -.SS salt.returners.local_cache +.UNINDENT +.SS salt.renderers.pydsl .sp -Return data to local job cache +A Python\-based DSL .INDENT 0.0 .TP -.B salt.returners.local_cache.clean_old_jobs() -Clean out the old jobs from the job cache +.B maintainer +Jack Kuan <\fI\%kjkuan@gmail.com\fP> +.TP +.B maturity +new +.TP +.B platform +all .UNINDENT +.sp +The \fIpydsl\fP renderer allows one to author salt formulas (.sls files) in pure +Python using a DSL that\(aqs easy to write and easy to read. Here\(aqs an example: .INDENT 0.0 -.TP -.B salt.returners.local_cache.get_endtime(jid) -Retrieve the stored endtime for a given job +.INDENT 3.5 .sp -Returns False if no endtime is present +.nf +.ft C +#!pydsl + +apache = state(\(aqapache\(aq) +apache.pkg.installed() +apache.service.running() +state(\(aq/var/www/index.html\(aq) \e + .file(\(aqmanaged\(aq, + source=\(aqsalt://webserver/index.html\(aq) \e + .require(pkg=\(aqapache\(aq) +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.get_jid(jid) -Return the information returned when the specified job id was executed .UNINDENT +.sp +Notice that any Python code is allow in the file as it\(aqs really a Python +module, so you have the full power of Python at your disposal. In this module, +a few objects are defined for you, including the usual (with \fB__\fP added) +\fB__salt__\fP dictionary, \fB__grains__\fP, \fB__pillar__\fP, \fB__opts__\fP, +\fB__env__\fP, and \fB__sls__\fP, plus a few more: .INDENT 0.0 -.TP -.B salt.returners.local_cache.get_jids() -Return a dict mapping all job ids to job information -.UNINDENT +.INDENT 3.5 +\fB__file__\fP .INDENT 0.0 -.TP -.B salt.returners.local_cache.get_jids_filter(count, filter_find_job=True) -Return a list of all jobs information filtered by the given criteria. -:param int count: show not more than the count of most recent jobs -:param bool filter_find_jobs: filter out \(aqsaltutil.find_job\(aq jobs +.INDENT 3.5 +local file system path to the sls module. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.get_load(jid) -Return the load data that marks a specified jid .UNINDENT +.sp +\fB__pydsl__\fP .INDENT 0.0 -.TP -.B salt.returners.local_cache.load_reg() -Load the register from msgpack files +.INDENT 3.5 +Salt PyDSL object, useful for configuring DSL behavior per sls rendering. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.prep_jid(nocache=False, passed_jid=None, recurse_count=0) -Return a job id and prepare the job id directory. -.sp -This is the function responsible for making sure jids don\(aqt collide (unless -it is passed a jid). -So do what you have to do to make sure that stays the case .UNINDENT +.sp +\fBinclude\fP .INDENT 0.0 -.TP -.B salt.returners.local_cache.returner(load) -Return data to the local job cache +.INDENT 3.5 +Salt PyDSL function for creating \fI\%Include declaration\fP\(aqs. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.save_load(jid, clear_load, minions=None, recurse_count=0) -Save the load to the specified jid -.sp -minions argument is to provide a pre\-computed list of matched minions for -the job, for cases when this function can\(aqt compute that list itself (such -as for salt\-ssh) .UNINDENT +.sp +\fBextend\fP .INDENT 0.0 -.TP -.B salt.returners.local_cache.save_minions(jid, minions, syndic_id=None) -Save/update the serialized list of minions for a given job +.INDENT 3.5 +Salt PyDSL function for creating \fI\%Extend declaration\fP\(aqs. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.save_reg(data) -Save the register to msgpack files .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.local_cache.update_endtime(jid, time) -Update (or store) the end time for a given job .sp -Endtime is stored as a plain text string +\fBstate\fP +.INDENT 0.0 +.INDENT 3.5 +Salt PyDSL function for creating \fI\%ID declaration\fP\(aqs. +.UNINDENT +.UNINDENT +.UNINDENT .UNINDENT -.SS salt.returners.mattermost_returner -.sp -Return salt data via mattermost -.sp -New in version 2017.7.0. - .sp -The following fields can be set in the minion conf file: +A state \fI\%ID declaration\fP is created with a \fBstate(id)\fP function call. +Subsequent \fBstate(id)\fP call with the same id returns the same object. This +singleton access pattern applies to all declaration objects created with the +DSL. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mattermost.hook (required) -mattermost.username (optional) -mattermost.channel (optional) +state(\(aqexample\(aq) +assert state(\(aqexample\(aq) is state(\(aqexample\(aq) +assert state(\(aqexample\(aq).cmd is state(\(aqexample\(aq).cmd +assert state(\(aqexample\(aq).cmd.running is state(\(aqexample\(aq).cmd.running .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +The \fIid\fP argument is optional. If omitted, an UUID will be generated and used as +the \fIid\fP\&. +.sp +\fBstate(id)\fP returns an object under which you can create a +\fI\%State declaration\fP object by accessing an attribute named after \fIany\fP +state module available in Salt. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mattermost.channel -mattermost.hook -mattermost.username +state(\(aqexample\(aq).cmd +state(\(aqexample\(aq).file +state(\(aqexample\(aq).pkg +\&... .ft P .fi .UNINDENT .UNINDENT .sp -mattermost settings may also be configured as: +Then, a \fI\%Function declaration\fP object can be created from a +\fI\%State declaration\fP object by one of the following two ways: +.INDENT 0.0 +.IP 1. 3 +by calling a method named after the state function on the \fI\%State declaration\fP object. +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mattermost: - channel: RoomName - hook: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - username: user +state(\(aqexample\(aq).file.managed(...) .ft P .fi .UNINDENT .UNINDENT -.sp -To use the mattermost returner, append \(aq\-\-return mattermost\(aq to the salt command. +.INDENT 0.0 +.IP 2. 3 +by directly calling the attribute named for the \fI\%State declaration\fP, and +supplying the state function name as the first argument. +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mattermost +state(\(aqexample\(aq).file(\(aqmanaged\(aq, ...) .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(aqkey:\(aq: \(aqvalue\(aq}\(aq to the salt command. +With either way of creating a \fI\%Function declaration\fP object, any +\fI\%Function arg declaration\fP\(aqs can be passed as keyword arguments to the +call. Subsequent calls of a \fI\%Function declaration\fP will update the arg +declarations. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mattermost \-\-return_kwargs \(aq{\(aqchannel\(aq: \(aq#random\(aq}\(aq +state(\(aqexample\(aq).file(\(aqmanaged\(aq, source=\(aqsalt://webserver/index.html\(aq) +state(\(aqexample\(aq).file.managed(source=\(aqsalt://webserver/index.html\(aq) .ft P .fi .UNINDENT .UNINDENT +.sp +As a shortcut, the special \fIname\fP argument can also be passed as the +first or second positional argument depending on the first or second +way of calling the \fI\%State declaration\fP object. In the following +two examples \fIls \-la\fP is the \fIname\fP argument. .INDENT 0.0 -.TP -.B salt.returners.mattermost_returner.event_return(events) -Send the events to a mattermost room. -.INDENT 7.0 -.TP -.B Parameters -\fBevents\fP \-\- List of events -.TP -.B Returns -Boolean if messages were sent successfully. -.UNINDENT -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mattermost_returner.post_message(channel, message, username, api_url, hook) -Send a message to a mattermost room. -.INDENT 7.0 -.TP -.B Parameters -.INDENT 7.0 -.IP \(bu 2 -\fBchannel\fP \-\- The room name. -.IP \(bu 2 -\fBmessage\fP \-\- The message to send to the mattermost room. -.IP \(bu 2 -\fBusername\fP \-\- Specify who the message is from. -.IP \(bu 2 -\fBhook\fP \-\- The mattermost hook, if not specified in the configuration. -.UNINDENT -.TP -.B Returns -Boolean if message was sent successfully. -.UNINDENT +.INDENT 3.5 +.sp +.nf +.ft C +state(\(aqexample\(aq).cmd.run(\(aqls \-la\(aq, cwd=\(aq/\(aq) +state(\(aqexample\(aq).cmd(\(aqrun\(aq, \(aqls \-la\(aq, cwd=\(aq/\(aq) +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mattermost_returner.returner(ret) -Send an mattermost message with the data .UNINDENT -.SS salt.returners.memcache_return .sp -Return data to a memcache server +Finally, a \fI\%Requisite declaration\fP object with its +\fI\%Requisite reference\fP\(aqs can be created by invoking one of the +requisite methods (see \fI\%State Requisites\fP) on either a +\fI\%Function declaration\fP object or a \fI\%State declaration\fP object. +The return value of a requisite call is also a \fI\%Function declaration\fP +object, so you can chain several requisite calls together. .sp -To enable this returner the minion will need the python client for memcache -installed and the following values configured in the minion or master -config, these are the defaults. +Arguments to a requisite call can be a list of \fI\%State declaration\fP objects +and/or a set of keyword arguments whose names are state modules and values are +IDs of \fI\%ID declaration\fP\(aqs or names of \fI\%Name declaration\fP\(aqs. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -memcache.host: \(aqlocalhost\(aq -memcache.port: \(aq11211\(aq +apache2 = state(\(aqapache2\(aq) +apache2.pkg.installed() +state(\(aqlibapache2\-mod\-wsgi\(aq).pkg.installed() + +# you can call requisites on function declaration +apache2.service.running() \e + .require(apache2.pkg, + pkg=\(aqlibapache2\-mod\-wsgi\(aq) \e + .watch(file=\(aq/etc/apache2/httpd.conf\(aq) + +# or you can call requisites on state declaration. +# this actually creates an anonymous function declaration object +# to add the requisites. +apache2.service.require(state(\(aqlibapache2\-mod\-wsgi\(aq).pkg, + pkg=\(aqapache2\(aq) \e + .watch(file=\(aq/etc/apache2/httpd.conf\(aq) + +# we still need to set the name of the function declaration. +apache2.service.running() .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location. +\fI\%Include declaration\fP objects can be created with the \fBinclude\fP function, +while \fI\%Extend declaration\fP objects can be created with the \fBextend\fP function, +whose arguments are just \fI\%Function declaration\fP objects. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.memcache.host: \(aqlocalhost\(aq -alternative.memcache.port: \(aq11211\(aq +include(\(aqedit.vim\(aq, \(aqhttp.server\(aq) +extend(state(\(aqapache2\(aq).service.watch(file=\(aq/etc/httpd/httpd.conf\(aq) .ft P .fi .UNINDENT .UNINDENT .sp -python2\-memcache uses \(aqlocalhost\(aq and \(aq11211\(aq as syntax on connection. -.sp -To use the memcache returner, append \(aq\-\-return memcache\(aq to the salt command. +The \fBinclude\fP function, by default, causes the included sls file to be rendered +as soon as the \fBinclude\fP function is called. It returns a list of rendered module +objects; sls files not rendered with the pydsl renderer return \fBNone\fP\(aqs. +This behavior creates no \fI\%Include declaration\fP\(aqs in the resulting high state +data structure. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return memcache +import types + +# including multiple sls returns a list. +_, mod = include(\(aqa\-non\-pydsl\-sls\(aq, \(aqa\-pydsl\-sls\(aq) + +assert _ is None +assert isinstance(slsmods[1], types.ModuleType) + +# including a single sls returns a single object +mod = include(\(aqa\-pydsl\-sls\(aq) + +# myfunc is a function that calls state(...) to create more states. +mod.myfunc(1, 2, \(dqthree\(dq) .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Notice how you can define a reusable function in your pydsl sls module and then +call it via the module returned by \fBinclude\fP\&. .sp -New in version 2015.5.0. - +It\(aqs still possible to do late includes by passing the \fBdelayed=True\fP keyword +argument to \fBinclude\fP\&. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return memcache \-\-return_config alternative +include(\(aqedit.vim\(aq, \(aqhttp.server\(aq, delayed=True) .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +Above will just create a \fI\%Include declaration\fP in the rendered result, and +such call always returns \fBNone\fP\&. +.SS Special integration with the \fIcmd\fP state .sp -New in version 2016.3.0. - +Taking advantage of rendering a Python module, PyDSL allows you to declare a +state that calls a pre\-defined Python function when the state is executed. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return memcache \-\-return_kwargs \(aq{\(dqhost\(dq: \(dqhostname.domain.com\(dq}\(aq +greeting = \(dqhello world\(dq +def helper(something, *args, **kws): + print greeting # hello world + print something, args, kws # test123 [\(aqa\(aq, \(aqb\(aq, \(aqc\(aq] {\(aqx\(aq: 1, \(aqy\(aq: 2} + +state().cmd.call(helper, \(dqtest123\(dq, \(aqa\(aq, \(aqb\(aq, \(aqc\(aq, x=1, y=2) .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.returner(ret) -Return data to a memcache data store -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.save_load(jid, load, minions=None) -Save the load to the specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.memcache_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.mongo_future_return .sp -Return data to a mongodb server +The \fIcmd.call\fP state function takes care of calling our \fBhelper\fP function +with the arguments we specified in the states, and translates the return value +of our function into a structure expected by the state system. +See \fI\%salt.states.cmd.call()\fP for more information. +.SS Implicit ordering of states .sp -Required python modules: pymongo +Salt states are explicitly ordered via \fI\%Requisite declaration\fP\(aqs. +However, with \fIpydsl\fP it\(aqs possible to let the renderer track the order +of creation for \fI\%Function declaration\fP objects, and implicitly add +\fBrequire\fP requisites for your states to enforce the ordering. This feature +is enabled by setting the \fBordered\fP option on \fB__pydsl__\fP\&. .sp -This returner will send data from the minions to a MongoDB server. MongoDB -server can be configured by using host, port, db, user and password settings -or by connection string URI (for pymongo > 2.3). To configure the settings -for your MongoDB server, add the following lines to the minion config files: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +this feature is only available if your minions are using Python >= 2.7. +.UNINDENT +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mongo.db: -mongo.host: -mongo.user: -mongo.password: -mongo.port: 27017 +include(\(aqsome.sls.file\(aq) + +A = state(\(aqA\(aq).cmd.run(cwd=\(aq/var/tmp\(aq) +extend(A) + +__pydsl__.set(ordered=True) + +for i in range(10): + i = str(i) + state(i).cmd.run(\(aqecho \(aq+i, cwd=\(aq/\(aq) +state(\(aq1\(aq).cmd.run(\(aqecho one\(aq) +state(\(aq2\(aq).cmd.run(name=\(aqecho two\(aq) .ft P .fi .UNINDENT .UNINDENT .sp -Or single URI: +Notice that the \fBordered\fP option needs to be set after any \fBextend\fP calls. +This is to prevent \fIpydsl\fP from tracking the creation of a state function that\(aqs +passed to an \fBextend\fP call. +.sp +Above example should create states from \fB0\fP to \fB9\fP that will output \fB0\fP, +\fBone\fP, \fBtwo\fP, \fB3\fP, ... \fB9\fP, in that order. +.sp +It\(aqs important to know that \fIpydsl\fP tracks the \fIcreations\fP of +\fI\%Function declaration\fP objects, and automatically adds a \fBrequire\fP requisite +to a \fI\%Function declaration\fP object that requires the last +\fI\%Function declaration\fP object created before it in the sls file. +.sp +This means later calls (perhaps to update the function\(aqs \fI\%Function arg declaration\fP) to a previously created function declaration will not change the +order. +.SS Render time state execution +.sp +When Salt processes a salt formula file, the file is rendered to salt\(aqs +high state data representation by a renderer before the states can be executed. +In the case of the \fIpydsl\fP renderer, the .sls file is executed as a python module +as it is being rendered which makes it easy to execute a state at render time. +In \fIpydsl\fP, executing one or more states at render time can be done by calling a +configured \fI\%ID declaration\fP object. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mongo.uri: URI +#!pydsl + +s = state() # save for later invocation + +# configure it +s.cmd.run(\(aqecho at render time\(aq, cwd=\(aq/\(aq) +s.file.managed(\(aqtarget.txt\(aq, source=\(aqsalt://source.txt\(aq) + +s() # execute the two states now .ft P .fi .UNINDENT .UNINDENT .sp -where uri is in the format: +Once an \fI\%ID declaration\fP is called at render time it is detached from the +sls module as if it was never defined. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]] -.ft P -.fi +If \fIimplicit ordering\fP is enabled (i.e., via \fB__pydsl__.set(ordered=True)\fP) then +the \fIfirst\fP invocation of a \fI\%ID declaration\fP object must be done before a +new \fI\%Function declaration\fP is created. .UNINDENT .UNINDENT +.SS Integration with the stateconf renderer .sp -Example: +The \fI\%salt.renderers.stateconf\fP renderer offers a few interesting features that +can be leveraged by the \fIpydsl\fP renderer. In particular, when using with the \fIpydsl\fP +renderer, we are interested in \fIstateconf\fP\(aqs sls namespacing feature (via dot\-prefixed +id declarations), as well as, the automatic \fIstart\fP and \fIgoal\fP states generation. +.sp +Now you can use \fIpydsl\fP with \fIstateconf\fP like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mongodb://db1.example.net:27017/mydatabase -mongodb://db1.example.net:27017,db2.example.net:2500/?replicaSet=test -mongodb://db1.example.net:27017,db2.example.net:2500/?replicaSet=test&connectTimeoutMS=300000 +#!pydsl|stateconf \-ps + +include(\(aqxxx\(aq, \(aqyyy\(aq) + +# ensure that states in xxx run BEFORE states in this file. +extend(state(\(aq.start\(aq).stateconf.require(stateconf=\(aqxxx::goal\(aq)) + +# ensure that states in yyy run AFTER states in this file. +extend(state(\(aq.goal\(aq).stateconf.require_in(stateconf=\(aqyyy::start\(aq)) + +__pydsl__.set(ordered=True) + +\&... .ft P .fi .UNINDENT .UNINDENT .sp -More information on URI format can be found in -\fI\%https://docs.mongodb.com/manual/reference/connection\-string/\fP +\fB\-s\fP enables the generation of a stateconf \fIstart\fP state, and \fB\-p\fP lets us pipe +high state data rendered by \fIpydsl\fP to \fIstateconf\fP\&. This example shows that by +\fBrequire\fP\-ing or \fBrequire_in\fP\-ing the included sls\(aq \fIstart\fP or \fIgoal\fP states, +it\(aqs possible to ensure that the included sls files can be made to execute before +or after a state in the including sls file. +.SS Importing custom Python modules .sp -You can also ask for indexes creation on the most common used fields, which -should greatly improve performance. Indexes are not created by default. +To use a custom Python module inside a PyDSL state, place the module somewhere that +it can be loaded by the Salt loader, such as \fI_modules\fP in the \fI/srv/salt\fP directory. +.sp +Then, copy it to any minions as necessary by using \fIsaltutil.sync_modules\fP\&. +.sp +To import into a PyDSL SLS, one must bypass the Python importer and insert it manually +by getting a reference from Python\(aqs \fIsys.modules\fP dictionary. +.sp +For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mongo.indexes: true +#!pydsl|stateconf \-ps + +def main(): + my_mod = sys.modules[\(aqsalt.loaded.ext.module.my_mod\(aq] .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B exception salt.renderers.pydsl.PyDslError +.UNINDENT +.INDENT 0.0 +.TP +.B exception salt.renderers.pydsl.SaltRenderError(message, line_num=None, buf=\(aq\(aq, marker=\(aq <======================\(aq, trace=None) +Used when a renderer needs to raise an explicit error. If a line number and +buffer string are passed, get_context will be invoked to get the location +of the error. +.UNINDENT +.INDENT 0.0 +.TP +.B salt.renderers.pydsl.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, tmplpath=None, rendered_sls=None, **kws) +.UNINDENT +.SS salt.renderers.pyobjects .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Python renderer that includes a Pythonic Object based interface +.INDENT 0.0 +.TP +.B maintainer +Evan Borgstrom <\fI\%evan@borgstrom.ca\fP> +.UNINDENT +.sp +Let\(aqs take a look at how you use pyobjects in a state file. Here\(aqs a quick +example that ensures the \fB/tmp\fP directory is in the correct state. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.mongo.db: -alternative.mongo.host: -alternative.mongo.user: -alternative.mongo.password: -alternative.mongo.port: 27017 + #!pyobjects + + File.managed(\(dq/tmp\(dq, user=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq1777\(aq) .ft P .fi .UNINDENT .UNINDENT .sp -Or single URI: +Nice and Pythonic! +.sp +By using the \(dqshebang\(dq syntax to switch to the pyobjects renderer we can now +write our state data using an object based interface that should feel at home +to python developers. You can import any module and do anything that you\(aqd +like (with caution, importing sqlalchemy, django or other large frameworks has +not been tested yet). Using the pyobjects renderer is exactly the same as +using the built\-in Python renderer with the exception that pyobjects provides +you with an object based interface for generating state data. +.SS Creating state data +.sp +Pyobjects takes care of creating an object for each of the available states on +the minion. Each state is represented by an object that is the CamelCase +version of its name (i.e. \fBFile\fP, \fBService\fP, \fBUser\fP, etc), and these +objects expose all of their available state functions (i.e. \fBFile.managed\fP, +\fBService.running\fP, etc). +.sp +The name of the state is split based upon underscores (\fB_\fP), then each part +is capitalized and finally the parts are joined back together. +.sp +Some examples: +.INDENT 0.0 +.IP \(bu 2 +\fBpostgres_user\fP becomes \fBPostgresUser\fP +.IP \(bu 2 +\fBssh_known_hosts\fP becomes \fBSshKnownHosts\fP +.UNINDENT +.SS Context Managers and requisites +.sp +How about something a little more complex. Here we\(aqre going to get into the +core of how to use pyobjects to write states. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.mongo.uri: URI + #!pyobjects + + with Pkg.installed(\(dqnginx\(dq): + Service.running(\(dqnginx\(dq, enable=True) + + with Service(\(dqnginx\(dq, \(dqwatch_in\(dq): + File.managed(\(dq/etc/nginx/conf.d/mysite.conf\(dq, + owner=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq0444\(aq, + source=\(aqsalt://nginx/mysite.conf\(aq) .ft P .fi .UNINDENT .UNINDENT .sp -This mongo returner is being developed to replace the default mongodb returner -in the future and should not be considered API stable yet. +The objects that are returned from each of the magic method calls are setup to +be used a Python context managers (\fBwith\fP) and when you use them as such all +declarations made within the scope will \fBautomatically\fP use the enclosing +state as a requisite! .sp -To use the mongo returner, append \(aq\-\-return mongo\(aq to the salt command. +The above could have also been written use direct requisite statements as. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo + #!pyobjects + + Pkg.installed(\(dqnginx\(dq) + Service.running(\(dqnginx\(dq, enable=True, require=Pkg(\(dqnginx\(dq)) + File.managed(\(dq/etc/nginx/conf.d/mysite.conf\(dq, + owner=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq0444\(aq, + source=\(aqsalt://nginx/mysite.conf\(aq, + watch_in=Service(\(dqnginx\(dq)) .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. -.sp -New in version 2015.5.0. - +You can use the direct requisite statement for referencing states that are +generated outside of the current file. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo \-\-return_config alternative + #!pyobjects + + # some\-other\-package is defined in some other state file + Pkg.installed(\(dqnginx\(dq, require=Pkg(\(dqsome\-other\-package\(dq)) .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +The last thing that direct requisites provide is the ability to select which +of the SaltStack requisites you want to use (require, require_in, watch, +watch_in, use & use_in) when using the requisite as a context manager. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq + #!pyobjects + + with Service(\(dqmy\-service\(dq, \(dqwatch_in\(dq): + ... .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.event_return(events) -Return events to Mongodb server -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.get_fun(fun) -Return the most recent jobs that have executed the named function -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.get_jid(jid) -Return the return information associated with a jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.get_jids() -Return a list of job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.get_load(jid) -Return the load associated with a given job id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.returner(ret) -Return data to a mongodb server -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.save_load(jid, load, minions=None) -Save the load for a given job id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_future_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.mongo_return .sp -Return data to a mongodb server +The above example would cause all declarations inside the scope of the context +manager to automatically have their \fBwatch_in\fP set to +\fBService(\(dqmy\-service\(dq)\fP\&. +.SS Including and Extending .sp -Required python modules: pymongo +To include other states use the \fBinclude()\fP function. It takes one name per +state to include. .sp -This returner will send data from the minions to a MongoDB server. To -configure the settings for your MongoDB server, add the following lines -to the minion config files. +To extend another state use the \fBextend()\fP function on the name when creating +a state. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mongo.db: -mongo.host: -mongo.user: -mongo.password: -mongo.port: 27017 + #!pyobjects + + include(\(aqhttp\(aq, \(aqssh\(aq) + + Service.running(extend(\(aqapache\(aq), + watch=[File(\(aq/etc/httpd/extra/httpd\-vhosts.conf\(aq)]) .ft P .fi .UNINDENT .UNINDENT +.SS Importing from other state files .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location. +Like any Python project that grows you will likely reach a point where you want +to create reusability in your state tree and share objects between state files, +Map Data (described below) is a perfect example of this. +.sp +To facilitate this Python\(aqs \fBimport\fP statement has been augmented to allow +for a special case when working with a Salt state tree. If you specify a Salt +url (\fBsalt://...\fP) as the target for importing from then the pyobjects +renderer will take care of fetching the file for you, parsing it with all of +the pyobjects features available and then place the requested objects in the +global scope of the template being rendered. +.sp +This works for all types of import statements; \fBimport X\fP, +\fBfrom X import Y\fP, and \fBfrom X import Y as Z\fP\&. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.mongo.db: -alternative.mongo.host: -alternative.mongo.user: -alternative.mongo.password: -alternative.mongo.port: 27017 + #!pyobjects + + import salt://myfile.sls + from salt://something/data.sls import Object + from salt://something/data.sls import Object as Other .ft P .fi .UNINDENT .UNINDENT .sp -To use the mongo returner, append \(aq\-\-return mongo\(aq to the salt command. +See the Map Data section for a more practical use. +.sp +Caveats: +.INDENT 0.0 +.IP \(bu 2 +Imported objects are ALWAYS put into the global scope of your template, +regardless of where your import statement is. +.UNINDENT +.SS Salt object +.sp +In the spirit of the object interface for creating state data pyobjects also +provides a simple object interface to the \fB__salt__\fP object. +.sp +A function named \fBsalt\fP exists in scope for your sls files and will dispatch +its attributes to the \fB__salt__\fP dictionary. +.sp +The following lines are functionally equivalent: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo_return + #!pyobjects + + ret = salt.cmd.run(bar) + ret = __salt__[\(aqcmd.run\(aq](bar) .ft P .fi .UNINDENT .UNINDENT +.SS Pillar, grain, mine & config data .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Pyobjects provides shortcut functions for calling \fBpillar.get\fP, +\fBgrains.get\fP, \fBmine.get\fP & \fBconfig.get\fP on the \fB__salt__\fP object. This +helps maintain the readability of your state files. .sp -New in version 2015.5.0. - +Each type of data can be access by a function of the same name: \fBpillar()\fP, +\fBgrains()\fP, \fBmine()\fP and \fBconfig()\fP\&. +.sp +The following pairs of lines are functionally equivalent: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo_return \-\-return_config alternative + #!pyobjects + + value = pillar(\(aqfoo:bar:baz\(aq, \(aqqux\(aq) + value = __salt__[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) + + value = grains(\(aqpkg:apache\(aq) + value = __salt__[\(aqgrains.get\(aq](\(aqpkg:apache\(aq) + + value = mine(\(aqos:Fedora\(aq, \(aqnetwork.interfaces\(aq, \(aqgrain\(aq) + value = __salt__[\(aqmine.get\(aq](\(aqos:Fedora\(aq, \(aqnetwork.interfaces\(aq, \(aqgrain\(aq) + + value = config(\(aqfoo:bar:baz\(aq, \(aqqux\(aq) + value = __salt__[\(aqconfig.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) .ft P .fi .UNINDENT .UNINDENT +.SS Opts dictionary and SLS name .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +Pyobjects provides variable access to the minion options dictionary and the SLS +name that the code resides in. These variables are the same as the \fIopts\fP and +\fIsls\fP variables available in the Jinja renderer. .sp -New in version 2016.3.0. - +The following lines show how to access that information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq + #!pyobjects + + test_mode = __opts__[\(dqtest\(dq] + sls_name = __sls__ .ft P .fi .UNINDENT .UNINDENT +.SS Map Data .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +When building complex states or formulas you often need a way of building up a +map of data based on grain data. The most common use of this is tracking the +package and service name differences between distributions. .sp -New in version 2016.3.0. - +To build map data using pyobjects we provide a class named Map that you use to +build your own classes with inner classes for each set of values for the +different grain matches. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mongo \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq + #!pyobjects + + class Samba(Map): + merge = \(aqsamba:lookup\(aq + # NOTE: priority is new to 2017.7.0 + priority = (\(aqos_family\(aq, \(aqos\(aq) + + class Ubuntu: + __grain__ = \(aqos\(aq + service = \(aqsmbd\(aq + + class Debian: + server = \(aqsamba\(aq + client = \(aqsamba\-client\(aq + service = \(aqsamba\(aq + + class RHEL: + __match__ = \(aqRedHat\(aq + server = \(aqsamba\(aq + client = \(aqsamba\(aq + service = \(aqsmb\(aq .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_return.get_fun(fun) -Return the most recent jobs that have executed the named function -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_return.get_jid(jid) -Return the return information associated with a jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_return.returner(ret) -Return data to a mongodb server -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mongo_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.multi_returner .sp -Read/Write multiple returners +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.multi_returner.clean_old_jobs() -Clean out the old jobs from all returners (if you have it) -.UNINDENT +.INDENT 3.5 +By default, the \fBos_family\fP grain will be used as the target for +matching. This can be overridden by specifying a \fB__grain__\fP attribute. +.sp +If a \fB__match__\fP attribute is defined for a given class, then that value +will be matched against the targeted grain, otherwise the class name\(aqs +value will be be matched. +.sp +Given the above example, the following is true: .INDENT 0.0 -.TP -.B salt.returners.multi_returner.get_jid(jid) -Merge the return data from all returners +.IP 1. 3 +Minions with an \fBos_family\fP of \fBDebian\fP will be assigned the +attributes defined in the \fBDebian\fP class. +.IP 2. 3 +Minions with an \fBos\fP grain of \fBUbuntu\fP will be assigned the +attributes defined in the \fBUbuntu\fP class. +.IP 3. 3 +Minions with an \fBos_family\fP grain of \fBRedHat\fP will be assigned the +attributes defined in the \fBRHEL\fP class. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.multi_returner.get_jids() -Return all job data from all returners +.sp +That said, sometimes a minion may match more than one class. For instance, +in the above example, Ubuntu minions will match both the \fBDebian\fP and +\fBUbuntu\fP classes, since Ubuntu has an \fBos_family\fP grain of \fBDebian\fP +and an \fBos\fP grain of \fBUbuntu\fP\&. As of the 2017.7.0 release, the order is +dictated by the order of declaration, with classes defined later overriding +earlier ones. Additionally, 2017.7.0 adds support for explicitly defining +the ordering using an optional attribute called \fBpriority\fP\&. +.sp +Given the above example, \fBos_family\fP matches will be processed first, +with \fBos\fP matches processed after. This would have the effect of +assigning \fBsmbd\fP as the \fBservice\fP attribute on Ubuntu minions. If the +\fBpriority\fP item was not defined, or if the order of the items in the +\fBpriority\fP tuple were reversed, Ubuntu minions would have a \fBservice\fP +attribute of \fBsamba\fP, since \fBos_family\fP matches would have been +processed second. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.multi_returner.get_load(jid) -Merge the load data from all returners .UNINDENT +.sp +To use this new data you can import it into your state file and then access +your attributes. To access the data in the map you simply access the attribute +name on the base class that is extending Map. Assuming the above Map was in the +file \fBsamba/map.sls\fP, you could do the following. .INDENT 0.0 -.TP -.B salt.returners.multi_returner.prep_jid(nocache=False, passed_jid=None) -Call both with prep_jid on all returners in multi_returner +.INDENT 3.5 .sp -TODO: finish this, what do do when you get different jids from 2 returners... -since our jids are time based, this make this problem hard, because they -aren\(aqt unique, meaning that we have to make sure that no one else got the jid -and if they did we spin to get a new one, which means \(dqlocking\(dq the jid in 2 -returners is non\-trivial +.nf +.ft C + #!pyobjects + + from salt://samba/map.sls import Samba + + with Pkg.installed(\(dqsamba\(dq, names=[Samba.server, Samba.client]): + Service.running(\(dqsamba\(dq, name=Samba.service) +.ft P +.fi +.UNINDENT .UNINDENT .INDENT 0.0 .TP -.B salt.returners.multi_returner.returner(load) -Write return to all returners in multi_returner +.B class salt.renderers.pyobjects.PyobjectsModule(name, attrs) +This provides a wrapper for bare imports. .UNINDENT .INDENT 0.0 .TP -.B salt.returners.multi_returner.save_load(jid, clear_load, minions=None) -Write load to all returners in multi_returner +.B salt.renderers.pyobjects.load_states() +This loads our states into the salt __context__ .UNINDENT .INDENT 0.0 .TP -.B salt.returners.multi_returner.save_minions(jid, minions, syndic_id=None) -Included for API consistency +.B salt.renderers.pyobjects.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, salt_data=True, **kwargs) .UNINDENT -.SS salt.returners.mysql -.sp -Return data to a mysql server +.SS salt.renderers.stateconf .INDENT 0.0 .TP .B maintainer -Dave Boucha <\fI\%dave@saltstack.com\fP>, Seth House <\fI\%shouse@saltstack.com\fP> +Jack Kuan <\fI\%kjkuan@gmail.com\fP> .TP .B maturity -mature -.TP -.B depends -python\-mysqldb +new .TP .B platform all .UNINDENT .sp -To enable this returner, the minion will need the python client for mysql -installed and the following values configured in the minion or master -config. These are the defaults: +This module provides a custom renderer that processes a salt file with a +specified templating engine (e.g. Jinja) and a chosen data renderer (e.g. YAML), +extracts arguments for any \fBstateconf.set\fP state, and provides the extracted +arguments (including Salt\-specific args, such as \fBrequire\fP, etc) as template +context. The goal is to make writing reusable/configurable/parameterized +salt files easier and cleaner. +.sp +To use this renderer, either set it as the default renderer via the +\fBrenderer\fP option in master/minion\(aqs config, or use the shebang line in each +individual sls file, like so: \fB#!stateconf\fP\&. Note, due to the way this +renderer works, it must be specified as the first renderer in a render +pipeline. That is, you cannot specify \fB#!mako|yaml|stateconf\fP, for example. +Instead, you specify them as renderer arguments: \fB#!stateconf mako . yaml\fP\&. +.sp +Here\(aqs a list of features enabled by this renderer. .INDENT 0.0 +.IP \(bu 2 +Prefixes any state id (declaration or reference) that starts with a dot (\fB\&.\fP) +to avoid duplicated state ids when the salt file is included by other salt +files. +.sp +For example, in the \fIsalt://some/file.sls\fP, a state id such as \fB\&.sls_params\fP +will be turned into \fBsome.file::sls_params\fP\&. Example: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -mysql.host: \(aqsalt\(aq -mysql.user: \(aqsalt\(aq -mysql.pass: \(aqsalt\(aq -mysql.db: \(aqsalt\(aq -mysql.port: 3306 +#!stateconf yaml . jinja + +\&.vim: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -SSL is optional. The defaults are set to None. If you do not want to use SSL, -either exclude these options or set them to None. -.INDENT 0.0 +Above will be translated into: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -mysql.ssl_ca: None -mysql.ssl_cert: None -mysql.ssl_key: None +some.file::vim: + pkg.installed: + \- name: vim .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration -with \fIalternative.\fP\&. Any values not found in the alternative configuration will -be pulled from the default location. As stated above, SSL configuration is -optional. The following ssl options are simply for illustration purposes: -.INDENT 0.0 +Notice how that if a state under a dot\-prefixed state id has no \fBname\fP +argument then one will be added automatically by using the state id with +the leading dot stripped off. +.sp +The leading dot trick can be used with extending state ids as well, +so you can include relatively and extend relatively. For example, when +extending a state in \fIsalt://some/other_file.sls\fP, e.g.: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -alternative.mysql.host: \(aqsalt\(aq -alternative.mysql.user: \(aqsalt\(aq -alternative.mysql.pass: \(aqsalt\(aq -alternative.mysql.db: \(aqsalt\(aq -alternative.mysql.port: 3306 -alternative.mysql.ssl_ca: \(aq/etc/pki/mysql/certs/localhost.pem\(aq -alternative.mysql.ssl_cert: \(aq/etc/pki/mysql/certs/localhost.crt\(aq -alternative.mysql.ssl_key: \(aq/etc/pki/mysql/certs/localhost.key\(aq +#!stateconf yaml . jinja + +include: + \- .file + +extend: + .file::sls_params: + stateconf.set: + \- name1: something .ft P .fi .UNINDENT .UNINDENT .sp -Should you wish the returner data to be cleaned out every so often, set -\fIkeep_jobs_seconds\fP to the number of hours for the jobs to live in the -tables. Setting it to \fI0\fP will cause the data to stay in the tables. The -default setting for \fIkeep_jobs_seconds\fP is set to \fI86400\fP\&. +Above will be pre\-processed into: +.INDENT 2.0 +.INDENT 3.5 .sp -Should you wish to archive jobs in a different table for later processing, -set \fIarchive_jobs\fP to True. Salt will create 3 archive tables -.INDENT 0.0 -.IP \(bu 2 -\fIjids_archive\fP +.nf +.ft C +include: + \- some.file + +extend: + some.file::sls_params: + stateconf.set: + \- name1: something +.ft P +.fi +.UNINDENT +.UNINDENT .IP \(bu 2 -\fIsalt_returns_archive\fP +Adds a \fBsls_dir\fP context variable that expands to the directory containing +the rendering salt file. So, you can write \fBsalt://{{sls_dir}}/...\fP to +reference templates files used by your salt file. .IP \(bu 2 -\fIsalt_events_archive\fP +Recognizes the special state function, \fBstateconf.set\fP, that configures a +default list of named arguments usable within the template context of +the salt file. Example: +.INDENT 2.0 +.INDENT 3.5 +.sp +.nf +.ft C +#!stateconf yaml . jinja + +\&.sls_params: + stateconf.set: + \- name1: value1 + \- name2: value2 + \- name3: + \- value1 + \- value2 + \- value3 + \- require_in: + \- cmd: output + +# \-\-\- end of state config \-\-\- + +\&.output: + cmd.run: + \- name: | + echo \(aqname1={{sls_params.name1}} + name2={{sls_params.name2}} + name3[1]={{sls_params.name3[1]}} + \(aq +.ft P +.fi +.UNINDENT .UNINDENT .sp -and move the contents of \fIjids\fP, \fIsalt_returns\fP, and \fIsalt_events\fP that are -more than \fIkeep_jobs_seconds\fP seconds old to these tables. +This even works with \fBinclude\fP + \fBextend\fP so that you can override +the default configured arguments by including the salt file and then +\fBextend\fP the \fBstateconf.set\fP states that come from the included salt +file. (\fIIMPORTANT: Both the included and the extending sls files must use the +stateconf renderer for this \(ga\(gaextend\(ga\(ga to work!\fP) .sp -Use the following mysql database schema: -.INDENT 0.0 +Notice that the end of configuration marker (\fB# \-\-\- end of state config \-\-\fP) +is needed to separate the use of \(aqstateconf.set\(aq form the rest of your salt +file. The regex that matches such marker can be configured via the +\fBstateconf_end_marker\fP option in your master or minion config file. +.sp +Sometimes, it is desirable to set a default argument value that\(aqs based on +earlier arguments in the same \fBstateconf.set\fP\&. For example, it may be +tempting to do something like this: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -CREATE DATABASE \(gasalt\(ga - DEFAULT CHARACTER SET utf8 - DEFAULT COLLATE utf8_general_ci; - -USE \(gasalt\(ga; +#!stateconf yaml . jinja -\-\- -\-\- Table structure for table \(gajids\(ga -\-\- +\&.apache: + stateconf.set: + \- host: localhost + \- port: 1234 + \- url: \(aqhttp://{{host}}:{{port}}/\(aq -DROP TABLE IF EXISTS \(gajids\(ga; -CREATE TABLE \(gajids\(ga ( - \(gajid\(ga varchar(255) NOT NULL, - \(gaload\(ga mediumtext NOT NULL, - UNIQUE KEY \(gajid\(ga (\(gajid\(ga) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +# \-\-\- end of state config \-\-\- -\-\- -\-\- Table structure for table \(gasalt_returns\(ga -\-\- +\&.test: + cmd.run: + \- name: echo \(aq{{apache.url}}\(aq + \- cwd: / +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +However, this won\(aqt work. It can however be worked around like so: +.INDENT 2.0 +.INDENT 3.5 +.sp +.nf +.ft C +#!stateconf yaml . jinja -DROP TABLE IF EXISTS \(gasalt_returns\(ga; -CREATE TABLE \(gasalt_returns\(ga ( - \(gafun\(ga varchar(50) NOT NULL, - \(gajid\(ga varchar(255) NOT NULL, - \(gareturn\(ga mediumtext NOT NULL, - \(gaid\(ga varchar(255) NOT NULL, - \(gasuccess\(ga varchar(10) NOT NULL, - \(gafull_ret\(ga mediumtext NOT NULL, - \(gaalter_time\(ga TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - KEY \(gaid\(ga (\(gaid\(ga), - KEY \(gajid\(ga (\(gajid\(ga), - KEY \(gafun\(ga (\(gafun\(ga) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +\&.apache: + stateconf.set: + \- host: localhost + \- port: 1234 +{# \- url: \(aqhttp://{{host}}:{{port}}/\(aq #} -\-\- -\-\- Table structure for table \(gasalt_events\(ga -\-\- +# \-\-\- end of state config \-\-\- +# {{ apache.setdefault(\(aqurl\(aq, \(dqhttp://%(host)s:%(port)s/\(dq % apache) }} -DROP TABLE IF EXISTS \(gasalt_events\(ga; -CREATE TABLE \(gasalt_events\(ga ( -\(gaid\(ga BIGINT NOT NULL AUTO_INCREMENT, -\(gatag\(ga varchar(255) NOT NULL, -\(gadata\(ga mediumtext NOT NULL, -\(gaalter_time\(ga TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -\(gamaster_id\(ga varchar(255) NOT NULL, -PRIMARY KEY (\(gaid\(ga), -KEY \(gatag\(ga (\(gatag\(ga) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +\&.test: + cmd.run: + \- name: echo \(aq{{apache.url}}\(aq + \- cwd: / .ft P .fi .UNINDENT .UNINDENT +.IP \(bu 2 +Adds support for relative include and exclude of .sls files. Example: +.INDENT 2.0 +.INDENT 3.5 .sp -Required python modules: MySQLdb +.nf +.ft C +#!stateconf yaml . jinja + +include: + \- .apache + \- .db.mysql + \- ..app.django + +exclude: + \- sls: .users +.ft P +.fi +.UNINDENT +.UNINDENT .sp -To use the mysql returner, append \(aq\-\-return mysql\(aq to the salt command. -.INDENT 0.0 +If the above is written in a salt file at \fIsalt://some/where.sls\fP then +it will include \fIsalt://some/apache.sls\fP, \fIsalt://some/db/mysql.sls\fP and +\fIsalt://app/django.sls\fP, and exclude \fIsalt://some/users.ssl\fP\&. Actually, +it does that by rewriting the above \fBinclude\fP and \fBexclude\fP into: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mysql +include: + \- some.apache + \- some.db.mysql + \- app.django + +exclude: + \- sls: some.users .ft P .fi .UNINDENT .UNINDENT +.IP \(bu 2 +Optionally (enabled by default, \fIdisable\fP via the \fI\-G\fP renderer option, +e.g. in the shebang line: \fB#!stateconf \-G\fP), generates a +\fBstateconf.set\fP goal state (state id named as \fB\&.goal\fP by default, +configurable via the master/minion config option, \fBstateconf_goal_state\fP) +that requires all other states in the salt file. Note, the \fB\&.goal\fP +state id is subject to dot\-prefix rename rule mentioned earlier. .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Such goal state is intended to be required by some state in an including +salt file. For example, in your webapp salt file, if you include a +sls file that is supposed to setup Tomcat, you might want to make sure that +all states in the Tomcat sls file will be executed before some state in +the webapp sls file. +.IP \(bu 2 +Optionally (enable via the \fI\-o\fP renderer option, e.g. in the shebang line: +\fB#!stateconf \-o\fP), orders the states in a sls file by adding a +\fBrequire\fP requisite to each state such that every state requires the +state defined just before it. The order of the states here is the order +they are defined in the sls file. (Note: this feature is only available +if your minions are using Python >= 2.7. For Python2.6, it should also +work if you install the \fIordereddict\fP module from PyPI) .sp -New in version 2015.5.0. - +By enabling this feature, you are basically agreeing to author your sls +files in a way that gives up the explicit (or implicit?) ordering imposed +by the use of \fBrequire\fP, \fBwatch\fP, \fBrequire_in\fP or \fBwatch_in\fP +requisites, and instead, you rely on the order of states you define in +the sls files. This may or may not be a better way for you. However, if +there are many states defined in a sls file, then it tends to be easier +to see the order they will be executed with this feature. +.sp +You are still allowed to use all the requisites, with a few restrictions. +You cannot \fBrequire\fP or \fBwatch\fP a state defined \fIafter\fP the current +state. Similarly, in a state, you cannot \fBrequire_in\fP or \fBwatch_in\fP +a state defined \fIbefore\fP it. Breaking any of the two restrictions above +will result in a state loop. The renderer will check for such incorrect +uses if this feature is enabled. +.sp +Additionally, \fBnames\fP declarations cannot be used with this feature +because the way they are compiled into low states make it impossible to +guarantee the order in which they will be executed. This is also checked +by the renderer. As a workaround for not being able to use \fBnames\fP, +you can achieve the same effect, by generate your states with the +template engine available within your sls file. +.sp +Finally, with the use of this feature, it becomes possible to easily make +an included sls file execute all its states \fIafter\fP some state (say, with +id \fBX\fP) in the including sls file. All you have to do is to make state, +\fBX\fP, \fBrequire_in\fP the first state defined in the included sls file. +.UNINDENT +.sp +When writing sls files with this renderer, one should avoid using what can be +defined in a \fBname\fP argument of a state as the state\(aqs id. That is, avoid +writing states like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mysql \-\-return_config alternative +/path/to/some/file: + file.managed: + \- source: salt://some/file + +cp /path/to/some/file file2: + cmd.run: + \- cwd: / + \- require: + \- file: /path/to/some/file .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +Instead, define the state id and the \fBname\fP argument separately for each +state. Also, the ID should be something meaningful and easy to reference within +a requisite (which is a good habit anyway, and such extra indirection would +also makes the sls file easier to modify later). Thus, the above states should +be written like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return mysql \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +add\-some\-file: + file.managed: + \- name: /path/to/some/file + \- source: salt://some/file + +copy\-files: + cmd.run: + \- name: cp /path/to/some/file file2 + \- cwd: / + \- require: + \- file: add\-some\-file .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.clean_old_jobs() -Called in the master\(aqs event loop every loop_interval. Archives and/or -deletes the events and job details from the database. -:return: -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.event_return(events) -Return event to mysql server .sp -Requires that configuration be enabled via \(aqevent_return\(aq -option in master config. -.UNINDENT +Moreover, when referencing a state from a requisite, you should reference the +state\(aqs id plus the state name rather than the state name plus its \fBname\fP +argument. (Yes, in the above example, you can actually \fBrequire\fP the +\fBfile: /path/to/some/file\fP, instead of the \fBfile: add\-some\-file\fP). The +reason is that this renderer will re\-write or rename state id\(aqs and their +references for state id\(aqs prefixed with \fB\&.\fP\&. So, if you reference \fBname\fP +then there\(aqs no way to reliably rewrite such reference. +.SS salt.renderers.toml .INDENT 0.0 .TP -.B salt.returners.mysql.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 +.B salt.renderers.tomlmod.render(sls_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts TOML as a string or as a file object and runs it through the +parser. +.INDENT 7.0 .TP -.B salt.returners.mysql.get_jid(jid) -Return the information returned when the specified job id was executed +.B Return type +A Python data structure .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.get_jids() -Return a list of all job ids .UNINDENT +.SS salt.renderers.wempy .INDENT 0.0 .TP -.B salt.returners.mysql.get_jids_filter(count, filter_find_job=True) -Return a list of all job ids -:param int count: show not more than the count of most recent jobs -:param bool filter_find_jobs: filter out \(aqsaltutil.find_job\(aq jobs -.UNINDENT -.INDENT 0.0 +.B salt.renderers.wempy.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, context=None, **kws) +Render the data passing the functions and grains into the rendering system +.INDENT 7.0 .TP -.B salt.returners.mysql.get_load(jid) -Return the load data that marks a specified jid +.B Return type +string .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.get_minions() -Return a list of minions .UNINDENT +.SS salt.renderers.yaml +.SS Understanding YAML +.sp +The default renderer for SLS files is the YAML renderer. YAML is a +markup language with many powerful features. However, Salt uses +a small subset of YAML that maps over very commonly used data structures, +like lists and dictionaries. It is the job of the YAML renderer to take +the YAML data structure and compile it into a Python data structure for +use by Salt. +.sp +Though YAML syntax may seem daunting and terse at first, there are only +three very simple rules to remember when writing YAML for SLS files. +.SS Rule One: Indentation +.sp +YAML uses a fixed indentation scheme to represent relationships between +data layers. Salt requires that the indentation for each level consists +of exactly two spaces. Do not use tabs. +.SS Rule Two: Colons +.sp +Python dictionaries are, of course, simply key\-value pairs. Users from other +languages may recognize this data type as hashes or associative arrays. +.sp +Dictionary keys are represented in YAML as strings terminated by a trailing colon. +Values are represented by either a string following the colon, separated by a space: .INDENT 0.0 -.TP -.B salt.returners.mysql.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.INDENT 3.5 +.sp +.nf +.ft C +my_key: my_value +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.returner(ret) -Return data to a mysql server .UNINDENT +.sp +In Python, the above maps to: .INDENT 0.0 -.TP -.B salt.returners.mysql.save_load(jid, load, minions=None) -Save the load to the specified jid id +.INDENT 3.5 +.sp +.nf +.ft C +{\(dqmy_key\(dq: \(dqmy_value\(dq} +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.mysql.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.nagios_nrdp_return -.sp -Return salt data to Nagios .sp -The following fields can be set in the minion conf file: +Dictionaries can be nested: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nagios.url (required) -nagios.token (required) -nagios.service (optional) -nagios.check_type (optional) +first_level_dict_key: + second_level_dict_key: value_in_second_level_dict .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +And in Python: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nagios.url -nagios.token -nagios.service +{\(dqfirst_level_dict_key\(dq: {\(dqsecond_level_dict_key\(dq: \(dqvalue_in_second_level_dict\(dq}} .ft P .fi .UNINDENT .UNINDENT +.SS Rule Three: Dashes .sp -Nagios settings may also be configured as: +To represent lists of items, a single dash followed by a space is used. Multiple +items are a part of the same list as a function of their having the same level of indentation. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - nagios: - url: http://localhost/nrdp - token: r4nd0mt0k3n - service: service\-check - - alternative.nagios: - url: http://localhost/nrdp - token: r4nd0mt0k3n - service: another\-service\-check - -To use the Nagios returner, append \(aq\-\-return nagios\(aq to the salt command. ex: - -\&.. code\-block:: bash - - salt \(aq*\(aq test.ping \-\-return nagios - -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. ex: - - salt \(aq*\(aq test.ping \-\-return nagios \-\-return_config alternative +\- list_value_one +\- list_value_two +\- list_value_three .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +Lists can be the value of a key\-value pair. This is quite common in Salt: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return nagios \-\-return_kwargs \(aq{\(dqservice\(dq: \(dqservice\-name\(dq}\(aq +my_dictionary: + \- list_value_one + \- list_value_two + \- list_value_three .ft P .fi .UNINDENT .UNINDENT +.SS Reference +.sp +YAML Renderer for Salt +.sp +For YAML usage information see \fI\%Understanding YAML\fP\&. .INDENT 0.0 .TP -.B salt.returners.nagios_nrdp_return.returner(ret) -Send a message to Nagios with the data +.B salt.renderers.yaml.get_yaml_loader(argline) +Return the ordered dict yaml loader .UNINDENT -.SS salt.returners.odbc -.sp -Return data to an ODBC compliant server. This driver was -developed with Microsoft SQL Server in mind, but theoretically -could be used to return data to any compliant ODBC database -as long as there is a working ODBC driver for it on your -minion platform. .INDENT 0.0 .TP -.B maintainer +.B salt.renderers.yaml.render(yaml_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kws) +Accepts YAML as a string or as a file object and runs it through the YAML +parser. .INDENT 7.0 -.IP C. 3 -.INDENT 3.0 -.IP R. 3 -Oldham (\fI\%cr@saltstack.com\fP) +.TP +.B Return type +A Python data structure .UNINDENT .UNINDENT -.TP -.B maturity -New -.TP -.B depends -unixodbc, pyodbc, freetds (for SQL Server) -.TP -.B platform -all -.UNINDENT -.sp -To enable this returner the minion will need +.SS salt.renderers.yamlex .sp -On Linux: +YAMLEX renderer is a replacement of the YAML renderer. +It\(aqs 100% YAML with a pinch of Salt magic: .INDENT 0.0 -.INDENT 3.5 -unixodbc (\fI\%http://www.unixodbc.org\fP) -pyodbc (\fIpip install pyodbc\fP) -The FreeTDS ODBC driver for SQL Server (\fI\%http://www.freetds.org\fP) -or another compatible ODBC driver -.UNINDENT +.IP \(bu 2 +All mappings are automatically OrderedDict +.IP \(bu 2 +All strings are automatically str obj +.IP \(bu 2 +data aggregation with !aggregation yaml tag, based on the \fBsalt.utils.aggregation\fP module. +.IP \(bu 2 +data aggregation over documents for pillar .UNINDENT .sp -On Windows: +Instructed aggregation within the \fB!aggregation\fP and the \fB!reset\fP tags: .INDENT 0.0 .INDENT 3.5 -TBD +.sp +.nf +.ft C +#!yamlex +foo: !aggregate first +foo: !aggregate second +bar: !aggregate {first: foo} +bar: !aggregate {second: bar} +baz: !aggregate 42 +qux: !aggregate default +!reset qux: !aggregate my custom data +.ft P +.fi .UNINDENT .UNINDENT .sp -unixODBC and FreeTDS need to be configured via /etc/odbcinst.ini and -/etc/odbc.ini. -.sp -/etc/odbcinst.ini: +is roughly equivalent to .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[TDS] -Description=TDS -Driver=/usr/lib/x86_64\-linux\-gnu/odbc/libtdsodbc.so +foo: [first, second] +bar: {first: foo, second: bar} +baz: [42] +qux: [my custom data] .ft P .fi .UNINDENT .UNINDENT +.SS Reference +.INDENT 0.0 +.TP +.B salt.renderers.yamlex.render(sls_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) +Accepts YAML_EX as a string or as a file object and runs it through the YAML_EX +parser. +.INDENT 7.0 +.TP +.B Return type +A Python data structure +.UNINDENT +.UNINDENT +.SH USING SALT .sp -(Note the above Driver line needs to point to the location of the FreeTDS -shared library. This example is for Ubuntu 14.04.) +This section describes the fundamental components and concepts that you need to understand to use Salt. +.SS Grains .sp -/etc/odbc.ini: +Salt comes with an interface to derive information about the underlying system. +This is called the grains interface, because it presents salt with grains of +information. Grains are collected for the operating system, domain name, +IP address, kernel, OS type, memory, and many other system properties. +.sp +The grains interface is made available to Salt modules and components so that +the right salt minion commands are automatically available on the right +systems. +.sp +Grain data is relatively static, though if system information changes +(for example, if network settings are changed), or if a new value is assigned +to a custom grain, grain data is refreshed. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -[TS] -Description = \(dqSalt Returner\(dq -Driver=TDS -Server = -Port = 1433 -Database = salt -Trace = No -.ft P -.fi +Grains resolve to lowercase letters. For example, \fBFOO\fP, and \fBfoo\fP +target the same grain. .UNINDENT .UNINDENT +.SS Listing Grains .sp -Also you need the following values configured in the minion or master config. -Configure as you see fit: +Available grains can be listed by using the \(aqgrains.ls\(aq module: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -returner.odbc.dsn: \(aqTS\(aq -returner.odbc.user: \(aqsalt\(aq -returner.odbc.passwd: \(aqsalt\(aq +salt \(aq*\(aq grains.ls .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Grains data can be listed by using the \(aqgrains.items\(aq module: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.returner.odbc.dsn: \(aqTS\(aq -alternative.returner.odbc.user: \(aqsalt\(aq -alternative.returner.odbc.passwd: \(aqsalt\(aq +salt \(aq*\(aq grains.items .ft P .fi .UNINDENT .UNINDENT +.SS Using grains in a state .sp -Running the following commands against Microsoft SQL Server in the desired -database as the appropriate user should create the database tables -correctly. Replace with equivalent SQL for other ODBC\-compliant servers +To use a grain in a state you can access it via \fI{{ grains[\(aqkey\(aq] }}\fP\&. +.SS Grains in the Minion Config +.sp +Grains can also be statically assigned within the minion configuration file. +Just add the option \fI\%grains\fP and pass options to it: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - \-\- - \-\- Table structure for table \(aqjids\(aq - \-\- - - if OBJECT_ID(\(aqdbo.jids\(aq, \(aqU\(aq) is not null - DROP TABLE dbo.jids - - CREATE TABLE dbo.jids ( - jid varchar(255) PRIMARY KEY, - load varchar(MAX) NOT NULL - ); - - \-\- - \-\- Table structure for table \(aqsalt_returns\(aq - \-\- - IF OBJECT_ID(\(aqdbo.salt_returns\(aq, \(aqU\(aq) IS NOT NULL - DROP TABLE dbo.salt_returns; - - CREATE TABLE dbo.salt_returns ( - added datetime not null default (getdate()), - fun varchar(100) NOT NULL, - jid varchar(255) NOT NULL, - retval varchar(MAX) NOT NULL, - id varchar(255) NOT NULL, - success bit default(0) NOT NULL, - full_ret varchar(MAX) - ); - - CREATE INDEX salt_returns_added on dbo.salt_returns(added); - CREATE INDEX salt_returns_id on dbo.salt_returns(id); - CREATE INDEX salt_returns_jid on dbo.salt_returns(jid); - CREATE INDEX salt_returns_fun on dbo.salt_returns(fun); - -To use this returner, append \(aq\-\-return odbc\(aq to the salt command. - -\&.. code\-block:: bash - - salt \(aq*\(aq status.diskusage \-\-return odbc - -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. - -\&.. versionadded:: 2015.5.0 - -\&.. code\-block:: bash - - salt \(aq*\(aq test.ping \-\-return odbc \-\-return_config alternative +grains: + roles: + \- webserver + \- memcache + deployment: datacenter4 + cabinet: 13 + cab_u: 14\-15 .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +Then status data specific to your servers can be retrieved via Salt, or used +inside of the State system for matching. It also makes it possible to target based on specific data about your deployment, as in the example above. +.SS Grains in /etc/salt/grains .sp -New in version 2016.3.0. - +If you do not want to place your custom static grains in the minion config +file, you can also put them in \fB/etc/salt/grains\fP on the minion. They are configured in the +same way as in the above example, only without a top\-level \fBgrains:\fP key: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return odbc \-\-return_kwargs \(aq{\(dqdsn\(dq: \(dqdsn\-name\(dq}\(aq +roles: + \- webserver + \- memcache +deployment: datacenter4 +cabinet: 13 +cab_u: 14\-15 .ft P .fi .UNINDENT .UNINDENT +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.odbc.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.returner(ret) -Return data to an odbc server -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.save_load(jid, load, minions=None) -Save the load to the specified jid id +.INDENT 3.5 +Grains in \fB/etc/salt/grains\fP are ignored if you specify the same grains in the minion config. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.odbc.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.pgjsonb .sp -Return data to a PostgreSQL server with json data stored in Pg\(aqs jsonb data type +\fBNOTE:\fP .INDENT 0.0 -.TP -.B maintainer -Dave Boucha <\fI\%dave@saltstack.com\fP>, Seth House <\fI\%shouse@saltstack.com\fP>, C. R. Oldham <\fI\%cr@saltstack.com\fP> -.TP -.B maturity -Stable -.TP -.B depends -python\-psycopg2 -.TP -.B platform -all +.INDENT 3.5 +Grains are static, and since they are not often changed, they will need a grains refresh when they are updated. You can do this by calling: \fBsalt minion saltutil.refresh_modules\fP +.UNINDENT .UNINDENT .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -There are three PostgreSQL returners. Any can function as an external -\fI\%master job cache\fP\&. but each has different -features. SaltStack recommends -\fI\%returners.pgjsonb\fP if you are working with -a version of PostgreSQL that has the appropriate native binary JSON types. -Otherwise, review -\fI\%returners.postgres\fP and -\fI\%returners.postgres_local_cache\fP -to see which module best suits your particular needs. +You can equally configure static grains for Proxy Minions. +As multiple Proxy Minion processes can run on the same machine, you need +to index the files using the Minion ID, under \fB/etc/salt/proxy.d//grains\fP\&. +For example, the grains for the Proxy Minion \fBrouter1\fP can be defined +under \fB/etc/salt/proxy.d/router1/grains\fP, while the grains for the +Proxy Minion \fBswitch7\fP can be put in \fB/etc/salt/proxy.d/switch7/grains\fP\&. .UNINDENT .UNINDENT +.SS Matching Grains in the Top File .sp -To enable this returner, the minion will need the python client for PostgreSQL -installed and the following values configured in the minion or master -config. These are the defaults: +With correctly configured grains on the Minion, the \fI\%top file\fP used in +Pillar or during Highstate can be made very efficient. For example, consider +the following configuration: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -returner.pgjsonb.host: \(aqsalt\(aq -returner.pgjsonb.user: \(aqsalt\(aq -returner.pgjsonb.pass: \(aqsalt\(aq -returner.pgjsonb.db: \(aqsalt\(aq -returner.pgjsonb.port: 5432 +\(aqroles:webserver\(aq: + \- match: grain + \- state0 + +\(aqroles:memcache\(aq: + \- match: grain + \- state1 + \- state2 .ft P .fi .UNINDENT .UNINDENT .sp -SSL is optional. The defaults are set to None. If you do not want to use SSL, -either exclude these options or set them to None. +For this example to work, you would need to have defined the grain +\fBrole\fP for the minions you wish to match. +.SS Writing Grains +.sp +\fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -returner.pgjsonb.sslmode: None -returner.pgjsonb.sslcert: None -returner.pgjsonb.sslkey: None -returner.pgjsonb.sslrootcert: None -returner.pgjsonb.sslcrl: None -.ft P -.fi +Grains can be set by users that have access to the minion configuration files on +the local system, making them less secure than other identifiers in Salt. Avoid +storing sensitive data, such as passwords or keys, on minions. Instead, make +use of \fI\%Storing Static Data in the Pillar\fP and/or \fI\%Storing Data in Other Databases\fP\&. .UNINDENT .UNINDENT .sp -New in version 2017.5.0. - +The grains are derived by executing all of the \(dqpublic\(dq functions (i.e. those +which do not begin with an underscore) found in the modules located in the +Salt\(aqs core grains code, followed by those in any custom grains modules. The +functions in a grains module must return a \fI\%Python dictionary\fP, where the dictionary keys are the names of grains, and +each key\(aqs value is that value for that grain. .sp -Alternative configuration values can be used by prefacing the configuration -with \fIalternative.\fP\&. Any values not found in the alternative configuration will -be pulled from the default location. As stated above, SSL configuration is -optional. The following ssl options are simply for illustration purposes: +Custom grains modules should be placed in a subdirectory named \fB_grains\fP +located under the \fI\%file_roots\fP specified by the master config +file. The default path would be \fB/srv/salt/_grains\fP\&. Custom grains modules +will be distributed to the minions when \fI\%state.highstate\fP is run, or by executing the +\fI\%saltutil.sync_grains\fP or +\fI\%saltutil.sync_all\fP functions. +.sp +Grains modules are easy to write, and (as noted above) only need to return a +dictionary. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.pgjsonb.host: \(aqsalt\(aq -alternative.pgjsonb.user: \(aqsalt\(aq -alternative.pgjsonb.pass: \(aqsalt\(aq -alternative.pgjsonb.db: \(aqsalt\(aq -alternative.pgjsonb.port: 5432 -alternative.pgjsonb.ssl_ca: \(aq/etc/pki/mysql/certs/localhost.pem\(aq -alternative.pgjsonb.ssl_cert: \(aq/etc/pki/mysql/certs/localhost.crt\(aq -alternative.pgjsonb.ssl_key: \(aq/etc/pki/mysql/certs/localhost.key\(aq +def yourfunction(): + # initialize a grains dictionary + grains = {} + # Some code for logic that sets grains like + grains[\(dqyourcustomgrain\(dq] = True + grains[\(dqanothergrain\(dq] = \(dqsomevalue\(dq + return grains .ft P .fi .UNINDENT .UNINDENT .sp -Should you wish the returner data to be cleaned out every so often, set -\fBkeep_jobs_seconds\fP to the number of seconds for the jobs to live in the tables. -Setting it to \fB0\fP or leaving it unset will cause the data to stay in the tables. -.sp -Should you wish to archive jobs in a different table for later processing, -set \fBarchive_jobs\fP to True. Salt will create 3 archive tables; -.INDENT 0.0 -.IP \(bu 2 -\fBjids_archive\fP -.IP \(bu 2 -\fBsalt_returns_archive\fP -.IP \(bu 2 -\fBsalt_events_archive\fP -.UNINDENT +The name of the function does not matter and will not factor into the grains +data at all; only the keys/values returned become part of the grains. +.SS When to Use a Custom Grain .sp -and move the contents of \fBjids\fP, \fBsalt_returns\fP, and \fBsalt_events\fP that are -more than \fBkeep_jobs_seconds\fP seconds old to these tables. +Before adding new grains, consider what the data is and remember that grains +should (for the most part) be static data. .sp -New in version 2019.2.0. - +If the data is something that is likely to change, consider using \fI\%Pillar\fP or an execution module instead. If it\(aqs a simple set of +key/value pairs, pillar is a good match. If compiling the information requires +that system commands be run, then putting this information in an execution +module is likely a better idea. .sp -Use the following Pg database schema: +Good candidates for grains are data that is useful for targeting minions in the +\fI\%top file\fP or the Salt CLI. The name and data structure of +the grain should be designed to support many platforms, operating systems or +applications. Also, keep in mind that Jinja templating in Salt supports +referencing pillar data as well as invoking functions from execution modules, +so there\(aqs no need to place information in grains to make it available to Jinja +templates. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -CREATE DATABASE salt - WITH ENCODING \(aqutf\-8\(aq; - -\-\- -\-\- Table structure for table \(gajids\(ga -\-\- -DROP TABLE IF EXISTS jids; -CREATE TABLE jids ( - jid varchar(255) NOT NULL primary key, - load jsonb NOT NULL -); -CREATE INDEX idx_jids_jsonb on jids - USING gin (load) - WITH (fastupdate=on); - -\-\- -\-\- Table structure for table \(gasalt_returns\(ga -\-\- - -DROP TABLE IF EXISTS salt_returns; -CREATE TABLE salt_returns ( - fun varchar(50) NOT NULL, - jid varchar(255) NOT NULL, - return jsonb NOT NULL, - id varchar(255) NOT NULL, - success varchar(10) NOT NULL, - full_ret jsonb NOT NULL, - alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW()); - -CREATE INDEX idx_salt_returns_id ON salt_returns (id); -CREATE INDEX idx_salt_returns_jid ON salt_returns (jid); -CREATE INDEX idx_salt_returns_fun ON salt_returns (fun); -CREATE INDEX idx_salt_returns_return ON salt_returns - USING gin (return) with (fastupdate=on); -CREATE INDEX idx_salt_returns_full_ret ON salt_returns - USING gin (full_ret) with (fastupdate=on); - -\-\- -\-\- Table structure for table \(gasalt_events\(ga -\-\- - -DROP TABLE IF EXISTS salt_events; -DROP SEQUENCE IF EXISTS seq_salt_events_id; -CREATE SEQUENCE seq_salt_events_id; -CREATE TABLE salt_events ( - id BIGINT NOT NULL UNIQUE DEFAULT nextval(\(aqseq_salt_events_id\(aq), - tag varchar(255) NOT NULL, - data jsonb NOT NULL, - alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - master_id varchar(255) NOT NULL); - -CREATE INDEX idx_salt_events_tag on - salt_events (tag); -CREATE INDEX idx_salt_events_data ON salt_events - USING gin (data) with (fastupdate=on); +\&... +\&... +{{ salt[\(aqmodule.function_name\(aq](\(aqargument_1\(aq, \(aqargument_2\(aq) }} +{{ pillar[\(aqmy_pillar_key\(aq] }} +\&... +\&... .ft P .fi .UNINDENT .UNINDENT .sp -Required python modules: Psycopg2 +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +Custom grains will not be available in the top file until after the first +\fI\%highstate\fP\&. To make custom grains available on a +minion\(aqs first highstate, it is recommended to use \fI\%this example\fP to ensure that the custom grains are synced when +the minion starts. +.UNINDENT +.UNINDENT +.SS Loading Custom Grains .sp -To use this returner, append \(aq\-\-return pgjsonb\(aq to the salt command. +If you have multiple functions specifying grains that are called from a \fBmain\fP +function, be sure to prepend grain function names with an underscore. This prevents +Salt from including the loaded grains from the grain functions in the final +grain data structure. For example, consider this custom grain file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return pgjsonb +#!/usr/bin/env python +def _my_custom_grain(): + my_grain = {\(dqfoo\(dq: \(dqbar\(dq, \(dqhello\(dq: \(dqworld\(dq} + return my_grain + + +def main(): + # initialize a grains dictionary + grains = {} + grains[\(dqmy_grains\(dq] = _my_custom_grain() + return grains .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. -.sp -New in version 2015.5.0. - +The output of this example renders like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return pgjsonb \-\-return_config alternative +# salt\-call \-\-local grains.items +local: + \-\-\-\-\-\-\-\-\-\- + + my_grains: + \-\-\-\-\-\-\-\-\-\- + foo: + bar + hello: + world .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. -.sp -New in version 2016.3.0. - +However, if you don\(aqt prepend the \fBmy_custom_grain\fP function with an underscore, +the function will be rendered twice by Salt in the items output: once for the +\fBmy_custom_grain\fP call itself, and again when it is called in the \fBmain\fP +function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return pgjsonb \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +# salt\-call \-\-local grains.items +local: +\-\-\-\-\-\-\-\-\-\- + + foo: + bar + + hello: + world + + my_grains: + \-\-\-\-\-\-\-\-\-\- + foo: + bar + hello: + world .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.clean_old_jobs() -Called in the master\(aqs event loop every loop_interval. Archives and/or -deletes the events and job details from the database. -:return: -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.event_return(events) -Return event to Pg server +.SS Precedence .sp -Requires that configuration be enabled via \(aqevent_return\(aq -option in master config. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT +Core grains can be overridden by custom grains. As there are several ways of +defining custom grains, there is an order of precedence which should be kept in +mind when defining them. The order of evaluation is as follows: .INDENT 0.0 -.TP -.B salt.returners.pgjsonb.returner(ret) -Return data to a Pg server +.IP 1. 3 +Core grains. +.IP 2. 3 +Custom grains in \fB/etc/salt/grains\fP\&. +.IP 3. 3 +Custom grains in \fB/etc/salt/minion\fP\&. +.IP 4. 3 +Custom grain modules in \fB_grains\fP directory, synced to minions. .UNINDENT +.sp +Each successive evaluation overrides the previous ones, so any grains defined +by custom grains modules synced to minions that have the same name as a core +grain will override that core grain. Similarly, grains from +\fB/etc/salt/minion\fP override both core grains and custom grain modules, and +grains in \fB_grains\fP will override \fIany\fP grains of the same name. +.sp +For custom grains, if the function takes an argument \fBgrains\fP, then the +previously rendered grains will be passed in. Because the rest of the grains +could be rendered in any order, the only grains that can be relied upon to be +passed in are \fBcore\fP grains. This was added in the 2019.2.0 release. +.SS Examples of Grains +.sp +The core module in the grains package is where the main grains are loaded by +the Salt minion and provides the principal example of how to write grains: +.sp +\fI\%salt/grains/core.py\fP +.SS Syncing Grains +.sp +Syncing grains can be done a number of ways. They are automatically synced when +\fI\%state.highstate\fP is called, or (as noted +above) the grains can be manually synced and reloaded by calling the +\fI\%saltutil.sync_grains\fP or +\fI\%saltutil.sync_all\fP functions. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.pgjsonb.save_load(jid, load, minions=None) -Save the load to the specified jid id +.INDENT 3.5 +When the \fI\%grains_cache\fP is set to False, the grains dictionary is built +and stored in memory on the minion. Every time the minion restarts or +\fBsaltutil.refresh_grains\fP is run, the grain dictionary is rebuilt from scratch. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.pgjsonb.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.postgres +.SS Storing Static Data in the Pillar .sp -Return data to a postgresql server +Pillar is an interface for Salt designed to offer global values that can be +distributed to minions. Pillar data is managed in a similar way as +the Salt State Tree. +.sp +Pillar was added to Salt in version 0.9.8 .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -There are three PostgreSQL returners. Any can function as an external -\fI\%master job cache\fP\&. but each has different -features. SaltStack recommends -\fI\%returners.pgjsonb\fP if you are working with -a version of PostgreSQL that has the appropriate native binary JSON types. -Otherwise, review -\fI\%returners.postgres\fP and -\fI\%returners.postgres_local_cache\fP -to see which module best suits your particular needs. -.UNINDENT +Storing sensitive data +.sp +Pillar data is compiled on the master. Additionally, pillar data for a +given minion is only accessible by the minion for which it is targeted in +the pillar configuration. This makes pillar useful for storing sensitive +data specific to a particular minion. .UNINDENT -.INDENT 0.0 -.TP -.B maintainer -None -.TP -.B maturity -New -.TP -.B depends -psycopg2 -.TP -.B platform -all .UNINDENT +.SS Declaring the Master Pillar .sp -To enable this returner the minion will need the psycopg2 installed and -the following values configured in the minion or master config: +The Salt Master server maintains a \fI\%pillar_roots\fP setup that +matches the structure of the \fI\%file_roots\fP used in the Salt file +server. Like \fI\%file_roots\fP, the \fI\%pillar_roots\fP option +maps environments to directories. The pillar data is then mapped to minions +based on matchers in a top file which is laid out in the same way as the state +top file. Salt pillars can use the same matcher types as the standard \fI\%top +file\fP\&. +.sp +conf_master:\fIpillar_roots\fP is configured just like \fI\%file_roots\fP\&. +For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -returner.postgres.host: \(aqsalt\(aq -returner.postgres.user: \(aqsalt\(aq -returner.postgres.passwd: \(aqsalt\(aq -returner.postgres.db: \(aqsalt\(aq -returner.postgres.port: 5432 +pillar_roots: + base: + \- /srv/pillar .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +This example configuration declares that the base environment will be located +in the \fB/srv/pillar\fP directory. It must not be in a subdirectory of the +state tree. +.sp +The top file used matches the name of the top file used for States, +and has the same structure: +.sp +\fB/srv/pillar/top.sls\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.returner.postgres.host: \(aqsalt\(aq -alternative.returner.postgres.user: \(aqsalt\(aq -alternative.returner.postgres.passwd: \(aqsalt\(aq -alternative.returner.postgres.db: \(aqsalt\(aq -alternative.returner.postgres.port: 5432 +base: + \(aq*\(aq: + \- packages .ft P .fi .UNINDENT .UNINDENT .sp -Running the following commands as the postgres user should create the database -correctly: +In the above top file, it is declared that in the \fBbase\fP environment, the +glob matching all minions will have the pillar data found in the \fBpackages\fP +pillar available to it. Assuming the \fBpillar_roots\fP value of \fB/srv/pillar\fP +taken from above, the \fBpackages\fP pillar would be located at +\fB/srv/pillar/packages.sls\fP\&. +.sp +Any number of matchers can be added to the base environment. For example, here +is an expanded version of the Pillar top file stated above: +.sp +/srv/pillar/top.sls: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -psql << EOF -CREATE ROLE salt WITH PASSWORD \(aqsalt\(aq; -CREATE DATABASE salt WITH OWNER salt; -EOF - -psql \-h localhost \-U salt << EOF -\-\- -\-\- Table structure for table \(aqjids\(aq -\-\- - -DROP TABLE IF EXISTS jids; -CREATE TABLE jids ( - jid varchar(20) PRIMARY KEY, - load text NOT NULL -); - -\-\- -\-\- Table structure for table \(aqsalt_returns\(aq -\-\- - -DROP TABLE IF EXISTS salt_returns; -CREATE TABLE salt_returns ( - fun varchar(50) NOT NULL, - jid varchar(255) NOT NULL, - return text NOT NULL, - full_ret text, - id varchar(255) NOT NULL, - success varchar(10) NOT NULL, - alter_time TIMESTAMP WITH TIME ZONE DEFAULT now() -); - -CREATE INDEX idx_salt_returns_id ON salt_returns (id); -CREATE INDEX idx_salt_returns_jid ON salt_returns (jid); -CREATE INDEX idx_salt_returns_fun ON salt_returns (fun); -CREATE INDEX idx_salt_returns_updated ON salt_returns (alter_time); - -\-\- -\-\- Table structure for table \(gasalt_events\(ga -\-\- - -DROP TABLE IF EXISTS salt_events; -DROP SEQUENCE IF EXISTS seq_salt_events_id; -CREATE SEQUENCE seq_salt_events_id; -CREATE TABLE salt_events ( - id BIGINT NOT NULL UNIQUE DEFAULT nextval(\(aqseq_salt_events_id\(aq), - tag varchar(255) NOT NULL, - data text NOT NULL, - alter_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - master_id varchar(255) NOT NULL -); - -CREATE INDEX idx_salt_events_tag on salt_events (tag); - -EOF +base: + \(aq*\(aq: + \- packages + \(aqweb*\(aq: + \- vim .ft P .fi .UNINDENT .UNINDENT .sp -Required python modules: psycopg2 +In this expanded top file, minions that match \fBweb*\fP will have access to the +\fB/srv/pillar/packages.sls\fP file, as well as the \fB/srv/pillar/vim.sls\fP file. .sp -To use the postgres returner, append \(aq\-\-return postgres\(aq to the salt command. +Another example shows how to use other standard top matching types +to deliver specific salt pillar data to minions with different properties. +.sp +Here is an example using the \fBgrains\fP matcher to target pillars to minions +by their \fBos\fP grain: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return postgres +dev: + \(aqos:Debian\(aq: + \- match: grain + \- servers .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Pillar definitions can also take a keyword argument \fBignore_missing\fP\&. +When the value of \fBignore_missing\fP is \fBTrue\fP, all errors for missing +pillar files are ignored. The default value for \fBignore_missing\fP is +\fBFalse\fP\&. .sp -New in version 2015.5.0. - +Here is an example using the \fBignore_missing\fP keyword parameter to ignore +errors for missing pillar files: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return postgres \-\-return_config alternative +base: + \(aq*\(aq: + \- servers + \- systems + \- ignore_missing: True .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +Assuming that the pillar \fBservers\fP exists in the fileserver backend +and the pillar \fBsystems\fP doesn\(aqt, all pillar data from \fBservers\fP +pillar is delivered to minions and no error for the missing pillar +\fBsystems\fP is noted under the key \fB_errors\fP in the pillar data +delivered to minions. .sp -New in version 2016.3.0. - +Should the \fBignore_missing\fP keyword parameter have the value \fBFalse\fP, +an error for the missing pillar \fBsystems\fP would produce the value +\fBSpecified SLS \(aqservers\(aq in environment \(aqbase\(aq is not available on the salt master\fP +under the key \fB_errors\fP in the pillar data delivered to minions. +.sp +\fB/srv/pillar/packages.sls\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return postgres \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +{% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} +apache: httpd +git: git +{% elif grains[\(aqos\(aq] == \(aqDebian\(aq %} +apache: apache2 +git: git\-core +{% endif %} + +company: Foo Industries .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.event_return(events) -Return event to Pg server -.sp -Requires that configuration be enabled via \(aqevent_return\(aq -option in master config. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.get_jids() -Return a list of all job ids -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.returner(ret) -Return data to a postgres server -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.save_load(jid, load, minions=None) -Save the load to the specified jid id -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres.save_minions(jid, minions, syndic_id=None) -Included for API consistency -.UNINDENT -.SS salt.returners.postgres_local_cache -.sp -Use a postgresql server for the master job cache. This helps the job cache to -cope with scale. .sp -\fBNOTE:\fP +\fBIMPORTANT:\fP .INDENT 0.0 .INDENT 3.5 -There are three PostgreSQL returners. Any can function as an external -\fI\%master job cache\fP\&. but each has different -features. SaltStack recommends -\fI\%returners.pgjsonb\fP if you are working with -a version of PostgreSQL that has the appropriate native binary JSON types. -Otherwise, review -\fI\%returners.postgres\fP and -\fI\%returners.postgres_local_cache\fP -to see which module best suits your particular needs. -.UNINDENT +See \fI\%Is Targeting using Grain Data Secure?\fP for +important security information. .UNINDENT -.INDENT 0.0 -.TP -.B maintainer -\fI\%gjredelinghuys@gmail.com\fP -.TP -.B maturity -Stable -.TP -.B depends -psycopg2 -.TP -.B platform -all .UNINDENT .sp -To enable this returner the minion will need the psycopg2 installed and -the following values configured in the master config: +The above pillar sets two key/value pairs. If a minion is running RedHat, then +the \fBapache\fP key is set to \fBhttpd\fP and the \fBgit\fP key is set to the value +of \fBgit\fP\&. If the minion is running Debian, those values are changed to +\fBapache2\fP and \fBgit\-core\fP respectively. All minions that have this pillar +targeting to them via a top file will have the key of \fBcompany\fP with a value +of \fBFoo Industries\fP\&. +.sp +Consequently this data can be used from within modules, renderers, State SLS +files, and more via the shared pillar dictionary: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -master_job_cache: postgres_local_cache -master_job_cache.postgres.host: \(aqsalt\(aq -master_job_cache.postgres.user: \(aqsalt\(aq -master_job_cache.postgres.passwd: \(aqsalt\(aq -master_job_cache.postgres.db: \(aqsalt\(aq -master_job_cache.postgres.port: 5432 +apache: + pkg.installed: + \- name: {{ pillar[\(aqapache\(aq] }} .ft P .fi .UNINDENT .UNINDENT -.sp -Running the following command as the postgres user should create the database -correctly: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -psql << EOF -CREATE ROLE salt WITH PASSWORD \(aqsalt\(aq; -CREATE DATABASE salt WITH OWNER salt; -EOF +git: + pkg.installed: + \- name: {{ pillar[\(aqgit\(aq] }} .ft P .fi .UNINDENT .UNINDENT .sp -In case the postgres database is a remote host, you\(aqll need this command also: +Finally, the above states can utilize the values provided to them via Pillar. +All pillar values targeted to a minion are available via the \(aqpillar\(aq +dictionary. As seen in the above example, Jinja substitution can then be +utilized to access the keys and values in the Pillar dictionary. +.sp +Note that you cannot just list key/value\-information in \fBtop.sls\fP\&. Instead, +target a minion to a pillar file and then list the keys and values in the +pillar. Here is an example top file that illustrates this point: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -ALTER ROLE salt WITH LOGIN; +base: + \(aq*\(aq: + \- common_pillar .ft P .fi .UNINDENT .UNINDENT .sp -and then: +And the actual pillar file at \(aq/srv/pillar/common_pillar.sls\(aq: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -psql \-h localhost \-U salt << EOF -\-\- -\-\- Table structure for table \(aqjids\(aq -\-\- - -DROP TABLE IF EXISTS jids; -CREATE TABLE jids ( - jid varchar(20) PRIMARY KEY, - started TIMESTAMP WITH TIME ZONE DEFAULT now(), - tgt_type text NOT NULL, - cmd text NOT NULL, - tgt text NOT NULL, - kwargs text NOT NULL, - ret text NOT NULL, - username text NOT NULL, - arg text NOT NULL, - fun text NOT NULL -); - -\-\- -\-\- Table structure for table \(aqsalt_returns\(aq -\-\- -\-\- note that \(aqsuccess\(aq must not have NOT NULL constraint, since -\-\- some functions don\(aqt provide it. - -DROP TABLE IF EXISTS salt_returns; -CREATE TABLE salt_returns ( - added TIMESTAMP WITH TIME ZONE DEFAULT now(), - fun text NOT NULL, - jid varchar(20) NOT NULL, - return text NOT NULL, - id text NOT NULL, - success boolean -); -CREATE INDEX ON salt_returns (added); -CREATE INDEX ON salt_returns (id); -CREATE INDEX ON salt_returns (jid); -CREATE INDEX ON salt_returns (fun); - -DROP TABLE IF EXISTS salt_events; -CREATE TABLE salt_events ( - id SERIAL, - tag text NOT NULL, - data text NOT NULL, - alter_time TIMESTAMP WITH TIME ZONE DEFAULT now(), - master_id text NOT NULL -); -CREATE INDEX ON salt_events (tag); -CREATE INDEX ON salt_events (data); -CREATE INDEX ON salt_events (id); -CREATE INDEX ON salt_events (master_id); -EOF +foo: bar +boo: baz .ft P .fi .UNINDENT .UNINDENT .sp -Required python modules: psycopg2 +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.clean_old_jobs() -Clean out the old jobs from the job cache -.UNINDENT +.INDENT 3.5 +When working with multiple pillar environments, assuming that each pillar +environment has its own top file, the jinja placeholder \fB{{ saltenv }}\fP +can be used in place of the environment name: .INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.event_return(events) -Return event to a postgres server +.INDENT 3.5 .sp -Require that configuration be enabled via \(aqevent_return\(aq -option in master config. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.get_jids() -Return a list of all job ids -For master job cache this also formats the output and returns a string +.nf +.ft C +{{ saltenv }}: + \(aq*\(aq: + \- common_pillar +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.get_load(jid) -Return the load data that marks a specified jid .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.prep_jid(nocache=False, passed_jid=None) -Return a job id and prepare the job id directory -This is the function responsible for making sure jids don\(aqt collide -(unless its passed a jid). So do what you have to do to make sure that -stays the case +.sp +Yes, this is \fB{{ saltenv }}\fP, and not \fB{{ pillarenv }}\fP\&. The reason for +this is because the Pillar top files are parsed using some of the same code +which parses top files when \fI\%running states\fP, so +the pillar environment takes the place of \fB{{ saltenv }}\fP in the jinja +context. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.returner(load) -Return data to a postgres server .UNINDENT +.SS Dynamic Pillar Environments +.sp +If environment \fB__env__\fP is specified in \fI\%pillar_roots\fP, all +environments that are not explicitly specified in \fI\%pillar_roots\fP +will map to the directories from \fB__env__\fP\&. This allows one to use dynamic +git branch based environments for state/pillar files with the same file\-based +pillar applying to all environments. For example: .INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.save_load(jid, clear_load, minions=None) -Save the load to the specified jid id +.INDENT 3.5 +.sp +.nf +.ft C +pillar_roots: + __env__: + \- /srv/pillar + +ext_pillar: + \- git: + \- __env__ https://example.com/git\-pillar.git +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.postgres_local_cache.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.pushover_returner -.sp -Return salt data via pushover (\fI\%http://www.pushover.net\fP) .sp -New in version 2016.3.0. +New in version 2017.7.5,2018.3.1. .sp -The following fields can be set in the minion conf file: +Taking it one step further, \fB__env__\fP can also be used in the \fBpillar_root\fP +filesystem path. It will be replaced with the actual \fBpillarenv\fP and searched +for Pillar data to provide to the minion. Note this substitution ONLY occurs for +the \fB__env__\fP environment. For instance, this configuration: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pushover.user (required) -pushover.token (required) -pushover.title (optional) -pushover.device (optional) -pushover.priority (optional) -pushover.expire (optional) -pushover.retry (optional) -pushover.profile (optional) +pillar_roots: + __env__: + \- /srv/__env__/pillar .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP +is equivalent to this static configuration: .INDENT 0.0 .INDENT 3.5 -The \fBuser\fP here is your \fBuser key\fP, \fInot\fP the email address you use to -login to pushover.net. +.sp +.nf +.ft C +pillar_roots: + dev: + \- /srv/dev/pillar + test: + \- /srv/test/pillar + prod: + \- /srv/prod/pillar +.ft P +.fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +New in version 3005. + +.SS Pillar Namespace Flattening +.sp +The separate pillar SLS files all merge down into a single dictionary of +key\-value pairs. When the same key is defined in multiple SLS files, this can +result in unexpected behavior if care is not taken to how the pillar SLS files +are laid out. +.sp +For example, given a \fBtop.sls\fP containing the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.pushover.user -alternative.pushover.token -alternative.pushover.title -alternative.pushover.device -alternative.pushover.priority -alternative.pushover.expire -alternative.pushover.retry +base: + \(aq*\(aq: + \- packages + \- services .ft P .fi .UNINDENT .UNINDENT .sp -PushOver settings may also be configured as: +with \fBpackages.sls\fP containing: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - pushover: - user: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - title: Salt Returner - device: phone - priority: \-1 - expire: 3600 - retry: 5 - - alternative.pushover: - user: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - title: Salt Returner - device: phone - priority: 1 - expire: 4800 - retry: 2 - - pushover_profile: - pushover.token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - - pushover: - user: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - profile: pushover_profile - - alternative.pushover: - user: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - profile: pushover_profile - -To use the PushOver returner, append \(aq\-\-return pushover\(aq to the salt command. ex: - -\&.. code\-block:: bash - - salt \(aq*\(aq test.ping \-\-return pushover - -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. ex: - - salt \(aq*\(aq test.ping \-\-return pushover \-\-return_config alternative +bind: bind9 .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +and \fBservices.sls\fP containing: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return pushover \-\-return_kwargs \(aq{\(dqtitle\(dq: \(dqSalt is awesome!\(dq}\(aq +bind: named .ft P .fi .UNINDENT .UNINDENT +.sp +Then a request for the \fBbind\fP pillar key will only return \fBnamed\fP\&. The +\fBbind9\fP value will be lost, because \fBservices.sls\fP was evaluated later. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.pushover_returner.returner(ret) -Send an PushOver message with the data +.INDENT 3.5 +Pillar files are applied in the order they are listed in the top file. +Therefore conflicting keys will be overwritten in a \(aqlast one wins\(aq manner! +For example, in the above scenario conflicting key values in \fBservices\fP +will overwrite those in \fBpackages\fP because it\(aqs at the bottom of the list. +.UNINDENT .UNINDENT -.SS salt.returners.rawfile_json -.sp -Take data from salt and \(dqreturn\(dq it into a raw file containing the json, with -one line per event. .sp -Add the following to the minion or master configuration file. +It can be better to structure your pillar files with more hierarchy. For +example the \fBpackage.sls\fP file could be configured like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -rawfile_json.filename: +packages: + bind: bind9 .ft P .fi .UNINDENT .UNINDENT .sp -Default is \fB/var/log/salt/events\fP\&. +This would make the \fBpackages\fP pillar key a nested dictionary containing a +\fBbind\fP key. +.SS Pillar Dictionary Merging .sp -Common use is to log all events on the master. This can generate a lot of -noise, so you may wish to configure batch processing and/or configure the -\fI\%event_return_whitelist\fP or \fI\%event_return_blacklist\fP -to restrict the events that are written. -.INDENT 0.0 -.TP -.B salt.returners.rawfile_json.event_return(events) -Write event data (return data and non\-return data) to file on the master. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.rawfile_json.returner(ret) -Write the return data to a file on the minion. -.UNINDENT -.SS salt.returners.redis_return +If the same pillar key is defined in multiple pillar SLS files, and the keys in +both files refer to nested dictionaries, then the content from these +dictionaries will be recursively merged. .sp -Return data to a redis server +For example, keeping the \fBtop.sls\fP the same, assume the following +modifications to the pillar SLS files: .sp -To enable this returner the minion will need the python client for redis -installed and the following values configured in the minion or master -config, these are the defaults: +\fBpackages.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -redis.db: \(aq0\(aq -redis.host: \(aqsalt\(aq -redis.port: 6379 +bind: + package\-name: bind9 + version: 9.9.5 .ft P .fi .UNINDENT .UNINDENT .sp -New in version 2018.3.1: Alternatively a UNIX socket can be specified by \fIunix_socket_path\fP: - +\fBservices.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -redis.db: \(aq0\(aq -redis.unix_socket_path: /var/run/redis/redis.sock +bind: + port: 53 + listen\-on: any .ft P .fi .UNINDENT .UNINDENT .sp -Cluster Mode Example: +The resulting pillar dictionary will be: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -redis.db: \(aq0\(aq -redis.cluster_mode: true -redis.cluster.skip_full_coverage_check: true -redis.cluster.startup_nodes: - \- host: redis\-member\-1 - port: 6379 - \- host: redis\-member\-2 - port: 6379 +$ salt\-call pillar.get bind +local: + \-\-\-\-\-\-\-\-\-\- + listen\-on: + any + package\-name: + bind9 + port: + 53 + version: + 9.9.5 .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Since both pillar SLS files contained a \fBbind\fP key which contained a nested +dictionary, the pillar dictionary\(aqs \fBbind\fP key contains the combined contents +of both SLS files\(aq \fBbind\fP keys. +.SS Including Other Pillars +.sp +New in version 0.16.0. + +.sp +Pillar SLS files may include other pillar files, similar to State files. Two +syntaxes are available for this purpose. The simple form simply includes the +additional pillar as if it were part of the same file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.redis.db: \(aq0\(aq -alternative.redis.host: \(aqsalt\(aq -alternative.redis.port: 6379 +include: + \- users .ft P .fi .UNINDENT .UNINDENT .sp -To use the redis returner, append \(aq\-\-return redis\(aq to the salt command. +The full include form allows two additional options \-\- passing default values +to the templating engine for the included pillar file as well as an optional +key under which to nest the results of the included pillar: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return redis +include: + \- users: + defaults: + sudo: [\(aqbob\(aq, \(aqpaul\(aq] + key: users .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +With this form, the included file (users.sls) will be nested within the \(aqusers\(aq +key of the compiled pillar. Additionally, the \(aqsudo\(aq value will be available +as a template variable to users.sls. +.SS In\-Memory Pillar Data vs. On\-Demand Pillar Data .sp -New in version 2015.5.0. - +Since compiling pillar data is computationally expensive, the minion will +maintain a copy of the pillar data in memory to avoid needing to ask the master +to recompile and send it a copy of the pillar data each time pillar data is +requested. This in\-memory pillar data is what is returned by the +\fI\%pillar.item\fP, \fI\%pillar.get\fP, and \fI\%pillar.raw\fP +functions. +.sp +Also, for those writing custom execution modules, or contributing to Salt\(aqs +existing execution modules, the in\-memory pillar data is available as the +\fB__pillar__\fP dunder dictionary. +.sp +The in\-memory pillar data is generated on minion start, and can be refreshed +using the \fI\%saltutil.refresh_pillar\fP function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return redis \-\-return_config alternative +salt \(aq*\(aq saltutil.refresh_pillar .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +This function triggers the minion to asynchronously refresh the in\-memory +pillar data and will always return \fBNone\fP\&. .sp -New in version 2016.3.0. - +In contrast to in\-memory pillar data, certain actions trigger pillar data to be +compiled to ensure that the most up\-to\-date pillar data is available. These +actions include: +.INDENT 0.0 +.IP \(bu 2 +Running states +.IP \(bu 2 +Running \fI\%pillar.items\fP +.UNINDENT +.sp +Performing these actions will \fInot\fP refresh the in\-memory pillar data. So, if +pillar data is modified, and then states are run, the states will see the +updated pillar data, but \fI\%pillar.item\fP, +\fI\%pillar.get\fP, and \fI\%pillar.raw\fP will not see this data unless refreshed using +\fI\%saltutil.refresh_pillar\fP\&. +.sp +If you are using the Pillar Cache and have set \fI\%pillar_cache\fP to \fITrue\fP, +the pillar cache can be updated either when you run \fI\%saltutil.refresh_pillar\fP, or using the pillar runner function +\fI\%pillar.clear_pillar_cache\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return redis \-\-return_kwargs \(aq{\(dqdb\(dq: \(dqanother\-salt\(dq}\(aq +salt\-run pillar.clear_pillar_cache \(aqminion\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Redis Cluster Mode Options: +The pillar will not be updated when running \fI\%pillar.items\fP or a state for example. If you are +using a Salt version before 3003, you would need to manually delete the cache +file, located in Salt\(aqs master cache. For example, on linux the file would be +in this directory: /var/cache/salt/master/pillar_cache/ +.SS How Pillar Environments Are Handled +.sp +When multiple pillar environments are used, the default behavior is for the +pillar data from all environments to be merged together. The pillar dictionary +will therefore contain keys from all configured environments. +.sp +The \fI\%pillarenv\fP minion config option can be used to force the +minion to only consider pillar configuration from a single environment. This +can be useful in cases where one needs to run states with alternate pillar +data, either in a testing/QA environment or to test changes to the pillar data +before pushing them live. +.sp +For example, assume that the following is set in the minion config file: .INDENT 0.0 -.TP -.B cluster_mode: \fBFalse\fP -Whether cluster_mode is enabled or not -.TP -.B cluster.startup_nodes: -A list of host, port dictionaries pointing to cluster members. At least one is required -but multiple nodes are better -.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -redis.cluster.startup_nodes - \- host: redis\-member\-1 - port: 6379 - \- host: redis\-member\-2 - port: 6379 +pillarenv: base .ft P .fi .UNINDENT .UNINDENT -.TP -.B cluster.skip_full_coverage_check: \fBFalse\fP -Some cluster providers restrict certain redis commands such as CONFIG for enhanced security. -Set this option to true to skip checks that required advanced privileges. .sp -\fBNOTE:\fP -.INDENT 7.0 -.INDENT 3.5 -Most cloud hosted redis clusters will require this to be set to \fBTrue\fP -.UNINDENT -.UNINDENT -.UNINDENT +This would cause that minion to ignore all other pillar environments besides +\fBbase\fP when compiling the in\-memory pillar data. Then, when running states, +the \fBpillarenv\fP CLI argument can be used to override the minion\(aqs +\fI\%pillarenv\fP config value: .INDENT 0.0 -.TP -.B salt.returners.redis_return.clean_old_jobs() -Clean out minions\(aqs return data for old jobs. +.INDENT 3.5 .sp -Normally, hset \(aqret:\(aq are saved with a TTL, and will eventually -get cleaned by redis.But for jobs with some very late minion return, the -corresponding hset\(aqs TTL will be refreshed to a too late timestamp, we\(aqll -do manually cleaning here. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.get_jid(jid) -Return the information returned when the specified job id was executed -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.get_jids() -Return a dict mapping all job ids to job information -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.get_load(jid) -Return the load data that marks a specified jid -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.get_minions() -Return a list of minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.nf +.ft C +salt \(aq*\(aq state.apply mystates pillarenv=testing +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.returner(ret) -Return data to a redis data store .UNINDENT +.sp +The above command will run the states with pillar data sourced exclusively from +the \fBtesting\fP environment, without modifying the in\-memory pillar data. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.redis_return.save_load(jid, load, minions=None) -Save the load to the specified jid +.INDENT 3.5 +When running states, the \fBpillarenv\fP CLI option does not require a +\fI\%pillarenv\fP option to be set in the minion config file. When +\fI\%pillarenv\fP is left unset, as mentioned above all configured +environments will be combined. Running states with \fBpillarenv=testing\fP in +this case would still restrict the states\(aq pillar data to just that of the +\fBtesting\fP pillar environment. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.redis_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.sentry_return -.sp -Salt returner that reports execution results back to sentry. The returner will -inspect the payload to identify errors and flag them as such. .sp -Pillar needs something like: +Starting in the 2017.7.0 release, it is possible to pin the pillarenv to the +effective saltenv, using the \fI\%pillarenv_from_saltenv\fP minion +config option. When this is set to \fBTrue\fP, if a specific saltenv is specified +when running states, the \fBpillarenv\fP will be the same. This essentially makes +the following two commands equivalent: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -raven: - servers: - \- http://192.168.1.1 - \- https://sentry.example.com - public_key: deadbeefdeadbeefdeadbeefdeadbeef - secret_key: beefdeadbeefdeadbeefdeadbeefdead - project: 1 - tags: - \- os - \- master - \- saltversion - \- cpuarch +salt \(aq*\(aq state.apply mystates saltenv=dev +salt \(aq*\(aq state.apply mystates saltenv=dev pillarenv=dev .ft P .fi .UNINDENT .UNINDENT .sp -or using a dsn: +However, if a pillarenv is specified, it will override this behavior. So, the +following command will use the \fBqa\fP pillar environment but source the SLS +files from the \fBdev\fP saltenv: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -raven: - dsn: https://aaaa:bbbb@app.getsentry.com/12345 - tags: - \- os - \- master - \- saltversion - \- cpuarch +salt \(aq*\(aq state.apply mystates saltenv=dev pillarenv=qa .ft P .fi .UNINDENT .UNINDENT .sp -\fI\%https://pypi.python.org/pypi/raven\fP must be installed. -.sp -The pillar can be hidden on sentry return by setting hide_pillar: true. -.sp -The tags list (optional) specifies grains items that will be used as sentry -tags, allowing tagging of events in the sentry ui. +So, if a \fBpillarenv\fP is set in the minion config file, +\fI\%pillarenv_from_saltenv\fP will be ignored, and passing a +\fBpillarenv\fP on the CLI will temporarily override +\fI\%pillarenv_from_saltenv\fP\&. +.SS Viewing Pillar Data .sp -To report only errors to sentry, set report_errors_only: true. -.INDENT 0.0 -.TP -.B salt.returners.sentry_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id -.UNINDENT +To view pillar data, use the \fI\%pillar\fP execution +module. This module includes several functions, each of them with their own +use. These functions include: .INDENT 0.0 -.TP -.B salt.returners.sentry_return.returner(ret) -Log outcome to sentry. The returner tries to identify errors and report -them as such. All other messages will be reported at info level. -Failed states will be appended as separate list for convenience. +.IP \(bu 2 +\fI\%pillar.item\fP \- Retrieves the value of +one or more keys from the \fI\%in\-memory pillar data\fP\&. +.IP \(bu 2 +\fI\%pillar.items\fP \- Compiles a fresh pillar +dictionary and returns it, leaving the \fI\%in\-memory pillar data\fP untouched. If pillar keys are passed to this function +however, this function acts like \fI\%pillar.item\fP and returns their values from the \fI\%in\-memory +pillar data\fP\&. +.IP \(bu 2 +\fI\%pillar.raw\fP \- Like \fI\%pillar.items\fP, it returns the entire pillar dictionary, but +from the \fI\%in\-memory pillar data\fP instead of compiling +fresh pillar data. +.IP \(bu 2 +\fI\%pillar.get\fP \- Described in detail below. .UNINDENT -.SS salt.returners.slack_returner -.sp -Return salt data via slack +.SS The \fI\%pillar.get\fP Function .sp -New in version 2015.5.0. +New in version 0.14.0. .sp -The following fields can be set in the minion conf file: +The \fI\%pillar.get\fP function works much in the same +way as the \fBget\fP method in a python dict, but with an enhancement: nested +dictionaries can be traversed using a colon as a delimiter. +.sp +If a structure like this is in pillar: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack.channel (required) -slack.api_key (required) -slack.username (required) -slack.as_user (required to see the profile picture of your bot) -slack.profile (optional) -slack.changes(optional, only show changes and failed states) -slack.only_show_failed(optional, only show failed states) -slack.yaml_format(optional, format the json in yaml format) +foo: + bar: + baz: qux .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Extracting it from the raw pillar in an sls formula or file template is done +this way: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack.channel -slack.api_key -slack.username -slack.as_user +{{ pillar[\(aqfoo\(aq][\(aqbar\(aq][\(aqbaz\(aq] }} .ft P .fi .UNINDENT .UNINDENT .sp -Slack settings may also be configured as: +Now, with the new \fI\%pillar.get\fP function the data +can be safely gathered and a default can be set, allowing the template to fall +back if the value is not available: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack: - channel: RoomName - api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - username: user - as_user: true - -alternative.slack: - room_id: RoomName - api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - from_name: user@email.com - -slack_profile: - slack.api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - slack.from_name: user@email.com - -slack: - profile: slack_profile - channel: RoomName - -alternative.slack: - profile: slack_profile - channel: RoomName +{{ salt[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -To use the Slack returner, append \(aq\-\-return slack\(aq to the salt command. +This makes handling nested structures much easier. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +\fBpillar.get()\fP vs \fBsalt[\(aqpillar.get\(aq]()\fP .sp -.nf -.ft C -salt \(aq*\(aq test.ping \-\-return slack -.ft P -.fi +It should be noted that within templating, the \fBpillar\fP variable is just +a dictionary. This means that calling \fBpillar.get()\fP inside of a +template will just use the default dictionary \fB\&.get()\fP function which +does not include the extra \fB:\fP delimiter functionality. It must be +called using the above syntax (\fBsalt[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, +\(aqqux\(aq)\fP) to get the salt function, instead of the default dictionary +behavior. .UNINDENT .UNINDENT +.SS Setting Pillar Data at the Command Line .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +Pillar data can be set at the command line like the following example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return slack \-\-return_config alternative +salt \(aq*\(aq state.apply pillar=\(aq{\(dqcheese\(dq: \(dqspam\(dq}\(aq .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +This will add a pillar key of \fBcheese\fP with its value set to \fBspam\fP\&. .sp -New in version 2016.3.0. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Be aware that when sending sensitive data via pillar on the command\-line +that the publication containing that data will be received by all minions +and will not be restricted to the targeted minions. This may represent +a security concern in some cases. +.UNINDENT +.UNINDENT +.SS Pillar Encryption +.sp +Salt\(aqs renderer system can be used to decrypt pillar data. This allows for +pillar items to be stored in an encrypted state, and decrypted during pillar +compilation. +.SS Encrypted Pillar SLS +.sp +New in version 2017.7.0. +.sp +Consider the following pillar SLS file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return slack \-\-return_kwargs \(aq{\(dqchannel\(dq: \(dq#random\(dq}\(aq +secrets: + vault: + foo: | + \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- + + hQEMAw2B674HRhwSAQgAhTrN8NizwUv/VunVrqa4/X8t6EUulrnhKcSeb8sZS4th + W1Qz3K2NjL4lkUHCQHKZVx/VoZY7zsddBIFvvoGGfj8+2wjkEDwFmFjGE4DEsS74 + ZLRFIFJC1iB/O0AiQ+oU745skQkU6OEKxqavmKMrKo3rvJ8ZCXDC470+i2/Hqrp7 + +KWGmaDOO422JaSKRm5D9bQZr9oX7KqnrPG9I1+UbJyQSJdsdtquPWmeIpamEVHb + VMDNQRjSezZ1yKC4kCWm3YQbBF76qTHzG1VlLF5qOzuGI9VkyvlMaLfMibriqY73 + zBbPzf6Bkp2+Y9qyzuveYMmwS4sEOuZL/PetqisWe9JGAWD/O+slQ2KRu9hNww06 + KMDPJRdyj5bRuBVE4hHkkP23KrYr7SuhW2vpe7O/MvWEJ9uDNegpMLhTWruGngJh + iFndxegN9w== + =bAuo + \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- + bar: this was unencrypted already + baz: | + \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- + + hQEMAw2B674HRhwSAQf+Ne+IfsP2IcPDrUWct8sTJrga47jQvlPCmO+7zJjOVcqz + gLjUKvMajrbI/jorBWxyAbF+5E7WdG9WHHVnuoywsyTB9rbmzuPqYCJCe+ZVyqWf + 9qgJ+oUjcvYIFmH3h7H68ldqbxaAUkAOQbTRHdr253wwaTIC91ZeX0SCj64HfTg7 + Izwk383CRWonEktXJpientApQFSUWNeLUWagEr/YPNFA3vzpPF5/Ia9X8/z/6oO2 + q+D5W5mVsns3i2HHbg2A8Y+pm4TWnH6mTSh/gdxPqssi9qIrzGQ6H1tEoFFOEq1V + kJBe0izlfudqMq62XswzuRB4CYT5Iqw1c97T+1RqENJCASG0Wz8AGhinTdlU5iQl + JkLKqBxcBz4L70LYWyHhYwYROJWjHgKAywX5T67ftq0wi8APuZl9olnOkwSK+wrY + 1OZi + =7epf + \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- + qux: + \- foo + \- bar + \- | + \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- + + hQEMAw2B674HRhwSAQgAg1YCmokrweoOI1c9HO0BLamWBaFPTMblOaTo0WJLZoTS + ksbQ3OJAMkrkn3BnnM/djJc5C7vNs86ZfSJ+pvE8Sp1Rhtuxh25EKMqGOn/SBedI + gR6N5vGUNiIpG5Tf3DuYAMNFDUqw8uY0MyDJI+ZW3o3xrMUABzTH0ew+Piz85FDA + YrVgwZfqyL+9OQuu6T66jOIdwQNRX2NPFZqvon8liZUPus5VzD8E5cAL9OPxQ3sF + f7/zE91YIXUTimrv3L7eCgU1dSxKhhfvA2bEUi+AskMWFXFuETYVrIhFJAKnkFmE + uZx+O9R9hADW3hM5hWHKH9/CRtb0/cC84I9oCWIQPdI+AaPtICxtsD2N8Q98hhhd + 4M7I0sLZhV+4ZJqzpUsOnSpaGyfh1Zy/1d3ijJi99/l+uVHuvmMllsNmgR+ZTj0= + =LrCQ + \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.slack_returner.returner(ret) -Send an slack message with the data -.UNINDENT -.SS salt.returners.slack_webhook_return -.sp -Return salt data via Slack using Incoming Webhooks -.INDENT 0.0 -.TP -.B codeauthor -\fICarlos D. Álvaro \fP -.UNINDENT .sp -The following fields can be set in the minion conf file: +When the pillar data is compiled, the results will be decrypted: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack_webhook.webhook (required, the webhook id. Just the part after: \(aqhttps://hooks.slack.com/services/\(aq) -slack_webhook.success_title (optional, short title for succeeded states. By default: \(aq{id} | Succeeded\(aq) -slack_webhook.failure_title (optional, short title for failed states. By default: \(aq{id} | Failed\(aq) -slack_webhook.author_icon (optional, a URL that with a small 16x16px image. Must be of type: GIF, JPEG, PNG, and BMP) -slack_webhook.show_tasks (optional, show identifiers for changed and failed tasks. By default: False) +# salt myminion pillar.items +myminion: + \-\-\-\-\-\-\-\-\-\- + secrets: + \-\-\-\-\-\-\-\-\-\- + vault: + \-\-\-\-\-\-\-\-\-\- + bar: + this was unencrypted already + baz: + rosebud + foo: + supersecret + qux: + \- foo + \- bar + \- baz .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Salt must be told what portions of the pillar data to decrypt. This is done +using the \fI\%decrypt_pillar\fP config option: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack_webhook.webhook -slack_webhook.success_title -slack_webhook.failure_title -slack_webhook.author_icon -slack_webhook.show_tasks +decrypt_pillar: + \- \(aqsecrets:vault\(aq: gpg .ft P .fi .UNINDENT .UNINDENT .sp -Slack settings may also be configured as: +The notation used to specify the pillar item(s) to be decrypted is the same as +the one used in \fI\%pillar.get\fP function. +.sp +If a different delimiter is needed, it can be specified using the +\fI\%decrypt_pillar_delimiter\fP config option: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -slack_webhook: - webhook: T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX - success_title: \(aq[{id}] | Success\(aq - failure_title: \(aq[{id}] | Failure\(aq - author_icon: https://platform.slack\-edge.com/img/default_application_icon.png - show_tasks: true +decrypt_pillar: + \- \(aqsecrets|vault\(aq: gpg -alternative.slack_webhook: - webhook: T00000000/C00000000/YYYYYYYYYYYYYYYYYYYYYYYY - show_tasks: false +decrypt_pillar_delimiter: \(aq|\(aq .ft P .fi .UNINDENT .UNINDENT .sp -To use the Slack returner, -append \(aq\-\-return slack_webhook\(aq to the salt command. +The name of the renderer used to decrypt a given pillar item can be omitted, +and if so it will fall back to the value specified by the +\fI\%decrypt_pillar_default\fP config option, which defaults to \fBgpg\fP\&. +So, the first example above could be rewritten as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return slack_webhook +decrypt_pillar: + \- \(aqsecrets:vault\(aq .ft P .fi .UNINDENT .UNINDENT +.SS Encrypted Pillar Data on the CLI .sp -To use the alternative configuration, -append \(aq\-\-return_config alternative\(aq to the salt command. +New in version 2016.3.0. + +.sp +The following functions support passing pillar data on the CLI via the +\fBpillar\fP argument: +.INDENT 0.0 +.IP \(bu 2 +\fI\%pillar.items\fP +.IP \(bu 2 +\fI\%state.apply\fP +.IP \(bu 2 +\fI\%state.highstate\fP +.IP \(bu 2 +\fI\%state.sls\fP +.UNINDENT +.sp +Triggering decryption of this CLI pillar data can be done in one of two ways: .INDENT 0.0 +.IP 1. 3 +Using the \fBpillar_enc\fP argument: +.INDENT 3.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return slack_webhook \-\-return_config alternative +# salt myminion pillar.items pillar_enc=gpg pillar=\(aq{foo: \(dq\-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\-\en\enhQEMAw2B674HRhwSAQf+OvPqEdDoA2fk15I5dYUTDoj1yf/pVolAma6iU4v8Zixn\enRDgWsaAnFz99FEiFACsAGDEFdZaVOxG80T0Lj+PnW4pVy0OXmXHnY2KjV9zx8FLS\enQxfvmhRR4t23WSFybozfMm0lsN8r1vfBBjbK+A72l0oxN78d1rybJ6PWNZiXi+aC\enmqIeunIbAKQ21w/OvZHhxH7cnIiGQIHc7N9nQH7ibyoKQzQMSZeilSMGr2abAHun\enmLzscr4wKMb+81Z0/fdBfP6g3bLWMJga3hSzSldU9ovu7KR8rDJI1qOlENj3Wm8C\enwTpDOB33kWIKMqiAjY3JFtb5MCHrafyggwQL7cX1+tI+AbSO6kZpbcDfzetb77LZ\enxc5NWnnGK4pGoqq4MAmZshw98RpecSHKMosto2gtiuWCuo9Zn5cV/FbjZ9CTWrQ=\en=0hO/\en\-\-\-\-\-END PGP MESSAGE\-\-\-\-\-\(dq}\(aq .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.slack_webhook_return.event_return(events) -Send event data to returner function -:param events: The Salt event return -:return: The result of the post -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.slack_webhook_return.returner(ret, **kwargs) -Send a slack message with the data through a webhook -:param ret: The Salt return -:return: The result of the post -.UNINDENT -.SS salt.returners.sms_return .sp -Return data by SMS. +The newlines in this example are specified using a literal \fB\en\fP\&. Newlines +can be replaced with a literal \fB\en\fP using \fBsed\fP: +.INDENT 3.0 +.INDENT 3.5 .sp -New in version 2015.5.0. - -.INDENT 0.0 -.TP -.B maintainer -Damian Myerscough -.TP -.B maturity -new -.TP -.B depends -twilio -.TP -.B platform -all +.nf +.ft C +$ echo \-n bar | gpg \-\-armor \-\-trust\-model always \-\-encrypt \-r user@domain.tld | sed \(aq:a;N;$!ba;s/\en/\e\en/g\(aq +.ft P +.fi +.UNINDENT .UNINDENT .sp -To enable this returner the minion will need the python twilio library -installed and the following values configured in the minion or master -config: +\fBNOTE:\fP +.INDENT 3.0 +.INDENT 3.5 +Using \fBpillar_enc\fP will perform the decryption minion\-side, so for +this to work it will be necessary to set up the keyring in +\fB/etc/salt/gpgkeys\fP on the minion just as one would typically do on +the master. The easiest way to do this is to first export the keys from +the master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -twilio.sid: \(aqXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\(aq -twilio.token: \(aqXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\(aq -twilio.to: \(aq+1415XXXXXXX\(aq -twilio.from: \(aq+1650XXXXXXX\(aq +# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-key \-a user@domain.tld >/tmp/keypair.gpg .ft P .fi .UNINDENT .UNINDENT .sp -To use the sms returner, append \(aq\-\-return sms\(aq to the salt command. +Then, copy the file to the minion, setup the keyring, and import: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return sms +# mkdir \-p /etc/salt/gpgkeys +# chmod 0700 /etc/salt/gpgkeys +# gpg \-\-homedir /etc/salt/gpgkeys \-\-list\-keys +# gpg \-\-homedir /etc/salt/gpgkeys \-\-import \-\-allow\-secret\-key\-import keypair.gpg .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sms_return.returner(ret) -Return a response in an SMS message -.UNINDENT -.SS salt.returners.smtp_return -.sp -Return salt data via email .sp -The following fields can be set in the minion conf file. Fields are optional -unless noted otherwise. -.INDENT 0.0 -.IP \(bu 2 -\fBfrom\fP (required) The name/address of the email sender. -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fBto\fP (required) The names/addresses of the email recipients; -comma\-delimited. For example: \fByou@example.com,someoneelse@example.com\fP\&. +The \fB\-\-list\-keys\fP command is run create a keyring in the newly\-created +directory. .UNINDENT -.IP \(bu 2 -\fBhost\fP (required) The SMTP server hostname or address. -.IP \(bu 2 -\fBport\fP The SMTP server port; defaults to \fB25\fP\&. -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fBusername\fP The username used to authenticate to the server. If specified a -password is also required. It is recommended but not required to also use -TLS with this option. .UNINDENT -.IP \(bu 2 -\fBpassword\fP The password used to authenticate to the server. -.IP \(bu 2 -\fBtls\fP Whether to secure the connection using TLS; defaults to \fBFalse\fP -.IP \(bu 2 -\fBsubject\fP The email subject line. -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fBfields\fP Which fields from the returned data to include in the subject line -of the email; comma\-delimited. For example: \fBid,fun\fP\&. Please note, \fIthe -subject line is not encrypted\fP\&. +.sp +Pillar data which is decrypted minion\-side will still be securely +transferred to the master, since the data sent between minion and master is +encrypted with the master\(aqs public key. +.IP 2. 3 +Use the \fI\%decrypt_pillar\fP option. This is less flexible in that +the pillar key passed on the CLI must be pre\-configured on the master, but +it doesn\(aqt require a keyring to be setup on the minion. One other caveat to +this method is that pillar decryption on the master happens at the end of +pillar compilation, so if the encrypted pillar data being passed on the CLI +needs to be referenced by pillar or ext_pillar \fIduring pillar compilation\fP, +it \fImust\fP be decrypted minion\-side. .UNINDENT +.SS Adding New Renderers for Decryption +.sp +Those looking to add new renderers for decryption should look at the \fI\%gpg\fP renderer for an example of how to do so. The function +that performs the decryption should be recursive and be able to traverse a +mutable type such as a dictionary, and modify the values in\-place. +.sp +Once the renderer has been written, \fI\%decrypt_pillar_renderers\fP +should be modified so that Salt allows it to be used for decryption. +.sp +If the renderer is being submitted upstream to the Salt project, the renderer +should be added in \fI\%salt/renderers/\fP\&. Additionally, the following should be +done: +.INDENT 0.0 .IP \(bu 2 -.INDENT 2.0 -.TP -.B \fBgpgowner\fP A user\(aqs \fB~/.gpg\fP directory. This must contain a gpg -public key matching the address the mail is sent to. If left unset, no -encryption will be used. Requires \fBpython\-gnupg\fP to be installed. -.UNINDENT +Both occurrences of \fI\%decrypt_pillar_renderers\fP in +\fI\%salt/config/__init__.py\fP should be updated to include the name of the new +renderer so that it is included in the default value for this config option. .IP \(bu 2 -\fBtemplate\fP The path to a file to be used as a template for the email body. +The documentation for the \fI\%decrypt_pillar_renderers\fP config +option in the \fI\%master config file\fP and \fI\%minion config file\fP should be +updated to show the correct new default value. .IP \(bu 2 -.INDENT 2.0 -.TP -.B \fBrenderer\fP A Salt renderer, or render\-pipe, to use to render the email -template. Default \fBjinja\fP\&. -.UNINDENT +The commented example for the \fI\%decrypt_pillar_renderers\fP config +option in the \fI\%master config template\fP should be updated to show the correct +new default value. .UNINDENT +.SS Binary Data in the Pillar .sp -Below is an example of the above settings in a Salt Minion configuration file: +Salt has partial support for binary pillar data. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -smtp.from: me@example.net -smtp.to: you@example.com -smtp.host: localhost -smtp.port: 1025 -.ft P -.fi +There are some situations (such as salt\-ssh) where only text (ASCII or +Unicode) is allowed. .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location. For example: +The simplest way to embed binary data in your pillar is to make use of YAML\(aqs +built\-in binary data type, which requires base64 encoded data. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.smtp.username: saltdev -alternative.smtp.password: saltdev -alternative.smtp.tls: True +salt_pic: !!binary + iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAMAAAC67D+PAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAA .ft P .fi .UNINDENT .UNINDENT .sp -To use the SMTP returner, append \(aq\-\-return smtp\(aq to the \fBsalt\fP command. +Then you can use it as a \fBcontents_pillar\fP in a state: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return smtp +/tmp/salt.png: + file.managed: + \- contents_pillar: salt_pic .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the \fBsalt\fP command. +It is also possible to add ASCII\-armored encrypted data to pillars, as +mentioned in the Pillar Encryption section. +.SS Master Config in Pillar .sp -New in version 2015.5.0. - +For convenience the data stored in the master configuration file can be made +available in all minion\(aqs pillars. This makes global configuration of services +and systems very easy but may not be desired if sensitive data is stored in the +master configuration. This option is disabled by default. +.sp +To enable the master config from being added to the pillar set +\fBpillar_opts\fP to \fBTrue\fP in the minion config file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return smtp \-\-return_config alternative +pillar_opts: True .ft P .fi .UNINDENT .UNINDENT +.SS Minion Config in Pillar .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the -\fBsalt\fP command. -.sp -New in version 2016.3.0. - +Minion configuration options can be set on pillars. Any option that you want +to modify, should be in the first level of the pillars, in the same way you set +the options in the config file. For example, to configure the MySQL root +password to be used by MySQL Salt execution module, set the following pillar +variable: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return smtp \-\-return_kwargs \(aq{\(dqto\(dq: \(dquser@domain.com\(dq}\(aq +mysql.pass: hardtoguesspassword .ft P .fi .UNINDENT .UNINDENT +.SS Master Provided Pillar Error .sp -An easy way to test the SMTP returner is to use the development SMTP server -built into Python. The command below will start a single\-threaded SMTP server -that prints any email it receives to the console. +By default if there is an error rendering a pillar, the detailed error is +hidden and replaced with: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -python \-m smtpd \-n \-c DebuggingServer localhost:1025 +Rendering SLS \(aqmy.sls\(aq failed. Please see master log for details. .ft P .fi .UNINDENT .UNINDENT .sp -New in version 2016.11.0. - +The error is protected because it\(aqs possible to contain templating data +which would give that minion information it shouldn\(aqt know, like a password! .sp -It is possible to send emails with selected Salt events by configuring \fBevent_return\fP option -for Salt Master. For example: +To have the master provide the detailed error that could potentially carry +protected data set \fBpillar_safe_render_error\fP to \fBFalse\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -event_return: smtp - -event_return_whitelist: - \- salt/key - -smtp.from: me@example.net -smtp.to: you@example.com -smtp.host: localhost -smtp.subject: \(aqSalt Master {{act}}ed key from Minion ID: {{id}}\(aq -smtp.template: /srv/salt/templates/email.j2 +pillar_safe_render_error: False .ft P .fi .UNINDENT .UNINDENT +.SS Pillar Walkthrough .sp -Also you need to create additional file \fB/srv/salt/templates/email.j2\fP with email body template: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -act: {{act}} -id: {{id}} -result: {{result}} -.ft P -.fi +This walkthrough assumes that the reader has already completed the initial +Salt \fI\%walkthrough\fP\&. .UNINDENT .UNINDENT .sp -This configuration enables Salt Master to send an email when accepting or rejecting minions keys. +Pillars are tree\-like structures of data defined on the Salt Master and passed +through to minions. They allow confidential, targeted data to be securely sent +only to the relevant minion. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.smtp_return.event_return(events) -Return event data via SMTP +.INDENT 3.5 +Grains and Pillar are sometimes confused, just remember that Grains +are data about a minion which is stored or generated from the minion. +This is why information like the OS and CPU type are found in Grains. +Pillar is information about a minion or many minions stored or generated +on the Salt Master. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.smtp_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id .UNINDENT +.sp +Pillar data is useful for: .INDENT 0.0 .TP -.B salt.returners.smtp_return.returner(ret) -Send an email with the data +.B Highly Sensitive Data: +Information transferred via pillar is guaranteed to only be presented to +the minions that are targeted, making Pillar suitable +for managing security information, such as cryptographic keys and +passwords. +.TP +.B Minion Configuration: +Minion modules such as the execution modules, states, and returners can +often be configured via data stored in pillar. +.TP +.B Variables: +Variables which need to be assigned to specific minions or groups of +minions can be defined in pillar and then accessed inside sls formulas +and template files. +.TP +.B Arbitrary Data: +Pillar can contain any basic data structure in dictionary format, +so a key/value store can be defined making it easy to iterate over a group +of values in sls formulas. .UNINDENT -.SS salt.returners.splunk .sp -Send json response data to Splunk via the HTTP Event Collector -Requires the following config values to be specified in config or pillar: +Pillar is therefore one of the most important systems when using Salt. This +walkthrough is designed to get a simple Pillar up and running in a few minutes +and then to dive into the capabilities of Pillar and where the data is +available. +.SS Setting Up Pillar +.sp +The pillar is already running in Salt by default. To see the minion\(aqs +pillar data: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -splunk_http_forwarder: - token: - indexer: - sourcetype: - index: - verify_ssl: true +salt \(aq*\(aq pillar.items .ft P .fi .UNINDENT .UNINDENT .sp -Run a test by using \fBsalt\-call test.ping \-\-return splunk\fP -.sp -Written by Scott Pack (github.com/scottjpack) -.INDENT 0.0 -.TP -.B salt.returners.splunk.event_return(events) -Return events to Splunk via the HTTP Event Collector. -Requires the Splunk HTTP Event Collector running on port 8088. -This is available on Splunk Enterprise version 6.3 or higher. -.UNINDENT +\fBNOTE:\fP .INDENT 0.0 -.TP -.B class salt.returners.splunk.http_event_collector(token, http_event_server, host=\(aq\(aq, http_event_port=\(aq8088\(aq, http_event_server_ssl=True, max_bytes=100000, verify_ssl=True) -.INDENT 7.0 -.TP -.B sendEvent(payload, eventtime=\(aq\(aq) -.UNINDENT +.INDENT 3.5 +Prior to version 0.16.2, this function is named \fBpillar.data\fP\&. This +function name is still supported for backwards compatibility. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.splunk.returner(ret) -Send a message to Splunk via the HTTP Event Collector. -Requires the Splunk HTTP Event Collector running on port 8088. -This is available on Splunk Enterprise version 6.3 or higher. .UNINDENT -.SS salt.returners.sqlite3 .sp -Insert minion return data into a sqlite3 database +By default, the contents of the master configuration file are not loaded into +pillar for all minions. This default is stored in the \fBpillar_opts\fP setting, +which defaults to \fBFalse\fP\&. +.sp +The contents of the master configuration file can be made available to minion +pillar files. This makes global configuration of services and systems very easy, +but note that this may not be desired or appropriate if sensitive data is stored +in the master\(aqs configuration file. To enable the master configuration file to be +available to minion as pillar, set \fBpillar_opts: True\fP in the master +configuration file, and then for appropriate minions also set \fBpillar_opts: True\fP +in the minion(s) configuration file. +.sp +Similar to the state tree, the pillar is comprised of sls files and has a top file. +The default location for the pillar is in /srv/pillar. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B maintainer -Mickey Malone <\fI\%mickey.malone@gmail.com\fP> -.TP -.B maturity -New -.TP -.B depends -None -.TP -.B platform -All +.INDENT 3.5 +The pillar location can be configured via the \fBpillar_roots\fP option inside +the master configuration file. It must not be in a subdirectory of the state +tree or file_roots. If the pillar is under file_roots, any pillar targeting +can be bypassed by minions. +.UNINDENT .UNINDENT .sp -Sqlite3 is a serverless database that lives in a single file. -In order to use this returner the database file must exist, -have the appropriate schema defined, and be accessible to the -user whom the minion process is running as. This returner -requires the following values configured in the master or -minion config: +To start setting up the pillar, the /srv/pillar directory needs to be present: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -sqlite3.database: /usr/lib/salt/salt.db -sqlite3.timeout: 5.0 +mkdir /srv/pillar .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +Now create a simple top file, following the same format as the top file used for +states: +.sp +\fB/srv/pillar/top.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.sqlite3.database: /usr/lib/salt/salt.db -alternative.sqlite3.timeout: 5.0 +base: + \(aq*\(aq: + \- data .ft P .fi .UNINDENT .UNINDENT .sp -Use the commands to create the sqlite3 database and tables: +This top file associates the data.sls file to all minions. Now the +\fB/srv/pillar/data.sls\fP file needs to be populated: +.sp +\fB/srv/pillar/data.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -sqlite3 /usr/lib/salt/salt.db << EOF -\-\- -\-\- Table structure for table \(aqjids\(aq -\-\- - -CREATE TABLE jids ( - jid TEXT PRIMARY KEY, - load TEXT NOT NULL - ); - -\-\- -\-\- Table structure for table \(aqsalt_returns\(aq -\-\- - -CREATE TABLE salt_returns ( - fun TEXT KEY, - jid TEXT KEY, - id TEXT KEY, - fun_args TEXT, - date TEXT NOT NULL, - full_ret TEXT NOT NULL, - success TEXT NOT NULL - ); -EOF +info: some data .ft P .fi .UNINDENT .UNINDENT .sp -To use the sqlite returner, append \(aq\-\-return sqlite3\(aq to the salt command. +To ensure that the minions have the new pillar data, issue a command +to them asking that they fetch their pillars from the master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return sqlite3 +salt \(aq*\(aq saltutil.refresh_pillar .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. -.sp -New in version 2015.5.0. - +Now that the minions have the new pillar, it can be retrieved: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return sqlite3 \-\-return_config alternative +salt \(aq*\(aq pillar.items .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +The key \fBinfo\fP should now appear in the returned pillar data. +.SS More Complex Data .sp -New in version 2016.3.0. - +Unlike states, pillar files do not need to define \fBformulas\fP\&. +This example sets up user data with a UID: +.sp +\fB/srv/pillar/users/init.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return sqlite3 \-\-return_kwargs \(aq{\(dqdb\(dq: \(dq/var/lib/salt/another\-salt.db\(dq}\(aq +users: + thatch: 1000 + shouse: 1001 + utahdave: 1002 + redbeard: 1003 .ft P .fi .UNINDENT .UNINDENT +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.get_fun(fun) -Return a dict of the last function called for all minions -.UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.get_jid(jid) -Return the information returned from a specified jid +.INDENT 3.5 +The same directory lookups that exist in states exist in pillar, so the +file \fBusers/init.sls\fP can be referenced with \fBusers\fP in the \fI\%top +file\fP\&. .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.get_jids() -Return a list of all job ids .UNINDENT +.sp +The top file will need to be updated to include this sls file: +.sp +\fB/srv/pillar/top.sls\fP: .INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.get_load(jid) -Return the load from a specified jid +.INDENT 3.5 +.sp +.nf +.ft C +base: + \(aq*\(aq: + \- data + \- users +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.get_minions() -Return a list of minions .UNINDENT +.sp +Now the data will be available to the minions. To use the pillar data in a +state, you can use Jinja: +.sp +\fB/srv/salt/users/init.sls\fP .INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.prep_jid(nocache=False, passed_jid=None) -Do any work necessary to prepare a JID, including sending a custom id +.INDENT 3.5 +.sp +.nf +.ft C +{% for user, uid in pillar.get(\(aqusers\(aq, {}).items() %} +{{user}}: + user.present: + \- uid: {{uid}} +{% endfor %} +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.returner(ret) -Insert minion return data into the sqlite3 database .UNINDENT +.sp +This approach allows for users to be safely defined in a pillar and then the +user data is applied in an sls file. +.SS Parameterizing States With Pillar +.sp +Pillar data can be accessed in state files to customise behavior for each +minion. All pillar (and grain) data applicable to each minion is substituted +into the state files through templating before being run. Typical uses +include setting directories appropriate for the minion and skipping states +that don\(aqt apply. +.sp +A simple example is to set up a mapping of package names in pillar for +separate Linux distributions: +.sp +\fB/srv/pillar/pkg/init.sls\fP: .INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.save_load(jid, load, minions=None) -Save the load to the specified jid +.INDENT 3.5 +.sp +.nf +.ft C +pkgs: + {% if grains[\(aqos_family\(aq] == \(aqRedHat\(aq %} + apache: httpd + vim: vim\-enhanced + {% elif grains[\(aqos_family\(aq] == \(aqDebian\(aq %} + apache: apache2 + vim: vim + {% elif grains[\(aqos\(aq] == \(aqArch\(aq %} + apache: apache + vim: vim + {% endif %} +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.sqlite3_return.save_minions(jid, minions, syndic_id=None) -Included for API consistency .UNINDENT -.SS salt.returners.syslog_return .sp -Return data to the host operating system\(aqs syslog facility +The new \fBpkg\fP sls needs to be added to the top file: .sp -To use the syslog returner, append \(aq\-\-return syslog\(aq to the -salt command. +\fB/srv/pillar/top.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return syslog +base: + \(aq*\(aq: + \- data + \- users + \- pkg .ft P .fi .UNINDENT .UNINDENT .sp -The following fields can be set in the minion conf file: +Now the minions will auto map values based on respective operating systems +inside of the pillar, so sls files can be safely parameterized: +.sp +\fB/srv/salt/apache/init.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -syslog.level (optional, Default: LOG_INFO) -syslog.facility (optional, Default: LOG_USER) -syslog.tag (optional, Default: salt\-minion) -syslog.options (list, optional, Default: []) +apache: + pkg.installed: + \- name: {{ pillar[\(aqpkgs\(aq][\(aqapache\(aq] }} .ft P .fi .UNINDENT .UNINDENT .sp -Available levels, facilities, and options can be found in the -\fBsyslog\fP docs for your python version. +Or, if no pillar is available a default can be set as well: .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The default tag comes from \fBsys.argv[0]\fP which is -usually \(dqsalt\-minion\(dq but could be different based on -the specific environment. +The function \fBpillar.get\fP used in this example was added to Salt in +version 0.14.0 .UNINDENT .UNINDENT .sp -Configuration example: +\fB/srv/salt/apache/init.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -syslog.level: \(aqLOG_ERR\(aq -syslog.facility: \(aqLOG_DAEMON\(aq -syslog.tag: \(aqmysalt\(aq -syslog.options: - \- LOG_PID +apache: + pkg.installed: + \- name: {{ salt[\(aqpillar.get\(aq](\(aqpkgs:apache\(aq, \(aqhttpd\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -Of course you can also nest the options: +In the above example, if the pillar value \fBpillar[\(aqpkgs\(aq][\(aqapache\(aq]\fP is not +set in the minion\(aqs pillar, then the default of \fBhttpd\fP will be used. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Under the hood, pillar is just a Python dict, so Python dict methods such +as \fBget\fP and \fBitems\fP can be used. +.UNINDENT +.UNINDENT +.SS Pillar Makes Simple States Grow Easily +.sp +One of the design goals of pillar is to make simple sls formulas easily grow +into more flexible formulas without refactoring or complicating the states. +.sp +A simple formula: +.sp +\fB/srv/salt/edit/vim.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -syslog: - level: \(aqLOG_ERR\(aq - facility: \(aqLOG_DAEMON\(aq - tag: \(aqmysalt\(aq - options: - \- LOG_PID +vim: + pkg.installed: [] + +/etc/vimrc: + file.managed: + \- source: salt://edit/vimrc + \- mode: 644 + \- user: root + \- group: root + \- require: + \- pkg: vim .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by -prefacing the configuration. Any values not found -in the alternative configuration will be pulled from -the default location: +Can be easily transformed into a powerful, parameterized formula: +.sp +\fB/srv/salt/edit/vim.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -alternative.syslog.level: \(aqLOG_WARN\(aq -alternative.syslog.facility: \(aqLOG_NEWS\(aq +vim: + pkg.installed: + \- name: {{ pillar[\(aqpkgs\(aq][\(aqvim\(aq] }} + +/etc/vimrc: + file.managed: + \- source: {{ pillar[\(aqvimrc\(aq] }} + \- mode: 644 + \- user: root + \- group: root + \- require: + \- pkg: vim .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append -\fB\-\-return_config alternative\fP to the salt command. +Where the vimrc source location can now be changed via pillar: .sp -New in version 2015.5.0. - +\fB/srv/pillar/edit/vim.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return syslog \-\-return_config alternative +{% if grains[\(aqid\(aq].startswith(\(aqdev\(aq) %} +vimrc: salt://edit/dev_vimrc +{% elif grains[\(aqid\(aq].startswith(\(aqqa\(aq) %} +vimrc: salt://edit/qa_vimrc +{% else %} +vimrc: salt://edit/vimrc +{% endif %} .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append -\-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +Ensuring that the right vimrc is sent out to the correct minions. .sp -New in version 2016.3.0. - +The pillar top file must include a reference to the new sls pillar file: +.sp +\fB/srv/pillar/top.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return syslog \-\-return_kwargs \(aq{\(dqlevel\(dq: \(dqLOG_DEBUG\(dq}\(aq +base: + \(aq*\(aq: + \- pkg + \- edit.vim .ft P .fi .UNINDENT .UNINDENT +.SS Setting Pillar Data on the Command Line .sp -\fBNOTE:\fP +Pillar data can be set on the command line when running \fBstate.apply += 1.3.1 +.INDENT 3.5 +.sp +.nf +.ft C +mysql.pass: hardtoguesspassword +.ft P +.fi +.UNINDENT .UNINDENT .sp -The following fields can be set in the minion conf file: +This is very convenient when you need some dynamic configuration change that +you want to be applied on the fly. For example, there is a chicken and the egg +problem if you do this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -xmpp.jid (required) -xmpp.password (required) -xmpp.recipient (required) -xmpp.profile (optional) +mysql\-admin\-passwd: + mysql_user.present: + \- name: root + \- password: somepasswd + +mydb: + mysql_db.present .ft P .fi .UNINDENT .UNINDENT .sp -Alternative configuration values can be used by prefacing the configuration. -Any values not found in the alternative configuration will be pulled from -the default location: +The second state will fail, because you changed the root password and the +minion didn\(aqt notice it. Setting mysql.pass in the pillar, will help to sort +out the issue. But always change the root admin password in the first place. +.sp +This is very helpful for any module that needs credentials to apply state +changes: mysql, keystone, etc. +.SS Targeting Minions +.sp +Targeting minions is specifying which minions should run a command or execute a +state by matching against hostnames, or system information, or defined groups, +or even combinations thereof. +.sp +For example the command \fBsalt web1 apache.signal restart\fP to restart the +Apache httpd server specifies the machine \fBweb1\fP as the target and the +command will only be run on that one minion. +.sp +Similarly when using States, the following \fI\%top file\fP specifies that only +the \fBweb1\fP minion should execute the contents of \fBwebserver.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -xmpp.jid -xmpp.password -xmpp.recipient -xmpp.profile +base: + \(aqweb1\(aq: + \- webserver .ft P .fi .UNINDENT .UNINDENT .sp -XMPP settings may also be configured as: +The simple target specifications, glob, regex, and list will cover many use +cases, and for some will cover all use cases, but more powerful options exist. +.SS Targeting with Grains +.sp +The Grains interface was built into Salt to allow minions to be targeted by +system properties. So minions running on a particular operating system can +be called to execute a function, or a specific kernel. +.sp +Calling via a grain is done by passing the \-G option to salt, specifying +a grain and a glob expression to match the value of the grain. The syntax for +the target is the grain key followed by a glob expression: \(dqos:Arch*\(dq. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -xmpp: - jid: user@xmpp.domain.com/resource - password: password - recipient: user@xmpp.example.com - -alternative.xmpp: - jid: user@xmpp.domain.com/resource - password: password - recipient: someone@xmpp.example.com - -xmpp_profile: - xmpp.jid: user@xmpp.domain.com/resource - xmpp.password: password - -xmpp: - profile: xmpp_profile - recipient: user@xmpp.example.com - -alternative.xmpp: - profile: xmpp_profile - recipient: someone\-else@xmpp.example.com +salt \-G \(aqos:Fedora\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -To use the XMPP returner, append \(aq\-\-return xmpp\(aq to the salt command. +Will return True from all of the minions running Fedora. +.sp +To discover what grains are available and what the values are, execute the +grains.item salt function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return xmpp +salt \(aq*\(aq grains.items .ft P .fi .UNINDENT .UNINDENT .sp -To use the alternative configuration, append \(aq\-\-return_config alternative\(aq to the salt command. +More info on using targeting with grains can be found \fI\%here\fP\&. +.SS Compound Targeting .sp -New in version 2015.5.0. +New in version 0.9.5. +.sp +Multiple target interfaces can be used in conjunction to determine the command +targets. These targets can then be combined using \fBand\fP or \fBor\fP statements. +This is well defined with an example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return xmpp \-\-return_config alternative +salt \-C \(aqG@os:Debian and webser* or E@db.*\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -To override individual configuration items, append \-\-return_kwargs \(aq{\(dqkey:\(dq: \(dqvalue\(dq}\(aq to the salt command. +In this example any minion who\(aqs id starts with \fBwebser\fP and is running +Debian, or any minion who\(aqs id starts with db will be matched. .sp -New in version 2016.3.0. +The type of matcher defaults to glob, but can be specified with the +corresponding letter followed by the \fB@\fP symbol. In the above example a grain +is used with \fBG@\fP as well as a regular expression with \fBE@\fP\&. The +\fBwebser*\fP target does not need to be prefaced with a target type specifier +because it is a glob. +.sp +More info on using compound targeting can be found \fI\%here\fP\&. +.SS Node Group Targeting +.sp +New in version 0.9.5. +.sp +For certain cases, it can be convenient to have a predefined group of minions +on which to execute commands. This can be accomplished using what are called +\fI\%nodegroups\fP\&. Nodegroups allow for predefined +compound targets to be declared in the master configuration file, as a sort of +shorthand for having to type out complicated compound expressions. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return xmpp \-\-return_kwargs \(aq{\(dqrecipient\(dq: \(dqsomeone\-else@xmpp.example.com\(dq}\(aq +nodegroups: + group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com and bl*.domain.com\(aq + group2: \(aqG@os:Debian and foo.domain.com\(aq + group3: \(aqG@os:Debian and N@group1\(aq .ft P .fi .UNINDENT .UNINDENT +.SS Advanced Targeting Methods +.sp +There are many ways to target individual minions or groups of minions in Salt: +.SS Matching the \fBminion id\fP +.sp +Each minion needs a unique identifier. By default when a minion starts for the +first time it chooses its FQDN as that +identifier. The minion id can be overridden via the minion\(aqs \fI\%id\fP +configuration setting. +.sp +\fBTIP:\fP .INDENT 0.0 -.TP -.B class salt.returners.xmpp_return.SendMsgBot(jid, password, recipient, msg) -.INDENT 7.0 -.TP -.B start(event) +.INDENT 3.5 +minion id and minion keys +.sp +The \fI\%minion id\fP is used to generate the minion\(aqs public/private keys +and if it ever changes the master must then accept the new key as though +the minion was a new host. .UNINDENT .UNINDENT +.SS Globbing +.sp +The default matching that Salt utilizes is \fBshell\-style globbing\fP around the \fI\%minion id\fP\&. This also works for states +in the \fI\%top file\fP\&. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.returners.xmpp_return.returner(ret) -Send an xmpp message with the data +.INDENT 3.5 +You must wrap \fBsalt\fP calls that use globbing in single\-quotes to +prevent the shell from expanding the globs before Salt is invoked. +.UNINDENT .UNINDENT -.SS salt.returners.zabbix_return -.sp -Return salt data to Zabbix .sp -The following Type: \(dqZabbix trapper\(dq with \(dqType of information\(dq Text items are required: +Match all minions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Key: salt.trap.info -Key: salt.trap.warning -Key: salt.trap.high +salt \(aq*\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -To use the Zabbix returner, append \(aq\-\-return zabbix\(aq to the salt command. ex: +Match all minions in the example.net domain or any of the example domains: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq test.ping \-\-return zabbix +salt \(aq*.example.net\(aq test.version +salt \(aq*.example.*\(aq test.version .ft P .fi .UNINDENT .UNINDENT +.sp +Match all the \fBwebN\fP minions in the example.net domain (\fBweb1.example.net\fP, +\fBweb2.example.net\fP … \fBwebN.example.net\fP): .INDENT 0.0 -.TP -.B salt.returners.zabbix_return.returner(ret) +.INDENT 3.5 +.sp +.nf +.ft C +salt \(aqweb?.example.net\(aq test.version +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.zabbix_return.save_load(jid, load, minions=None) -Included for API consistency .UNINDENT +.sp +Match the \fBweb1\fP through \fBweb5\fP minions: .INDENT 0.0 -.TP -.B salt.returners.zabbix_return.zabbix_send(key, output) +.INDENT 3.5 +.sp +.nf +.ft C +salt \(aqweb[1\-5]\(aq test.version +.ft P +.fi .UNINDENT -.INDENT 0.0 -.TP -.B salt.returners.zabbix_return.zbx() .UNINDENT -.SS Renderers -.sp -The Salt state system operates by gathering information from common data types -such as lists, dictionaries, and strings that would be familiar to any -developer. -.sp -Salt Renderers translate input from the format in which it is written into -Python data structures. -.sp -The default renderer is set in the master/minion configuration file using the -\fI\%renderer\fP config option, which defaults to \fBjinja|yaml\fP\&. -.SS Two Kinds of Renderers .sp -Renderers fall into one of two categories, based on what they output: text or -data. Some exceptions to this would be the \fI\%pure python\fP and \fI\%gpg\fP renderers which could be used in either capacity. -.SS Text Renderers -.sp -\fBIMPORTANT:\fP +Match the \fBweb1\fP and \fBweb3\fP minions: .INDENT 0.0 .INDENT 3.5 -\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt -takes advantage of. Other text \fI\%Renderers\fP do not support this -functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. +.sp +.nf +.ft C +salt \(aqweb[1,3]\(aq test.version +.ft P +.fi .UNINDENT .UNINDENT .sp -A text renderer returns text. These include templating engines such as -\fI\%jinja\fP, \fI\%mako\fP, and -\fI\%genshi\fP, as well as the \fI\%gpg\fP renderer. The following are all text renderers: +Match the \fBweb\-x\fP, \fBweb\-y\fP, and \fBweb\-z\fP minions: .INDENT 0.0 -.IP \(bu 2 -\fI\%aws_kms\fP -.IP \(bu 2 -\fI\%cheetah\fP -.IP \(bu 2 -\fI\%genshi\fP -.IP \(bu 2 -\fI\%gpg\fP -.IP \(bu 2 -\fI\%jinja\fP -.IP \(bu 2 -\fI\%mako\fP -.IP \(bu 2 -\fI\%nacl\fP -.IP \(bu 2 -\fI\%pass\fP -.IP \(bu 2 -\fI\%py\fP -.IP \(bu 2 -\fI\%wempy\fP +.INDENT 3.5 +.sp +.nf +.ft C +salt \(aqweb\-[x\-z]\(aq test.version +.ft P +.fi +.UNINDENT .UNINDENT -.SS Data Renderers .sp -A data renderer returns a Python data structure (typically a dictionary). The -following are all data renderers: +\fBNOTE:\fP .INDENT 0.0 -.IP \(bu 2 -\fI\%dson\fP -.IP \(bu 2 -\fI\%hjson\fP -.IP \(bu 2 -\fI\%json5\fP -.IP \(bu 2 -\fI\%json\fP -.IP \(bu 2 -\fI\%pydsl\fP -.IP \(bu 2 -\fI\%pyobjects\fP -.IP \(bu 2 -\fI\%py\fP -.IP \(bu 2 -\fI\%stateconf\fP -.IP \(bu 2 -\fI\%yamlex\fP -.IP \(bu 2 -\fI\%yaml\fP -.IP \(bu 2 -\fI\%gpg\fP +.INDENT 3.5 +For additional targeting methods please review the +\fI\%compound matchers\fP documentation. .UNINDENT -.SS Overriding the Default Renderer +.UNINDENT +.SS Regular Expressions .sp -It can sometimes be beneficial to write an SLS file using a renderer other than -the default one. This can be done by using a \(dqshebang\(dq\-like syntax on the first -line of the SLS file: +Minions can be matched using Perl\-compatible \fBregular expressions\fP (which is globbing on steroids and a ton of caffeine). .sp -Here is an example of using the \fI\%pure python\fP renderer -to install a package: +Match both \fBweb1\-prod\fP and \fBweb1\-devel\fP minions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!py - - -def run(): - \(dq\(dq\(dq - Install version 1.5\-1.el7 of package \(dqpython\-foo\(dq - \(dq\(dq\(dq - return { - \(dqinclude\(dq: [\(dqpython\(dq], - \(dqpython\-foo\(dq: {\(dqpkg.installed\(dq: [{\(dqversion\(dq: \(dq1.5\-1.el7\(dq}]}, - } +salt \-E \(aqweb1\-(prod|devel)\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -This would be equivalent to the following: +When using regular expressions in a State\(aqs \fI\%top file\fP, you must specify +the matcher as the first option. The following example executes the contents of +\fBwebserver.sls\fP on the above\-mentioned minions. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include: - \- python - -python\-foo: - pkg.installed: - \- version: \(aq1.5\-1.el7\(aq +base: + \(aqweb1\-(prod|devel)\(aq: + \- match: pcre + \- webserver .ft P .fi .UNINDENT .UNINDENT -.SS Composing Renderers (a.k.a. The \(dqRender Pipeline\(dq) +.SS Lists .sp -A render pipeline can be composed from other renderers by connecting them in a -series of \(dqpipes\(dq (i.e. \fB|\fP). The renderers will be evaluated from left to -right, with each renderer receiving the result of the previous renderer\(aqs -execution. +At the most basic level, you can specify a flat list of minion IDs: +.INDENT 0.0 +.INDENT 3.5 .sp -Take for example the default renderer (\fBjinja|yaml\fP). The file is evaluated -first a jinja template, and the result of that template is evaluated as a YAML -document. +.nf +.ft C +salt \-L \(aqweb1,web2,web3\(aq test.version +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Targeting using Grains .sp -Other render pipeline combinations include: +Grain data can be used when targeting minions. +.sp +For example, the following matches all CentOS minions: .INDENT 0.0 .INDENT 3.5 -.INDENT 0.0 -.TP -.B \fByaml\fP -Just YAML, no templating. -.TP -.B \fBmako|yaml\fP -This passes the input to the \fBmako\fP renderer, with its output fed into -the \fByaml\fP renderer. -.TP -.B \fBjinja|mako|yaml\fP -This one allows you to use both jinja and mako templating syntax in the -input and then parse the final rendered output as YAML. -.UNINDENT +.sp +.nf +.ft C +salt \-G \(aqos:CentOS\(aq test.version +.ft P +.fi .UNINDENT .UNINDENT .sp -The following is a contrived example SLS file using the \fBjinja|mako|yaml\fP -render pipeline: +Match all minions with 64\-bit CPUs, and return number of CPU cores for each +matching minion: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!jinja|mako|yaml - -An_Example: - cmd.run: - \- name: | - echo \(dqUsing Salt ${grains[\(aqsaltversion\(aq]}\(dq \e - \(dqfrom path {{grains[\(aqsaltpath\(aq]}}.\(dq - \- cwd: / - -<%doc> ${...} is Mako\(aqs notation, and so is this comment. -{# Similarly, {{...}} is Jinja\(aqs notation, and so is this comment. #} +salt \-G \(aqcpuarch:x86_64\(aq grains.item num_cpus .ft P .fi .UNINDENT .UNINDENT .sp -\fBIMPORTANT:\fP +Additionally, globs can be used in grain matches, and grains that are nested in +a dictionary can be matched by adding a colon for each level that is traversed. +For example, the following will match hosts that have a grain called +\fBec2_tags\fP, which itself is a dictionary with a key named \fBenvironment\fP, +which has a value that contains the word \fBproduction\fP: .INDENT 0.0 .INDENT 3.5 -Keep in mind that not all renderers can be used alone or with any other -renderers. For example, text renderers shouldn\(aqt be used alone as their -outputs are just strings, which still need to be parsed by another renderer -to turn them into Python data structures. -.sp -For example, it would not make sense to use \fByaml|jinja\fP because the -output of the \fI\%yaml\fP renderer is a Python data -structure, and the \fI\%jinja\fP renderer only -accepts text as input. .sp -Therefore, when combining renderers, you should know what each renderer -accepts as input and what it returns as output. One way of thinking about -it is that you can chain together multiple text renderers, but the pipeline -\fImust\fP end in a data renderer. Similarly, since the text renderers in Salt -don\(aqt accept data structures as input, a text renderer should usually not -come after a data renderer. It\(aqs technically \fIpossible\fP to write a renderer -that takes a data structure as input and returns a string, but no such -renderer is distributed with Salt. +.nf +.ft C +salt \-G \(aqec2_tags:environment:*production*\(aq +.ft P +.fi .UNINDENT .UNINDENT -.SS Writing Renderers .sp -A custom renderer must be a Python module which implements a \fBrender\fP -function. This function must implement three positional arguments: +\fBIMPORTANT:\fP .INDENT 0.0 -.IP 1. 3 -\fBdata\fP \- Can be called whatever you like. This is the input to be -rendered. -.IP 2. 3 -\fBsaltenv\fP -.IP 3. 3 -\fBsls\fP +.INDENT 3.5 +See \fI\%Is Targeting using Grain Data Secure?\fP for +important security information. +.UNINDENT .UNINDENT +.SS Targeting using Pillar .sp -The first is the important one, and the 2nd and 3rd must be included since Salt -needs to pass this info to each render, even though it is only used by template -renderers. +Pillar data can be used when targeting minions. This allows for ultimate +control and flexibility when targeting minions. .sp -Renderers should be written so that the \fBdata\fP argument can accept either -strings or file\-like objects as input. For example: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +To start using Pillar targeting it is required to make a Pillar +data cache on Salt Master for each Minion via following commands: +\fBsalt \(aq*\(aq saltutil.refresh_pillar\fP or \fBsalt \(aq*\(aq saltutil.sync_all\fP\&. +Also Pillar data cache will be populated during the +\fI\%highstate\fP run. Once Pillar data changes, you +must refresh the cache by running above commands for this targeting +method to work correctly. +.UNINDENT +.UNINDENT +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import mycoolmodule -from salt.ext import six - - -def render(data, saltenv=\(dqbase\(dq, sls=\(dq\(dq, **kwargs): - if not isinstance(data, six.string_types): - # Read from file\-like object - data = data.read() - - return mycoolmodule.do_something(data) +salt \-I \(aqsomekey:specialvalue\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Custom renderers should be placed within \fBsalt://_renderers/\fP, so that they -can be synced to minions. They are synced when any of the following are run: +Like with \fI\%Grains\fP, it is possible to use globbing +as well as match nested values in Pillar, by adding colons for each level that +is being traversed. The below example would match minions with a pillar named +\fBfoo\fP, which is a dict containing a key \fBbar\fP, with a value beginning with +\fBbaz\fP: .INDENT 0.0 -.IP \(bu 2 -\fI\%state.apply\fP -.IP \(bu 2 -\fI\%saltutil.sync_renderers\fP -.IP \(bu 2 -\fI\%saltutil.sync_all\fP -.UNINDENT +.INDENT 3.5 .sp -Any custom renderers which have been synced to a minion, that are named the -same as one of Salt\(aqs default set of renderers, will take the place of the -default renderer with the same name. +.nf +.ft C +salt \-I \(aqfoo:bar:baz*\(aq test.version +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Subnet/IP Address Matching .sp -\fBNOTE:\fP +Minions can easily be matched based on IP address, or by subnet (using \fI\%CIDR\fP +notation). .INDENT 0.0 .INDENT 3.5 -Renderers can also be synced from \fBsalt://_renderers/\fP to the Master -using either the \fI\%saltutil.sync_renderers\fP or \fI\%saltutil.sync_all\fP runner function. +.sp +.nf +.ft C +salt \-S 192.168.40.20 test.version +salt \-S 2001:db8::/64 test.version +.ft P +.fi .UNINDENT .UNINDENT -.SS Examples -.sp -The best place to find examples of renderers is in the Salt source code. .sp -Documentation for renderers included with Salt can be found here: -.sp -\fI\%salt/renderers\fP -.sp -Here is a simple YAML renderer example: +Ipcidr matching can also be used in compound matches .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import salt.utils.yaml -from salt.utils.yamlloader import SaltYamlSafeLoader -from salt.ext import six - - -def render(yaml_data, saltenv=\(dq\(dq, sls=\(dq\(dq, **kws): - if not isinstance(yaml_data, six.string_types): - yaml_data = yaml_data.read() - data = salt.utils.yaml.safe_load(yaml_data) - return data if data else {} +salt \-C \(aqS@10.0.0.0/24 and G@os:Debian\(aq test.version .ft P .fi .UNINDENT .UNINDENT -.SS Full List of Renderers -.SS renderer modules .sp -\fBIMPORTANT:\fP +It is also possible to use in both pillar and state\-matching .INDENT 0.0 .INDENT 3.5 -\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt -takes advantage of. Other text \fI\%Renderers\fP do not support this -functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. +.sp +.nf +.ft C +\(aq172.16.0.0/12\(aq: + \- match: ipcidr + \- internal +.ft P +.fi .UNINDENT .UNINDENT +.SS Compound matchers +.sp +Compound matchers allow very granular minion targeting using any of Salt\(aqs +matchers. The default matcher is a \fBglob\fP match, just as +with CLI and \fI\%top file\fP matching. To match using anything other than a +glob, prefix the match string with the appropriate letter from the table below, +followed by an \fB@\fP sign. .TS center; -|l|l|. +|l|l|l|l|. _ T{ -\fI\%aws_kms\fP +Letter T} T{ -T} -_ -T{ -\fI\%cheetah\fP +Match Type T} T{ -Cheetah Renderer for Salt -T} -_ -T{ -\fI\%dson\fP +Example T} T{ -DSON Renderer for Salt +\fI\%Alt Delimiter?\fP T} _ T{ -\fI\%genshi\fP +G T} T{ -Genshi Renderer for Salt -T} -_ -T{ -\fI\%gpg\fP +Grains glob T} T{ -Renderer that will decrypt GPG ciphers -T} -_ -T{ -\fI\%hjson\fP +\fBG@os:Ubuntu\fP T} T{ -hjson renderer for Salt +Yes T} _ T{ -\fI\%jinja\fP +E T} T{ -Jinja loading utils to enable a more powerful backend for jinja templates -T} -_ -T{ -\fI\%json\fP +PCRE Minion ID T} T{ -JSON Renderer for Salt -T} -_ -T{ -\fI\%json5\fP +\fBE@web\ed+\e.(dev|qa|prod)\e.loc\fP T} T{ -JSON5 Renderer for Salt +No T} _ T{ -\fI\%mako\fP +P T} T{ -Mako Renderer for Salt -T} -_ -T{ -\fI\%msgpack\fP +Grains PCRE T} T{ -T} -_ -T{ -\fI\%nacl\fP +\fBP@os:(RedHat|Fedora|CentOS)\fP T} T{ -Renderer that will decrypt NACL ciphers +Yes T} _ T{ -\fI\%pass\fP +L T} T{ -Pass Renderer for Salt -T} -_ -T{ -\fI\%py\fP +List of minions T} T{ -Pure python state renderer -T} -_ -T{ -\fI\%pydsl\fP +\fBL@minion1.example.com,minion3.domain.com or bl*.domain.com\fP T} T{ -A Python\-based DSL +No T} _ T{ -\fI\%pyobjects\fP +I T} T{ -Python renderer that includes a Pythonic Object based interface -T} -_ -T{ -\fI\%stateconf\fP +Pillar glob T} T{ -A flexible renderer that takes a templating engine and a data format +\fBI@pdata:foobar\fP +T} T{ +Yes T} _ T{ -\fI\%tomlmod\fP +J +T} T{ +Pillar PCRE +T} T{ +\fBJ@pdata:^(foo|bar)$\fP T} T{ +Yes T} _ T{ -\fI\%wempy\fP +S +T} T{ +Subnet/IP address +T} T{ +\fBS@192.168.1.0/24\fP or \fBS@192.168.1.100\fP T} T{ +No T} _ T{ -\fI\%yaml\fP +R T} T{ -YAML Renderer for Salt +Range cluster +T} T{ +\fBR@%foo.bar\fP +T} T{ +No T} _ T{ -\fI\%yamlex\fP +N +T} T{ +Nodegroups +T} T{ +\fBN@group1\fP T} T{ +No T} _ .TE -.SS salt.renderers.aws_kms -.sp -Renderer that will decrypt ciphers encrypted using \fI\%AWS KMS Envelope Encryption\fP\&. -.sp -Any key in the data to be rendered can be a urlsafe_b64encoded string, and this renderer will attempt -to decrypt it before passing it off to Salt. This allows you to safely store secrets in -source control, in such a way that only your Salt master can decrypt them and -distribute them only to the minions that need them. .sp -The typical use\-case would be to use ciphers in your pillar data, and keep the encrypted -data key on your master. This way developers with appropriate AWS IAM privileges can add new secrets -quickly and easily. -.sp -This renderer requires the \fI\%boto3\fP Python library. -.SS Setup +Matchers can be joined using boolean \fBand\fP, \fBor\fP, and \fBnot\fP operators. .sp -First, set up your AWS client. For complete instructions on configuration the AWS client, -please read the \fI\%boto3 configuration documentation\fP\&. By default, this renderer will use -the default AWS profile. You can override the profile name in salt configuration. -For example, if you have a profile in your aws client configuration named \(dqsalt\(dq, -you can add the following salt configuration: +For example, the following string matches all Debian minions with a hostname +that begins with \fBwebserv\fP, as well as any minions that have a hostname which +matches the \fBregular expression\fP \fBweb\-dc1\-srv.*\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -aws_kms: - profile_name: salt +salt \-C \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -The rest of these instructions assume that you will use the default profile for key generation -and setup. If not, export AWS_PROFILE and set it to the desired value. -.sp -Once the aws client is configured, generate a KMS customer master key and use that to generate -a local data key. +That same example expressed in a \fI\%top file\fP looks like the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# data_key=$(aws kms generate\-data\-key \-\-key\-id your\-key\-id \-\-key\-spec AES_256 - \-\-query \(aqCiphertextBlob\(aq \-\-output text) -# echo \(aqaws_kms:\(aq -# echo \(aq data_key: !!binary \(dq%s\(dq\en\(aq \(dq$data_key\(dq >> config/master +base: + \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq: + \- match: compound + \- webserver .ft P .fi .UNINDENT .UNINDENT .sp -To apply the renderer on a file\-by\-file basis add the following line to the -top of any pillar with gpg data in it: +New in version 2015.8.0. + +.sp +Excluding a minion based on its ID is also possible: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yaml|aws_kms +salt \-C \(aqnot web\-dc1\-srv\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Now with your renderer configured, you can include your ciphers in your pillar -data like so: +Versions prior to 2015.8.0 a leading \fBnot\fP was not supported in compound +matches. Instead, something like the following was required: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yaml|aws_kms - -a\-secret: gAAAAABaj5uzShPI3PEz6nL5Vhk2eEHxGXSZj8g71B84CZsVjAAtDFY1mfjNRl\-1Su9YVvkUzNjI4lHCJJfXqdcTvwczBYtKy0Pa7Ri02s10Wn1tF0tbRwk= +salt \-C \(aq* and not G@kernel:Darwin\(aq test.version .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.aws_kms.render(data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) -Decrypt the data to be rendered that was encrypted using AWS KMS envelope encryption. -.UNINDENT -.SS salt.renderers.cheetah .sp -Cheetah Renderer for Salt +Excluding a minion based on its ID was also possible: .INDENT 0.0 -.TP -.B salt.renderers.cheetah.render(cheetah_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, method=\(aqxml\(aq, **kws) -Render a Cheetah template. -.INDENT 7.0 -.TP -.B Return type -A Python data structure +.INDENT 3.5 +.sp +.nf +.ft C +salt \-C \(aq* and not web\-dc1\-srv\(aq test.version +.ft P +.fi .UNINDENT .UNINDENT -.SS salt.renderers.dson -.sp -DSON Renderer for Salt -.sp -This renderer is intended for demonstration purposes. Information on the DSON -spec can be found \fI\%here\fP\&. +.SS Precedence Matching .sp -This renderer requires \fI\%Dogeon\fP (installable via pip) +Matchers can be grouped together with parentheses to explicitly declare precedence amongst groups. .INDENT 0.0 -.TP -.B salt.renderers.dson.render(dson_input, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kwargs) -Accepts DSON data as a string or as a file object and runs it through the -JSON parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure +.INDENT 3.5 +.sp +.nf +.ft C +salt \-C \(aq( ms\-1 or G@id:ms\-3 ) and G@id:ms\-3\(aq test.version +.ft P +.fi .UNINDENT .UNINDENT -.SS salt.renderers.genshi .sp -Genshi Renderer for Salt +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.renderers.genshi.render(genshi_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, method=\(aqxml\(aq, **kws) -Render a Genshi template. A method should be passed in as part of the -kwargs. If no method is passed in, xml is assumed. Valid methods are: -.sp -Note that the \fBtext\fP method will call \fBNewTextTemplate\fP\&. If \fBoldtext\fP -is desired, it must be called explicitly -.INDENT 7.0 -.TP -.B Return type -A Python data structure +.INDENT 3.5 +Be certain to note that spaces are required between the parentheses and targets. Failing to obey this +rule may result in incorrect targeting! .UNINDENT .UNINDENT -.SS salt.renderers.gpg -.sp -Renderer that will decrypt GPG ciphers -.sp -Any value in the SLS file can be a GPG cipher, and this renderer will decrypt it -before passing it off to Salt. This allows you to safely store secrets in -source control, in such a way that only your Salt master can decrypt them and -distribute them only to the minions that need them. -.sp -The typical use\-case would be to use ciphers in your pillar data, and keep a -secret key on your master. You can put the public key in source control so that -developers can add new secrets quickly and easily. -.sp -This renderer requires the \fI\%gpg\fP binary. No python libraries are required as of -the 2015.8.0 release. -.SS GPG Homedir +.SS Alternate Delimiters .sp -The default \fIGPG Homedir \fP is \fB~/.gnupg\fP and needs to be set using -\fBgpg \-\-homedir\fP\&. Be very careful to not forget this option. It is also important -to run \fBgpg\fP commands as the user that owns the keys directory. If the salt\-master -runs as user \fBsalt\fP, then use \fBsu \- salt\fP before running any gpg commands. +New in version 2015.8.0. + .sp -In some cases, it\(aqs preferable to have gpg keys stored on removable media or -other non\-standard locations. This can be done using the \fBgpg_keydir\fP option -on the salt master. This will also require using a different path to \fB\-\-homedir\fP\&. +Matchers that target based on a key value pair use a colon (\fB:\fP) as +a delimiter. Matchers with a \fBYes\fP in the \fBAlt Delimiters\fP column +in the previous table support specifying an alternate delimiter character. .sp -The \fB\-\-homedir\fP argument can be configured for the current user using -\fBecho \(aqhomedir /etc/salt/gpgkeys\(aq >> ~/.gnupg\fP, but this should be used with -caution to avoid potential confusion. +This is done by specifying an alternate delimiter character between the leading +matcher character and the \fB@\fP pattern separator character. This avoids +incorrect interpretation of the pattern in the case that \fB:\fP is part of the +grain or pillar data structure traversal. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -gpg_keydir: +salt \-C \(aqJ|@foo|bar|^foo:bar$ or J!@gitrepo!https://github.com:example/project.git\(aq test.ping .ft P .fi .UNINDENT .UNINDENT -.SS GPG Keys -.sp -GPG key pairs include both a public and private key. The private key is akin to -a password and should be kept secure by the owner. A public key is used to -encrypt data being sent to the owner of the private key. +.SS Node groups .sp -This means that the public key will be freely distributed so that others can -encrypt pillar data without access to the secret key. -.SS New Key Pair +Nodegroups are declared using a compound target specification. The compound +target documentation can be found \fI\%here\fP\&. .sp -To create a new GPG key pair for encrypting data, log in to the master as root -and run the following: +The \fI\%nodegroups\fP master config file parameter is used to define +nodegroups. Here\(aqs an example nodegroup configuration within +\fB/etc/salt/master\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# mkdir \-p /etc/salt/gpgkeys -# chmod 0700 /etc/salt/gpgkeys -# gpg \-\-homedir /etc/salt/gpgkeys \-\-gen\-key +nodegroups: + group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com\(aq + group2: \(aqG@os:Debian and foo.domain.com\(aq + group3: \(aqG@os:Debian and N@group1\(aq + group4: + \- \(aqG@foo:bar\(aq + \- \(aqor\(aq + \- \(aqG@foo:baz\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Do not supply a password for the keypair and use a name that makes sense for -your application. -.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -In some situations, gpg may be starved of entropy and will take an incredibly -long time to finish. Two common tools to generate (less secure) pseudo\-random -data are \fBrng\-tools\fP and \fBhaveged\fP\&. -.UNINDENT -.UNINDENT +The \fBL\fP within group1 is matching a list of minions, while the \fBG\fP in +group2 is matching specific grains. See the \fI\%compound matchers\fP documentation for more details. .sp -The new keys can be seen and verified using \fB\-\-list\-secret\-keys\fP: +As of the 2017.7.0 release of Salt, group names can also be prepended with +a dash. This brings the usage in line with many other areas of Salt. For +example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# gpg \-\-homedir /etc/salt/gpgkeys \-\-list\-secret\-keys -/etc/salt/gpgkeys/pubring.kbx -\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -sec rsa4096 2002\-05\-12 [SC] [expires: 2012\-05\-10] - 2DC47B416EE8C3484450B450A4D44406274AF44E -uid [ultimate] salt\-master (gpg key for salt) -ssb rsa4096 2002\-05\-12 [E] [expires: 2012\-05\-10] +nodegroups: + \- group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com\(aq .ft P .fi .UNINDENT .UNINDENT +.UNINDENT +.UNINDENT .sp -In the example above, our KEY\-ID is \fB2DC47B416EE8C3484450B450A4D44406274AF44E\fP\&. -.SS Export Public Key +New in version 2015.8.0. + .sp -To export a public key suitable for public distribution: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -# gpg \-\-homedir /etc/salt/gpgkeys \-\-armor \-\-export > exported_pubkey.asc -.ft P -.fi +Nodegroups can reference other nodegroups as seen in \fBgroup3\fP\&. Ensure +that you do not have circular references. Circular references will be +detected and cause partial expansion with a logged error message. .UNINDENT .UNINDENT -.SS Import Public Key .sp -Users wishing to import the public key into their local keychain may run: +New in version 2015.8.0. + +.sp +Compound nodegroups can be either string values or lists of string values. +When the nodegroup is A string value will be tokenized by splitting on +whitespace. This may be a problem if whitespace is necessary as part of a +pattern. When a nodegroup is a list of strings then tokenization will +happen for each list element as a whole. +.sp +To match a nodegroup on the CLI, use the \fB\-N\fP command\-line option: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ gpg \-\-import exported_pubkey.asc +salt \-N group1 test.version .ft P .fi .UNINDENT .UNINDENT -.SS Export (Save) Private Key .sp -This key protects all gpg\-encrypted pillar data and should be backed up to a -safe and secure location. This command will generate a backup of secret keys -in the \fB/etc/salt/gpgkeys\fP directory to the \fBgpgkeys.secret\fP file: +New in version 2019.2.0. + +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-keys \-\-export\-options export\-backup \-o gpgkeys.secret -.ft P -.fi +The \fBN@\fP classifier historically could not be used in compound matches +within the CLI or \fI\%top file\fP, it was only recognized in the +\fI\%nodegroups\fP master config file parameter. As of the 2019.2.0 +release, this limitation no longer exists. .UNINDENT .UNINDENT .sp -Salt does not support password\-protected private keys, which means this file -is essentially a clear\-text password (just add \fB\-\-armor\fP). Fortunately, it -is trivial to pass this export back to gpg to be encrypted with symmetric key: +To match a nodegroup in your \fI\%top file\fP, make sure to put \fB\- match: +nodegroup\fP on the line directly following the nodegroup name. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-keys \-\-export\-options export\-backup | gpg \-\-symmetric \-o gpgkeys.gpg +base: + group1: + \- match: nodegroup + \- webserver .ft P .fi .UNINDENT @@ -30739,4909 +30892,4041 @@ is trivial to pass this export back to gpg to be encrypted with symmetric key: \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -In some cases, particularly when using su/sudo, gpg gets confused and needs -to be told which TTY to use; this can be done with: \fBexport GPG_TTY=$(tty)\fP\&. +When adding or modifying nodegroups to a master configuration file, the +master must be restarted for those changes to be fully recognized. +.sp +A limited amount of functionality, such as targeting with \-N from the +command\-line may be available without a restart. .UNINDENT .UNINDENT -.SS Import (Restore) Private Key +.SS Defining Nodegroups as Lists of Minion IDs .sp -To import/restore a private key, create a directory with the correct permissions -and import using gpg. +A simple list of minion IDs would traditionally be defined like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# mkdir \-p /etc/salt/gpgkeys -# chmod 0700 /etc/salt/gpgkeys -# gpg \-\-homedir /etc/salt/gpgkeys \-\-import gpgkeys.secret +nodegroups: + group1: L@host1,host2,host3 .ft P .fi .UNINDENT .UNINDENT .sp -If the export was encrypted using a symmetric key, then decrypt first with: +They can now also be defined as a YAML list, like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# gpg \-\-decrypt gpgkeys.gpg | gpg \-\-homedir /etc/salt/gpgkeys \-\-import +nodegroups: + group1: + \- host1 + \- host2 + \- host3 .ft P .fi .UNINDENT .UNINDENT -.SS Adjust trust level of imported keys .sp -In some cases, importing existing keys may not be enough and the trust level of -the key needs to be adjusted. This can be done by editing the key. The \fBKEY\-ID\fP -and the actual trust level of the key can be seen by listing the already imported -keys. +New in version 2016.11.0. + +.SS Batch Size .sp -If the trust\-level is not \fBultimate\fP it needs to be changed by running +The \fB\-b\fP (or \fB\-\-batch\-size\fP) option allows commands to be executed on only +a specified number of minions at a time. Both percentages and finite numbers are +supported. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -gpg \-\-homedir /etc/salt/gpgkeys \-\-edit\-key +salt \(aq*\(aq \-b 10 test.version + +salt \-G \(aqos:RedHat\(aq \-\-batch\-size 25% apache.signal restart .ft P .fi .UNINDENT .UNINDENT .sp -This will open an interactive shell for the management of the GPG encryption key. -Type \fBtrust\fP to be able to set the trust level for the key and then select \fB5 -(I trust ultimately)\fP\&. Then quit the shell by typing \fBsave\fP\&. -.SS Encrypting Data +This will only run test.version on 10 of the targeted minions at a time and then +restart apache on 25% of the minions matching \fBos:RedHat\fP at a time and work +through them all until the task is complete. This makes jobs like rolling web +server restarts behind a load balancer or doing maintenance on BSD firewalls +using carp much easier with salt. .sp -In order to encrypt data to a recipient (salt), the public key must be imported -into the local keyring. Importing the public key is described above in the -\fIImport Public Key \fP section. +The batch system maintains a window of running minions, so, if there are a +total of 150 minions targeted and the batch size is 10, then the command is +sent to 10 minions, when one minion returns then the command is sent to one +additional minion, so that the job is constantly running on 10 minions. .sp -To generate a cipher from a secret: +New in version 2016.3. + +.sp +The \fB\-\-batch\-wait\fP argument can be used to specify a number of seconds to +wait after a minion returns, before sending the command to a new minion. +.SS SECO Range +.sp +SECO range is a cluster\-based metadata store developed and maintained by Yahoo! +.sp +The Range project is hosted here: +.sp +\fI\%https://github.com/ytoolshed/range\fP +.sp +Learn more about range here: +.sp +\fI\%https://github.com/ytoolshed/range/wiki/\fP +.SS Prerequisites +.sp +To utilize range support in Salt, a range server is required. Setting up a +range server is outside the scope of this document. Apache modules are included +in the range distribution. +.sp +With a working range server, cluster files must be defined. These files are +written in YAML and define hosts contained inside a cluster. Full documentation +on writing YAML range files is here: +.sp +\fI\%https://github.com/ytoolshed/range/wiki/%22yamlfile%22\-module\-file\-spec\fP +.sp +Additionally, the Python seco range libraries must be installed on the salt +master. One can verify that they have been installed correctly via the +following command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ echo \-n \(aqsupersecret\(aq | gpg \-\-trust\-model always \-ear +python \-c \(aqimport seco.range\(aq .ft P .fi .UNINDENT .UNINDENT .sp -To apply the renderer on a file\-by\-file basis add the following line to the -top of any pillar with gpg data in it: +If no errors are returned, range is installed successfully on the salt master. +.SS Preparing Salt +.sp +Range support must be enabled on the salt master by setting the hostname and +port of the range server inside the master configuration file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yaml|gpg +range_server: my.range.server.com:80 .ft P .fi .UNINDENT .UNINDENT .sp -Now with your renderer configured, you can include your ciphers in your pillar -data like so: +Following this, the master must be restarted for the change to have an effect. +.SS Targeting with Range +.sp +Once a cluster has been defined, it can be targeted with a salt command by +using the \fB\-R\fP or \fB\-\-range\fP flags. +.sp +For example, given the following range YAML file being served from a range +server: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yaml|gpg - -a\-secret: | - \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- - Version: GnuPG v1 - - hQEMAweRHKaPCfNeAQf9GLTN16hCfXAbPwU6BbBK0unOc7i9/etGuVc5CyU9Q6um - QuetdvQVLFO/HkrC4lgeNQdM6D9E8PKonMlgJPyUvC8ggxhj0/IPFEKmrsnv2k6+ - cnEfmVexS7o/U1VOVjoyUeliMCJlAz/30RXaME49Cpi6No2+vKD8a4q4nZN1UZcG - RhkhC0S22zNxOXQ38TBkmtJcqxnqT6YWKTUsjVubW3bVC+u2HGqJHu79wmwuN8tz - m4wBkfCAd8Eyo2jEnWQcM4TcXiF01XPL4z4g1/9AAxh+Q4d8RIRP4fbw7ct4nCJv - Gr9v2DTF7HNigIMl4ivMIn9fp+EZurJNiQskLgNbktJGAeEKYkqX5iCuB1b693hJ - FKlwHiJt5yA8X2dDtfk8/Ph1Jx2TwGS+lGjlZaNqp3R1xuAZzXzZMLyZDe5+i3RJ - skqmFTbOiA===Eqsm - \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- +$ cat /etc/range/test.yaml +CLUSTER: host1..100.test.com +APPS: + \- frontend + \- backend + \- mysql .ft P .fi .UNINDENT .UNINDENT -.SS Encrypted CLI Pillar Data -.sp -New in version 2016.3.0. - .sp -Functions like \fI\%state.highstate\fP and -\fI\%state.sls\fP allow for pillar data to be -passed on the CLI. +One might target host1 through host100 in the test.com domain with Salt as follows: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion state.highstate pillar=\(dq{\(aqmypillar\(aq: \(aqfoo\(aq}\(dq +salt \-\-range %test:CLUSTER test.version .ft P .fi .UNINDENT .UNINDENT .sp -Starting with the 2016.3.0 release of Salt, it is now possible for this pillar -data to be GPG\-encrypted, and to use the GPG renderer to decrypt it. -.SS Replacing Newlines -.sp -To pass encrypted pillar data on the CLI, the ciphertext must have its newlines -replaced with a literal backslash\-n (\fB\en\fP), as newlines are not supported -within Salt CLI arguments. There are a number of ways to do this: -.sp -With awk or Perl: +The following salt command would target three hosts: \fBfrontend\fP, \fBbackend\fP, and \fBmysql\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# awk -ciphertext=\(gaecho \-n \(dqsupersecret\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | awk \(aq{printf \(dq%s\e\en\(dq,$0} END {print \(dq\(dq}\(aq\(ga -# Perl -ciphertext=\(gaecho \-n \(dqsupersecret\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | perl \-pe \(aqs/\en/\e\en/g\(aq\(ga +salt \-\-range %test:APPS test.version .ft P .fi .UNINDENT .UNINDENT +.SS Loadable Matchers .sp -With Python: +New in version 2019.2.0. + +.sp +Internally targeting is implemented with chunks of code called Matchers. As of +the 2019.2.0 release, matchers can be loaded dynamically. Currently new matchers +cannot be created, but existing matchers can have their functionality altered or +extended. For more information on Matchers see +.SS Matchers +.sp +New in version 3000. + +.sp +Matchers are modules that provide Salt\(aqs targeting abilities. As of the +3000 release, matchers can be dynamically loaded. Currently new matchers +cannot be created because the required plumbing for the CLI does not exist yet. +Existing matchers may have their functionality altered or extended. +.sp +For details of targeting methods, see the \fI\%Targeting\fP topic. +.sp +A matcher module must have a function called \fBmatch()\fP\&. This function ends up +becoming a method on the Matcher class. All matcher functions require at least +two arguments, \fBself\fP (because the function will be turned into a method), and +\fBtgt\fP, which is the actual target string. The grains and pillar matchers also +take a \fBdelimiter\fP argument and should default to \fBDEFAULT_TARGET_DELIM\fP\&. +.sp +Like other Salt loadable modules, modules that override built\-in functionality +can be placed in \fBfile_roots\fP in a special directory and then copied to the +minion through the normal sync process. \fI\%saltutil.sync_all\fP +will transfer all loadable modules, and the 3000 release introduces +\fI\%saltutil.sync_matchers\fP\&. For matchers, the directory is +\fB/srv/salt/_matchers\fP (assuming your \fBfile_roots\fP is set to the default +\fB/srv/salt\fP). +.sp +As an example, let\(aqs modify the \fBlist\fP matcher to have the separator be a +\(aq\fB/\fP\(aq instead of the default \(aq\fB,\fP\(aq. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import subprocess +from __future__ import absolute_import, print_function, unicode_literals +from salt.ext import six # pylint: disable=3rd\-party\-module\-not\-gated -secret, stderr = subprocess.Popen( - [\(aqgpg\(aq, \(aq\-\-armor\(aq, \(aq\-\-batch\(aq, \(aq\-\-trust\-model\(aq, \(aqalways\(aq, \(aq\-\-encrypt\(aq, - \(aq\-r\(aq, \(aquser@domain.com\(aq], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate(input=\(aqsupersecret\(aq) -if secret: - print(secret.replace(\(aq\en\(aq, r\(aq\en\(aq)) -else: - raise ValueError(\(aqNo ciphertext found: {0}\(aq.format(stderr)) +def match(self, tgt): + \(dq\(dq\(dq + Determines if this host is on the list + \(dq\(dq\(dq + if isinstance(tgt, six.string_types): + # The stock matcher splits on \(ga,\(ga. Change to \(ga/\(ga below. + tgt = tgt.split(\(dq/\(dq) + return bool(self.opts[\(dqid\(dq] in tgt) .ft P .fi .UNINDENT .UNINDENT +.sp +Place this code in a file called \fBlist_match.py\fP in a \fB_matchers\fP directory in your +\fBfile_roots\fP\&. Sync this down to your minions with +\fI\%saltutil.sync_matchers\fP\&. +Then attempt to match with the following, replacing \fBminionX\fP with three of your minions. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -ciphertext=\(gapython /path/to/script.py\(ga +salt \-L \(aqminion1/minion2/minion3\(aq test.ping .ft P .fi .UNINDENT .UNINDENT .sp -The ciphertext can be included in the CLI pillar data like so: +Three of your minions should respond. +.sp +The current supported matchers and associated filenames are +.TS +center; +|l|l|l|. +_ +T{ +Salt CLI Switch +T} T{ +Match Type +T} T{ +Filename +T} +_ +T{ + +T} T{ +Glob +T} T{ +glob_match.py +T} +_ +T{ +\-C +T} T{ +Compound +T} T{ +compound_match.py +T} +_ +T{ +\-E +T} T{ +Perl\-Compatible +Regular Expressions +T} T{ +pcre_match.py +T} +_ +T{ +\-L +T} T{ +List +T} T{ +list_match.py +T} +_ +T{ +\-G +T} T{ +Grain +T} T{ +grain_match.py +T} +_ +T{ +\-P +T} T{ +Grain Perl\-Compatible +Regular Expressions +T} T{ +grain_pcre_match.py +T} +_ +T{ +\-N +T} T{ +Nodegroup +T} T{ +nodegroup_match.py +T} +_ +T{ +\-R +T} T{ +Range +T} T{ +range_match.py +T} +_ +T{ +\-I +T} T{ +Pillar +T} T{ +pillar_match.py +T} +_ +T{ +\-J +T} T{ +Pillar Perl\-Compatible +Regular Expressions +T} T{ +pillar_pcre.py +T} +_ +T{ +\-S +T} T{ +IP\-Classless Internet +Domain Routing +T} T{ +ipcidr_match.py +T} +_ +.TE +.SS The Salt Mine +.sp +The Salt Mine is used to collect arbitrary data from Minions and store it on +the Master. This data is then made available to all Minions via the +\fI\%salt.modules.mine\fP module. +.sp +Mine data is gathered on the Minion and sent back to the Master where only the +most recent data is maintained (if long term data is required use returners or +the external job cache). +.SS Mine vs Grains +.sp +Mine data is designed to be much more up\-to\-date than grain data. Grains are +refreshed on a very limited basis and are largely static data. Mines are +designed to replace slow peer publishing calls when Minions need data from +other Minions. Rather than having a Minion reach out to all the other Minions +for a piece of data, the Salt Mine, running on the Master, can collect it from +all the Minions every \fI\%Mine Interval\fP, resulting in +almost fresh data at any given time, with much less overhead. +.SS Mine Functions +.sp +To enable the Salt Mine the \fBmine_functions\fP option needs to be applied to a +Minion. This option can be applied via the Minion\(aqs configuration file, or the +Minion\(aqs Pillar. The \fBmine_functions\fP option dictates what functions are +being executed and allows for arguments to be passed in. The list of +functions are available in the \fBsalt.module\fP\&. If no arguments +are passed, an empty list must be added like in the \fBtest.ping\fP function in +the example below: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion state.sls secretstuff pillar_enc=gpg pillar=\(dq{secret_pillar: \(aq$ciphertext\(aq}\(dq +mine_functions: + test.ping: [] + network.ip_addrs: + interface: eth0 + cidr: 10.0.0.0/8 .ft P .fi .UNINDENT .UNINDENT .sp -The \fBpillar_enc=gpg\fP argument tells Salt that there is GPG\-encrypted pillar -data, so that the CLI pillar data is passed through the GPG renderer, which -will iterate recursively though the CLI pillar dictionary to decrypt any -encrypted values. -.SS Encrypting the Entire CLI Pillar Dictionary +In the example above \fI\%salt.modules.network.ip_addrs\fP has additional +filters to help narrow down the results. In the above example IP addresses +are only returned if they are on a eth0 interface and in the 10.0.0.0/8 IP +range. .sp -If several values need to be encrypted, it may be more convenient to encrypt -the entire CLI pillar dictionary. Again, this can be done in several ways: +Changed in version 3000. + .sp -With awk or Perl: +The format to define mine_functions has been changed to allow the same format +as used for module.run. The old format (above) will still be supported. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# awk -ciphertext=\(gaecho \-n \(dq{\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, \(aqsecret_b\(aq: \(aqGPG is fun!\(aq}\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | awk \(aq{printf \(dq%s\e\en\(dq,$0} END {print \(dq\(dq}\(aq\(ga -# Perl -ciphertext=\(gaecho \-n \(dq{\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, \(aqsecret_b\(aq: \(aqGPG is fun!\(aq}\(dq | gpg \-\-armor \-\-batch \-\-trust\-model always \-\-encrypt \-r user@domain.com | perl \-pe \(aqs/\en/\e\en/g\(aq\(ga +mine_functions: + test.ping: [] + network.ip_addrs: + \- interface: eth0 + \- cidr: 10.0.0.0/8 + test.arg: + \- isn\(aqt + \- this + \- fun + \- this: that + \- salt: stack .ft P .fi .UNINDENT .UNINDENT +.SS Minion\-side Access Control .sp -With Python: +New in version 3000. + +.sp +Mine functions can be targeted to only be available to specific minions. This +uses the same targeting parameters as \fI\%Targeting Minions\fP but with keywords \fBallow_tgt\fP +and \fBallow_tgt_type\fP\&. When a minion requests a function from the salt mine that +is not allowed to be requested by that minion (i.e. when looking up the combination +of \fBallow_tgt\fP and \fBallow_tgt_type\fP and the requesting minion is not in the list) +it will get no data, just as if the requested function is not present in the salt mine. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import subprocess - -pillar_data = {\(aqsecret_a\(aq: \(aqCorrectHorseBatteryStaple\(aq, - \(aqsecret_b\(aq: \(aqGPG is fun!\(aq} - -secret, stderr = subprocess.Popen( - [\(aqgpg\(aq, \(aq\-\-armor\(aq, \(aq\-\-batch\(aq, \(aq\-\-trust\-model\(aq, \(aqalways\(aq, \(aq\-\-encrypt\(aq, - \(aq\-r\(aq, \(aquser@domain.com\(aq], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate(input=repr(pillar_data)) - -if secret: - print(secret.replace(\(aq\en\(aq, r\(aq\en\(aq)) -else: - raise ValueError(\(aqNo ciphertext found: {0}\(aq.format(stderr)) +mine_functions: + network.ip_addrs: + \- interface: eth0 + \- cidr: 10.0.0.0/8 + \- allow_tgt: \(aqG@role:master\(aq + \- allow_tgt_type: \(aqcompound\(aq .ft P .fi .UNINDENT .UNINDENT +.SS Mine Functions Aliases +.sp +Function aliases can be used to provide friendly names, usage intentions or to +allow multiple calls of the same function with different arguments. There is a +different syntax for passing positional and key\-value arguments. Mixing +positional and key\-value arguments is not supported. +.sp +New in version 2014.7.0. + .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -ciphertext=\(gapython /path/to/script.py\(ga +mine_functions: + network.ip_addrs: [eth0] + networkplus.internal_ip_addrs: [] + internal_ip_addrs: + mine_function: network.ip_addrs + cidr: 192.168.0.0/16 + ip_list: + \- mine_function: grains.get + \- ip_interfaces .ft P .fi .UNINDENT .UNINDENT .sp -With the entire pillar dictionary now encrypted, it can be included in the CLI -pillar data like so: +Changed in version 3000. + +.sp +With the addition of the module.run\-like format for defining mine_functions, the +method of adding aliases remains similar. Just add a \fBmine_function\fP kwarg with +the name of the real function to call, making the key below \fBmine_functions\fP +the alias: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion state.sls secretstuff pillar_enc=gpg pillar=\(dq$ciphertext\(dq +mine_functions: + alias_name: + \- mine_function: network.ip_addrs + \- eth0 + internal_ip_addrs: + \- mine_function: network.ip_addrs + \- cidr: 192.168.0.0/16 + ip_list: + \- mine_function: grains.get + \- ip_interfaces .ft P .fi .UNINDENT .UNINDENT -.SS Configuration +.SS Mine Interval .sp -The default behaviour of this renderer is to log a warning if a block could not -be decrypted; in other words, it just returns the ciphertext rather than the -encrypted secret. +The Salt Mine functions are executed when the Minion starts and at a given +interval by the scheduler. The default interval is every 60 minutes and can +be adjusted for the Minion via the \fBmine_interval\fP option in the minion +config: +.INDENT 0.0 +.INDENT 3.5 .sp -This behaviour can be changed via the \fIgpg_decrypt_must_succeed\fP configuration -option. If set to \fITrue\fP, any gpg block that cannot be decrypted raises a -\fISaltRenderError\fP exception, which registers an error in \fB_errors\fP during -rendering. +.nf +.ft C +mine_interval: 60 +.ft P +.fi +.UNINDENT +.UNINDENT +.SS Mine in Salt\-SSH .sp -In the Chlorine release, the default behavior will be reversed and an error -message will be added to \fB_errors\fP by default. +As of the 2015.5.0 release of salt, salt\-ssh supports \fBmine.get\fP\&. +.sp +Because the Minions cannot provide their own \fBmine_functions\fP configuration, +we retrieve the args for specified mine functions in one of three places, +searched in the following order: .INDENT 0.0 -.TP -.B salt.renderers.gpg.render(gpg_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) -Create a gpg object given a gpg_keydir, and then use it to try to decrypt -the data to be rendered. +.IP 1. 3 +Roster data +.IP 2. 3 +Pillar +.IP 3. 3 +Master config .UNINDENT -.SS salt.renderers.hjson -.sp -hjson renderer for Salt .sp -See the \fI\%hjson\fP documentation for more information +The \fBmine_functions\fP are formatted exactly the same as in normal salt, just +stored in a different location. Here is an example of a flat roster containing +\fBmine_functions\fP: .INDENT 0.0 -.TP -.B salt.renderers.hjson.render(hjson_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts HJSON as a string or as a file object and runs it through the HJSON -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure +.INDENT 3.5 +.sp +.nf +.ft C +test: + host: 104.237.131.248 + user: root + mine_functions: + cmd.run: [\(aqecho \(dqhello!\(dq\(aq] + network.ip_addrs: + interface: eth0 +.ft P +.fi .UNINDENT .UNINDENT -.SS salt.renderers.jinja -.sp -Jinja loading utils to enable a more powerful backend for jinja templates .sp -\fBIMPORTANT:\fP +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt -takes advantage of. Other text \fI\%Renderers\fP do not support this -functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. +Because of the differences in the architecture of salt\-ssh, \fBmine.get\fP +calls are somewhat inefficient. Salt must make a new salt\-ssh call to each +of the Minions in question to retrieve the requested data, much like a +publish call. However, unlike publish, it must run the requested function +as a wrapper function, so we can retrieve the function args from the pillar +of the Minion in question. This results in a non\-trivial delay in +retrieving the requested data. .UNINDENT .UNINDENT +.SS Minions Targeting with Mine +.sp +The \fBmine.get\fP function supports various methods of \fI\%Minions targeting\fP to fetch Mine data from particular hosts, such as glob or regular +expression matching on Minion id (name), grains, pillars and \fI\%compound +matches\fP\&. See the \fI\%salt.modules.mine\fP module +documentation for the reference. +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B salt.renderers.jinja.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, context=None, tmplpath=None, **kws) -Render the template_file, passing the functions and grains into the -Jinja rendering system. -.INDENT 7.0 -.TP -.B Return type -string +.INDENT 3.5 +Pillar data needs to be cached on Master for pillar targeting to work with +Mine. Read the note in \fI\%relevant section\fP\&. .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B class salt.utils.jinja.SerializerExtension(environment) -Yaml and Json manipulation. +.SS Example .sp -\fBFormat filters\fP +One way to use data from Salt Mine is in a State. The values can be retrieved +via Jinja and used in the SLS file. The following example is a partial HAProxy +configuration file and pulls IP addresses from all Minions with the \(dqweb\(dq grain +to add them to the pool of load balanced servers. .sp -Allows jsonifying or yamlifying any data structure. For example, this dataset: -.INDENT 7.0 +\fB/srv/pillar/top.sls\fP: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -data = { - \(aqfoo\(aq: True, - \(aqbar\(aq: 42, - \(aqbaz\(aq: [1, 2, 3], - \(aqqux\(aq: 2.0 -} +base: + \(aqG@roles:web\(aq: + \- web .ft P .fi .UNINDENT .UNINDENT -.INDENT 7.0 +.sp +\fB/srv/pillar/web.sls\fP: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -yaml = {{ data|yaml }} -json = {{ data|json }} -python = {{ data|python }} -xml = {{ {\(aqroot_node\(aq: data}|xml }} +mine_functions: + network.ip_addrs: [eth0] .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +Then trigger the minions to refresh their pillar data by running: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -yaml = {bar: 42, baz: [1, 2, 3], foo: true, qux: 2.0} -json = {\(dqbaz\(dq: [1, 2, 3], \(dqfoo\(dq: true, \(dqbar\(dq: 42, \(dqqux\(dq: 2.0} -python = {\(aqbar\(aq: 42, \(aqbaz\(aq: [1, 2, 3], \(aqfoo\(aq: True, \(aqqux\(aq: 2.0} -xml = \(dq\(dq\(dq< - - 1 - 2 - 3 - \(dq\(dq\(dq +salt \(aq*\(aq saltutil.refresh_pillar .ft P .fi .UNINDENT .UNINDENT .sp -The yaml filter takes an optional flow_style parameter to control the -default\-flow\-style parameter of the YAML dumper. -.INDENT 7.0 +Verify that the results are showing up in the pillar on the minions by +executing the following and checking for \fBnetwork.ip_addrs\fP in the output: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ data|yaml(False) }} +salt \(aq*\(aq pillar.items .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +Which should show that the function is present on the minion, but not include +the output: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -bar: 42 -baz: - \- 1 - \- 2 - \- 3 -foo: true -qux: 2.0 +minion1.example.com: + \-\-\-\-\-\-\-\-\-\- + mine_functions: + \-\-\-\-\-\-\-\-\-\- + network.ip_addrs: + \- eth0 .ft P .fi .UNINDENT .UNINDENT .sp -\fBLoad filters\fP +Mine data is typically only updated on the master every 60 minutes, this can +be modified by setting: .sp -Strings and variables can be deserialized with \fBload_yaml\fP and -\fBload_json\fP tags and filters. It allows one to manipulate data directly -in templates, easily: -.INDENT 7.0 +\fB/etc/salt/minion.d/mine.conf\fP: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{%\- set yaml_src = \(dq{foo: it works}\(dq|load_yaml %} -{%\- set json_src = \(aq{\(dqbar\(dq: \(dqfor real\(dq}\(aq|load_json %} -Dude, {{ yaml_src.foo }} {{ json_src.bar }}! +mine_interval: 5 .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +To force the mine data to update immediately run: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Dude, it works for real! +salt \(aq*\(aq mine.update .ft P .fi .UNINDENT .UNINDENT .sp -\fBLoad tags\fP -.sp -Salt implements \fBload_yaml\fP and \fBload_json\fP tags. They work like -the \fI\%import tag\fP, except that the document is also deserialized. -.sp -Syntaxes are \fB{% load_yaml as [VARIABLE] %}[YOUR DATA]{% endload %}\fP -and \fB{% load_json as [VARIABLE] %}[YOUR DATA]{% endload %}\fP -.sp -For example: -.INDENT 7.0 +Setup the \fI\%salt.states.file.managed\fP state in +\fB/srv/salt/haproxy.sls\fP: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% load_yaml as yaml_src %} - foo: it works -{% endload %} -{% load_json as json_src %} - { - \(dqbar\(dq: \(dqfor real\(dq - } -{% endload %} -Dude, {{ yaml_src.foo }} {{ json_src.bar }}! +haproxy_config: + file.managed: + \- name: /etc/haproxy/config + \- source: salt://haproxy_config + \- template: jinja .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +Create the Jinja template in \fB/srv/salt/haproxy_config\fP: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Dude, it works for real! +<...file contents snipped...> + +{% for server, addrs in salt[\(aqmine.get\(aq](\(aqroles:web\(aq, \(aqnetwork.ip_addrs\(aq, tgt_type=\(aqgrain\(aq) | dictsort() %} +server {{ server }} {{ addrs[0] }}:80 check +{% endfor %} + +<...file contents snipped...> .ft P .fi .UNINDENT .UNINDENT .sp -\fBImport tags\fP +In the above example, \fBserver\fP will be expanded to the \fBminion_id\fP\&. .sp -External files can be imported and made available as a Jinja variable. -.INDENT 7.0 +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The expr_form argument will be renamed to \fBtgt_type\fP in the 2017.7.0 +release of Salt. +.UNINDENT +.UNINDENT +.SS Runners +.sp +Salt runners are convenience applications executed with the salt\-run command. +.sp +Salt runners work similarly to Salt execution modules however they execute on the +Salt master itself instead of remote Salt minions. +.sp +A Salt runner can be a simple client call or a complex application. +.sp +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%The full list of runners\fP +.UNINDENT +.UNINDENT +.SS Writing Salt Runners +.sp +A Salt runner is written in a similar manner to a Salt execution module. +Both are Python modules which contain functions and each public function +is a runner which may be executed via the \fIsalt\-run\fP command. +.sp +For example, if a Python module named \fBtest.py\fP is created in the runners +directory and contains a function called \fBfoo\fP, the \fBtest\fP runner could be +invoked with the following command: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% import_yaml \(dqmyfile.yml\(dq as myfile %} -{% import_json \(dqdefaults.json\(dq as defaults %} -{% import_text \(dqcompleteworksofshakespeare.txt\(dq as poems %} +# salt\-run test.foo .ft P .fi .UNINDENT .UNINDENT .sp -\fBCatalog\fP -.sp -\fBimport_*\fP and \fBload_*\fP tags will automatically expose their -target variable to import. This feature makes catalog of data to -handle. +Runners have several options for controlling output. .sp -for example: -.INDENT 7.0 +Any \fBprint\fP statement in a runner is automatically also +fired onto the master event bus where. For example: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# doc1.sls -{% load_yaml as var1 %} - foo: it works -{% endload %} -{% load_yaml as var2 %} - bar: for real -{% endload %} +def a_runner(outputter=None, display_progress=False): + print(\(dqHello world\(dq) + ... .ft P .fi .UNINDENT .UNINDENT -.INDENT 7.0 +.sp +The above would result in an event fired as follows: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# doc2.sls -{% from \(dqdoc1.sls\(dq import var1, var2 as local2 %} -{{ var1.foo }} {{ local2.bar }} +Event fired at Tue Jan 13 15:26:45 2015 +************************* +Tag: salt/run/20150113152644070246/print +Data: +{\(aq_stamp\(aq: \(aq2015\-01\-13T15:26:45.078707\(aq, + \(aqdata\(aq: \(aqhello\(aq, + \(aqoutputter\(aq: \(aqpprint\(aq} .ft P .fi .UNINDENT .UNINDENT .sp -** Escape Filters ** -.sp -New in version 2017.7.0. - -.sp -Allows escaping of strings so they can be interpreted literally by another -function. +A runner may also send a progress event, which is displayed to the user during +runner execution and is also passed across the event bus if the \fBdisplay_progress\fP +argument to a runner is set to True. .sp -For example: -.INDENT 7.0 +A custom runner may send its own progress event by using the +\fB__jid_event_.fire_event()\fP method as shown here: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -regex_escape = {{ \(aqhttps://example.com?foo=bar%20baz\(aq | regex_escape }} +if display_progress: + __jid_event__.fire_event({\(dqmessage\(dq: \(dqA progress message\(dq}, \(dqprogress\(dq) .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +The above would produce output on the console reading: \fBA progress message\fP +as well as an event on the event similar to: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -regex_escape = https\e:\e/\e/example\e.com\e?foo\e=bar\e%20baz +Event fired at Tue Jan 13 15:21:20 2015 +************************* +Tag: salt/run/20150113152118341421/progress +Data: +{\(aq_stamp\(aq: \(aq2015\-01\-13T15:21:20.390053\(aq, + \(aqmessage\(aq: \(dqA progress message\(dq} .ft P .fi .UNINDENT .UNINDENT .sp -** Set Theory Filters ** +A runner could use the same approach to send an event with a customized tag +onto the event bus by replacing the second argument (\fBprogress\fP) with +whatever tag is desired. However, this will not be shown on the command\-line +and will only be fired onto the event bus. +.SS Synchronous vs. Asynchronous .sp -New in version 2017.7.0. - +A runner may be fired asynchronously which will immediately return control. In +this case, no output will be display to the user if \fBsalt\-run\fP is being used +from the command\-line. If used programmatically, no results will be returned. +If results are desired, they must be gathered either by firing events on the +bus from the runner and then watching for them or by some other means. .sp -Performs set math using Jinja filters. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +When running a runner in asynchronous mode, the \fB\-\-progress\fP flag will +not deliver output to the salt\-run CLI. However, progress events will +still be fired on the bus. +.UNINDENT +.UNINDENT .sp -For example: -.INDENT 7.0 +In synchronous mode, which is the default, control will not be returned until +the runner has finished executing. +.sp +To add custom runners, put them in a directory and add it to +\fI\%runner_dirs\fP in the master configuration file. +.SS Examples +.sp +Examples of runners can be found in the Salt distribution: +.sp +\fI\%salt/runners\fP +.sp +A simple runner that returns a well\-formatted list of the minions that are +responding to Salt calls could look like this: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -unique = {{ [\(aqfoo\(aq, \(aqfoo\(aq, \(aqbar\(aq] | unique }} +# Import salt modules +import salt.client + + +def up(): + \(dq\(dq\(dq + Print a list of all of the minions that are up + \(dq\(dq\(dq + client = salt.client.LocalClient(__opts__[\(dqconf_file\(dq]) + minions = client.cmd(\(dq*\(dq, \(dqtest.version\(dq, timeout=1) + for minion in sorted(minions): + print(minion) .ft P .fi .UNINDENT .UNINDENT +.SS Salt Engines .sp -will be rendered as: -.INDENT 7.0 +New in version 2015.8.0. + +.sp +Salt Engines are long\-running, external system processes that leverage Salt. +.INDENT 0.0 +.IP \(bu 2 +Engines have access to Salt configuration, execution modules, and runners (\fB__opts__\fP, \fB__salt__\fP, and \fB__runners__\fP). +.IP \(bu 2 +Engines are executed in a separate process that is monitored by Salt. If a Salt engine stops, it is restarted automatically. +.IP \(bu 2 +Engines can run on the Salt master and on Salt minions. +.UNINDENT +.sp +Salt engines enhance and replace the \fI\%external processes\fP functionality. +.SS Configuration +.sp +Salt engines are configured under an \fBengines\fP top\-level section in your Salt master or Salt minion configuration. Provide a list of engines and parameters under this section. +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -unique = [\(aqfoo\(aq, \(aqbar\(aq] +engines: + \- logstash: + host: log.my_network.com + port: 5959 + proto: tcp .ft P .fi .UNINDENT .UNINDENT .sp -** Salt State Parameter Format Filters ** -.sp -New in version 3005. +New in version 3000. .sp -Renders a formatted multi\-line YAML string from a Python dictionary. Each -key/value pair in the dictionary will be added as a single\-key dictionary -to a list that will then be sent to the YAML formatter. -.sp -For example: -.INDENT 7.0 +Multiple copies of a particular Salt engine can be configured by including the \fBengine_module\fP parameter in the engine configuration. +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set thing_params = { - \(dqname\(dq: \(dqthing\(dq, - \(dqchanges\(dq: True, - \(dqwarnings\(dq: \(dqOMG! Stuff is happening!\(dq - } -%} - -thing: - test.configurable_test_state: - {{ thing_params | dict_to_sls_yaml_params | indent }} +engines: + \- production_logstash: + host: production_log.my_network.com + port: 5959 + proto: tcp + engine_module: logstash + \- develop_logstash: + host: develop_log.my_network.com + port: 5959 + proto: tcp + engine_module: logstash .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: -.INDENT 7.0 +Salt engines must be in the Salt path, or you can add the \fBengines_dirs\fP option in your Salt master configuration with a list of directories under which Salt attempts to find Salt engines. This option should be formatted as a list of directories to search, such as: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\&.. code\-block:: yaml +engines_dirs: + \- /home/bob/engines .ft P .fi .UNINDENT .UNINDENT -.INDENT 7.0 -.INDENT 3.5 -.INDENT 0.0 -.TP -.B thing: -.INDENT 7.0 -.TP -.B test.configurable_test_state: -.INDENT 7.0 -.IP \(bu 2 -name: thing -.IP \(bu 2 -changes: true -.IP \(bu 2 -warnings: OMG! Stuff is happening! -.UNINDENT -.UNINDENT -.UNINDENT -.UNINDENT -.UNINDENT -.UNINDENT -.SS salt.renderers.json -.sp -JSON Renderer for Salt -.INDENT 0.0 -.TP -.B salt.renderers.json.render(json_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts JSON as a string or as a file object and runs it through the JSON -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure -.UNINDENT -.UNINDENT -.SS salt.renderers.json5 -.sp -JSON5 Renderer for Salt -.sp -New in version 2016.3.0. - -.sp -JSON5 is an unofficial extension to JSON. See \fI\%http://json5.org/\fP for more -information. +.SS Writing an Engine .sp -This renderer requires the \fI\%json5 python bindings\fP, installable via pip. -.INDENT 0.0 -.TP -.B salt.renderers.json5.render(json_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts JSON as a string or as a file object and runs it through the JSON -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure -.UNINDENT -.UNINDENT -.SS salt.renderers.mako +An example Salt engine, \fI\%salt/engines/test.py\fP, is available in the Salt source. To develop an engine, the only requirement is that your module implement the \fBstart()\fP function. +.SS What is YAML and How To Use It .sp -Mako Renderer for Salt +The default renderer for SLS files is the YAML renderer. +.SS What is YAML .sp -This renderer requires the Mako library. +What does YAML stand for? It\(aqs an acronym for \fIYAML Ain\(aqt Markup Language\fP\&. .sp -To install Mako, do the following: -.INDENT 0.0 -.TP -.B salt.renderers.mako.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, context=None, tmplpath=None, **kws) -Render the template_file, passing the functions and grains into the -Mako rendering system. -.INDENT 7.0 -.TP -.B Return type -string -.UNINDENT -.UNINDENT -.SS salt.renderers.msgpack +\fI\%The Official YAML Website\fP defines YAML as: .INDENT 0.0 -.TP -.B salt.renderers.msgpack.render(msgpack_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts a message pack string or a file object, renders said data back to -a python dict. -.INDENT 7.0 -.TP -.B Return type -A Python data structure +.INDENT 3.5 +\fI\&...a human friendly data serialization\fP +\fIstandard for all programming languages.\fP .UNINDENT .UNINDENT -.SS salt.renderers.nacl .sp -Renderer that will decrypt NACL ciphers +However, Salt uses a small subset of YAML that maps over very commonly used data +structures, like lists and dictionaries. It is the job of the YAML renderer to +take the YAML data structure and compile it into a Python data structure for use +by Salt. +.SS Defining YAML .sp -Any key in the SLS file can be an NACL cipher, and this renderer will decrypt it -before passing it off to Salt. This allows you to safely store secrets in -source control, in such a way that only your Salt master can decrypt them and -distribute them only to the minions that need them. +Though YAML syntax may seem daunting and terse at first, there are only +three very simple rules to remember when writing YAML for SLS files. +.SS Rule One: Indentation .sp -The typical use\-case would be to use ciphers in your pillar data, and keep a -secret key on your master. You can put the public key in source control so that -developers can add new secrets quickly and easily. +YAML uses a fixed indentation scheme to represent relationships between +data layers. Salt requires that the indentation for each level consists +of exactly two spaces. Do not use tabs. +.SS Rule Two: Colons .sp -This renderer requires the libsodium library binary and PyNacl >= 1.0 -.SS Setup +Python dictionaries are, of course, simply key\-value pairs. Users from other +languages may recognize this data type as hashes or associative arrays. .sp -To set things up, first generate a keypair. On the master, run the following: +Dictionary keys are represented in YAML as strings terminated by a trailing +colon. Values are represented by either a string following the colon, +separated by a space: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-call \-\-local nacl.keygen sk_file=/root/.nacl +my_key: my_value .ft P .fi .UNINDENT .UNINDENT -.SS Using encrypted pillar .sp -To encrypt secrets, copy the public key to your local machine and run: +In Python, the above maps to: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt\-call \-\-local nacl.enc datatoenc pk_file=/root/.nacl.pub +{\(dqmy_key\(dq: \(dqmy_value\(dq} .ft P .fi .UNINDENT .UNINDENT .sp -To apply the renderer on a file\-by\-file basis add the following line to the -top of any pillar with nacl encrypted data in it: +Alternatively, a value can be associated with a key through indentation. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yaml|nacl +my_key: + my_value .ft P .fi .UNINDENT .UNINDENT .sp -Now with your renderer configured, you can include your ciphers in your pillar -data like so: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -#!yaml|nacl - -a\-secret: \(dqNACL[MRN3cc+fmdxyQbz6WMF+jq1hKdU5X5BBI7OjK+atvHo1ll+w1gZ7XyWtZVfq9gK9rQaMfkDxmidJKwE0Mw==]\(dq -.ft P -.fi -.UNINDENT +The above syntax is valid YAML but is uncommon in SLS files because most often, +the value for a key is not singular but instead is a \fIlist\fP of values. .UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.nacl.render(nacl_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) -Decrypt the data to be rendered using the given nacl key or the one given -in config .UNINDENT -.SS salt.renderers.pass -.SS Pass Renderer for Salt -.sp -\fI\%pass\fP is an encrypted on\-disk password store. -.sp -New in version 2017.7.0. - -.SS Setup -.sp -\fINote\fP: \fB\fP needs to be replaced with the user salt\-master will be -running as. .sp -Have private gpg loaded into \fBuser\fP\(aqs gpg keyring +In Python, the above maps to: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -load_private_gpg_key: - cmd.run: - \- name: gpg \-\-import - \- unless: gpg \-\-list\-keys \(aq\(aq +{\(dqmy_key\(dq: \(dqmy_value\(dq} .ft P .fi .UNINDENT .UNINDENT .sp -Said private key\(aqs public key should have been used when encrypting pass entries -that are of interest for pillar data. -.sp -Fetch and keep local pass git repo up\-to\-date +Dictionaries can be nested: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -update_pass: - git.latest: - \- force_reset: True - \- name: - \- target: //.password\-store - \- identity: - \- require: - \- cmd: load_private_gpg_key +first_level_dict_key: + second_level_dict_key: value_in_second_level_dict .ft P .fi .UNINDENT .UNINDENT .sp -Install pass binary +And in Python: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pass: - pkg.installed +{\(dqfirst_level_dict_key\(dq: {\(dqsecond_level_dict_key\(dq: \(dqvalue_in_second_level_dict\(dq}} .ft P .fi .UNINDENT .UNINDENT +.SS Rule Three: Dashes .sp -Salt master configuration options +To represent lists of items, a single dash followed by a space is used. Multiple +items are a part of the same list as a function of their having the same level of indentation. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# If the prefix is *not* set (default behavior), all template variables are -# considered for fetching secrets from Pass. Those that cannot be resolved -# to a secret are passed through. -# -# If the prefix is set, only the template variables with matching prefix are -# considered for fetching the secrets, other variables are passed through. -# -# For ease of use it is recommended to set the following options as well: -# renderer: \(aqjinja|yaml|pass\(aq -# pass_strict_fetch: true -# -pass_variable_prefix: \(aqpass:\(aq - -# If set to \(aqtrue\(aq, error out when unable to fetch a secret for a template variable. -pass_strict_fetch: true - -# Set GNUPGHOME env for Pass. -# Defaults to: ~/.gnupg -pass_gnupghome: - -# Set PASSWORD_STORE_DIR env for Pass. -# Defaults to: ~/.password\-store -pass_dir: +\- list_value_one +\- list_value_two +\- list_value_three .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.pass.render(pass_info, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kwargs) -Fetch secret from pass based on pass_path -.UNINDENT -.SS salt.renderers.py -.SS Pure python state renderer -.sp -To use this renderer, the SLS file should contain a function called \fBrun\fP -which returns highstate data. .sp -The highstate data is a dictionary containing identifiers as keys, and execution -dictionaries as values. For example the following state declaration in YAML: +Lists can be the value of a key\-value pair. This is quite common in Salt: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -common_packages: - pkg.installed: - \- pkgs: - \- curl - \- vim +my_dictionary: + \- list_value_one + \- list_value_two + \- list_value_three .ft P .fi .UNINDENT .UNINDENT .sp -translates to: +In Python, the above maps to: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(aqcommon_packages\(aq: {\(aqpkg.installed\(aq: [{\(aqpkgs\(aq: [\(aqcurl\(aq, \(aqvim\(aq]}]}} +{\(dqmy_dictionary\(dq: [\(dqlist_value_one\(dq, \(dqlist_value_two\(dq, \(dqlist_value_three\(dq]} .ft P .fi .UNINDENT .UNINDENT +.SS Learning more about YAML .sp -In this module, a few objects are defined for you, giving access to Salt\(aqs -execution functions, grains, pillar, etc. They are: +One easy way to learn more about how YAML gets rendered into Python data structures is +to use an online YAML parser to see the Python output. +.sp +Here are some excellent links for experimenting with and referencing YAML: .INDENT 0.0 .IP \(bu 2 -\fB__salt__\fP \- \fI\%Execution functions\fP (i.e. -\fB__salt__[\(aqtest.echo\(aq](\(aqfoo\(aq)\fP) -.IP \(bu 2 -\fB__grains__\fP \- \fI\%Grains\fP (i.e. \fB__grains__[\(aqos\(aq]\fP) -.IP \(bu 2 -\fB__pillar__\fP \- \fI\%Pillar data\fP (i.e. \fB__pillar__[\(aqfoo\(aq]\fP) -.IP \(bu 2 -\fB__opts__\fP \- Minion configuration options -.IP \(bu 2 -\fB__env__\fP \- The effective salt fileserver environment (i.e. \fBbase\fP). Also -referred to as a \(dqsaltenv\(dq. \fB__env__\fP should not be modified in a pure -python SLS file. To use a different environment, the environment should be -set when executing the state. This can be done in a couple different ways: -.INDENT 2.0 +\fI\%Online YAML Parser\fP: Convert YAML +to JSON or Python data structures. .IP \(bu 2 -Using the \fBsaltenv\fP argument on the salt CLI (i.e. \fBsalt \(aq*\(aq state.sls -foo.bar.baz saltenv=env_name\fP). +\fI\%The Official YAML Specification\fP .IP \(bu 2 -By adding a \fBsaltenv\fP argument to an individual state within the SLS -file. In other words, adding a line like this to the state\(aqs data -structure: \fB{\(aqsaltenv\(aq: \(aqenv_name\(aq}\fP +\fI\%The Wikipedia page for YAML\fP .UNINDENT -.IP \(bu 2 -\fB__sls__\fP \- The SLS path of the file. For example, if the root of the base -environment is \fB/srv/salt\fP, and the SLS file is -\fB/srv/salt/foo/bar/baz.sls\fP, then \fB__sls__\fP in that file will be -\fBfoo.bar.baz\fP\&. +.SS Templating +.sp +Jinja statements and expressions are allowed by default in SLS files. See +\fI\%Understanding Jinja\fP\&. +.SS Understanding Jinja +.sp +\fI\%Jinja\fP is the default templating language in SLS files. +.sp +\fBIMPORTANT:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt +takes advantage of. Other text \fI\%Renderers\fP do not support this +functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. .UNINDENT +.UNINDENT +.SS Jinja in States .sp -When used in a scenario where additional user\-provided context data is supplied -(such as with \fI\%file.managed\fP), the additional -data will typically be injected into the script as one or more global -variables: +Jinja is evaluated before YAML, which means it is evaluated before the States +are run. +.sp +The most basic usage of Jinja in state files is using control structures to +wrap conditional or redundant state elements: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/etc/http/conf/http.conf: +{% if grains[\(aqos\(aq] != \(aqFreeBSD\(aq %} +tcsh: + pkg: + \- installed +{% endif %} + +motd: file.managed: - \- source: salt://apache/generate_http_conf.py - \- template: py - \- context: - # Will be injected as the global variable \(dqsite_name\(dq. - site_name: {{ site_name }} + {% if grains[\(aqos\(aq] == \(aqFreeBSD\(aq %} + \- name: /etc/motd + {% elif grains[\(aqos\(aq] == \(aqDebian\(aq %} + \- name: /etc/motd.tail + {% endif %} + \- source: salt://motd .ft P .fi .UNINDENT .UNINDENT .sp -When writing a reactor SLS file the global context \fBdata\fP (same as context -\fB{{ data }}\fP for states written with Jinja + YAML) is available. The -following YAML + Jinja state declaration: +In this example, the first \fBif\fP block will only be evaluated on minions that +aren\(aqt running FreeBSD, and the second block changes the file name based on the +\fIos\fP grain. +.sp +Writing \fBif\-else\fP blocks can lead to very redundant state files however. In +this case, using \fI\%pillars\fP, or using a previously +defined variable might be easier: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% if data[\(aqid\(aq] == \(aqmysql1\(aq %} -highstate_run: - local.state.apply: - \- tgt: mysql1 +{% set motd = [\(aq/etc/motd\(aq] %} +{% if grains[\(aqos\(aq] == \(aqDebian\(aq %} + {% set motd = [\(aq/etc/motd.tail\(aq, \(aq/var/run/motd\(aq] %} {% endif %} + +{% for motdfile in motd %} +{{ motdfile }}: + file.managed: + \- source: salt://motd +{% endfor %} .ft P .fi .UNINDENT .UNINDENT .sp -translates to: +Using a variable set by the template, the \fI\%for loop\fP will iterate over the +list of MOTD files to update, adding a state block for each file. +.sp +The filter_by function can also be used to set variables based on grains: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -if data[\(aqid\(aq] == \(aqmysql1\(aq: - return {\(aqhighstate_run\(aq: {\(aqlocal.state.apply\(aq: [{\(aqtgt\(aq: \(aqmysql1\(aq}]}} +{% set auditd = salt[\(aqgrains.filter_by\(aq]({ +\(aqRedHat\(aq: { \(aqpackage\(aq: \(aqaudit\(aq }, +\(aqDebian\(aq: { \(aqpackage\(aq: \(aqauditd\(aq }, +}) %} .ft P .fi .UNINDENT .UNINDENT -.SS Full Example +.SS Include and Import +.sp +Includes and \fI\%imports\fP can be used to share common, reusable state configuration +between state files and between files. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!py - - def run(): - config = {} - - if __grains__[\(aqos\(aq] == \(aqUbuntu\(aq: - user = \(aqubuntu\(aq - group = \(aqubuntu\(aq - home = \(aq/home/{0}\(aq.format(user) - else: - user = \(aqroot\(aq - group = \(aqroot\(aq - home = \(aq/root/\(aq - - config[\(aqs3cmd\(aq] = { - \(aqpkg\(aq: [ - \(aqinstalled\(aq, - {\(aqname\(aq: \(aqs3cmd\(aq}, - ], - } - - config[home + \(aq/.s3cfg\(aq] = { - \(aqfile.managed\(aq: [ - {\(aqsource\(aq: \(aqsalt://s3cfg/templates/s3cfg\(aq}, - {\(aqtemplate\(aq: \(aqjinja\(aq}, - {\(aquser\(aq: user}, - {\(aqgroup\(aq: group}, - {\(aqmode\(aq: 600}, - {\(aqcontext\(aq: { - \(aqaws_key\(aq: __pillar__[\(aqAWS_ACCESS_KEY_ID\(aq], - \(aqaws_secret_key\(aq: __pillar__[\(aqAWS_SECRET_ACCESS_KEY\(aq], - }, - }, - ], - } - - return config +{% from \(aqlib.sls\(aq import test %} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.py.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, tmplpath=None, **kws) -Render the python module\(aqs components -.INDENT 7.0 -.TP -.B Return type -string -.UNINDENT -.UNINDENT -.SS salt.renderers.pydsl -.sp -A Python\-based DSL -.INDENT 0.0 -.TP -.B maintainer -Jack Kuan <\fI\%kjkuan@gmail.com\fP> -.TP -.B maturity -new -.TP -.B platform -all -.UNINDENT .sp -The \fIpydsl\fP renderer allows one to author salt formulas (.sls files) in pure -Python using a DSL that\(aqs easy to write and easy to read. Here\(aqs an example: +This would import the \fBtest\fP template variable or macro, not the \fBtest\fP +state element, from the file \fBlib.sls\fP\&. In the case that the included file +performs checks against grains, or something else that requires context, passing +the context into the included file is required: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!pydsl - -apache = state(\(aqapache\(aq) -apache.pkg.installed() -apache.service.running() -state(\(aq/var/www/index.html\(aq) \e - .file(\(aqmanaged\(aq, - source=\(aqsalt://webserver/index.html\(aq) \e - .require(pkg=\(aqapache\(aq) +{% from \(aqlib.sls\(aq import test with context %} .ft P .fi .UNINDENT .UNINDENT .sp -Notice that any Python code is allow in the file as it\(aqs really a Python -module, so you have the full power of Python at your disposal. In this module, -a few objects are defined for you, including the usual (with \fB__\fP added) -\fB__salt__\fP dictionary, \fB__grains__\fP, \fB__pillar__\fP, \fB__opts__\fP, -\fB__env__\fP, and \fB__sls__\fP, plus a few more: -.INDENT 0.0 -.INDENT 3.5 -\fB__file__\fP -.INDENT 0.0 -.INDENT 3.5 -local file system path to the sls module. -.UNINDENT -.UNINDENT +Includes must use full paths, like so: .sp -\fB__pydsl__\fP +spam/eggs.jinja .INDENT 0.0 .INDENT 3.5 -Salt PyDSL object, useful for configuring DSL behavior per sls rendering. -.UNINDENT -.UNINDENT .sp -\fBinclude\fP -.INDENT 0.0 -.INDENT 3.5 -Salt PyDSL function for creating \fI\%Include declaration\fP\(aqs. +.nf +.ft C + {% include \(aqspam/foobar.jinja\(aq %} +.ft P +.fi .UNINDENT .UNINDENT +.SS Including Context During Include/Import .sp -\fBextend\fP +By adding \fBwith context\fP to the include/import directive, the +current context can be passed to an included/imported template. .INDENT 0.0 .INDENT 3.5 -Salt PyDSL function for creating \fI\%Extend declaration\fP\(aqs. +.sp +.nf +.ft C +{% import \(aqopenssl/vars.sls\(aq as ssl with context %} +.ft P +.fi .UNINDENT .UNINDENT +.SS Macros .sp -\fBstate\fP +\fI\%Macros\fP are helpful for eliminating redundant code. Macros are most useful as +mini\-templates to repeat blocks of strings with a few parameterized variables. +Be aware that stripping whitespace from the template block, as well as +contained blocks, may be necessary to emulate a variable return from the macro. .INDENT 0.0 .INDENT 3.5 -Salt PyDSL function for creating \fI\%ID declaration\fP\(aqs. -.UNINDENT -.UNINDENT +.sp +.nf +.ft C +# init.sls +{% from \(aqlib.sls\(aq import pythonpkg with context %} + +python\-virtualenv: + pkg.installed: + \- name: {{ pythonpkg(\(aqvirtualenv\(aq) }} + +python\-fabric: + pkg.installed: + \- name: {{ pythonpkg(\(aqfabric\(aq) }} +.ft P +.fi .UNINDENT .UNINDENT -.sp -A state \fI\%ID declaration\fP is created with a \fBstate(id)\fP function call. -Subsequent \fBstate(id)\fP call with the same id returns the same object. This -singleton access pattern applies to all declaration objects created with the -DSL. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq) -assert state(\(aqexample\(aq) is state(\(aqexample\(aq) -assert state(\(aqexample\(aq).cmd is state(\(aqexample\(aq).cmd -assert state(\(aqexample\(aq).cmd.running is state(\(aqexample\(aq).cmd.running +# lib.sls +{% macro pythonpkg(pkg) \-%} + {%\- if grains[\(aqos\(aq] == \(aqFreeBSD\(aq \-%} + py27\-{{ pkg }} + {%\- elif grains[\(aqos\(aq] == \(aqDebian\(aq \-%} + python\-{{ pkg }} + {%\- endif \-%} +{%\- endmacro %} .ft P .fi .UNINDENT .UNINDENT .sp -The \fIid\fP argument is optional. If omitted, an UUID will be generated and used as -the \fIid\fP\&. +This would define a \fI\%macro\fP that would return a string of the full package name, +depending on the packaging system\(aqs naming convention. The whitespace of the +macro was eliminated, so that the macro would return a string without line +breaks, using \fI\%whitespace control\fP\&. +.SS Template Inheritance .sp -\fBstate(id)\fP returns an object under which you can create a -\fI\%State declaration\fP object by accessing an attribute named after \fIany\fP -state module available in Salt. +\fI\%Template inheritance\fP works fine from state files and files. The search path +starts at the root of the state tree or pillar. +.SS Errors +.sp +Saltstack allows raising custom errors using the \fBraise\fP jinja function. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq).cmd -state(\(aqexample\(aq).file -state(\(aqexample\(aq).pkg -\&... +{{ raise(\(aqCustom Error\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -Then, a \fI\%Function declaration\fP object can be created from a -\fI\%State declaration\fP object by one of the following two ways: -.INDENT 0.0 -.IP 1. 3 -by calling a method named after the state function on the \fI\%State declaration\fP object. -.UNINDENT +When rendering the template containing the above statement, a \fBTemplateError\fP +exception is raised, causing the rendering to fail with the following message: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq).file.managed(...) +TemplateError: Custom Error .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP 2. 3 -by directly calling the attribute named for the \fI\%State declaration\fP, and -supplying the state function name as the first argument. -.UNINDENT +.SS Filters +.sp +Saltstack extends \fI\%builtin filters\fP with these custom filters: +.SS \fBstrftime\fP +.sp +Converts any time related object into a time based string. It requires valid +strftime directives. An exhaustive list can be found \fI\%here\fP in the Python documentation. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq).file(\(aqmanaged\(aq, ...) +{% set curtime = None | strftime() %} .ft P .fi .UNINDENT .UNINDENT .sp -With either way of creating a \fI\%Function declaration\fP object, any -\fI\%Function arg declaration\fP\(aqs can be passed as keyword arguments to the -call. Subsequent calls of a \fI\%Function declaration\fP will update the arg -declarations. +Fuzzy dates require the \fI\%timelib\fP Python module is installed. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq).file(\(aqmanaged\(aq, source=\(aqsalt://webserver/index.html\(aq) -state(\(aqexample\(aq).file.managed(source=\(aqsalt://webserver/index.html\(aq) +{{ \(dq2002/12/25\(dq|strftime(\(dq%y\(dq) }} +{{ \(dq1040814000\(dq|strftime(\(dq%Y\-%m\-%d\(dq) }} +{{ datetime|strftime(\(dq%u\(dq) }} +{{ \(dqtomorrow\(dq|strftime }} .ft P .fi .UNINDENT .UNINDENT +.SS \fBsequence\fP .sp -As a shortcut, the special \fIname\fP argument can also be passed as the -first or second positional argument depending on the first or second -way of calling the \fI\%State declaration\fP object. In the following -two examples \fIls \-la\fP is the \fIname\fP argument. +Ensure that parsed data is a sequence. +.SS \fByaml_encode\fP +.sp +Serializes a single object into a YAML scalar with any necessary +handling for escaping special characters. This will work for any +scalar YAML data type: ints, floats, timestamps, booleans, strings, +unicode. It will \fInot\fP work for multi\-objects such as sequences or +maps. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -state(\(aqexample\(aq).cmd.run(\(aqls \-la\(aq, cwd=\(aq/\(aq) -state(\(aqexample\(aq).cmd(\(aqrun\(aq, \(aqls \-la\(aq, cwd=\(aq/\(aq) +{%\- set bar = 7 %} +{%\- set baz = none %} +{%\- set zip = true %} +{%\- set zap = \(aqThe word of the day is \(dqsalty\(dq\(aq %} + +{%\- load_yaml as foo %} +bar: {{ bar|yaml_encode }} +baz: {{ baz|yaml_encode }} +zip: {{ zip|yaml_encode }} +zap: {{ zap|yaml_encode }} +{%\- endload %} .ft P .fi .UNINDENT .UNINDENT .sp -Finally, a \fI\%Requisite declaration\fP object with its -\fI\%Requisite reference\fP\(aqs can be created by invoking one of the -requisite methods (see \fI\%State Requisites\fP) on either a -\fI\%Function declaration\fP object or a \fI\%State declaration\fP object. -The return value of a requisite call is also a \fI\%Function declaration\fP -object, so you can chain several requisite calls together. +In the above case \fB{{ bar }}\fP and \fB{{ foo.bar }}\fP should be +identical and \fB{{ baz }}\fP and \fB{{ foo.baz }}\fP should be +identical. +.SS \fByaml_dquote\fP .sp -Arguments to a requisite call can be a list of \fI\%State declaration\fP objects -and/or a set of keyword arguments whose names are state modules and values are -IDs of \fI\%ID declaration\fP\(aqs or names of \fI\%Name declaration\fP\(aqs. +Serializes a string into a properly\-escaped YAML double\-quoted +string. This is useful when the contents of a string are unknown +and may contain quotes or unicode that needs to be preserved. The +resulting string will be emitted with opening and closing double +quotes. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -apache2 = state(\(aqapache2\(aq) -apache2.pkg.installed() -state(\(aqlibapache2\-mod\-wsgi\(aq).pkg.installed() - -# you can call requisites on function declaration -apache2.service.running() \e - .require(apache2.pkg, - pkg=\(aqlibapache2\-mod\-wsgi\(aq) \e - .watch(file=\(aq/etc/apache2/httpd.conf\(aq) - -# or you can call requisites on state declaration. -# this actually creates an anonymous function declaration object -# to add the requisites. -apache2.service.require(state(\(aqlibapache2\-mod\-wsgi\(aq).pkg, - pkg=\(aqapache2\(aq) \e - .watch(file=\(aq/etc/apache2/httpd.conf\(aq) +{%\- set bar = \(aq\(dqThe quick brown fox . . .\(dq\(aq %} +{%\- set baz = \(aqThe word of the day is \(dqsalty\(dq.\(aq %} -# we still need to set the name of the function declaration. -apache2.service.running() +{%\- load_yaml as foo %} +bar: {{ bar|yaml_dquote }} +baz: {{ baz|yaml_dquote }} +{%\- endload %} .ft P .fi .UNINDENT .UNINDENT .sp -\fI\%Include declaration\fP objects can be created with the \fBinclude\fP function, -while \fI\%Extend declaration\fP objects can be created with the \fBextend\fP function, -whose arguments are just \fI\%Function declaration\fP objects. +In the above case \fB{{ bar }}\fP and \fB{{ foo.bar }}\fP should be +identical and \fB{{ baz }}\fP and \fB{{ foo.baz }}\fP should be +identical. If variable contents are not guaranteed to be a string +then it is better to use \fByaml_encode\fP which handles all YAML +scalar types. +.SS \fByaml_squote\fP +.sp +Similar to the \fByaml_dquote\fP filter but with single quotes. Note +that YAML only allows special escapes inside double quotes so +\fByaml_squote\fP is not nearly as useful (viz. you likely want to +use \fByaml_encode\fP or \fByaml_dquote\fP). +.SS \fBdict_to_sls_yaml_params\fP +.sp +New in version 3005. + +.sp +Renders a formatted multi\-line YAML string from a Python dictionary. Each +key/value pair in the dictionary will be added as a single\-key dictionary +to a list that will then be sent to the YAML formatter. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include(\(aqedit.vim\(aq, \(aqhttp.server\(aq) -extend(state(\(aqapache2\(aq).service.watch(file=\(aq/etc/httpd/httpd.conf\(aq) +{% set thing_params = { + \(dqname\(dq: \(dqthing\(dq, + \(dqchanges\(dq: True, + \(dqwarnings\(dq: \(dqOMG! Stuff is happening!\(dq + } +%} + +thing: + test.configurable_test_state: + {{ thing_params | dict_to_sls_yaml_params | indent }} .ft P .fi .UNINDENT .UNINDENT .sp -The \fBinclude\fP function, by default, causes the included sls file to be rendered -as soon as the \fBinclude\fP function is called. It returns a list of rendered module -objects; sls files not rendered with the pydsl renderer return \fBNone\fP\(aqs. -This behavior creates no \fI\%Include declaration\fP\(aqs in the resulting high state -data structure. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import types - -# including multiple sls returns a list. -_, mod = include(\(aqa\-non\-pydsl\-sls\(aq, \(aqa\-pydsl\-sls\(aq) - -assert _ is None -assert isinstance(slsmods[1], types.ModuleType) - -# including a single sls returns a single object -mod = include(\(aqa\-pydsl\-sls\(aq) - -# myfunc is a function that calls state(...) to create more states. -mod.myfunc(1, 2, \(dqthree\(dq) +thing: + test.configurable_test_state: + \- name: thing + \- changes: true + \- warnings: OMG! Stuff is happening! .ft P .fi .UNINDENT .UNINDENT +.SS \fBto_bool\fP .sp -Notice how you can define a reusable function in your pydsl sls module and then -call it via the module returned by \fBinclude\fP\&. +New in version 2017.7.0. + .sp -It\(aqs still possible to do late includes by passing the \fBdelayed=True\fP keyword -argument to \fBinclude\fP\&. +Returns the logical value of an element. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include(\(aqedit.vim\(aq, \(aqhttp.server\(aq, delayed=True) +{{ \(aqyes\(aq | to_bool }} +{{ \(aqtrue\(aq | to_bool }} +{{ 1 | to_bool }} +{{ \(aqno\(aq | to_bool }} .ft P .fi .UNINDENT .UNINDENT .sp -Above will just create a \fI\%Include declaration\fP in the rendered result, and -such call always returns \fBNone\fP\&. -.SS Special integration with the \fIcmd\fP state -.sp -Taking advantage of rendering a Python module, PyDSL allows you to declare a -state that calls a pre\-defined Python function when the state is executed. +Will be rendered as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -greeting = \(dqhello world\(dq -def helper(something, *args, **kws): - print greeting # hello world - print something, args, kws # test123 [\(aqa\(aq, \(aqb\(aq, \(aqc\(aq] {\(aqx\(aq: 1, \(aqy\(aq: 2} - -state().cmd.call(helper, \(dqtest123\(dq, \(aqa\(aq, \(aqb\(aq, \(aqc\(aq, x=1, y=2) +True +True +True +False .ft P .fi .UNINDENT .UNINDENT +.SS \fBexactly_n_true\fP .sp -The \fIcmd.call\fP state function takes care of calling our \fBhelper\fP function -with the arguments we specified in the states, and translates the return value -of our function into a structure expected by the state system. -See \fI\%salt.states.cmd.call()\fP for more information. -.SS Implicit ordering of states +New in version 2017.7.0. + .sp -Salt states are explicitly ordered via \fI\%Requisite declaration\fP\(aqs. -However, with \fIpydsl\fP it\(aqs possible to let the renderer track the order -of creation for \fI\%Function declaration\fP objects, and implicitly add -\fBrequire\fP requisites for your states to enforce the ordering. This feature -is enabled by setting the \fBordered\fP option on \fB__pydsl__\fP\&. +Tests that exactly N items in an iterable are \(dqtruthy\(dq (neither None, False, nor 0). .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -this feature is only available if your minions are using Python >= 2.7. -.UNINDENT -.UNINDENT +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include(\(aqsome.sls.file\(aq) - -A = state(\(aqA\(aq).cmd.run(cwd=\(aq/var/tmp\(aq) -extend(A) - -__pydsl__.set(ordered=True) - -for i in range(10): - i = str(i) - state(i).cmd.run(\(aqecho \(aq+i, cwd=\(aq/\(aq) -state(\(aq1\(aq).cmd.run(\(aqecho one\(aq) -state(\(aq2\(aq).cmd.run(name=\(aqecho two\(aq) +{{ [\(aqyes\(aq, 0, False, \(aqTrue\(aq] | exactly_n_true(2) }} .ft P .fi .UNINDENT .UNINDENT .sp -Notice that the \fBordered\fP option needs to be set after any \fBextend\fP calls. -This is to prevent \fIpydsl\fP from tracking the creation of a state function that\(aqs -passed to an \fBextend\fP call. -.sp -Above example should create states from \fB0\fP to \fB9\fP that will output \fB0\fP, -\fBone\fP, \fBtwo\fP, \fB3\fP, ... \fB9\fP, in that order. -.sp -It\(aqs important to know that \fIpydsl\fP tracks the \fIcreations\fP of -\fI\%Function declaration\fP objects, and automatically adds a \fBrequire\fP requisite -to a \fI\%Function declaration\fP object that requires the last -\fI\%Function declaration\fP object created before it in the sls file. -.sp -This means later calls (perhaps to update the function\(aqs \fI\%Function arg declaration\fP) to a previously created function declaration will not change the -order. -.SS Render time state execution -.sp -When Salt processes a salt formula file, the file is rendered to salt\(aqs -high state data representation by a renderer before the states can be executed. -In the case of the \fIpydsl\fP renderer, the .sls file is executed as a python module -as it is being rendered which makes it easy to execute a state at render time. -In \fIpydsl\fP, executing one or more states at render time can be done by calling a -configured \fI\%ID declaration\fP object. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!pydsl - -s = state() # save for later invocation - -# configure it -s.cmd.run(\(aqecho at render time\(aq, cwd=\(aq/\(aq) -s.file.managed(\(aqtarget.txt\(aq, source=\(aqsalt://source.txt\(aq) - -s() # execute the two states now +True .ft P .fi .UNINDENT .UNINDENT +.SS \fBexactly_one_true\fP .sp -Once an \fI\%ID declaration\fP is called at render time it is detached from the -sls module as if it was never defined. +New in version 2017.7.0. + .sp -\fBNOTE:\fP +Tests that exactly one item in an iterable is \(dqtruthy\(dq (neither None, False, nor 0). +.sp +Example: .INDENT 0.0 .INDENT 3.5 -If \fIimplicit ordering\fP is enabled (i.e., via \fB__pydsl__.set(ordered=True)\fP) then -the \fIfirst\fP invocation of a \fI\%ID declaration\fP object must be done before a -new \fI\%Function declaration\fP is created. +.sp +.nf +.ft C +{{ [\(aqyes\(aq, False, 0, None] | exactly_one_true }} +.ft P +.fi .UNINDENT .UNINDENT -.SS Integration with the stateconf renderer -.sp -The \fI\%salt.renderers.stateconf\fP renderer offers a few interesting features that -can be leveraged by the \fIpydsl\fP renderer. In particular, when using with the \fIpydsl\fP -renderer, we are interested in \fIstateconf\fP\(aqs sls namespacing feature (via dot\-prefixed -id declarations), as well as, the automatic \fIstart\fP and \fIgoal\fP states generation. .sp -Now you can use \fIpydsl\fP with \fIstateconf\fP like this: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!pydsl|stateconf \-ps - -include(\(aqxxx\(aq, \(aqyyy\(aq) - -# ensure that states in xxx run BEFORE states in this file. -extend(state(\(aq.start\(aq).stateconf.require(stateconf=\(aqxxx::goal\(aq)) - -# ensure that states in yyy run AFTER states in this file. -extend(state(\(aq.goal\(aq).stateconf.require_in(stateconf=\(aqyyy::start\(aq)) - -__pydsl__.set(ordered=True) - -\&... +True .ft P .fi .UNINDENT .UNINDENT +.SS \fBquote\fP .sp -\fB\-s\fP enables the generation of a stateconf \fIstart\fP state, and \fB\-p\fP lets us pipe -high state data rendered by \fIpydsl\fP to \fIstateconf\fP\&. This example shows that by -\fBrequire\fP\-ing or \fBrequire_in\fP\-ing the included sls\(aq \fIstart\fP or \fIgoal\fP states, -it\(aqs possible to ensure that the included sls files can be made to execute before -or after a state in the including sls file. -.SS Importing custom Python modules +New in version 2017.7.0. + .sp -To use a custom Python module inside a PyDSL state, place the module somewhere that -it can be loaded by the Salt loader, such as \fI_modules\fP in the \fI/srv/salt\fP directory. +This text will be wrapped in quotes. +.SS \fBregex_search\fP .sp -Then, copy it to any minions as necessary by using \fIsaltutil.sync_modules\fP\&. +New in version 2017.7.0. + .sp -To import into a PyDSL SLS, one must bypass the Python importer and insert it manually -by getting a reference from Python\(aqs \fIsys.modules\fP dictionary. +Scan through string looking for a location where this regular expression +produces a match. Returns \fBNone\fP in case there were no matches found .sp -For example: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!pydsl|stateconf \-ps - -def main(): - my_mod = sys.modules[\(aqsalt.loaded.ext.module.my_mod\(aq] +{{ \(aqabcdefabcdef\(aq | regex_search(\(aqBC(.*)\(aq, ignorecase=True) }} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B exception salt.renderers.pydsl.PyDslError -.UNINDENT -.INDENT 0.0 -.TP -.B exception salt.renderers.pydsl.SaltRenderError(message, line_num=None, buf=\(aq\(aq, marker=\(aq <======================\(aq, trace=None) -Used when a renderer needs to raise an explicit error. If a line number and -buffer string are passed, get_context will be invoked to get the location -of the error. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.pydsl.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, tmplpath=None, rendered_sls=None, **kws) -.UNINDENT -.SS salt.renderers.pyobjects -.sp -Python renderer that includes a Pythonic Object based interface -.INDENT 0.0 -.TP -.B maintainer -Evan Borgstrom <\fI\%evan@borgstrom.ca\fP> -.UNINDENT .sp -Let\(aqs take a look at how you use pyobjects in a state file. Here\(aqs a quick -example that ensures the \fB/tmp\fP directory is in the correct state. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - File.managed(\(dq/tmp\(dq, user=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq1777\(aq) +(\(dqdefabcdef\(dq,) .ft P .fi .UNINDENT .UNINDENT +.SS \fBregex_match\fP .sp -Nice and Pythonic! -.sp -By using the \(dqshebang\(dq syntax to switch to the pyobjects renderer we can now -write our state data using an object based interface that should feel at home -to python developers. You can import any module and do anything that you\(aqd -like (with caution, importing sqlalchemy, django or other large frameworks has -not been tested yet). Using the pyobjects renderer is exactly the same as -using the built\-in Python renderer with the exception that pyobjects provides -you with an object based interface for generating state data. -.SS Creating state data -.sp -Pyobjects takes care of creating an object for each of the available states on -the minion. Each state is represented by an object that is the CamelCase -version of its name (i.e. \fBFile\fP, \fBService\fP, \fBUser\fP, etc), and these -objects expose all of their available state functions (i.e. \fBFile.managed\fP, -\fBService.running\fP, etc). -.sp -The name of the state is split based upon underscores (\fB_\fP), then each part -is capitalized and finally the parts are joined back together. +New in version 2017.7.0. + .sp -Some examples: -.INDENT 0.0 -.IP \(bu 2 -\fBpostgres_user\fP becomes \fBPostgresUser\fP -.IP \(bu 2 -\fBssh_known_hosts\fP becomes \fBSshKnownHosts\fP -.UNINDENT -.SS Context Managers and requisites +If zero or more characters at the beginning of string match this regular +expression, otherwise returns \fBNone\fP\&. .sp -How about something a little more complex. Here we\(aqre going to get into the -core of how to use pyobjects to write states. +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - with Pkg.installed(\(dqnginx\(dq): - Service.running(\(dqnginx\(dq, enable=True) - - with Service(\(dqnginx\(dq, \(dqwatch_in\(dq): - File.managed(\(dq/etc/nginx/conf.d/mysite.conf\(dq, - owner=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq0444\(aq, - source=\(aqsalt://nginx/mysite.conf\(aq) +{{ \(aqabcdefabcdef\(aq | regex_match(\(aqBC(.*)\(aq, ignorecase=True) }} .ft P .fi .UNINDENT .UNINDENT .sp -The objects that are returned from each of the magic method calls are setup to -be used a Python context managers (\fBwith\fP) and when you use them as such all -declarations made within the scope will \fBautomatically\fP use the enclosing -state as a requisite! -.sp -The above could have also been written use direct requisite statements as. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - Pkg.installed(\(dqnginx\(dq) - Service.running(\(dqnginx\(dq, enable=True, require=Pkg(\(dqnginx\(dq)) - File.managed(\(dq/etc/nginx/conf.d/mysite.conf\(dq, - owner=\(aqroot\(aq, group=\(aqroot\(aq, mode=\(aq0444\(aq, - source=\(aqsalt://nginx/mysite.conf\(aq, - watch_in=Service(\(dqnginx\(dq)) +None .ft P .fi .UNINDENT .UNINDENT +.SS \fBregex_replace\fP .sp -You can use the direct requisite statement for referencing states that are -generated outside of the current file. +New in version 2017.7.0. + +.sp +Searches for a pattern and replaces with a sequence of characters. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - # some\-other\-package is defined in some other state file - Pkg.installed(\(dqnginx\(dq, require=Pkg(\(dqsome\-other\-package\(dq)) +{% set my_text = \(aqyes, this is a TEST\(aq %} +{{ my_text | regex_replace(\(aq ([a\-z])\(aq, \(aq__\e\e1\(aq, ignorecase=True) }} .ft P .fi .UNINDENT .UNINDENT .sp -The last thing that direct requisites provide is the ability to select which -of the SaltStack requisites you want to use (require, require_in, watch, -watch_in, use & use_in) when using the requisite as a context manager. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - with Service(\(dqmy\-service\(dq, \(dqwatch_in\(dq): - ... +yes,__this__is__a__TEST .ft P .fi .UNINDENT .UNINDENT +.SS \fBuuid\fP .sp -The above example would cause all declarations inside the scope of the context -manager to automatically have their \fBwatch_in\fP set to -\fBService(\(dqmy\-service\(dq)\fP\&. -.SS Including and Extending +New in version 2017.7.0. + .sp -To include other states use the \fBinclude()\fP function. It takes one name per -state to include. +Return a UUID. .sp -To extend another state use the \fBextend()\fP function on the name when creating -a state. +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - include(\(aqhttp\(aq, \(aqssh\(aq) - - Service.running(extend(\(aqapache\(aq), - watch=[File(\(aq/etc/httpd/extra/httpd\-vhosts.conf\(aq)]) +{{ \(aqrandom\(aq | uuid }} .ft P .fi .UNINDENT .UNINDENT -.SS Importing from other state files -.sp -Like any Python project that grows you will likely reach a point where you want -to create reusability in your state tree and share objects between state files, -Map Data (described below) is a perfect example of this. -.sp -To facilitate this Python\(aqs \fBimport\fP statement has been augmented to allow -for a special case when working with a Salt state tree. If you specify a Salt -url (\fBsalt://...\fP) as the target for importing from then the pyobjects -renderer will take care of fetching the file for you, parsing it with all of -the pyobjects features available and then place the requested objects in the -global scope of the template being rendered. .sp -This works for all types of import statements; \fBimport X\fP, -\fBfrom X import Y\fP, and \fBfrom X import Y as Z\fP\&. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - import salt://myfile.sls - from salt://something/data.sls import Object - from salt://something/data.sls import Object as Other +3652b285\-26ad\-588e\-a5dc\-c2ee65edc804 .ft P .fi .UNINDENT .UNINDENT +.SS \fBis_list\fP .sp -See the Map Data section for a more practical use. -.sp -Caveats: -.INDENT 0.0 -.IP \(bu 2 -Imported objects are ALWAYS put into the global scope of your template, -regardless of where your import statement is. -.UNINDENT -.SS Salt object -.sp -In the spirit of the object interface for creating state data pyobjects also -provides a simple object interface to the \fB__salt__\fP object. +New in version 2017.7.0. + .sp -A function named \fBsalt\fP exists in scope for your sls files and will dispatch -its attributes to the \fB__salt__\fP dictionary. +Return if an object is list. .sp -The following lines are functionally equivalent: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - ret = salt.cmd.run(bar) - ret = __salt__[\(aqcmd.run\(aq](bar) +{{ [1, 2, 3] | is_list }} .ft P .fi .UNINDENT .UNINDENT -.SS Pillar, grain, mine & config data .sp -Pyobjects provides shortcut functions for calling \fBpillar.get\fP, -\fBgrains.get\fP, \fBmine.get\fP & \fBconfig.get\fP on the \fB__salt__\fP object. This -helps maintain the readability of your state files. -.sp -Each type of data can be access by a function of the same name: \fBpillar()\fP, -\fBgrains()\fP, \fBmine()\fP and \fBconfig()\fP\&. -.sp -The following pairs of lines are functionally equivalent: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - value = pillar(\(aqfoo:bar:baz\(aq, \(aqqux\(aq) - value = __salt__[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) - - value = grains(\(aqpkg:apache\(aq) - value = __salt__[\(aqgrains.get\(aq](\(aqpkg:apache\(aq) - - value = mine(\(aqos:Fedora\(aq, \(aqnetwork.interfaces\(aq, \(aqgrain\(aq) - value = __salt__[\(aqmine.get\(aq](\(aqos:Fedora\(aq, \(aqnetwork.interfaces\(aq, \(aqgrain\(aq) - - value = config(\(aqfoo:bar:baz\(aq, \(aqqux\(aq) - value = __salt__[\(aqconfig.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) +True .ft P .fi .UNINDENT .UNINDENT -.SS Opts dictionary and SLS name +.SS \fBis_iter\fP .sp -Pyobjects provides variable access to the minion options dictionary and the SLS -name that the code resides in. These variables are the same as the \fIopts\fP and -\fIsls\fP variables available in the Jinja renderer. +New in version 2017.7.0. + .sp -The following lines show how to access that information. +Return if an object is iterable. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - test_mode = __opts__[\(dqtest\(dq] - sls_name = __sls__ +{{ [1, 2, 3] | is_iter }} .ft P .fi .UNINDENT .UNINDENT -.SS Map Data .sp -When building complex states or formulas you often need a way of building up a -map of data based on grain data. The most common use of this is tracking the -package and service name differences between distributions. -.sp -To build map data using pyobjects we provide a class named Map that you use to -build your own classes with inner classes for each set of values for the -different grain matches. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - class Samba(Map): - merge = \(aqsamba:lookup\(aq - # NOTE: priority is new to 2017.7.0 - priority = (\(aqos_family\(aq, \(aqos\(aq) - - class Ubuntu: - __grain__ = \(aqos\(aq - service = \(aqsmbd\(aq - - class Debian: - server = \(aqsamba\(aq - client = \(aqsamba\-client\(aq - service = \(aqsamba\(aq - - class RHEL: - __match__ = \(aqRedHat\(aq - server = \(aqsamba\(aq - client = \(aqsamba\(aq - service = \(aqsmb\(aq +True .ft P .fi .UNINDENT .UNINDENT +.SS \fBmin\fP .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -By default, the \fBos_family\fP grain will be used as the target for -matching. This can be overridden by specifying a \fB__grain__\fP attribute. +New in version 2017.7.0. + .sp -If a \fB__match__\fP attribute is defined for a given class, then that value -will be matched against the targeted grain, otherwise the class name\(aqs -value will be be matched. +Return the minimum value from a list. .sp -Given the above example, the following is true: +Example: .INDENT 0.0 -.IP 1. 3 -Minions with an \fBos_family\fP of \fBDebian\fP will be assigned the -attributes defined in the \fBDebian\fP class. -.IP 2. 3 -Minions with an \fBos\fP grain of \fBUbuntu\fP will be assigned the -attributes defined in the \fBUbuntu\fP class. -.IP 3. 3 -Minions with an \fBos_family\fP grain of \fBRedHat\fP will be assigned the -attributes defined in the \fBRHEL\fP class. -.UNINDENT -.sp -That said, sometimes a minion may match more than one class. For instance, -in the above example, Ubuntu minions will match both the \fBDebian\fP and -\fBUbuntu\fP classes, since Ubuntu has an \fBos_family\fP grain of \fBDebian\fP -and an \fBos\fP grain of \fBUbuntu\fP\&. As of the 2017.7.0 release, the order is -dictated by the order of declaration, with classes defined later overriding -earlier ones. Additionally, 2017.7.0 adds support for explicitly defining -the ordering using an optional attribute called \fBpriority\fP\&. +.INDENT 3.5 .sp -Given the above example, \fBos_family\fP matches will be processed first, -with \fBos\fP matches processed after. This would have the effect of -assigning \fBsmbd\fP as the \fBservice\fP attribute on Ubuntu minions. If the -\fBpriority\fP item was not defined, or if the order of the items in the -\fBpriority\fP tuple were reversed, Ubuntu minions would have a \fBservice\fP -attribute of \fBsamba\fP, since \fBos_family\fP matches would have been -processed second. +.nf +.ft C +{{ [1, 2, 3] | min }} +.ft P +.fi .UNINDENT .UNINDENT .sp -To use this new data you can import it into your state file and then access -your attributes. To access the data in the map you simply access the attribute -name on the base class that is extending Map. Assuming the above Map was in the -file \fBsamba/map.sls\fP, you could do the following. +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - #!pyobjects - - from salt://samba/map.sls import Samba - - with Pkg.installed(\(dqsamba\(dq, names=[Samba.server, Samba.client]): - Service.running(\(dqsamba\(dq, name=Samba.service) +1 .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B class salt.renderers.pyobjects.PyobjectsModule(name, attrs) -This provides a wrapper for bare imports. -.UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.pyobjects.load_states() -This loads our states into the salt __context__ -.UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.pyobjects.render(template, saltenv=\(aqbase\(aq, sls=\(aq\(aq, salt_data=True, **kwargs) -.UNINDENT -.SS salt.renderers.stateconf -.INDENT 0.0 -.TP -.B maintainer -Jack Kuan <\fI\%kjkuan@gmail.com\fP> -.TP -.B maturity -new -.TP -.B platform -all -.UNINDENT +.SS \fBmax\fP .sp -This module provides a custom renderer that processes a salt file with a -specified templating engine (e.g. Jinja) and a chosen data renderer (e.g. YAML), -extracts arguments for any \fBstateconf.set\fP state, and provides the extracted -arguments (including Salt\-specific args, such as \fBrequire\fP, etc) as template -context. The goal is to make writing reusable/configurable/parameterized -salt files easier and cleaner. +New in version 2017.7.0. + .sp -To use this renderer, either set it as the default renderer via the -\fBrenderer\fP option in master/minion\(aqs config, or use the shebang line in each -individual sls file, like so: \fB#!stateconf\fP\&. Note, due to the way this -renderer works, it must be specified as the first renderer in a render -pipeline. That is, you cannot specify \fB#!mako|yaml|stateconf\fP, for example. -Instead, you specify them as renderer arguments: \fB#!stateconf mako . yaml\fP\&. +Returns the maximum value from a list. .sp -Here\(aqs a list of features enabled by this renderer. +Example: .INDENT 0.0 -.IP \(bu 2 -Prefixes any state id (declaration or reference) that starts with a dot (\fB\&.\fP) -to avoid duplicated state ids when the salt file is included by other salt -files. -.sp -For example, in the \fIsalt://some/file.sls\fP, a state id such as \fB\&.sls_params\fP -will be turned into \fBsome.file::sls_params\fP\&. Example: -.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -#!stateconf yaml . jinja - -\&.vim: - pkg.installed +{{ [1, 2, 3] | max }} .ft P .fi .UNINDENT .UNINDENT .sp -Above will be translated into: -.INDENT 2.0 +Returns: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -some.file::vim: - pkg.installed: - \- name: vim +3 .ft P .fi .UNINDENT .UNINDENT +.SS \fBavg\fP .sp -Notice how that if a state under a dot\-prefixed state id has no \fBname\fP -argument then one will be added automatically by using the state id with -the leading dot stripped off. +New in version 2017.7.0. + .sp -The leading dot trick can be used with extending state ids as well, -so you can include relatively and extend relatively. For example, when -extending a state in \fIsalt://some/other_file.sls\fP, e.g.: -.INDENT 2.0 +Returns the average value of the elements of a list +.sp +Example: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!stateconf yaml . jinja - -include: - \- .file - -extend: - .file::sls_params: - stateconf.set: - \- name1: something +{{ [1, 2, 3] | avg }} .ft P .fi .UNINDENT .UNINDENT .sp -Above will be pre\-processed into: -.INDENT 2.0 +Returns: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include: - \- some.file - -extend: - some.file::sls_params: - stateconf.set: - \- name1: something +2 .ft P .fi .UNINDENT .UNINDENT -.IP \(bu 2 -Adds a \fBsls_dir\fP context variable that expands to the directory containing -the rendering salt file. So, you can write \fBsalt://{{sls_dir}}/...\fP to -reference templates files used by your salt file. -.IP \(bu 2 -Recognizes the special state function, \fBstateconf.set\fP, that configures a -default list of named arguments usable within the template context of -the salt file. Example: -.INDENT 2.0 -.INDENT 3.5 +.SS \fBunion\fP .sp -.nf -.ft C -#!stateconf yaml . jinja - -\&.sls_params: - stateconf.set: - \- name1: value1 - \- name2: value2 - \- name3: - \- value1 - \- value2 - \- value3 - \- require_in: - \- cmd: output - -# \-\-\- end of state config \-\-\- +New in version 2017.7.0. -\&.output: - cmd.run: - \- name: | - echo \(aqname1={{sls_params.name1}} - name2={{sls_params.name2}} - name3[1]={{sls_params.name3[1]}} - \(aq -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -This even works with \fBinclude\fP + \fBextend\fP so that you can override -the default configured arguments by including the salt file and then -\fBextend\fP the \fBstateconf.set\fP states that come from the included salt -file. (\fIIMPORTANT: Both the included and the extending sls files must use the -stateconf renderer for this \(ga\(gaextend\(ga\(ga to work!\fP) .sp -Notice that the end of configuration marker (\fB# \-\-\- end of state config \-\-\fP) -is needed to separate the use of \(aqstateconf.set\(aq form the rest of your salt -file. The regex that matches such marker can be configured via the -\fBstateconf_end_marker\fP option in your master or minion config file. +Return the union of two lists. .sp -Sometimes, it is desirable to set a default argument value that\(aqs based on -earlier arguments in the same \fBstateconf.set\fP\&. For example, it may be -tempting to do something like this: -.INDENT 2.0 +Example: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!stateconf yaml . jinja - -\&.apache: - stateconf.set: - \- host: localhost - \- port: 1234 - \- url: \(aqhttp://{{host}}:{{port}}/\(aq - -# \-\-\- end of state config \-\-\- - -\&.test: - cmd.run: - \- name: echo \(aq{{apache.url}}\(aq - \- cwd: / +{{ [1, 2, 3] | union([2, 3, 4]) | join(\(aq, \(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -However, this won\(aqt work. It can however be worked around like so: -.INDENT 2.0 +Returns: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!stateconf yaml . jinja - -\&.apache: - stateconf.set: - \- host: localhost - \- port: 1234 -{# \- url: \(aqhttp://{{host}}:{{port}}/\(aq #} - -# \-\-\- end of state config \-\-\- -# {{ apache.setdefault(\(aqurl\(aq, \(dqhttp://%(host)s:%(port)s/\(dq % apache) }} - -\&.test: - cmd.run: - \- name: echo \(aq{{apache.url}}\(aq - \- cwd: / +1, 2, 3, 4 .ft P .fi .UNINDENT .UNINDENT -.IP \(bu 2 -Adds support for relative include and exclude of .sls files. Example: -.INDENT 2.0 +.SS \fBintersect\fP +.sp +New in version 2017.7.0. + +.sp +Return the intersection of two lists. +.sp +Example: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!stateconf yaml . jinja - -include: - \- .apache - \- .db.mysql - \- ..app.django - -exclude: - \- sls: .users +{{ [1, 2, 3] | intersect([2, 3, 4]) | join(\(aq, \(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -If the above is written in a salt file at \fIsalt://some/where.sls\fP then -it will include \fIsalt://some/apache.sls\fP, \fIsalt://some/db/mysql.sls\fP and -\fIsalt://app/django.sls\fP, and exclude \fIsalt://some/users.ssl\fP\&. Actually, -it does that by rewriting the above \fBinclude\fP and \fBexclude\fP into: -.INDENT 2.0 +Returns: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include: - \- some.apache - \- some.db.mysql - \- app.django - -exclude: - \- sls: some.users +2, 3 .ft P .fi .UNINDENT .UNINDENT -.IP \(bu 2 -Optionally (enabled by default, \fIdisable\fP via the \fI\-G\fP renderer option, -e.g. in the shebang line: \fB#!stateconf \-G\fP), generates a -\fBstateconf.set\fP goal state (state id named as \fB\&.goal\fP by default, -configurable via the master/minion config option, \fBstateconf_goal_state\fP) -that requires all other states in the salt file. Note, the \fB\&.goal\fP -state id is subject to dot\-prefix rename rule mentioned earlier. -.sp -Such goal state is intended to be required by some state in an including -salt file. For example, in your webapp salt file, if you include a -sls file that is supposed to setup Tomcat, you might want to make sure that -all states in the Tomcat sls file will be executed before some state in -the webapp sls file. -.IP \(bu 2 -Optionally (enable via the \fI\-o\fP renderer option, e.g. in the shebang line: -\fB#!stateconf \-o\fP), orders the states in a sls file by adding a -\fBrequire\fP requisite to each state such that every state requires the -state defined just before it. The order of the states here is the order -they are defined in the sls file. (Note: this feature is only available -if your minions are using Python >= 2.7. For Python2.6, it should also -work if you install the \fIordereddict\fP module from PyPI) -.sp -By enabling this feature, you are basically agreeing to author your sls -files in a way that gives up the explicit (or implicit?) ordering imposed -by the use of \fBrequire\fP, \fBwatch\fP, \fBrequire_in\fP or \fBwatch_in\fP -requisites, and instead, you rely on the order of states you define in -the sls files. This may or may not be a better way for you. However, if -there are many states defined in a sls file, then it tends to be easier -to see the order they will be executed with this feature. -.sp -You are still allowed to use all the requisites, with a few restrictions. -You cannot \fBrequire\fP or \fBwatch\fP a state defined \fIafter\fP the current -state. Similarly, in a state, you cannot \fBrequire_in\fP or \fBwatch_in\fP -a state defined \fIbefore\fP it. Breaking any of the two restrictions above -will result in a state loop. The renderer will check for such incorrect -uses if this feature is enabled. +.SS \fBdifference\fP .sp -Additionally, \fBnames\fP declarations cannot be used with this feature -because the way they are compiled into low states make it impossible to -guarantee the order in which they will be executed. This is also checked -by the renderer. As a workaround for not being able to use \fBnames\fP, -you can achieve the same effect, by generate your states with the -template engine available within your sls file. +New in version 2017.7.0. + .sp -Finally, with the use of this feature, it becomes possible to easily make -an included sls file execute all its states \fIafter\fP some state (say, with -id \fBX\fP) in the including sls file. All you have to do is to make state, -\fBX\fP, \fBrequire_in\fP the first state defined in the included sls file. -.UNINDENT +Return the difference of two lists. .sp -When writing sls files with this renderer, one should avoid using what can be -defined in a \fBname\fP argument of a state as the state\(aqs id. That is, avoid -writing states like this: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/path/to/some/file: - file.managed: - \- source: salt://some/file - -cp /path/to/some/file file2: - cmd.run: - \- cwd: / - \- require: - \- file: /path/to/some/file +{{ [1, 2, 3] | difference([2, 3, 4]) | join(\(aq, \(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -Instead, define the state id and the \fBname\fP argument separately for each -state. Also, the ID should be something meaningful and easy to reference within -a requisite (which is a good habit anyway, and such extra indirection would -also makes the sls file easier to modify later). Thus, the above states should -be written like this: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -add\-some\-file: - file.managed: - \- name: /path/to/some/file - \- source: salt://some/file - -copy\-files: - cmd.run: - \- name: cp /path/to/some/file file2 - \- cwd: / - \- require: - \- file: add\-some\-file +1 .ft P .fi .UNINDENT .UNINDENT +.SS \fBsymmetric_difference\fP .sp -Moreover, when referencing a state from a requisite, you should reference the -state\(aqs id plus the state name rather than the state name plus its \fBname\fP -argument. (Yes, in the above example, you can actually \fBrequire\fP the -\fBfile: /path/to/some/file\fP, instead of the \fBfile: add\-some\-file\fP). The -reason is that this renderer will re\-write or rename state id\(aqs and their -references for state id\(aqs prefixed with \fB\&.\fP\&. So, if you reference \fBname\fP -then there\(aqs no way to reliably rewrite such reference. -.SS salt.renderers.toml -.INDENT 0.0 -.TP -.B salt.renderers.tomlmod.render(sls_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts TOML as a string or as a file object and runs it through the -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure -.UNINDENT -.UNINDENT -.SS salt.renderers.wempy -.INDENT 0.0 -.TP -.B salt.renderers.wempy.render(template_file, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, context=None, **kws) -Render the data passing the functions and grains into the rendering system -.INDENT 7.0 -.TP -.B Return type -string -.UNINDENT -.UNINDENT -.SS salt.renderers.yaml -.SS Understanding YAML -.sp -The default renderer for SLS files is the YAML renderer. YAML is a -markup language with many powerful features. However, Salt uses -a small subset of YAML that maps over very commonly used data structures, -like lists and dictionaries. It is the job of the YAML renderer to take -the YAML data structure and compile it into a Python data structure for -use by Salt. -.sp -Though YAML syntax may seem daunting and terse at first, there are only -three very simple rules to remember when writing YAML for SLS files. -.SS Rule One: Indentation -.sp -YAML uses a fixed indentation scheme to represent relationships between -data layers. Salt requires that the indentation for each level consists -of exactly two spaces. Do not use tabs. -.SS Rule Two: Colons +New in version 2017.7.0. + .sp -Python dictionaries are, of course, simply key\-value pairs. Users from other -languages may recognize this data type as hashes or associative arrays. +Return the symmetric difference of two lists. .sp -Dictionary keys are represented in YAML as strings terminated by a trailing colon. -Values are represented by either a string following the colon, separated by a space: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_key: my_value +{{ [1, 2, 3] | symmetric_difference([2, 3, 4]) | join(\(aq, \(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -In Python, the above maps to: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqmy_key\(dq: \(dqmy_value\(dq} +1, 4 .ft P .fi .UNINDENT .UNINDENT +.SS \fBflatten\fP .sp -Dictionaries can be nested: +New in version 3005. + +.sp +Flatten a list. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -first_level_dict_key: - second_level_dict_key: value_in_second_level_dict +{{ [3, [4, 2] ] | flatten }} +# => [3, 4, 2] .ft P .fi .UNINDENT .UNINDENT .sp -And in Python: +Flatten only the first level of a list: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqfirst_level_dict_key\(dq: {\(dqsecond_level_dict_key\(dq: \(dqvalue_in_second_level_dict\(dq}} +{{ [3, [4, [2]] ] | flatten(levels=1) }} +# => [3, 4, [2]] .ft P .fi .UNINDENT .UNINDENT -.SS Rule Three: Dashes .sp -To represent lists of items, a single dash followed by a space is used. Multiple -items are a part of the same list as a function of their having the same level of indentation. +Preserve nulls in a list, by default \fBflatten\fP removes them. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\- list_value_one -\- list_value_two -\- list_value_three +{{ [3, None, [4, [2]] ] | flatten(levels=1, preserve_nulls=True) }} +# => [3, None, 4, [2]] .ft P .fi .UNINDENT .UNINDENT +.SS \fBcombinations\fP .sp -Lists can be the value of a key\-value pair. This is quite common in Salt: +New in version 3005. + +.sp +Invokes the \fBcombinations\fP function from the \fBitertools\fP library. +.sp +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_dictionary: - \- list_value_one - \- list_value_two - \- list_value_three +{% for one, two in \(dqABCD\(dq | combinations(2) %}{{ one~two }} {% endfor %} +# => AB AC AD BC BD CD .ft P .fi .UNINDENT .UNINDENT -.SS Reference -.sp -YAML Renderer for Salt +.SS \fBcombinations_with_replacement\fP .sp -For YAML usage information see \fI\%Understanding YAML\fP\&. -.INDENT 0.0 -.TP -.B salt.renderers.yaml.get_yaml_loader(argline) -Return the ordered dict yaml loader -.UNINDENT -.INDENT 0.0 -.TP -.B salt.renderers.yaml.render(yaml_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, argline=\(aq\(aq, **kws) -Accepts YAML as a string or as a file object and runs it through the YAML -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure -.UNINDENT -.UNINDENT -.SS salt.renderers.yamlex +New in version 3005. + .sp -YAMLEX renderer is a replacement of the YAML renderer. -It\(aqs 100% YAML with a pinch of Salt magic: -.INDENT 0.0 -.IP \(bu 2 -All mappings are automatically OrderedDict -.IP \(bu 2 -All strings are automatically str obj -.IP \(bu 2 -data aggregation with !aggregation yaml tag, based on the \fBsalt.utils.aggregation\fP module. -.IP \(bu 2 -data aggregation over documents for pillar -.UNINDENT +Invokes the \fBcombinations_with_replacement\fP function from the \fBitertools\fP library. .sp -Instructed aggregation within the \fB!aggregation\fP and the \fB!reset\fP tags: +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!yamlex -foo: !aggregate first -foo: !aggregate second -bar: !aggregate {first: foo} -bar: !aggregate {second: bar} -baz: !aggregate 42 -qux: !aggregate default -!reset qux: !aggregate my custom data +{% for one, two in \(dqABC\(dq | combinations_with_replacement(2) %}{{ one~two }} {% endfor %} +# => AA AB AC BB BC CC .ft P .fi .UNINDENT .UNINDENT +.SS \fBcompress\fP .sp -is roughly equivalent to +New in version 3005. + +.sp +Invokes the \fBcompress\fP function from the \fBitertools\fP library. +.sp +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -foo: [first, second] -bar: {first: foo, second: bar} -baz: [42] -qux: [my custom data] +{% for val in \(dqABCDEF\(dq | compress([1,0,1,0,1,1]) %}{{ val }} {% endfor %} +# => A C E F .ft P .fi .UNINDENT .UNINDENT -.SS Reference -.INDENT 0.0 -.TP -.B salt.renderers.yamlex.render(sls_data, saltenv=\(aqbase\(aq, sls=\(aq\(aq, **kws) -Accepts YAML_EX as a string or as a file object and runs it through the YAML_EX -parser. -.INDENT 7.0 -.TP -.B Return type -A Python data structure -.UNINDENT -.UNINDENT -.SH USING SALT -.sp -This section describes the fundamental components and concepts that you need to understand to use Salt. -.SS Grains -.sp -Salt comes with an interface to derive information about the underlying system. -This is called the grains interface, because it presents salt with grains of -information. Grains are collected for the operating system, domain name, -IP address, kernel, OS type, memory, and many other system properties. -.sp -The grains interface is made available to Salt modules and components so that -the right salt minion commands are automatically available on the right -systems. +.SS \fBpermutations\fP .sp -Grain data is relatively static, though if system information changes -(for example, if network settings are changed), or if a new value is assigned -to a custom grain, grain data is refreshed. +New in version 3005. + .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Grains resolve to lowercase letters. For example, \fBFOO\fP, and \fBfoo\fP -target the same grain. -.UNINDENT -.UNINDENT -.SS Listing Grains +Invokes the \fBpermutations\fP function from the \fBitertools\fP library. .sp -Available grains can be listed by using the \(aqgrains.ls\(aq module: +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq grains.ls +{% for one, two in \(dqABCD\(dq | permutations(2) %}{{ one~two }} {% endfor %} +# => AB AC AD BA BC BD CA CB CD DA DB DC .ft P .fi .UNINDENT .UNINDENT +.SS \fBproduct\fP .sp -Grains data can be listed by using the \(aqgrains.items\(aq module: +New in version 3005. + +.sp +Invokes the \fBproduct\fP function from the \fBitertools\fP library. +.sp +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq grains.items +{% for one, two in \(dqABCD\(dq | product(\(dqxy\(dq) %}{{ one~two }} {% endfor %} +# => Ax Ay Bx By Cx Cy Dx Dy .ft P .fi .UNINDENT .UNINDENT -.SS Using grains in a state +.SS \fBzip\fP .sp -To use a grain in a state you can access it via \fI{{ grains[\(aqkey\(aq] }}\fP\&. -.SS Grains in the Minion Config +New in version 3005. + .sp -Grains can also be statically assigned within the minion configuration file. -Just add the option \fI\%grains\fP and pass options to it: -.INDENT 0.0 -.INDENT 3.5 +Invokes the native Python \fBzip\fP function. .sp -.nf -.ft C -grains: - roles: - \- webserver - \- memcache - deployment: datacenter4 - cabinet: 13 - cab_u: 14\-15 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -Then status data specific to your servers can be retrieved via Salt, or used -inside of the State system for matching. It also makes it possible to target based on specific data about your deployment, as in the example above. -.SS Grains in /etc/salt/grains +The \fBzip\fP function returns a zip object, which is an iterator of tuples where +the first item in each passed iterator is paired together, and then the second +item in each passed iterator are paired together etc. .sp -If you do not want to place your custom static grains in the minion config -file, you can also put them in \fB/etc/salt/grains\fP on the minion. They are configured in the -same way as in the above example, only without a top\-level \fBgrains:\fP key: +If the passed iterators have different lengths, the iterator with the least +items decides the length of the new iterator. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -roles: - \- webserver - \- memcache -deployment: datacenter4 -cabinet: 13 -cab_u: 14\-15 +{% for one, two in \(dqABCD\(dq | zip(\(dqxy\(dq) %}{{ one~two }} {% endfor %} +# => Ax By .ft P .fi .UNINDENT .UNINDENT +.SS \fBzip_longest\fP .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Grains in \fB/etc/salt/grains\fP are ignored if you specify the same grains in the minion config. -.UNINDENT -.UNINDENT -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Grains are static, and since they are not often changed, they will need a grains refresh when they are updated. You can do this by calling: \fBsalt minion saltutil.refresh_modules\fP -.UNINDENT -.UNINDENT +New in version 3005. + .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -You can equally configure static grains for Proxy Minions. -As multiple Proxy Minion processes can run on the same machine, you need -to index the files using the Minion ID, under \fB/etc/salt/proxy.d//grains\fP\&. -For example, the grains for the Proxy Minion \fBrouter1\fP can be defined -under \fB/etc/salt/proxy.d/router1/grains\fP, while the grains for the -Proxy Minion \fBswitch7\fP can be put in \fB/etc/salt/proxy.d/switch7/grains\fP\&. -.UNINDENT -.UNINDENT -.SS Matching Grains in the Top File +Invokes the \fBzip_longest\fP function from the \fBitertools\fP library. .sp -With correctly configured grains on the Minion, the \fI\%top file\fP used in -Pillar or during Highstate can be made very efficient. For example, consider -the following configuration: +See the \fI\%itertools documentation\fP for more information. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\(aqroles:webserver\(aq: - \- match: grain - \- state0 - -\(aqroles:memcache\(aq: - \- match: grain - \- state1 - \- state2 +{% for one, two in \(dqABCD\(dq | zip_longest(\(dqxy\(dq, fillvalue=\(dq\-\(dq) %}{{ one~two }} {% endfor %} +# => Ax By C\- D\- .ft P .fi .UNINDENT .UNINDENT +.SS \fBmethod_call\fP .sp -For this example to work, you would need to have defined the grain -\fBrole\fP for the minions you wish to match. -.SS Writing Grains -.sp -\fBWARNING:\fP -.INDENT 0.0 -.INDENT 3.5 -Grains can be set by users that have access to the minion configuration files on -the local system, making them less secure than other identifiers in Salt. Avoid -storing sensitive data, such as passwords or keys, on minions. Instead, make -use of \fI\%Storing Static Data in the Pillar\fP and/or \fI\%Storing Data in Other Databases\fP\&. -.UNINDENT -.UNINDENT -.sp -The grains are derived by executing all of the \(dqpublic\(dq functions (i.e. those -which do not begin with an underscore) found in the modules located in the -Salt\(aqs core grains code, followed by those in any custom grains modules. The -functions in a grains module must return a \fI\%Python dictionary\fP, where the dictionary keys are the names of grains, and -each key\(aqs value is that value for that grain. +New in version 3001. + .sp -Custom grains modules should be placed in a subdirectory named \fB_grains\fP -located under the \fI\%file_roots\fP specified by the master config -file. The default path would be \fB/srv/salt/_grains\fP\&. Custom grains modules -will be distributed to the minions when \fI\%state.highstate\fP is run, or by executing the -\fI\%saltutil.sync_grains\fP or -\fI\%saltutil.sync_all\fP functions. +Returns a result of object\(aqs method call. .sp -Grains modules are easy to write, and (as noted above) only need to return a -dictionary. For example: +Example #1: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -def yourfunction(): - # initialize a grains dictionary - grains = {} - # Some code for logic that sets grains like - grains[\(dqyourcustomgrain\(dq] = True - grains[\(dqanothergrain\(dq] = \(dqsomevalue\(dq - return grains +{{ [1, 2, 1, 3, 4] | method_call(\(aqindex\(aq, 1, 1, 3) }} .ft P .fi .UNINDENT .UNINDENT .sp -The name of the function does not matter and will not factor into the grains -data at all; only the keys/values returned become part of the grains. -.SS When to Use a Custom Grain -.sp -Before adding new grains, consider what the data is and remember that grains -should (for the most part) be static data. -.sp -If the data is something that is likely to change, consider using \fI\%Pillar\fP or an execution module instead. If it\(aqs a simple set of -key/value pairs, pillar is a good match. If compiling the information requires -that system commands be run, then putting this information in an execution -module is likely a better idea. -.sp -Good candidates for grains are data that is useful for targeting minions in the -\fI\%top file\fP or the Salt CLI. The name and data structure of -the grain should be designed to support many platforms, operating systems or -applications. Also, keep in mind that Jinja templating in Salt supports -referencing pillar data as well as invoking functions from execution modules, -so there\(aqs no need to place information in grains to make it available to Jinja -templates. For example: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\&... -\&... -{{ salt[\(aqmodule.function_name\(aq](\(aqargument_1\(aq, \(aqargument_2\(aq) }} -{{ pillar[\(aqmy_pillar_key\(aq] }} -\&... -\&... +2 .ft P .fi .UNINDENT .UNINDENT .sp -\fBWARNING:\fP -.INDENT 0.0 -.INDENT 3.5 -Custom grains will not be available in the top file until after the first -\fI\%highstate\fP\&. To make custom grains available on a -minion\(aqs first highstate, it is recommended to use \fI\%this example\fP to ensure that the custom grains are synced when -the minion starts. -.UNINDENT -.UNINDENT -.SS Loading Custom Grains +This filter can be used with the \fI\%map filter\fP to apply object methods without +using loop constructs or temporary variables. .sp -If you have multiple functions specifying grains that are called from a \fBmain\fP -function, be sure to prepend grain function names with an underscore. This prevents -Salt from including the loaded grains from the grain functions in the final -grain data structure. For example, consider this custom grain file: +Example #2: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -#!/usr/bin/env python -def _my_custom_grain(): - my_grain = {\(dqfoo\(dq: \(dqbar\(dq, \(dqhello\(dq: \(dqworld\(dq} - return my_grain - - -def main(): - # initialize a grains dictionary - grains = {} - grains[\(dqmy_grains\(dq] = _my_custom_grain() - return grains +{% set host_list = [\(aqweb01.example.com\(aq, \(aqdb01.example.com\(aq] %} +{% set host_list_split = [] %} +{% for item in host_list %} + {% do host_list_split.append(item.split(\(aq.\(aq, 1)) %} +{% endfor %} +{{ host_list_split }} .ft P .fi .UNINDENT .UNINDENT .sp -The output of this example renders like so: +Example #3: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-call \-\-local grains.items -local: - \-\-\-\-\-\-\-\-\-\- - - my_grains: - \-\-\-\-\-\-\-\-\-\- - foo: - bar - hello: - world +{{ host_list|map(\(aqmethod_call\(aq, \(aqsplit\(aq, \(aq.\(aq, 1)|list }} .ft P .fi .UNINDENT .UNINDENT .sp -However, if you don\(aqt prepend the \fBmy_custom_grain\fP function with an underscore, -the function will be rendered twice by Salt in the items output: once for the -\fBmy_custom_grain\fP call itself, and again when it is called in the \fBmain\fP -function: +Return of examples #2 and #3: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-call \-\-local grains.items -local: -\-\-\-\-\-\-\-\-\-\- - - foo: - bar - - hello: - world - - my_grains: - \-\-\-\-\-\-\-\-\-\- - foo: - bar - hello: - world +[[web01, example.com], [db01, example.com]] .ft P .fi .UNINDENT .UNINDENT -.SS Precedence -.sp -Core grains can be overridden by custom grains. As there are several ways of -defining custom grains, there is an order of precedence which should be kept in -mind when defining them. The order of evaluation is as follows: -.INDENT 0.0 -.IP 1. 3 -Core grains. -.IP 2. 3 -Custom grains in \fB/etc/salt/grains\fP\&. -.IP 3. 3 -Custom grains in \fB/etc/salt/minion\fP\&. -.IP 4. 3 -Custom grain modules in \fB_grains\fP directory, synced to minions. -.UNINDENT -.sp -Each successive evaluation overrides the previous ones, so any grains defined -by custom grains modules synced to minions that have the same name as a core -grain will override that core grain. Similarly, grains from -\fB/etc/salt/minion\fP override both core grains and custom grain modules, and -grains in \fB_grains\fP will override \fIany\fP grains of the same name. +.SS \fBis_sorted\fP .sp -For custom grains, if the function takes an argument \fBgrains\fP, then the -previously rendered grains will be passed in. Because the rest of the grains -could be rendered in any order, the only grains that can be relied upon to be -passed in are \fBcore\fP grains. This was added in the 2019.2.0 release. -.SS Examples of Grains +New in version 2017.7.0. + .sp -The core module in the grains package is where the main grains are loaded by -the Salt minion and provides the principal example of how to write grains: +Return \fBTrue\fP if an iterable object is already sorted. .sp -\fI\%salt/grains/core.py\fP -.SS Syncing Grains +Example: +.INDENT 0.0 +.INDENT 3.5 .sp -Syncing grains can be done a number of ways. They are automatically synced when -\fI\%state.highstate\fP is called, or (as noted -above) the grains can be manually synced and reloaded by calling the -\fI\%saltutil.sync_grains\fP or -\fI\%saltutil.sync_all\fP functions. +.nf +.ft C +{{ [1, 2, 3] | is_sorted }} +.ft P +.fi +.UNINDENT +.UNINDENT .sp -\fBNOTE:\fP +Returns: .INDENT 0.0 .INDENT 3.5 -When the \fI\%grains_cache\fP is set to False, the grains dictionary is built -and stored in memory on the minion. Every time the minion restarts or -\fBsaltutil.refresh_grains\fP is run, the grain dictionary is rebuilt from scratch. +.sp +.nf +.ft C +True +.ft P +.fi .UNINDENT .UNINDENT -.SS Storing Static Data in the Pillar +.SS \fBcompare_lists\fP .sp -Pillar is an interface for Salt designed to offer global values that can be -distributed to minions. Pillar data is managed in a similar way as -the Salt State Tree. +New in version 2017.7.0. + .sp -Pillar was added to Salt in version 0.9.8 +Compare two lists and return a dictionary with the changes. .sp -\fBNOTE:\fP +Example: .INDENT 0.0 .INDENT 3.5 -Storing sensitive data .sp -Pillar data is compiled on the master. Additionally, pillar data for a -given minion is only accessible by the minion for which it is targeted in -the pillar configuration. This makes pillar useful for storing sensitive -data specific to a particular minion. +.nf +.ft C +{{ [1, 2, 3] | compare_lists([1, 2, 4]) }} +.ft P +.fi .UNINDENT .UNINDENT -.SS Declaring the Master Pillar -.sp -The Salt Master server maintains a \fI\%pillar_roots\fP setup that -matches the structure of the \fI\%file_roots\fP used in the Salt file -server. Like \fI\%file_roots\fP, the \fI\%pillar_roots\fP option -maps environments to directories. The pillar data is then mapped to minions -based on matchers in a top file which is laid out in the same way as the state -top file. Salt pillars can use the same matcher types as the standard \fI\%top -file\fP\&. .sp -conf_master:\fIpillar_roots\fP is configured just like \fI\%file_roots\fP\&. -For example: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_roots: - base: - \- /srv/pillar +{\(dqnew\(dq: [4], \(dqold\(dq: [3]} .ft P .fi .UNINDENT .UNINDENT +.SS \fBcompare_dicts\fP .sp -This example configuration declares that the base environment will be located -in the \fB/srv/pillar\fP directory. It must not be in a subdirectory of the -state tree. +New in version 2017.7.0. + .sp -The top file used matches the name of the top file used for States, -and has the same structure: +Compare two dictionaries and return a dictionary with the changes. .sp -\fB/srv/pillar/top.sls\fP +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- packages +{{ {\(aqa\(aq: \(aqb\(aq} | compare_dicts({\(aqa\(aq: \(aqc\(aq}) }} .ft P .fi .UNINDENT .UNINDENT .sp -In the above top file, it is declared that in the \fBbase\fP environment, the -glob matching all minions will have the pillar data found in the \fBpackages\fP -pillar available to it. Assuming the \fBpillar_roots\fP value of \fB/srv/pillar\fP -taken from above, the \fBpackages\fP pillar would be located at -\fB/srv/pillar/packages.sls\fP\&. -.sp -Any number of matchers can be added to the base environment. For example, here -is an expanded version of the Pillar top file stated above: -.sp -/srv/pillar/top.sls: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- packages - \(aqweb*\(aq: - \- vim +{\(dqa\(dq: {\(dqnew\(dq: \(dqc\(dq, \(dqold\(dq: \(dqb\(dq}} .ft P .fi .UNINDENT .UNINDENT +.SS \fBis_hex\fP .sp -In this expanded top file, minions that match \fBweb*\fP will have access to the -\fB/srv/pillar/packages.sls\fP file, as well as the \fB/srv/pillar/vim.sls\fP file. +New in version 2017.7.0. + .sp -Another example shows how to use other standard top matching types -to deliver specific salt pillar data to minions with different properties. +Return \fBTrue\fP if the value is hexadecimal. .sp -Here is an example using the \fBgrains\fP matcher to target pillars to minions -by their \fBos\fP grain: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -dev: - \(aqos:Debian\(aq: - \- match: grain - \- servers +{{ \(aq0xabcd\(aq | is_hex }} +{{ \(aqxyzt\(aq | is_hex }} .ft P .fi .UNINDENT .UNINDENT .sp -Pillar definitions can also take a keyword argument \fBignore_missing\fP\&. -When the value of \fBignore_missing\fP is \fBTrue\fP, all errors for missing -pillar files are ignored. The default value for \fBignore_missing\fP is -\fBFalse\fP\&. -.sp -Here is an example using the \fBignore_missing\fP keyword parameter to ignore -errors for missing pillar files: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- servers - \- systems - \- ignore_missing: True +True +False .ft P .fi .UNINDENT .UNINDENT +.SS \fBcontains_whitespace\fP .sp -Assuming that the pillar \fBservers\fP exists in the fileserver backend -and the pillar \fBsystems\fP doesn\(aqt, all pillar data from \fBservers\fP -pillar is delivered to minions and no error for the missing pillar -\fBsystems\fP is noted under the key \fB_errors\fP in the pillar data -delivered to minions. +New in version 2017.7.0. + .sp -Should the \fBignore_missing\fP keyword parameter have the value \fBFalse\fP, -an error for the missing pillar \fBsystems\fP would produce the value -\fBSpecified SLS \(aqservers\(aq in environment \(aqbase\(aq is not available on the salt master\fP -under the key \fB_errors\fP in the pillar data delivered to minions. +Return \fBTrue\fP if a text contains whitespaces. .sp -\fB/srv/pillar/packages.sls\fP +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} -apache: httpd -git: git -{% elif grains[\(aqos\(aq] == \(aqDebian\(aq %} -apache: apache2 -git: git\-core -{% endif %} - -company: Foo Industries +{{ \(aqabcd\(aq | contains_whitespace }} +{{ \(aqab cd\(aq | contains_whitespace }} .ft P .fi .UNINDENT .UNINDENT .sp -\fBIMPORTANT:\fP +Returns: .INDENT 0.0 .INDENT 3.5 -See \fI\%Is Targeting using Grain Data Secure?\fP for -important security information. +.sp +.nf +.ft C +False +True +.ft P +.fi .UNINDENT .UNINDENT +.SS \fBsubstring_in_list\fP .sp -The above pillar sets two key/value pairs. If a minion is running RedHat, then -the \fBapache\fP key is set to \fBhttpd\fP and the \fBgit\fP key is set to the value -of \fBgit\fP\&. If the minion is running Debian, those values are changed to -\fBapache2\fP and \fBgit\-core\fP respectively. All minions that have this pillar -targeting to them via a top file will have the key of \fBcompany\fP with a value -of \fBFoo Industries\fP\&. +New in version 2017.7.0. + .sp -Consequently this data can be used from within modules, renderers, State SLS -files, and more via the shared pillar dictionary: +Return \fBTrue\fP if a substring is found in a list of string values. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -apache: - pkg.installed: - \- name: {{ pillar[\(aqapache\(aq] }} +{{ \(aqabcd\(aq | substring_in_list([\(aqthis\(aq, \(aqis\(aq, \(aqan abcd example\(aq]) }} .ft P .fi .UNINDENT .UNINDENT +.sp +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -git: - pkg.installed: - \- name: {{ pillar[\(aqgit\(aq] }} +True .ft P .fi .UNINDENT .UNINDENT +.SS \fBcheck_whitelist_blacklist\fP .sp -Finally, the above states can utilize the values provided to them via Pillar. -All pillar values targeted to a minion are available via the \(aqpillar\(aq -dictionary. As seen in the above example, Jinja substitution can then be -utilized to access the keys and values in the Pillar dictionary. +New in version 2017.7.0. + .sp -Note that you cannot just list key/value\-information in \fBtop.sls\fP\&. Instead, -target a minion to a pillar file and then list the keys and values in the -pillar. Here is an example top file that illustrates this point: +Check a whitelist and/or blacklist to see if the value matches it. +.sp +This filter can be used with either a whitelist or a blacklist individually, +or a whitelist and a blacklist can be passed simultaneously. +.sp +If whitelist is used alone, value membership is checked against the +whitelist only. If the value is found, the function returns \fBTrue\fP\&. +Otherwise, it returns \fBFalse\fP\&. +.sp +If blacklist is used alone, value membership is checked against the +blacklist only. If the value is found, the function returns \fBFalse\fP\&. +Otherwise, it returns \fBTrue\fP\&. +.sp +If both a whitelist and a blacklist are provided, value membership in the +blacklist will be examined first. If the value is not found in the blacklist, +then the whitelist is checked. If the value isn\(aqt found in the whitelist, +the function returns \fBFalse\fP\&. +.sp +Whitelist Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- common_pillar +{{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} .ft P .fi .UNINDENT .UNINDENT .sp -And the actual pillar file at \(aq/srv/pillar/common_pillar.sls\(aq: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -foo: bar -boo: baz +True .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -When working with multiple pillar environments, assuming that each pillar -environment has its own top file, the jinja placeholder \fB{{ saltenv }}\fP -can be used in place of the environment name: +Blacklist Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ saltenv }}: - \(aq*\(aq: - \- common_pillar +{{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} .ft P .fi .UNINDENT .UNINDENT -.sp -Yes, this is \fB{{ saltenv }}\fP, and not \fB{{ pillarenv }}\fP\&. The reason for -this is because the Pillar top files are parsed using some of the same code -which parses top files when \fI\%running states\fP, so -the pillar environment takes the place of \fB{{ saltenv }}\fP in the jinja -context. -.UNINDENT -.UNINDENT -.SS Dynamic Pillar Environments -.sp -If environment \fB__env__\fP is specified in \fI\%pillar_roots\fP, all -environments that are not explicitly specified in \fI\%pillar_roots\fP -will map to the directories from \fB__env__\fP\&. This allows one to use dynamic -git branch based environments for state/pillar files with the same file\-based -pillar applying to all environments. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_roots: - __env__: - \- /srv/pillar - -ext_pillar: - \- git: - \- __env__ https://example.com/git\-pillar.git +False .ft P .fi .UNINDENT .UNINDENT +.SS \fBdate_format\fP .sp -New in version 2017.7.5,2018.3.1. +New in version 2017.7.0. .sp -Taking it one step further, \fB__env__\fP can also be used in the \fBpillar_root\fP -filesystem path. It will be replaced with the actual \fBpillarenv\fP and searched -for Pillar data to provide to the minion. Note this substitution ONLY occurs for -the \fB__env__\fP environment. For instance, this configuration: +Converts unix timestamp into human\-readable string. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_roots: - __env__: - \- /srv/__env__/pillar +{{ 1457456400 | date_format }} +{{ 1457456400 | date_format(\(aq%d.%m.%Y %H:%M\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -is equivalent to this static configuration: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_roots: - dev: - \- /srv/dev/pillar - test: - \- /srv/test/pillar - prod: - \- /srv/prod/pillar +2017\-03\-08 +08.03.2017 17:00 .ft P .fi .UNINDENT .UNINDENT +.SS \fBto_num\fP .sp -New in version 3005. +New in version 2017.7.0. -.SS Pillar Namespace Flattening .sp -The separate pillar SLS files all merge down into a single dictionary of -key\-value pairs. When the same key is defined in multiple SLS files, this can -result in unexpected behavior if care is not taken to how the pillar SLS files -are laid out. +New in version 2018.3.0: Renamed from \fBstr_to_num\fP to \fBto_num\fP\&. + .sp -For example, given a \fBtop.sls\fP containing the following: +Converts a string to its numerical value. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- packages - \- services +{{ \(aq5\(aq | to_num }} .ft P .fi .UNINDENT .UNINDENT .sp -with \fBpackages.sls\fP containing: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -bind: bind9 +5 .ft P .fi .UNINDENT .UNINDENT +.SS \fBto_bytes\fP .sp -and \fBservices.sls\fP containing: +New in version 2017.7.0. + +.sp +Converts string\-type object to bytes. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -bind: named +{{ \(aqwall of text\(aq | to_bytes }} .ft P .fi .UNINDENT .UNINDENT .sp -Then a request for the \fBbind\fP pillar key will only return \fBnamed\fP\&. The -\fBbind9\fP value will be lost, because \fBservices.sls\fP was evaluated later. -.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Pillar files are applied in the order they are listed in the top file. -Therefore conflicting keys will be overwritten in a \(aqlast one wins\(aq manner! -For example, in the above scenario conflicting key values in \fBservices\fP -will overwrite those in \fBpackages\fP because it\(aqs at the bottom of the list. +This option may have adverse effects when using the default renderer, +\fBjinja|yaml\fP\&. This is due to the fact that YAML requires proper handling +in regard to special characters. Please see the section on \fI\%YAML ASCII +support\fP in the \fI\%YAML Idiosyncrasies\fP documentation for more information. .UNINDENT .UNINDENT +.SS \fBjson_encode_list\fP .sp -It can be better to structure your pillar files with more hierarchy. For -example the \fBpackage.sls\fP file could be configured like so: +New in version 2017.7.0. + +.sp +New in version 2018.3.0: Renamed from \fBjson_decode_list\fP to \fBjson_encode_list\fP\&. When you encode +something you get bytes, and when you decode, you get your locale\(aqs +encoding (usually a \fBunicode\fP type). This filter was incorrectly\-named +when it was added. \fBjson_decode_list\fP will be supported until the 3003 +release. + +.sp +Deprecated since version 2018.3.3,2019.2.0: The \fI\%tojson\fP filter accomplishes what this filter was designed +to do, making this filter redundant. + +.sp +Recursively encodes all string elements of the list to bytes. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -packages: - bind: bind9 +{{ [1, 2, 3] | json_encode_list }} .ft P .fi .UNINDENT .UNINDENT .sp -This would make the \fBpackages\fP pillar key a nested dictionary containing a -\fBbind\fP key. -.SS Pillar Dictionary Merging -.sp -If the same pillar key is defined in multiple pillar SLS files, and the keys in -both files refer to nested dictionaries, then the content from these -dictionaries will be recursively merged. -.sp -For example, keeping the \fBtop.sls\fP the same, assume the following -modifications to the pillar SLS files: -.sp -\fBpackages.sls\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -bind: - package\-name: bind9 - version: 9.9.5 +[1, 2, 3] .ft P .fi .UNINDENT .UNINDENT +.SS \fBjson_encode_dict\fP .sp -\fBservices.sls\fP: +New in version 2017.7.0. + +.sp +New in version 2018.3.0: Renamed from \fBjson_decode_dict\fP to \fBjson_encode_dict\fP\&. When you encode +something you get bytes, and when you decode, you get your locale\(aqs +encoding (usually a \fBunicode\fP type). This filter was incorrectly\-named +when it was added. \fBjson_decode_dict\fP will be supported until the 3003 +release. + +.sp +Deprecated since version 2018.3.3,2019.2.0: The \fI\%tojson\fP filter accomplishes what this filter was designed +to do, making this filter redundant. + +.sp +Recursively encodes all string items in the dictionary to bytes. +.sp +Example: +.sp +Assuming that \fBpillar[\(aqfoo\(aq]\fP contains \fB{u\(aqa\(aq: u\(aq\eu0414\(aq}\fP, and your locale +is \fBen_US.UTF\-8\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -bind: - port: 53 - listen\-on: any +{{ pillar[\(aqfoo\(aq] | json_encode_dict }} .ft P .fi .UNINDENT .UNINDENT .sp -The resulting pillar dictionary will be: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt\-call pillar.get bind -local: - \-\-\-\-\-\-\-\-\-\- - listen\-on: - any - package\-name: - bind9 - port: - 53 - version: - 9.9.5 +{\(dqa\(dq: \(dq\exd0\ex94\(dq} .ft P .fi .UNINDENT .UNINDENT +.SS \fBtojson\fP .sp -Since both pillar SLS files contained a \fBbind\fP key which contained a nested -dictionary, the pillar dictionary\(aqs \fBbind\fP key contains the combined contents -of both SLS files\(aq \fBbind\fP keys. -.SS Including Other Pillars +New in version 2018.3.3,2019.2.0. + .sp -New in version 0.16.0. +Dumps a data structure to JSON. +.sp +This filter was added to provide this functionality to hosts which have a +Jinja release older than version 2.9 installed. If Jinja 2.9 or newer is +installed, then the upstream version of the filter will be used. See the +\fI\%upstream docs\fP for more information. +.SS \fBrandom_hash\fP +.sp +New in version 2017.7.0. .sp -Pillar SLS files may include other pillar files, similar to State files. Two -syntaxes are available for this purpose. The simple form simply includes the -additional pillar as if it were part of the same file: +New in version 2018.3.0: Renamed from \fBrand_str\fP to \fBrandom_hash\fP to more accurately describe +what the filter does. \fBrand_str\fP will be supported to ensure backwards +compatibility but please use the preferred \fBrandom_hash\fP\&. + +.sp +Generates a random number between 1 and the number passed to the filter, and +then hashes it. The default hash type is the one specified by the minion\(aqs +\fI\%hash_type\fP config option, but an alternate hash type can be +passed to the filter as an argument. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include: - \- users +{% set num_range = 99999999 %} +{{ num_range | random_hash }} +{{ num_range | random_hash(\(aqsha512\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -The full include form allows two additional options \-\- passing default values -to the templating engine for the included pillar file as well as an optional -key under which to nest the results of the included pillar: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -include: - \- users: - defaults: - sudo: [\(aqbob\(aq, \(aqpaul\(aq] - key: users +43ec517d68b6edd3015b3edc9a11367b +d94a45acd81f8e3107d237dbc0d5d195f6a52a0d188bc0284c0763ece1eac9f9496fb6a531a296074c87b3540398dace1222b42e150e67c9301383fde3d66ae5 .ft P .fi .UNINDENT .UNINDENT +.SS \fBrandom_sample\fP .sp -With this form, the included file (users.sls) will be nested within the \(aqusers\(aq -key of the compiled pillar. Additionally, the \(aqsudo\(aq value will be available -as a template variable to users.sls. -.SS In\-Memory Pillar Data vs. On\-Demand Pillar Data -.sp -Since compiling pillar data is computationally expensive, the minion will -maintain a copy of the pillar data in memory to avoid needing to ask the master -to recompile and send it a copy of the pillar data each time pillar data is -requested. This in\-memory pillar data is what is returned by the -\fI\%pillar.item\fP, \fI\%pillar.get\fP, and \fI\%pillar.raw\fP -functions. +New in version 3005. + .sp -Also, for those writing custom execution modules, or contributing to Salt\(aqs -existing execution modules, the in\-memory pillar data is available as the -\fB__pillar__\fP dunder dictionary. +Returns a given sample size from a list. The \fBseed\fP parameter can be used to +return a predictable outcome. .sp -The in\-memory pillar data is generated on minion start, and can be refreshed -using the \fI\%saltutil.refresh_pillar\fP function: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq saltutil.refresh_pillar +{% set my_list = [\(dqone\(dq, \(dqtwo\(dq, \(dqthree\(dq, \(dqfour\(dq] %} +{{ my_list | random_sample(2) }} .ft P .fi .UNINDENT .UNINDENT .sp -This function triggers the minion to asynchronously refresh the in\-memory -pillar data and will always return \fBNone\fP\&. -.sp -In contrast to in\-memory pillar data, certain actions trigger pillar data to be -compiled to ensure that the most up\-to\-date pillar data is available. These -actions include: -.INDENT 0.0 -.IP \(bu 2 -Running states -.IP \(bu 2 -Running \fI\%pillar.items\fP -.UNINDENT -.sp -Performing these actions will \fInot\fP refresh the in\-memory pillar data. So, if -pillar data is modified, and then states are run, the states will see the -updated pillar data, but \fI\%pillar.item\fP, -\fI\%pillar.get\fP, and \fI\%pillar.raw\fP will not see this data unless refreshed using -\fI\%saltutil.refresh_pillar\fP\&. -.sp -If you are using the Pillar Cache and have set \fI\%pillar_cache\fP to \fITrue\fP, -the pillar cache can be updated either when you run \fI\%saltutil.refresh_pillar\fP, or using the pillar runner function -\fI\%pillar.clear_pillar_cache\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run pillar.clear_pillar_cache \(aqminion\(aq +[\(dqfour\(dq, \(dqone\(dq] .ft P .fi .UNINDENT .UNINDENT +.SS \fBrandom_shuffle\fP .sp -The pillar will not be updated when running \fI\%pillar.items\fP or a state for example. If you are -using a Salt version before 3003, you would need to manually delete the cache -file, located in Salt\(aqs master cache. For example, on linux the file would be -in this directory: /var/cache/salt/master/pillar_cache/ -.SS How Pillar Environments Are Handled -.sp -When multiple pillar environments are used, the default behavior is for the -pillar data from all environments to be merged together. The pillar dictionary -will therefore contain keys from all configured environments. +New in version 3005. + .sp -The \fI\%pillarenv\fP minion config option can be used to force the -minion to only consider pillar configuration from a single environment. This -can be useful in cases where one needs to run states with alternate pillar -data, either in a testing/QA environment or to test changes to the pillar data -before pushing them live. +Returns a shuffled copy of an input list. The \fBseed\fP parameter can be used to +return a predictable outcome. .sp -For example, assume that the following is set in the minion config file: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillarenv: base +{% set my_list = [\(dqone\(dq, \(dqtwo\(dq, \(dqthree\(dq, \(dqfour\(dq] %} +{{ my_list | random_shuffle }} .ft P .fi .UNINDENT .UNINDENT .sp -This would cause that minion to ignore all other pillar environments besides -\fBbase\fP when compiling the in\-memory pillar data. Then, when running states, -the \fBpillarenv\fP CLI argument can be used to override the minion\(aqs -\fI\%pillarenv\fP config value: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.apply mystates pillarenv=testing +[\(dqfour\(dq, \(dqthree\(dq, \(dqone\(dq, \(dqtwo\(dq] .ft P .fi .UNINDENT .UNINDENT +.SS \fBset_dict_key_value\fP .sp -The above command will run the states with pillar data sourced exclusively from -the \fBtesting\fP environment, without modifying the in\-memory pillar data. +New in version 3000. + .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -When running states, the \fBpillarenv\fP CLI option does not require a -\fI\%pillarenv\fP option to be set in the minion config file. When -\fI\%pillarenv\fP is left unset, as mentioned above all configured -environments will be combined. Running states with \fBpillarenv=testing\fP in -this case would still restrict the states\(aq pillar data to just that of the -\fBtesting\fP pillar environment. -.UNINDENT -.UNINDENT +Allows you to set a value in a nested dictionary without having to worry if all the nested keys actually exist. +Missing keys will be automatically created if they do not exist. +The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. .sp -Starting in the 2017.7.0 release, it is possible to pin the pillarenv to the -effective saltenv, using the \fI\%pillarenv_from_saltenv\fP minion -config option. When this is set to \fBTrue\fP, if a specific saltenv is specified -when running states, the \fBpillarenv\fP will be the same. This essentially makes -the following two commands equivalent: +Examples: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.apply mystates saltenv=dev -salt \(aq*\(aq state.apply mystates saltenv=dev pillarenv=dev + .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B Example 1: +{%\- set foo = {} %} +{{ foo | set_dict_key_value(\(aqbar:baz\(aq, 42) }} +.TP +.B Example 2: +{{ {} | set_dict_key_value(\(aqbar.baz.qux\(aq, 42, delimiter=\(aq.\(aq) }} +.UNINDENT .sp -However, if a pillarenv is specified, it will override this behavior. So, the -following command will use the \fBqa\fP pillar environment but source the SLS -files from the \fBdev\fP saltenv: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.apply mystates saltenv=dev pillarenv=qa + .ft P .fi .UNINDENT .UNINDENT -.sp -So, if a \fBpillarenv\fP is set in the minion config file, -\fI\%pillarenv_from_saltenv\fP will be ignored, and passing a -\fBpillarenv\fP on the CLI will temporarily override -\fI\%pillarenv_from_saltenv\fP\&. -.SS Viewing Pillar Data -.sp -To view pillar data, use the \fI\%pillar\fP execution -module. This module includes several functions, each of them with their own -use. These functions include: .INDENT 0.0 -.IP \(bu 2 -\fI\%pillar.item\fP \- Retrieves the value of -one or more keys from the \fI\%in\-memory pillar data\fP\&. -.IP \(bu 2 -\fI\%pillar.items\fP \- Compiles a fresh pillar -dictionary and returns it, leaving the \fI\%in\-memory pillar data\fP untouched. If pillar keys are passed to this function -however, this function acts like \fI\%pillar.item\fP and returns their values from the \fI\%in\-memory -pillar data\fP\&. -.IP \(bu 2 -\fI\%pillar.raw\fP \- Like \fI\%pillar.items\fP, it returns the entire pillar dictionary, but -from the \fI\%in\-memory pillar data\fP instead of compiling -fresh pillar data. -.IP \(bu 2 -\fI\%pillar.get\fP \- Described in detail below. +.TP +.B Example 1: +{\(aqbar\(aq: {\(aqbaz\(aq: 42}} +.TP +.B Example 2: +{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 42}}} .UNINDENT -.SS The \fI\%pillar.get\fP Function +.SS \fBappend_dict_key_value\fP .sp -New in version 0.14.0. +New in version 3000. .sp -The \fI\%pillar.get\fP function works much in the same -way as the \fBget\fP method in a python dict, but with an enhancement: nested -dictionaries can be traversed using a colon as a delimiter. +Allows you to append to a list nested (deep) in a dictionary without having to worry if all the nested keys (or the list itself) actually exist. +Missing keys will automatically be created if they do not exist. +The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. .sp -If a structure like this is in pillar: +Examples: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -foo: - bar: - baz: qux + .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B Example 1: +{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: [1, 2]}} %} +{{ foo | append_dict_key_value(\(aqbar:baz\(aq, 42) }} +.TP +.B Example 2: +{%\- set foo = {} %} +{{ foo | append_dict_key_value(\(aqbar:baz:qux\(aq, 42) }} +.UNINDENT .sp -Extracting it from the raw pillar in an sls formula or file template is done -this way: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ pillar[\(aqfoo\(aq][\(aqbar\(aq][\(aqbaz\(aq] }} + .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B Example 1: +{\(aqbar\(aq: {\(aqbaz\(aq: [1, 2, 42]}} +.TP +.B Example 2: +{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: [42]}}} +.UNINDENT +.SS \fBextend_dict_key_value\fP .sp -Now, with the new \fI\%pillar.get\fP function the data -can be safely gathered and a default can be set, allowing the template to fall -back if the value is not available: +New in version 3000. + +.sp +Allows you to extend a list nested (deep) in a dictionary without having to worry if all the nested keys (or the list itself) actually exist. +Missing keys will automatically be created if they do not exist. +The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +.sp +Examples: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ salt[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, \(aqqux\(aq) }} + .ft P .fi .UNINDENT .UNINDENT -.sp -This makes handling nested structures much easier. -.sp -\fBNOTE:\fP .INDENT 0.0 -.INDENT 3.5 -\fBpillar.get()\fP vs \fBsalt[\(aqpillar.get\(aq]()\fP -.sp -It should be noted that within templating, the \fBpillar\fP variable is just -a dictionary. This means that calling \fBpillar.get()\fP inside of a -template will just use the default dictionary \fB\&.get()\fP function which -does not include the extra \fB:\fP delimiter functionality. It must be -called using the above syntax (\fBsalt[\(aqpillar.get\(aq](\(aqfoo:bar:baz\(aq, -\(aqqux\(aq)\fP) to get the salt function, instead of the default dictionary -behavior. -.UNINDENT +.TP +.B Example 1: +{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: [1, 2]}} %} +{{ foo | extend_dict_key_value(\(aqbar:baz\(aq, [42, 42]) }} +.TP +.B Example 2: +{{ {} | extend_dict_key_value(\(aqbar:baz:qux\(aq, [42]) }} .UNINDENT -.SS Setting Pillar Data at the Command Line .sp -Pillar data can be set at the command line like the following example: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq state.apply pillar=\(aq{\(dqcheese\(dq: \(dqspam\(dq}\(aq + .ft P .fi .UNINDENT .UNINDENT -.sp -This will add a pillar key of \fBcheese\fP with its value set to \fBspam\fP\&. -.sp -\fBNOTE:\fP .INDENT 0.0 -.INDENT 3.5 -Be aware that when sending sensitive data via pillar on the command\-line -that the publication containing that data will be received by all minions -and will not be restricted to the targeted minions. This may represent -a security concern in some cases. -.UNINDENT +.TP +.B Example 1: +{\(aqbar\(aq: {\(aqbaz\(aq: [1, 2, 42, 42]}} +.TP +.B Example 2: +{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: [42]}}} .UNINDENT -.SS Pillar Encryption -.sp -Salt\(aqs renderer system can be used to decrypt pillar data. This allows for -pillar items to be stored in an encrypted state, and decrypted during pillar -compilation. -.SS Encrypted Pillar SLS +.SS \fBupdate_dict_key_value\fP .sp -New in version 2017.7.0. +New in version 3000. .sp -Consider the following pillar SLS file: +Allows you to update a dictionary nested (deep) in another dictionary without having to worry if all the nested keys actually exist. +Missing keys will automatically be created if they do not exist. +The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +.sp +Examples: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -secrets: - vault: - foo: | - \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- - - hQEMAw2B674HRhwSAQgAhTrN8NizwUv/VunVrqa4/X8t6EUulrnhKcSeb8sZS4th - W1Qz3K2NjL4lkUHCQHKZVx/VoZY7zsddBIFvvoGGfj8+2wjkEDwFmFjGE4DEsS74 - ZLRFIFJC1iB/O0AiQ+oU745skQkU6OEKxqavmKMrKo3rvJ8ZCXDC470+i2/Hqrp7 - +KWGmaDOO422JaSKRm5D9bQZr9oX7KqnrPG9I1+UbJyQSJdsdtquPWmeIpamEVHb - VMDNQRjSezZ1yKC4kCWm3YQbBF76qTHzG1VlLF5qOzuGI9VkyvlMaLfMibriqY73 - zBbPzf6Bkp2+Y9qyzuveYMmwS4sEOuZL/PetqisWe9JGAWD/O+slQ2KRu9hNww06 - KMDPJRdyj5bRuBVE4hHkkP23KrYr7SuhW2vpe7O/MvWEJ9uDNegpMLhTWruGngJh - iFndxegN9w== - =bAuo - \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- - bar: this was unencrypted already - baz: | - \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- - hQEMAw2B674HRhwSAQf+Ne+IfsP2IcPDrUWct8sTJrga47jQvlPCmO+7zJjOVcqz - gLjUKvMajrbI/jorBWxyAbF+5E7WdG9WHHVnuoywsyTB9rbmzuPqYCJCe+ZVyqWf - 9qgJ+oUjcvYIFmH3h7H68ldqbxaAUkAOQbTRHdr253wwaTIC91ZeX0SCj64HfTg7 - Izwk383CRWonEktXJpientApQFSUWNeLUWagEr/YPNFA3vzpPF5/Ia9X8/z/6oO2 - q+D5W5mVsns3i2HHbg2A8Y+pm4TWnH6mTSh/gdxPqssi9qIrzGQ6H1tEoFFOEq1V - kJBe0izlfudqMq62XswzuRB4CYT5Iqw1c97T+1RqENJCASG0Wz8AGhinTdlU5iQl - JkLKqBxcBz4L70LYWyHhYwYROJWjHgKAywX5T67ftq0wi8APuZl9olnOkwSK+wrY - 1OZi - =7epf - \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- - qux: - \- foo - \- bar - \- | - \-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\- +.ft P +.fi +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B Example 1: +{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 1}}} %} +{{ foo | update_dict_key_value(\(aqbar:baz\(aq, {\(aqquux\(aq: 3}) }} +.TP +.B Example 2: +{{ {} | update_dict_key_value(\(aqbar:baz:qux\(aq, {\(aqquux\(aq: 3}) }} +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C - hQEMAw2B674HRhwSAQgAg1YCmokrweoOI1c9HO0BLamWBaFPTMblOaTo0WJLZoTS - ksbQ3OJAMkrkn3BnnM/djJc5C7vNs86ZfSJ+pvE8Sp1Rhtuxh25EKMqGOn/SBedI - gR6N5vGUNiIpG5Tf3DuYAMNFDUqw8uY0MyDJI+ZW3o3xrMUABzTH0ew+Piz85FDA - YrVgwZfqyL+9OQuu6T66jOIdwQNRX2NPFZqvon8liZUPus5VzD8E5cAL9OPxQ3sF - f7/zE91YIXUTimrv3L7eCgU1dSxKhhfvA2bEUi+AskMWFXFuETYVrIhFJAKnkFmE - uZx+O9R9hADW3hM5hWHKH9/CRtb0/cC84I9oCWIQPdI+AaPtICxtsD2N8Q98hhhd - 4M7I0sLZhV+4ZJqzpUsOnSpaGyfh1Zy/1d3ijJi99/l+uVHuvmMllsNmgR+ZTj0= - =LrCQ - \-\-\-\-\-END PGP MESSAGE\-\-\-\-\- .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.TP +.B Example 1: +{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 1, \(aqquux\(aq: 3}}} +.TP +.B Example 2: +{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: {\(aqquux\(aq: 3}}}} +.UNINDENT +.SS \fBmd5\fP .sp -When the pillar data is compiled, the results will be decrypted: +New in version 2017.7.0. + +.sp +Return the md5 digest of a string. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt myminion pillar.items -myminion: - \-\-\-\-\-\-\-\-\-\- - secrets: - \-\-\-\-\-\-\-\-\-\- - vault: - \-\-\-\-\-\-\-\-\-\- - bar: - this was unencrypted already - baz: - rosebud - foo: - supersecret - qux: - \- foo - \- bar - \- baz +{{ \(aqrandom\(aq | md5 }} .ft P .fi .UNINDENT .UNINDENT .sp -Salt must be told what portions of the pillar data to decrypt. This is done -using the \fI\%decrypt_pillar\fP config option: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -decrypt_pillar: - \- \(aqsecrets:vault\(aq: gpg +7ddf32e17a6ac5ce04a8ecbf782ca509 .ft P .fi .UNINDENT .UNINDENT +.SS \fBsha256\fP .sp -The notation used to specify the pillar item(s) to be decrypted is the same as -the one used in \fI\%pillar.get\fP function. +New in version 2017.7.0. + .sp -If a different delimiter is needed, it can be specified using the -\fI\%decrypt_pillar_delimiter\fP config option: +Return the sha256 digest of a string. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -decrypt_pillar: - \- \(aqsecrets|vault\(aq: gpg - -decrypt_pillar_delimiter: \(aq|\(aq +{{ \(aqrandom\(aq | sha256 }} .ft P .fi .UNINDENT .UNINDENT .sp -The name of the renderer used to decrypt a given pillar item can be omitted, -and if so it will fall back to the value specified by the -\fI\%decrypt_pillar_default\fP config option, which defaults to \fBgpg\fP\&. -So, the first example above could be rewritten as: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -decrypt_pillar: - \- \(aqsecrets:vault\(aq +a441b15fe9a3cf56661190a0b93b9dec7d04127288cc87250967cf3b52894d11 .ft P .fi .UNINDENT .UNINDENT -.SS Encrypted Pillar Data on the CLI +.SS \fBsha512\fP .sp -New in version 2016.3.0. +New in version 2017.7.0. .sp -The following functions support passing pillar data on the CLI via the -\fBpillar\fP argument: -.INDENT 0.0 -.IP \(bu 2 -\fI\%pillar.items\fP -.IP \(bu 2 -\fI\%state.apply\fP -.IP \(bu 2 -\fI\%state.highstate\fP -.IP \(bu 2 -\fI\%state.sls\fP -.UNINDENT +Return the sha512 digest of a string. .sp -Triggering decryption of this CLI pillar data can be done in one of two ways: +Example: .INDENT 0.0 -.IP 1. 3 -Using the \fBpillar_enc\fP argument: -.INDENT 3.0 .INDENT 3.5 .sp .nf .ft C -# salt myminion pillar.items pillar_enc=gpg pillar=\(aq{foo: \(dq\-\-\-\-\-BEGIN PGP MESSAGE\-\-\-\-\-\en\enhQEMAw2B674HRhwSAQf+OvPqEdDoA2fk15I5dYUTDoj1yf/pVolAma6iU4v8Zixn\enRDgWsaAnFz99FEiFACsAGDEFdZaVOxG80T0Lj+PnW4pVy0OXmXHnY2KjV9zx8FLS\enQxfvmhRR4t23WSFybozfMm0lsN8r1vfBBjbK+A72l0oxN78d1rybJ6PWNZiXi+aC\enmqIeunIbAKQ21w/OvZHhxH7cnIiGQIHc7N9nQH7ibyoKQzQMSZeilSMGr2abAHun\enmLzscr4wKMb+81Z0/fdBfP6g3bLWMJga3hSzSldU9ovu7KR8rDJI1qOlENj3Wm8C\enwTpDOB33kWIKMqiAjY3JFtb5MCHrafyggwQL7cX1+tI+AbSO6kZpbcDfzetb77LZ\enxc5NWnnGK4pGoqq4MAmZshw98RpecSHKMosto2gtiuWCuo9Zn5cV/FbjZ9CTWrQ=\en=0hO/\en\-\-\-\-\-END PGP MESSAGE\-\-\-\-\-\(dq}\(aq +{{ \(aqrandom\(aq | sha512 }} .ft P .fi .UNINDENT .UNINDENT .sp -The newlines in this example are specified using a literal \fB\en\fP\&. Newlines -can be replaced with a literal \fB\en\fP using \fBsed\fP: -.INDENT 3.0 +Returns: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ echo \-n bar | gpg \-\-armor \-\-trust\-model always \-\-encrypt \-r user@domain.tld | sed \(aq:a;N;$!ba;s/\en/\e\en/g\(aq +811a90e1c8e86c7b4c0eef5b2c0bf0ec1b19c4b1b5a242e6455be93787cb473cb7bc9b0fdeb960d00d5c6881c2094dd63c5c900ce9057255e2a4e271fc25fef1 .ft P .fi .UNINDENT .UNINDENT +.SS \fBbase64_encode\fP .sp -\fBNOTE:\fP -.INDENT 3.0 -.INDENT 3.5 -Using \fBpillar_enc\fP will perform the decryption minion\-side, so for -this to work it will be necessary to set up the keyring in -\fB/etc/salt/gpgkeys\fP on the minion just as one would typically do on -the master. The easiest way to do this is to first export the keys from -the master: +New in version 2017.7.0. + +.sp +Encode a string as base64. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# gpg \-\-homedir /etc/salt/gpgkeys \-\-export\-secret\-key \-a user@domain.tld >/tmp/keypair.gpg +{{ \(aqrandom\(aq | base64_encode }} .ft P .fi .UNINDENT .UNINDENT .sp -Then, copy the file to the minion, setup the keyring, and import: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# mkdir \-p /etc/salt/gpgkeys -# chmod 0700 /etc/salt/gpgkeys -# gpg \-\-homedir /etc/salt/gpgkeys \-\-list\-keys -# gpg \-\-homedir /etc/salt/gpgkeys \-\-import \-\-allow\-secret\-key\-import keypair.gpg +cmFuZG9t .ft P .fi .UNINDENT .UNINDENT +.SS \fBbase64_decode\fP .sp -The \fB\-\-list\-keys\fP command is run create a keyring in the newly\-created -directory. -.UNINDENT -.UNINDENT -.sp -Pillar data which is decrypted minion\-side will still be securely -transferred to the master, since the data sent between minion and master is -encrypted with the master\(aqs public key. -.IP 2. 3 -Use the \fI\%decrypt_pillar\fP option. This is less flexible in that -the pillar key passed on the CLI must be pre\-configured on the master, but -it doesn\(aqt require a keyring to be setup on the minion. One other caveat to -this method is that pillar decryption on the master happens at the end of -pillar compilation, so if the encrypted pillar data being passed on the CLI -needs to be referenced by pillar or ext_pillar \fIduring pillar compilation\fP, -it \fImust\fP be decrypted minion\-side. -.UNINDENT -.SS Adding New Renderers for Decryption -.sp -Those looking to add new renderers for decryption should look at the \fI\%gpg\fP renderer for an example of how to do so. The function -that performs the decryption should be recursive and be able to traverse a -mutable type such as a dictionary, and modify the values in\-place. -.sp -Once the renderer has been written, \fI\%decrypt_pillar_renderers\fP -should be modified so that Salt allows it to be used for decryption. -.sp -If the renderer is being submitted upstream to the Salt project, the renderer -should be added in \fI\%salt/renderers/\fP\&. Additionally, the following should be -done: -.INDENT 0.0 -.IP \(bu 2 -Both occurrences of \fI\%decrypt_pillar_renderers\fP in -\fI\%salt/config/__init__.py\fP should be updated to include the name of the new -renderer so that it is included in the default value for this config option. -.IP \(bu 2 -The documentation for the \fI\%decrypt_pillar_renderers\fP config -option in the \fI\%master config file\fP and \fI\%minion config file\fP should be -updated to show the correct new default value. -.IP \(bu 2 -The commented example for the \fI\%decrypt_pillar_renderers\fP config -option in the \fI\%master config template\fP should be updated to show the correct -new default value. -.UNINDENT -.SS Binary Data in the Pillar -.sp -Salt has partial support for binary pillar data. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -There are some situations (such as salt\-ssh) where only text (ASCII or -Unicode) is allowed. -.UNINDENT -.UNINDENT +New in version 2017.7.0. + .sp -The simplest way to embed binary data in your pillar is to make use of YAML\(aqs -built\-in binary data type, which requires base64 encoded data. +Decode a base64\-encoded string. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt_pic: !!binary - iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAMAAAC67D+PAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAA +{{ \(aqZ2V0IHNhbHRlZA==\(aq | base64_decode }} .ft P .fi .UNINDENT .UNINDENT .sp -Then you can use it as a \fBcontents_pillar\fP in a state: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/tmp/salt.png: - file.managed: - \- contents_pillar: salt_pic +get salted .ft P .fi .UNINDENT .UNINDENT +.SS \fBhmac\fP .sp -It is also possible to add ASCII\-armored encrypted data to pillars, as -mentioned in the Pillar Encryption section. -.SS Master Config in Pillar +New in version 2017.7.0. + .sp -For convenience the data stored in the master configuration file can be made -available in all minion\(aqs pillars. This makes global configuration of services -and systems very easy but may not be desired if sensitive data is stored in the -master configuration. This option is disabled by default. +Verify a challenging hmac signature against a string / shared\-secret. Returns +a boolean value. .sp -To enable the master config from being added to the pillar set -\fBpillar_opts\fP to \fBTrue\fP in the minion config file: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_opts: True +{{ \(aqget salted\(aq | hmac(\(aqshared secret\(aq, \(aqeBWf9bstXg+NiP5AOwppB5HMvZiYMPzEM9W5YMm/AmQ=\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.SS Minion Config in Pillar .sp -Minion configuration options can be set on pillars. Any option that you want -to modify, should be in the first level of the pillars, in the same way you set -the options in the config file. For example, to configure the MySQL root -password to be used by MySQL Salt execution module, set the following pillar -variable: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mysql.pass: hardtoguesspassword +True .ft P .fi .UNINDENT .UNINDENT -.SS Master Provided Pillar Error +.SS \fBhttp_query\fP .sp -By default if there is an error rendering a pillar, the detailed error is -hidden and replaced with: +New in version 2017.7.0. + +.sp +Return the HTTP reply object from a URL. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Rendering SLS \(aqmy.sls\(aq failed. Please see master log for details. +{{ \(aqhttp://jsonplaceholder.typicode.com/posts/1\(aq | http_query }} .ft P .fi .UNINDENT .UNINDENT .sp -The error is protected because it\(aqs possible to contain templating data -which would give that minion information it shouldn\(aqt know, like a password! -.sp -To have the master provide the detailed error that could potentially carry -protected data set \fBpillar_safe_render_error\fP to \fBFalse\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pillar_safe_render_error: False +{ + \(aqbody\(aq: \(aq{ + \(dquserId\(dq: 1, + \(dqid\(dq: 1, + \(dqtitle\(dq: \(dqsunt aut facere repellat provident occaecati excepturi option reprehenderit\(dq, + \(dqbody\(dq: \(dqquia et suscipit\e\ensuscipit recusandae consequuntur expedita et cum\e\enreprehenderit molestiae ut ut quas totam\e\ennostrum rerum est autem sunt rem eveniet architecto\(dq + }\(aq +} .ft P .fi .UNINDENT .UNINDENT -.SS Pillar Walkthrough +.SS \fBtraverse\fP .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -This walkthrough assumes that the reader has already completed the initial -Salt \fI\%walkthrough\fP\&. -.UNINDENT -.UNINDENT +New in version 2018.3.3. + .sp -Pillars are tree\-like structures of data defined on the Salt Master and passed -through to minions. They allow confidential, targeted data to be securely sent -only to the relevant minion. +Traverse a dict or list using a colon\-delimited target string. +The target \(aqfoo:bar:0\(aq will return data[\(aqfoo\(aq][\(aqbar\(aq][0] if this value exists, +and will otherwise return the provided default value. .sp -\fBNOTE:\fP +Example: .INDENT 0.0 .INDENT 3.5 -Grains and Pillar are sometimes confused, just remember that Grains -are data about a minion which is stored or generated from the minion. -This is why information like the OS and CPU type are found in Grains. -Pillar is information about a minion or many minions stored or generated -on the Salt Master. +.sp +.nf +.ft C +{{ {\(aqa1\(aq: {\(aqb1\(aq: {\(aqc1\(aq: \(aqfoo\(aq}}, \(aqa2\(aq: \(aqbar\(aq} | traverse(\(aqa1:b1\(aq, \(aqdefault\(aq) }} +.ft P +.fi .UNINDENT .UNINDENT .sp -Pillar data is useful for: +Returns: .INDENT 0.0 -.TP -.B Highly Sensitive Data: -Information transferred via pillar is guaranteed to only be presented to -the minions that are targeted, making Pillar suitable -for managing security information, such as cryptographic keys and -passwords. -.TP -.B Minion Configuration: -Minion modules such as the execution modules, states, and returners can -often be configured via data stored in pillar. -.TP -.B Variables: -Variables which need to be assigned to specific minions or groups of -minions can be defined in pillar and then accessed inside sls formulas -and template files. -.TP -.B Arbitrary Data: -Pillar can contain any basic data structure in dictionary format, -so a key/value store can be defined making it easy to iterate over a group -of values in sls formulas. -.UNINDENT -.sp -Pillar is therefore one of the most important systems when using Salt. This -walkthrough is designed to get a simple Pillar up and running in a few minutes -and then to dive into the capabilities of Pillar and where the data is -available. -.SS Setting Up Pillar +.INDENT 3.5 .sp -The pillar is already running in Salt by default. To see the minion\(aqs -pillar data: +.nf +.ft C +{\(dqc1\(dq: \(dqfoo\(dq} +.ft P +.fi +.UNINDENT +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq pillar.items +{{ {\(aqa1\(aq: {\(aqb1\(aq: {\(aqc1\(aq: \(aqfoo\(aq}}, \(aqa2\(aq: \(aqbar\(aq} | traverse(\(aqa2:b2\(aq, \(aqdefault\(aq) }} .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP +Returns: .INDENT 0.0 .INDENT 3.5 -Prior to version 0.16.2, this function is named \fBpillar.data\fP\&. This -function name is still supported for backwards compatibility. +.sp +.nf +.ft C +\(dqdefault\(dq +.ft P +.fi .UNINDENT .UNINDENT +.SS \fBjson_query\fP .sp -By default, the contents of the master configuration file are not loaded into -pillar for all minions. This default is stored in the \fBpillar_opts\fP setting, -which defaults to \fBFalse\fP\&. -.sp -The contents of the master configuration file can be made available to minion -pillar files. This makes global configuration of services and systems very easy, -but note that this may not be desired or appropriate if sensitive data is stored -in the master\(aqs configuration file. To enable the master configuration file to be -available to minion as pillar, set \fBpillar_opts: True\fP in the master -configuration file, and then for appropriate minions also set \fBpillar_opts: True\fP -in the minion(s) configuration file. +New in version 3000. + .sp -Similar to the state tree, the pillar is comprised of sls files and has a top file. -The default location for the pillar is in /srv/pillar. +A port of Ansible \fBjson_query\fP Jinja filter to make queries against JSON data using \fI\%JMESPath language\fP\&. +Could be used to filter \fBpillar\fP data, \fByaml\fP maps, and together with \fI\%http_query\fP\&. +Depends on the \fI\%jmespath\fP Python module. .sp -\fBNOTE:\fP +Examples: .INDENT 0.0 .INDENT 3.5 -The pillar location can be configured via the \fBpillar_roots\fP option inside -the master configuration file. It must not be in a subdirectory of the state -tree or file_roots. If the pillar is under file_roots, any pillar targeting -can be bypassed by minions. +.sp +.nf +.ft C +Example 1: {{ [1, 2, 3, 4, [5, 6]] | json_query(\(aq[]\(aq) }} + +Example 2: {{ +{\(dqmachines\(dq: [ + {\(dqname\(dq: \(dqa\(dq, \(dqstate\(dq: \(dqrunning\(dq}, + {\(dqname\(dq: \(dqb\(dq, \(dqstate\(dq: \(dqstopped\(dq}, + {\(dqname\(dq: \(dqc\(dq, \(dqstate\(dq: \(dqrunning\(dq} +]} | json_query(\(dqmachines[?state==\(aqrunning\(aq].name\(dq) }} + +Example 3: {{ +{\(dqservices\(dq: [ + {\(dqname\(dq: \(dqhttp\(dq, \(dqhost\(dq: \(dq1.2.3.4\(dq, \(dqport\(dq: 80}, + {\(dqname\(dq: \(dqsmtp\(dq, \(dqhost\(dq: \(dq1.2.3.5\(dq, \(dqport\(dq: 25}, + {\(dqname\(dq: \(dqssh\(dq, \(dqhost\(dq: \(dq1.2.3.6\(dq, \(dqport\(dq: 22}, +]} | json_query(\(dqservices[].port\(dq) }} +.ft P +.fi .UNINDENT .UNINDENT .sp -To start setting up the pillar, the /srv/pillar directory needs to be present: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mkdir /srv/pillar +Example 1: [1, 2, 3, 4, 5, 6] + +Example 2: [\(aqa\(aq, \(aqc\(aq] + +Example 3: [80, 25, 22] .ft P .fi .UNINDENT .UNINDENT +.SS \fBto_entries\fP .sp -Now create a simple top file, following the same format as the top file used for -states: +New in version 3007.0. + .sp -\fB/srv/pillar/top.sls\fP: +A port of the \fBto_entries\fP function from \fBjq\fP\&. This function converts between an object and an array of key\-value +pairs. If \fBto_entries\fP is passed an object, then for each \fBk: v\fP entry in the input, the output array includes +\fB{\(dqkey\(dq: k, \(dqvalue\(dq: v}\fP\&. The \fBfrom_entries\fP function performs the opposite conversion. \fBfrom_entries\fP accepts +\(dqkey\(dq, \(dqKey\(dq, \(dqname\(dq, \(dqName\(dq, \(dqvalue\(dq, and \(dqValue\(dq as keys. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- data +{{ {\(dqa\(dq: 1, \(dqb\(dq: 2} | to_entries }} .ft P .fi .UNINDENT .UNINDENT .sp -This top file associates the data.sls file to all minions. Now the -\fB/srv/pillar/data.sls\fP file needs to be populated: -.sp -\fB/srv/pillar/data.sls\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -info: some data +[{\(dqkey\(dq:\(dqa\(dq, \(dqvalue\(dq:1}, {\(dqkey\(dq:\(dqb\(dq, \(dqvalue\(dq:2}] .ft P .fi .UNINDENT .UNINDENT +.SS \fBfrom_entries\fP .sp -To ensure that the minions have the new pillar data, issue a command -to them asking that they fetch their pillars from the master: +New in version 3007.0. + +.sp +A port of the \fBfrom_entries\fP function from \fBjq\fP\&. This function converts between an array of key\-value pairs and an +object. If \fBfrom_entries\fP is passed an object, then the input is expected to be an array of dictionaries in the format +of \fB{\(dqkey\(dq: k, \(dqvalue\(dq: v}\fP\&. The output will be be key\-value pairs \fBk: v\fP\&. \fBfrom_entries\fP accepts \(dqkey\(dq, \(dqKey\(dq, +\(dqname\(dq, \(dqName\(dq, \(dqvalue\(dq, and \(dqValue\(dq as keys. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq saltutil.refresh_pillar +{{ [{\(dqkey\(dq:\(dqa\(dq, \(dqvalue\(dq:1}, {\(dqkey\(dq:\(dqb\(dq, \(dqvalue\(dq:2}] | from_entries }} .ft P .fi .UNINDENT .UNINDENT .sp -Now that the minions have the new pillar, it can be retrieved: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq pillar.items +{\(dqa\(dq: 1, \(dqb\(dq: 2} .ft P .fi .UNINDENT .UNINDENT +.SS \fBto_snake_case\fP .sp -The key \fBinfo\fP should now appear in the returned pillar data. -.SS More Complex Data -.sp -Unlike states, pillar files do not need to define \fBformulas\fP\&. -This example sets up user data with a UID: +New in version 3000. + .sp -\fB/srv/pillar/users/init.sls\fP: +Converts a string from camelCase (or CamelCase) to snake_case. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -users: - thatch: 1000 - shouse: 1001 - utahdave: 1002 - redbeard: 1003 +Example: {{ camelsWillLoveThis | to_snake_case }} .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP +Returns: .INDENT 0.0 .INDENT 3.5 -The same directory lookups that exist in states exist in pillar, so the -file \fBusers/init.sls\fP can be referenced with \fBusers\fP in the \fI\%top -file\fP\&. +.sp +.nf +.ft C +Example: camels_will_love_this +.ft P +.fi .UNINDENT .UNINDENT +.SS \fBto_camelcase\fP .sp -The top file will need to be updated to include this sls file: +New in version 3000. + .sp -\fB/srv/pillar/top.sls\fP: +Converts a string from snake_case to camelCase (or UpperCamelCase if so indicated). .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- data - \- users +Example 1: {{ snake_case_for_the_win | to_camelcase }} + +Example 2: {{ snake_case_for_the_win | to_camelcase(uppercamel=True) }} .ft P .fi .UNINDENT .UNINDENT .sp -Now the data will be available to the minions. To use the pillar data in a -state, you can use Jinja: -.sp -\fB/srv/salt/users/init.sls\fP +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for user, uid in pillar.get(\(aqusers\(aq, {}).items() %} -{{user}}: - user.present: - \- uid: {{uid}} -{% endfor %} +Example 1: snakeCaseForTheWin +Example 2: SnakeCaseForTheWin .ft P .fi .UNINDENT .UNINDENT +.SS \fBhuman_to_bytes\fP .sp -This approach allows for users to be safely defined in a pillar and then the -user data is applied in an sls file. -.SS Parameterizing States With Pillar -.sp -Pillar data can be accessed in state files to customise behavior for each -minion. All pillar (and grain) data applicable to each minion is substituted -into the state files through templating before being run. Typical uses -include setting directories appropriate for the minion and skipping states -that don\(aqt apply. -.sp -A simple example is to set up a mapping of package names in pillar for -separate Linux distributions: +New in version 3005. + .sp -\fB/srv/pillar/pkg/init.sls\fP: +Given a human\-readable byte string (e.g. 2G, 30MB, 64KiB), return the number of bytes. +Will return 0 if the argument has unexpected form. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pkgs: - {% if grains[\(aqos_family\(aq] == \(aqRedHat\(aq %} - apache: httpd - vim: vim\-enhanced - {% elif grains[\(aqos_family\(aq] == \(aqDebian\(aq %} - apache: apache2 - vim: vim - {% elif grains[\(aqos\(aq] == \(aqArch\(aq %} - apache: apache - vim: vim - {% endif %} +Example 1: {{ \(dq32GB\(dq | human_to_bytes }} + +Example 2: {{ \(dq32GB\(dq | human_to_bytes(handle_metric=True) }} + +Example 3: {{ \(dq32\(dq | human_to_bytes(default_unit=\(dqGiB\(dq) }} .ft P .fi .UNINDENT .UNINDENT .sp -The new \fBpkg\fP sls needs to be added to the top file: -.sp -\fB/srv/pillar/top.sls\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- data - \- users - \- pkg +Example 1: 34359738368 +Example 2: 32000000000 +Example 3: 34359738368 .ft P .fi .UNINDENT .UNINDENT +.SS Networking Filters .sp -Now the minions will auto map values based on respective operating systems -inside of the pillar, so sls files can be safely parameterized: +The following networking\-related filters are supported: +.SS \fBis_ip\fP .sp -\fB/srv/salt/apache/init.sls\fP: +New in version 2017.7.0. + +.sp +Return if a string is a valid IP Address. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -apache: - pkg.installed: - \- name: {{ pillar[\(aqpkgs\(aq][\(aqapache\(aq] }} +{{ \(aq192.168.0.1\(aq | is_ip }} .ft P .fi .UNINDENT .UNINDENT .sp -Or, if no pillar is available a default can be set as well: -.sp -\fBNOTE:\fP +Additionally accepts the following options: .INDENT 0.0 -.INDENT 3.5 -The function \fBpillar.get\fP used in this example was added to Salt in -version 0.14.0 -.UNINDENT +.IP \(bu 2 +global +.IP \(bu 2 +link\-local +.IP \(bu 2 +loopback +.IP \(bu 2 +multicast +.IP \(bu 2 +private +.IP \(bu 2 +public +.IP \(bu 2 +reserved +.IP \(bu 2 +site\-local +.IP \(bu 2 +unspecified .UNINDENT .sp -\fB/srv/salt/apache/init.sls\fP: +Example \- test if a string is a valid loopback IP address. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -apache: - pkg.installed: - \- name: {{ salt[\(aqpillar.get\(aq](\(aqpkgs:apache\(aq, \(aqhttpd\(aq) }} +{{ \(aq192.168.0.1\(aq | is_ip(options=\(aqloopback\(aq) }} .ft P .fi .UNINDENT .UNINDENT +.SS \fBis_ipv4\fP .sp -In the above example, if the pillar value \fBpillar[\(aqpkgs\(aq][\(aqapache\(aq]\fP is not -set in the minion\(aqs pillar, then the default of \fBhttpd\fP will be used. +New in version 2017.7.0. + .sp -\fBNOTE:\fP +Returns if a string is a valid IPv4 address. Supports the same options +as \fBis_ip\fP\&. .INDENT 0.0 .INDENT 3.5 -Under the hood, pillar is just a Python dict, so Python dict methods such -as \fBget\fP and \fBitems\fP can be used. +.sp +.nf +.ft C +{{ \(aq192.168.0.1\(aq | is_ipv4 }} +.ft P +.fi .UNINDENT .UNINDENT -.SS Pillar Makes Simple States Grow Easily -.sp -One of the design goals of pillar is to make simple sls formulas easily grow -into more flexible formulas without refactoring or complicating the states. +.SS \fBis_ipv6\fP .sp -A simple formula: +New in version 2017.7.0. + .sp -\fB/srv/salt/edit/vim.sls\fP: +Returns if a string is a valid IPv6 address. Supports the same options +as \fBis_ip\fP\&. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -vim: - pkg.installed: [] - -/etc/vimrc: - file.managed: - \- source: salt://edit/vimrc - \- mode: 644 - \- user: root - \- group: root - \- require: - \- pkg: vim +{{ \(aqfe80::\(aq | is_ipv6 }} .ft P .fi .UNINDENT .UNINDENT +.SS \fBipaddr\fP .sp -Can be easily transformed into a powerful, parameterized formula: +New in version 2017.7.0. + .sp -\fB/srv/salt/edit/vim.sls\fP: +From a list, returns only valid IP entries. Supports the same options +as \fBis_ip\fP\&. The list can contains also IP interfaces/networks. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -vim: - pkg.installed: - \- name: {{ pillar[\(aqpkgs\(aq][\(aqvim\(aq] }} - -/etc/vimrc: - file.managed: - \- source: {{ pillar[\(aqvimrc\(aq] }} - \- mode: 644 - \- user: root - \- group: root - \- require: - \- pkg: vim +{{ [\(aq192.168.0.1\(aq, \(aqfoo\(aq, \(aqbar\(aq, \(aqfe80::\(aq] | ipaddr }} .ft P .fi .UNINDENT .UNINDENT .sp -Where the vimrc source location can now be changed via pillar: -.sp -\fB/srv/pillar/edit/vim.sls\fP: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% if grains[\(aqid\(aq].startswith(\(aqdev\(aq) %} -vimrc: salt://edit/dev_vimrc -{% elif grains[\(aqid\(aq].startswith(\(aqqa\(aq) %} -vimrc: salt://edit/qa_vimrc -{% else %} -vimrc: salt://edit/vimrc -{% endif %} +[\(dq192.168.0.1\(dq, \(dqfe80::\(dq] .ft P .fi .UNINDENT .UNINDENT +.SS \fBipv4\fP .sp -Ensuring that the right vimrc is sent out to the correct minions. +New in version 2017.7.0. + .sp -The pillar top file must include a reference to the new sls pillar file: +From a list, returns only valid IPv4 entries. Supports the same options +as \fBis_ip\fP\&. The list can contains also IP interfaces/networks. .sp -\fB/srv/pillar/top.sls\fP: +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aq*\(aq: - \- pkg - \- edit.vim +{{ [\(aq192.168.0.1\(aq, \(aqfoo\(aq, \(aqbar\(aq, \(aqfe80::\(aq] | ipv4 }} .ft P .fi .UNINDENT .UNINDENT -.SS Setting Pillar Data on the Command Line .sp -Pillar data can be set on the command line when running \fBstate.apply - +{% endif %} .ft P .fi .UNINDENT .UNINDENT -.SS Compound matchers -.sp -Compound matchers allow very granular minion targeting using any of Salt\(aqs -matchers. The default matcher is a \fBglob\fP match, just as -with CLI and \fI\%top file\fP matching. To match using anything other than a -glob, prefix the match string with the appropriate letter from the table below, -followed by an \fB@\fP sign. -.TS -center; -|l|l|l|l|. -_ -T{ -Letter -T} T{ -Match Type -T} T{ -Example -T} T{ -\fI\%Alt Delimiter?\fP -T} -_ -T{ -G -T} T{ -Grains glob -T} T{ -\fBG@os:Ubuntu\fP -T} T{ -Yes -T} -_ -T{ -E -T} T{ -PCRE Minion ID -T} T{ -\fBE@web\ed+\e.(dev|qa|prod)\e.loc\fP -T} T{ -No -T} -_ -T{ -P -T} T{ -Grains PCRE -T} T{ -\fBP@os:(RedHat|Fedora|CentOS)\fP -T} T{ -Yes -T} -_ -T{ -L -T} T{ -List of minions -T} T{ -\fBL@minion1.example.com,minion3.domain.com or bl*.domain.com\fP -T} T{ -No -T} -_ -T{ -I -T} T{ -Pillar glob -T} T{ -\fBI@pdata:foobar\fP -T} T{ -Yes -T} -_ -T{ -J -T} T{ -Pillar PCRE -T} T{ -\fBJ@pdata:^(foo|bar)$\fP -T} T{ -Yes -T} -_ -T{ -S -T} T{ -Subnet/IP address -T} T{ -\fBS@192.168.1.0/24\fP or \fBS@192.168.1.100\fP -T} T{ -No -T} -_ -T{ -R -T} T{ -Range cluster -T} T{ -\fBR@%foo.bar\fP -T} T{ -No -T} -_ -T{ -N -T} T{ -Nodegroups -T} T{ -\fBN@group1\fP -T} T{ -No -T} -_ -.TE .sp -Matchers can be joined using boolean \fBand\fP, \fBor\fP, and \fBnot\fP operators. +If clause evaluates to \fBTrue\fP .sp -For example, the following string matches all Debian minions with a hostname -that begins with \fBwebserv\fP, as well as any minions that have a hostname which -matches the \fBregular expression\fP \fBweb\-dc1\-srv.*\fP: +or with the \fBselectattr\fP filter: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq test.version +{{ [{\(aqvalue\(aq: 1}, {\(aqvalue\(aq: 2} , {\(aqvalue\(aq: 3}] | selectattr(\(aqvalue\(aq, \(aqequalto\(aq, 3) | list }} .ft P .fi .UNINDENT .UNINDENT .sp -That same example expressed in a \fI\%top file\fP looks like the following: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aqwebserv* and G@os:Debian or E@web\-dc1\-srv.*\(aq: - \- match: compound - \- webserver +[{\(dqvalue\(dq: 3}] .ft P .fi .UNINDENT .UNINDENT +.SS \fBmatch\fP .sp -New in version 2015.8.0. - +Tests that a string matches the regex passed as an argument. .sp -Excluding a minion based on its ID is also possible: +Can be used in a \fBif\fP statement directly: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aqnot web\-dc1\-srv\(aq test.version +{% if \(aqa\(aq is match(\(aq[a\-b]\(aq) %} + < statements > +{% endif %} .ft P .fi .UNINDENT .UNINDENT .sp -Versions prior to 2015.8.0 a leading \fBnot\fP was not supported in compound -matches. Instead, something like the following was required: +If clause evaluates to \fBTrue\fP +.sp +or with the \fBselectattr\fP filter: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aq* and not G@kernel:Darwin\(aq test.version +{{ [{\(aqvalue\(aq: \(aqa\(aq}, {\(aqvalue\(aq: \(aqb\(aq}, {\(aqvalue\(aq: \(aqc\(aq}] | selectattr(\(aqvalue\(aq, \(aqmatch\(aq, \(aq[b\-e]\(aq) | list }} .ft P .fi .UNINDENT .UNINDENT .sp -Excluding a minion based on its ID was also possible: +Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aq* and not web\-dc1\-srv\(aq test.version +[{\(dqvalue\(dq: \(dqb\(dq}, {\(dqvalue\(dq: \(dqc\(dq}] .ft P .fi .UNINDENT .UNINDENT -.SS Precedence Matching .sp -Matchers can be grouped together with parentheses to explicitly declare precedence amongst groups. +Test supports additional optional arguments: \fBignorecase\fP, \fBmultiline\fP +.SS Escape filters +.SS \fBregex_escape\fP +.sp +New in version 2017.7.0. + +.sp +Allows escaping of strings so they can be interpreted literally by another function. +.sp +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aq( ms\-1 or G@id:ms\-3 ) and G@id:ms\-3\(aq test.version +regex_escape = {{ \(aqhttps://example.com?foo=bar%20baz\(aq | regex_escape }} .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP +will be rendered as: .INDENT 0.0 .INDENT 3.5 -Be certain to note that spaces are required between the parentheses and targets. Failing to obey this -rule may result in incorrect targeting! +.sp +.nf +.ft C +regex_escape = https\e:\e/\e/example\e.com\e?foo\e=bar\e%20baz +.ft P +.fi .UNINDENT .UNINDENT -.SS Alternate Delimiters +.SS Set Theory Filters +.SS \fBunique\fP .sp -New in version 2015.8.0. +New in version 2017.7.0. .sp -Matchers that target based on a key value pair use a colon (\fB:\fP) as -a delimiter. Matchers with a \fBYes\fP in the \fBAlt Delimiters\fP column -in the previous table support specifying an alternate delimiter character. +Performs set math using Jinja filters. .sp -This is done by specifying an alternate delimiter character between the leading -matcher character and the \fB@\fP pattern separator character. This avoids -incorrect interpretation of the pattern in the case that \fB:\fP is part of the -grain or pillar data structure traversal. +Example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-C \(aqJ|@foo|bar|^foo:bar$ or J!@gitrepo!https://github.com:example/project.git\(aq test.ping +unique = {{ [\(aqfoo\(aq, \(aqfoo\(aq, \(aqbar\(aq] | unique }} .ft P .fi .UNINDENT .UNINDENT -.SS Node groups -.sp -Nodegroups are declared using a compound target specification. The compound -target documentation can be found \fI\%here\fP\&. .sp -The \fI\%nodegroups\fP master config file parameter is used to define -nodegroups. Here\(aqs an example nodegroup configuration within -\fB/etc/salt/master\fP: +will be rendered as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nodegroups: - group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com\(aq - group2: \(aqG@os:Debian and foo.domain.com\(aq - group3: \(aqG@os:Debian and N@group1\(aq - group4: - \- \(aqG@foo:bar\(aq - \- \(aqor\(aq - \- \(aqG@foo:baz\(aq +unique = [\(aqfoo\(aq, \(aqbar\(aq] .ft P .fi .UNINDENT .UNINDENT +.SS Global Functions .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -The \fBL\fP within group1 is matching a list of minions, while the \fBG\fP in -group2 is matching specific grains. See the \fI\%compound matchers\fP documentation for more details. +Salt Project extends \fI\%builtin global functions\fP with these custom global functions: +.SS \fBifelse\fP .sp -As of the 2017.7.0 release of Salt, group names can also be prepended with -a dash. This brings the usage in line with many other areas of Salt. For -example: +Evaluate each pair of arguments up to the last one as a (matcher, value) +tuple, returning \fBvalue\fP if matched. If none match, returns the last +argument. +.sp +The \fBifelse\fP function is like a multi\-level if\-else statement. It was +inspired by CFEngine\(aqs \fBifelse\fP function which in turn was inspired by +Oracle\(aqs \fBDECODE\fP function. It must have an odd number of arguments (from +1 to N). The last argument is the default value, like the \fBelse\fP clause in +standard programming languages. Every pair of arguments before the last one +are evaluated as a pair. If the first one evaluates true then the second one +is returned, as if you had used the first one in a compound match +expression. Boolean values can also be used as the first item in a pair, as it +will be translated to a match that will always match (\(dq*\(dq) or never match +(\(dqSALT_IFELSE_MATCH_NOTHING\(dq) a target system. +.sp +This is essentially another way to express the \fBmatch.filter_by\fP functionality +in way that\(aqs familiar to CFEngine or Oracle users. Consider using +\fBmatch.filter_by\fP unless this function fits your workflow. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nodegroups: - \- group1: \(aqL@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com\(aq +{{ ifelse(\(aqfoo*\(aq, \(aqfooval\(aq, \(aqbar*\(aq, \(aqbarval\(aq, \(aqdefaultval\(aq, minion_id=\(aqbar03\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.UNINDENT -.UNINDENT -.sp -New in version 2015.8.0. - -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Nodegroups can reference other nodegroups as seen in \fBgroup3\fP\&. Ensure -that you do not have circular references. Circular references will be -detected and cause partial expansion with a logged error message. -.UNINDENT -.UNINDENT -.sp -New in version 2015.8.0. - -.sp -Compound nodegroups can be either string values or lists of string values. -When the nodegroup is A string value will be tokenized by splitting on -whitespace. This may be a problem if whitespace is necessary as part of a -pattern. When a nodegroup is a list of strings then tokenization will -happen for each list element as a whole. +.SS Jinja in Files .sp -To match a nodegroup on the CLI, use the \fB\-N\fP command\-line option: +\fI\%Jinja\fP can be used in the same way in managed files: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-N group1 test.version +# redis.sls +/etc/redis/redis.conf: + file.managed: + \- source: salt://redis.conf + \- template: jinja + \- context: + bind: 127.0.0.1 .ft P .fi .UNINDENT .UNINDENT -.sp -New in version 2019.2.0. - -.sp -\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The \fBN@\fP classifier historically could not be used in compound matches -within the CLI or \fI\%top file\fP, it was only recognized in the -\fI\%nodegroups\fP master config file parameter. As of the 2019.2.0 -release, this limitation no longer exists. +.sp +.nf +.ft C +# lib.sls +{% set port = 6379 %} +.ft P +.fi .UNINDENT .UNINDENT -.sp -To match a nodegroup in your \fI\%top file\fP, make sure to put \fB\- match: -nodegroup\fP on the line directly following the nodegroup name. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - group1: - \- match: nodegroup - \- webserver +# redis.conf +{% from \(aqlib.sls\(aq import port with context %} +port {{ port }} +bind {{ bind }} .ft P .fi .UNINDENT .UNINDENT .sp +As an example, configuration was pulled from the file context and from an +external template file. +.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -When adding or modifying nodegroups to a master configuration file, the -master must be restarted for those changes to be fully recognized. -.sp -A limited amount of functionality, such as targeting with \-N from the -command\-line may be available without a restart. +Macros and variables can be shared across templates. They should not start +with one or more underscores, and should be managed by one of the +following tags: \fImacro\fP, \fIset\fP, \fIload_yaml\fP, \fIload_json\fP, \fIimport_yaml\fP and +\fIimport_json\fP\&. .UNINDENT .UNINDENT -.SS Defining Nodegroups as Lists of Minion IDs +.SS Escaping Jinja .sp -A simple list of minion IDs would traditionally be defined like this: +Occasionally, it may be necessary to escape Jinja syntax. There are two ways +to do this in Jinja. One is escaping individual variables or strings and the +other is to escape entire blocks. +.sp +To escape a string commonly used in Jinja syntax such as \fB{{\fP, you can use the +following syntax: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nodegroups: - group1: L@host1,host2,host3 +{{ \(aq{{\(aq }} .ft P .fi .UNINDENT .UNINDENT .sp -They can now also be defined as a YAML list, like this: +For larger blocks that contain Jinja syntax that needs to be escaped, you can use +raw blocks: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -nodegroups: - group1: - \- host1 - \- host2 - \- host3 +{% raw %} + some text that contains jinja characters that need to be escaped +{% endraw %} .ft P .fi .UNINDENT .UNINDENT .sp -New in version 2016.11.0. - -.SS Batch Size +See the \fI\%Escaping\fP section of Jinja\(aqs documentation to learn more. .sp -The \fB\-b\fP (or \fB\-\-batch\-size\fP) option allows commands to be executed on only -a specified number of minions at a time. Both percentages and finite numbers are -supported. +A real\-word example of needing to use raw tags to escape a larger block of code +is when using \fBfile.managed\fP with the \fBcontents_pillar\fP option to manage +files that contain something like consul\-template, which shares a syntax subset +with Jinja. Raw blocks are necessary here because the Jinja in the pillar would +be rendered before the file.managed is ever called, so the Jinja syntax must be +escaped: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq \-b 10 test.version +{% raw %} +\- contents_pillar: | + job \(dqexample\-job\(dq { + + task \(dqexample\(dq { + driver = \(dqdocker\(dq -salt \-G \(aqos:RedHat\(aq \-\-batch\-size 25% apache.signal restart + config { + image = \(dqdocker\-registry.service.consul:5000/example\-job:{{key \(dqnomad/jobs/example\-job/version\(dq}}\(dq + +{% endraw %} .ft P .fi .UNINDENT .UNINDENT +.SS Calling Salt Functions .sp -This will only run test.version on 10 of the targeted minions at a time and then -restart apache on 25% of the minions matching \fBos:RedHat\fP at a time and work -through them all until the task is complete. This makes jobs like rolling web -server restarts behind a load balancer or doing maintenance on BSD firewalls -using carp much easier with salt. -.sp -The batch system maintains a window of running minions, so, if there are a -total of 150 minions targeted and the batch size is 10, then the command is -sent to 10 minions, when one minion returns then the command is sent to one -additional minion, so that the job is constantly running on 10 minions. +The Jinja renderer provides a shorthand lookup syntax for the \fBsalt\fP +dictionary of \fI\%execution function\fP\&. .sp -New in version 2016.3. +New in version 2014.7.0. -.sp -The \fB\-\-batch\-wait\fP argument can be used to specify a number of seconds to -wait after a minion returns, before sending the command to a new minion. -.SS SECO Range -.sp -SECO range is a cluster\-based metadata store developed and maintained by Yahoo! -.sp -The Range project is hosted here: -.sp -\fI\%https://github.com/ytoolshed/range\fP -.sp -Learn more about range here: -.sp -\fI\%https://github.com/ytoolshed/range/wiki/\fP -.SS Prerequisites -.sp -To utilize range support in Salt, a range server is required. Setting up a -range server is outside the scope of this document. Apache modules are included -in the range distribution. -.sp -With a working range server, cluster files must be defined. These files are -written in YAML and define hosts contained inside a cluster. Full documentation -on writing YAML range files is here: -.sp -\fI\%https://github.com/ytoolshed/range/wiki/%22yamlfile%22\-module\-file\-spec\fP -.sp -Additionally, the Python seco range libraries must be installed on the salt -master. One can verify that they have been installed correctly via the -following command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -python \-c \(aqimport seco.range\(aq +# The following two function calls are equivalent. +{{ salt[\(aqcmd.run\(aq](\(aqwhoami\(aq) }} +{{ salt.cmd.run(\(aqwhoami\(aq) }} .ft P .fi .UNINDENT .UNINDENT +.SS Debugging .sp -If no errors are returned, range is installed successfully on the salt master. -.SS Preparing Salt +The \fBshow_full_context\fP function can be used to output all variables present +in the current Jinja context. .sp -Range support must be enabled on the salt master by setting the hostname and -port of the range server inside the master configuration file: +New in version 2014.7.0. + .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -range_server: my.range.server.com:80 +Context is: {{ show_full_context()|yaml(False) }} .ft P .fi .UNINDENT .UNINDENT +.SS Logs .sp -Following this, the master must be restarted for the change to have an effect. -.SS Targeting with Range -.sp -Once a cluster has been defined, it can be targeted with a salt command by -using the \fB\-R\fP or \fB\-\-range\fP flags. -.sp -For example, given the following range YAML file being served from a range -server: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ cat /etc/range/test.yaml -CLUSTER: host1..100.test.com -APPS: - \- frontend - \- backend - \- mysql -.ft P -.fi -.UNINDENT -.UNINDENT +New in version 2017.7.0. + .sp -One might target host1 through host100 in the test.com domain with Salt as follows: +Yes, in Salt, one is able to debug a complex Jinja template using the logs. +For example, making the call: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-\-range %test:CLUSTER test.version +{%\- do salt.log.error(\(aqtesting jinja logging\(aq) \-%} .ft P .fi .UNINDENT .UNINDENT .sp -The following salt command would target three hosts: \fBfrontend\fP, \fBbackend\fP, and \fBmysql\fP: +Will insert the following message in the minion logs: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-\-range %test:APPS test.version +2017\-02\-01 01:24:40,728 [salt.module.logmod][ERROR ][3779] testing jinja logging .ft P .fi .UNINDENT .UNINDENT -.SS Loadable Matchers -.sp -New in version 2019.2.0. - -.sp -Internally targeting is implemented with chunks of code called Matchers. As of -the 2019.2.0 release, matchers can be loaded dynamically. Currently new matchers -cannot be created, but existing matchers can have their functionality altered or -extended. For more information on Matchers see -.SS Matchers +.SS Profiling .sp -New in version 3000. +New in version 3002. .sp -Matchers are modules that provide Salt\(aqs targeting abilities. As of the -3000 release, matchers can be dynamically loaded. Currently new matchers -cannot be created because the required plumbing for the CLI does not exist yet. -Existing matchers may have their functionality altered or extended. -.sp -For details of targeting methods, see the \fI\%Targeting\fP topic. -.sp -A matcher module must have a function called \fBmatch()\fP\&. This function ends up -becoming a method on the Matcher class. All matcher functions require at least -two arguments, \fBself\fP (because the function will be turned into a method), and -\fBtgt\fP, which is the actual target string. The grains and pillar matchers also -take a \fBdelimiter\fP argument and should default to \fBDEFAULT_TARGET_DELIM\fP\&. -.sp -Like other Salt loadable modules, modules that override built\-in functionality -can be placed in \fBfile_roots\fP in a special directory and then copied to the -minion through the normal sync process. \fI\%saltutil.sync_all\fP -will transfer all loadable modules, and the 3000 release introduces -\fI\%saltutil.sync_matchers\fP\&. For matchers, the directory is -\fB/srv/salt/_matchers\fP (assuming your \fBfile_roots\fP is set to the default -\fB/srv/salt\fP). +When working with a very large codebase, it becomes increasingly imperative to +trace inefficiencies with state and pillar render times. The \fIprofile\fP jinja +block enables the user to get finely detailed information on the most expensive +areas in the codebase. +.SS Profiling blocks .sp -As an example, let\(aqs modify the \fBlist\fP matcher to have the separator be a -\(aq\fB/\fP\(aq instead of the default \(aq\fB,\fP\(aq. +Any block of jinja code can be wrapped in a \fBprofile\fP block. The syntax for +a profile block is \fB{% profile as \(aq\(aq %}{% endprofile %}\fP, +where \fB\fP can be any string. The \fB\fP token will appear in the +log at the \fBprofile\fP level along with the render time of the block. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -from __future__ import absolute_import, print_function, unicode_literals -from salt.ext import six # pylint: disable=3rd\-party\-module\-not\-gated - +# /srv/salt/example.sls +{%\- profile as \(aqlocal data\(aq %} + {%\- set local_data = {\(aqcounter\(aq: 0} %} + {%\- for i in range(313377) %} + {%\- do local_data.update({\(aqcounter\(aq: i}) %} + {%\- endfor %} +{%\- endprofile %} -def match(self, tgt): - \(dq\(dq\(dq - Determines if this host is on the list - \(dq\(dq\(dq - if isinstance(tgt, six.string_types): - # The stock matcher splits on \(ga,\(ga. Change to \(ga/\(ga below. - tgt = tgt.split(\(dq/\(dq) - return bool(self.opts[\(dqid\(dq] in tgt) +test: + cmd.run: + \- name: |\- + printf \(aqdata: %s\(aq \(aq{{ local_data[\(aqcounter\(aq] }}\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Place this code in a file called \fBlist_match.py\fP in a \fB_matchers\fP directory in your -\fBfile_roots\fP\&. Sync this down to your minions with -\fI\%saltutil.sync_matchers\fP\&. -Then attempt to match with the following, replacing \fBminionX\fP with three of your minions. +The \fBprofile\fP block in the \fBexample.sls\fP state will emit the following log +statement: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \-L \(aqminion1/minion2/minion3\(aq test.ping +# salt\-call \-\-local \-l profile state.apply example +[...] +[PROFILE ] Time (in seconds) to render profile block \(aqlocal data\(aq: 0.9385035037994385 +[...] .ft P .fi .UNINDENT .UNINDENT +.SS Profiling imports .sp -Three of your minions should respond. -.sp -The current supported matchers and associated filenames are -.TS -center; -|l|l|l|. -_ -T{ -Salt CLI Switch -T} T{ -Match Type -T} T{ -Filename -T} -_ -T{ - -T} T{ -Glob -T} T{ -glob_match.py -T} -_ -T{ -\-C -T} T{ -Compound -T} T{ -compound_match.py -T} -_ -T{ -\-E -T} T{ -Perl\-Compatible -Regular Expressions -T} T{ -pcre_match.py -T} -_ -T{ -\-L -T} T{ -List -T} T{ -list_match.py -T} -_ -T{ -\-G -T} T{ -Grain -T} T{ -grain_match.py -T} -_ -T{ -\-P -T} T{ -Grain Perl\-Compatible -Regular Expressions -T} T{ -grain_pcre_match.py -T} -_ -T{ -\-N -T} T{ -Nodegroup -T} T{ -nodegroup_match.py -T} -_ -T{ -\-R -T} T{ -Range -T} T{ -range_match.py -T} -_ -T{ -\-I -T} T{ -Pillar -T} T{ -pillar_match.py -T} -_ -T{ -\-J -T} T{ -Pillar Perl\-Compatible -Regular Expressions -T} T{ -pillar_pcre.py -T} -_ -T{ -\-S -T} T{ -IP\-Classless Internet -Domain Routing -T} T{ -ipcidr_match.py -T} -_ -.TE -.SS The Salt Mine -.sp -The Salt Mine is used to collect arbitrary data from Minions and store it on -the Master. This data is then made available to all Minions via the -\fI\%salt.modules.mine\fP module. -.sp -Mine data is gathered on the Minion and sent back to the Master where only the -most recent data is maintained (if long term data is required use returners or -the external job cache). -.SS Mine vs Grains -.sp -Mine data is designed to be much more up\-to\-date than grain data. Grains are -refreshed on a very limited basis and are largely static data. Mines are -designed to replace slow peer publishing calls when Minions need data from -other Minions. Rather than having a Minion reach out to all the other Minions -for a piece of data, the Salt Mine, running on the Master, can collect it from -all the Minions every \fI\%Mine Interval\fP, resulting in -almost fresh data at any given time, with much less overhead. -.SS Mine Functions -.sp -To enable the Salt Mine the \fBmine_functions\fP option needs to be applied to a -Minion. This option can be applied via the Minion\(aqs configuration file, or the -Minion\(aqs Pillar. The \fBmine_functions\fP option dictates what functions are -being executed and allows for arguments to be passed in. The list of -functions are available in the \fBsalt.module\fP\&. If no arguments -are passed, an empty list must be added like in the \fBtest.ping\fP function in -the example below: +Using the same logic as the \fBprofile\fP block, the \fBimport_yaml\fP, +\fBimport_json\fP, and \fBimport_text\fP blocks will emit similar statements at the +\fBprofile\fP log level. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_functions: - test.ping: [] - network.ip_addrs: - interface: eth0 - cidr: 10.0.0.0/8 -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -In the example above \fI\%salt.modules.network.ip_addrs\fP has additional -filters to help narrow down the results. In the above example IP addresses -are only returned if they are on a eth0 interface and in the 10.0.0.0/8 IP -range. -.sp -Changed in version 3000. +# /srv/salt/data.sls +{%\- set values = {\(aqcounter\(aq: 0} %} +{%\- for i in range(524288) %} + {%\- do values.update({\(aqcounter\(aq: i}) %} +{%\- endfor %} -.sp -The format to define mine_functions has been changed to allow the same format -as used for module.run. The old format (above) will still be supported. -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -mine_functions: - test.ping: [] - network.ip_addrs: - \- interface: eth0 - \- cidr: 10.0.0.0/8 - test.arg: - \- isn\(aqt - \- this - \- fun - \- this: that - \- salt: stack +data: {{ values[\(aqcounter\(aq] }} .ft P .fi .UNINDENT .UNINDENT -.SS Minion\-side Access Control -.sp -New in version 3000. - -.sp -Mine functions can be targeted to only be available to specific minions. This -uses the same targeting parameters as \fI\%Targeting Minions\fP but with keywords \fBallow_tgt\fP -and \fBallow_tgt_type\fP\&. When a minion requests a function from the salt mine that -is not allowed to be requested by that minion (i.e. when looking up the combination -of \fBallow_tgt\fP and \fBallow_tgt_type\fP and the requesting minion is not in the list) -it will get no data, just as if the requested function is not present in the salt mine. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_functions: - network.ip_addrs: - \- interface: eth0 - \- cidr: 10.0.0.0/8 - \- allow_tgt: \(aqG@role:master\(aq - \- allow_tgt_type: \(aqcompound\(aq +# /srv/salt/example.sls +{%\- import_yaml \(aqdata.sls\(aq as imported %} + +test: + cmd.run: + \- name: |\- + printf \(aqdata: %s\(aq \(aq{{ imported[\(aqdata\(aq] }}\(aq .ft P .fi .UNINDENT .UNINDENT -.SS Mine Functions Aliases -.sp -Function aliases can be used to provide friendly names, usage intentions or to -allow multiple calls of the same function with different arguments. There is a -different syntax for passing positional and key\-value arguments. Mixing -positional and key\-value arguments is not supported. .sp -New in version 2014.7.0. - +For \fBimport_*\fP blocks, the \fBprofile\fP log statement has the following form: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_functions: - network.ip_addrs: [eth0] - networkplus.internal_ip_addrs: [] - internal_ip_addrs: - mine_function: network.ip_addrs - cidr: 192.168.0.0/16 - ip_list: - \- mine_function: grains.get - \- ip_interfaces +# salt\-call \-\-local \-l profile state.apply example +[...] +[PROFILE ] Time (in seconds) to render import_yaml \(aqdata.sls\(aq: 1.5500736236572266 +[...] .ft P .fi .UNINDENT .UNINDENT +.SS Python Methods .sp -Changed in version 3000. - -.sp -With the addition of the module.run\-like format for defining mine_functions, the -method of adding aliases remains similar. Just add a \fBmine_function\fP kwarg with -the name of the real function to call, making the key below \fBmine_functions\fP -the alias: +A powerful feature of jinja that is only hinted at in the official jinja +documentation is that you can use the native python methods of the +variable type. Here is the python documentation for \fI\%string methods\fP\&. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_functions: - alias_name: - \- mine_function: network.ip_addrs - \- eth0 - internal_ip_addrs: - \- mine_function: network.ip_addrs - \- cidr: 192.168.0.0/16 - ip_list: - \- mine_function: grains.get - \- ip_interfaces +{% set hostname,domain = grains.id.partition(\(aq.\(aq)[::2] %}{{ hostname }} .ft P .fi .UNINDENT .UNINDENT -.SS Mine Interval -.sp -The Salt Mine functions are executed when the Minion starts and at a given -interval by the scheduler. The default interval is every 60 minutes and can -be adjusted for the Minion via the \fBmine_interval\fP option in the minion -config: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_interval: 60 +{% set strings = grains.id.split(\(aq\-\(aq) %}{{ strings[0] }} .ft P .fi .UNINDENT .UNINDENT -.SS Mine in Salt\-SSH -.sp -As of the 2015.5.0 release of salt, salt\-ssh supports \fBmine.get\fP\&. +.SS Custom Execution Modules .sp -Because the Minions cannot provide their own \fBmine_functions\fP configuration, -we retrieve the args for specified mine functions in one of three places, -searched in the following order: -.INDENT 0.0 -.IP 1. 3 -Roster data -.IP 2. 3 -Pillar -.IP 3. 3 -Master config -.UNINDENT +Custom execution modules can be used to supplement or replace complex Jinja. Many +tasks that require complex looping and logic are trivial when using Python +in a Salt execution module. Salt execution modules are easy to write and +distribute to Salt minions. .sp -The \fBmine_functions\fP are formatted exactly the same as in normal salt, just -stored in a different location. Here is an example of a flat roster containing -\fBmine_functions\fP: +Functions in custom execution modules are available in the Salt execution +module dictionary just like the built\-in execution modules: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -test: - host: 104.237.131.248 - user: root - mine_functions: - cmd.run: [\(aqecho \(dqhello!\(dq\(aq] - network.ip_addrs: - interface: eth0 +{{ salt[\(aqmy_custom_module.my_custom_function\(aq]() }} .ft P .fi .UNINDENT .UNINDENT -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Because of the differences in the architecture of salt\-ssh, \fBmine.get\fP -calls are somewhat inefficient. Salt must make a new salt\-ssh call to each -of the Minions in question to retrieve the requested data, much like a -publish call. However, unlike publish, it must run the requested function -as a wrapper function, so we can retrieve the function args from the pillar -of the Minion in question. This results in a non\-trivial delay in -retrieving the requested data. -.UNINDENT -.UNINDENT -.SS Minions Targeting with Mine -.sp -The \fBmine.get\fP function supports various methods of \fI\%Minions targeting\fP to fetch Mine data from particular hosts, such as glob or regular -expression matching on Minion id (name), grains, pillars and \fI\%compound -matches\fP\&. See the \fI\%salt.modules.mine\fP module -documentation for the reference. -.sp -\fBNOTE:\fP .INDENT 0.0 -.INDENT 3.5 -Pillar data needs to be cached on Master for pillar targeting to work with -Mine. Read the note in \fI\%relevant section\fP\&. -.UNINDENT +.IP \(bu 2 +\fI\%How to Convert Jinja Logic to an Execution Module\fP +.IP \(bu 2 +\fI\%Writing Execution Modules\fP .UNINDENT -.SS Example +.SS Custom Jinja filters .sp -One way to use data from Salt Mine is in a State. The values can be retrieved -via Jinja and used in the SLS file. The following example is a partial HAProxy -configuration file and pulls IP addresses from all Minions with the \(dqweb\(dq grain -to add them to the pool of load balanced servers. +Given that all execution modules are available in the Jinja template, +one can easily define a custom module as in the previous paragraph +and use it as a Jinja filter. +However, please note that it will not be accessible through the pipe. .sp -\fB/srv/pillar/top.sls\fP: +For example, instead of: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -base: - \(aqG@roles:web\(aq: - \- web +{{ my_variable | my_jinja_filter }} .ft P .fi .UNINDENT .UNINDENT .sp -\fB/srv/pillar/web.sls\fP: +The user will need to define \fBmy_jinja_filter\fP function under an extension +module, say \fBmy_filters\fP and use as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mine_functions: - network.ip_addrs: [eth0] +{{ salt.my_filters.my_jinja_filter(my_variable) }} .ft P .fi .UNINDENT .UNINDENT .sp -Then trigger the minions to refresh their pillar data by running: +The greatest benefit is that you are able to access thousands of existing functions, e.g.: .INDENT 0.0 +.IP \(bu 2 +get the DNS AAAA records for a specific address using the \fI\%dnsutil\fP: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq saltutil.refresh_pillar +{{ salt.dnsutil.AAAA(\(aqwww.google.com\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.sp -Verify that the results are showing up in the pillar on the minions by -executing the following and checking for \fBnetwork.ip_addrs\fP in the output: -.INDENT 0.0 +.IP \(bu 2 +retrieve a specific field value from a \fBRedis\fP hash: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq pillar.items +{{ salt.redis.hget(\(aqfoo_hash\(aq, \(aqbar_field\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.sp -Which should show that the function is present on the minion, but not include -the output: -.INDENT 0.0 +.IP \(bu 2 +get the routes to \fB0.0.0.0/0\fP using the \fI\%NAPALM route\fP: +.INDENT 2.0 .INDENT 3.5 .sp .nf .ft C -minion1.example.com: - \-\-\-\-\-\-\-\-\-\- - mine_functions: - \-\-\-\-\-\-\-\-\-\- - network.ip_addrs: - \- eth0 +{{ salt.route.show(\(aq0.0.0.0/0\(aq) }} .ft P .fi .UNINDENT .UNINDENT +.UNINDENT +.SS Tutorials Index +.SS Autoaccept minions from Grains .sp -Mine data is typically only updated on the master every 60 minutes, this can -be modified by setting: +New in version 2018.3.0. + .sp -\fB/etc/salt/minion.d/mine.conf\fP: +To automatically accept minions based on certain characteristics, e.g. the \fBuuid\fP +you can specify certain grain values on the salt master. Minions with matching grains +will have their keys automatically accepted. .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -mine_interval: 5 -.ft P -.fi -.UNINDENT +.IP 1. 3 +Configure the autosign_grains_dir in the master config file: .UNINDENT -.sp -To force the mine data to update immediately run: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq mine.update +autosign_grains_dir: /etc/salt/autosign_grains .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.IP 2. 3 +Configure the grain values to be accepted +.UNINDENT .sp -Setup the \fI\%salt.states.file.managed\fP state in -\fB/srv/salt/haproxy.sls\fP: +Place a file named like the grain in the autosign_grains_dir and write the values that +should be accepted automatically inside that file. For example to automatically +accept minions based on their \fBuuid\fP create a file named \fB/etc/salt/autosign_grains/uuid\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -haproxy_config: - file.managed: - \- name: /etc/haproxy/config - \- source: salt://haproxy_config - \- template: jinja +8f7d68e2\-30c5\-40c6\-b84a\-df7e978a03ee +1d3c5473\-1fbc\-479e\-b0c7\-877705a0730f .ft P .fi .UNINDENT .UNINDENT .sp -Create the Jinja template in \fB/srv/salt/haproxy_config\fP: +If already running, the master must be restarted for these config changes to take effect. +.sp +The master is now setup to accept minions with either of the two specified uuids. +Multiple values must always be written into separate lines. +Lines starting with a \fB#\fP are ignored. +.INDENT 0.0 +.IP 3. 3 +Configure the minion to send the specific grains to the master in the minion config file: +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -<...file contents snipped...> - -{% for server, addrs in salt[\(aqmine.get\(aq](\(aqroles:web\(aq, \(aqnetwork.ip_addrs\(aq, tgt_type=\(aqgrain\(aq) | dictsort() %} -server {{ server }} {{ addrs[0] }}:80 check -{% endfor %} - -<...file contents snipped...> +autosign_grains: + \- uuid .ft P .fi .UNINDENT .UNINDENT .sp -In the above example, \fBserver\fP will be expanded to the \fBminion_id\fP\&. +Now you should be able to start salt\-minion and run \fBsalt\-call +state.apply\fP or any other salt commands that require master authentication. +.SS Salt as a Cloud Controller +.sp +In Salt 0.14.0, an advanced cloud control system was introduced, allowing +private cloud VMs to be managed directly with Salt. This system is generally +referred to as \fBSalt Virt\fP\&. +.sp +The Salt Virt system already exists and is installed within Salt itself. This +means that besides setting up Salt, no additional salt code needs to be +deployed. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The expr_form argument will be renamed to \fBtgt_type\fP in the 2017.7.0 -release of Salt. +The \fBlibvirt\fP python module and the \fBcerttool\fP binary are required. .UNINDENT .UNINDENT -.SS Runners .sp -Salt runners are convenience applications executed with the salt\-run command. +The main goal of Salt Virt is to facilitate a very fast and simple cloud that +can scale and is fully featured. Salt Virt comes with the ability to set up and +manage complex virtual machine networking, powerful image and disk management, +and virtual machine migration with and without shared storage. .sp -Salt runners work similarly to Salt execution modules however they execute on the -Salt master itself instead of remote Salt minions. +This means that Salt Virt can be used to create a cloud from a blade center +and a SAN, but can also create a cloud out of a swarm of Linux Desktops +without a single shared storage system. Salt Virt can make clouds from +truly commodity hardware, but can also stand up the power of specialized +hardware as well. +.SS Setting up Hypervisors .sp -A Salt runner can be a simple client call or a complex application. +The first step to set up the hypervisors involves getting the correct software +installed and setting up the hypervisor network interfaces. +.SS Installing Hypervisor Software .sp -\fBSEE ALSO:\fP +Salt Virt is made to be hypervisor agnostic but currently, the only fully +implemented hypervisor is KVM via libvirt. +.sp +The required software for a hypervisor is libvirt and kvm. For advanced +features, install libguestfs or qemu\-nbd. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -\fI\%The full list of runners\fP +Libguestfs and qemu\-nbd allow for virtual machine images to be mounted +before startup and get pre\-seeded with configurations and a salt minion. .UNINDENT .UNINDENT -.SS Writing Salt Runners .sp -A Salt runner is written in a similar manner to a Salt execution module. -Both are Python modules which contain functions and each public function -is a runner which may be executed via the \fIsalt\-run\fP command. +This sls will set up the needed software for a hypervisor, and run the routines +to set up the libvirt pki keys. .sp -For example, if a Python module named \fBtest.py\fP is created in the runners -directory and contains a function called \fBfoo\fP, the \fBtest\fP runner could be -invoked with the following command: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -# salt\-run test.foo -.ft P -.fi +Package names and setup used is Red Hat specific. Different package names +will be required for different platforms. .UNINDENT .UNINDENT -.sp -Runners have several options for controlling output. -.sp -Any \fBprint\fP statement in a runner is automatically also -fired onto the master event bus where. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -def a_runner(outputter=None, display_progress=False): - print(\(dqHello world\(dq) - ... +libvirt: + pkg.installed: [] + file.managed: + \- name: /etc/sysconfig/libvirtd + \- contents: \(aqLIBVIRTD_ARGS=\(dq\-\-listen\(dq\(aq + \- require: + \- pkg: libvirt + virt.keys: + \- require: + \- pkg: libvirt + service.running: + \- name: libvirtd + \- require: + \- pkg: libvirt + \- network: br0 + \- libvirt: libvirt + \- watch: + \- file: libvirt + +libvirt\-python: + pkg.installed: [] + +libguestfs: + pkg.installed: + \- pkgs: + \- libguestfs + \- libguestfs\-tools .ft P .fi .UNINDENT .UNINDENT +.SS Hypervisor Network Setup .sp -The above would result in an event fired as follows: +The hypervisors will need to be running a network bridge to serve up network +devices for virtual machines. This formula will set up a standard bridge on +a hypervisor connecting the bridge to eth0: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Event fired at Tue Jan 13 15:26:45 2015 -************************* -Tag: salt/run/20150113152644070246/print -Data: -{\(aq_stamp\(aq: \(aq2015\-01\-13T15:26:45.078707\(aq, - \(aqdata\(aq: \(aqhello\(aq, - \(aqoutputter\(aq: \(aqpprint\(aq} +eth0: + network.managed: + \- enabled: True + \- type: eth + \- bridge: br0 + +br0: + network.managed: + \- enabled: True + \- type: bridge + \- proto: dhcp + \- require: + \- network: eth0 .ft P .fi .UNINDENT .UNINDENT +.SS Virtual Machine Network Setup .sp -A runner may also send a progress event, which is displayed to the user during -runner execution and is also passed across the event bus if the \fBdisplay_progress\fP -argument to a runner is set to True. +Salt Virt comes with a system to model the network interfaces used by the +deployed virtual machines. By default, a single interface is created for the +deployed virtual machine and is bridged to \fBbr0\fP\&. To get going with the +default networking setup, ensure that the bridge interface named \fBbr0\fP exists +on the hypervisor and is bridged to an active network device. .sp -A custom runner may send its own progress event by using the -\fB__jid_event_.fire_event()\fP method as shown here: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +To use more advanced networking in Salt Virt, read the \fISalt Virt +Networking\fP document: .sp -.nf -.ft C -if display_progress: - __jid_event__.fire_event({\(dqmessage\(dq: \(dqA progress message\(dq}, \(dqprogress\(dq) -.ft P -.fi +\fI\%Salt Virt Networking\fP .UNINDENT .UNINDENT +.SS Libvirt State .sp -The above would produce output on the console reading: \fBA progress message\fP -as well as an event on the event similar to: +One of the challenges of deploying a libvirt based cloud is the distribution +of libvirt certificates. These certificates allow for virtual machine +migration. Salt comes with a system used to auto deploy these certificates. +Salt manages the signing authority key and generates keys for libvirt clients +on the master, signs them with the certificate authority, and uses pillar to +distribute them. This is managed via the \fBlibvirt\fP state. Simply execute this +formula on the minion to ensure that the certificate is in place and up to +date: +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The above formula includes the calls needed to set up libvirt keys. +.UNINDENT +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Event fired at Tue Jan 13 15:21:20 2015 -************************* -Tag: salt/run/20150113152118341421/progress -Data: -{\(aq_stamp\(aq: \(aq2015\-01\-13T15:21:20.390053\(aq, - \(aqmessage\(aq: \(dqA progress message\(dq} +libvirt_keys: + virt.keys .ft P .fi .UNINDENT .UNINDENT +.SS Getting Virtual Machine Images Ready .sp -A runner could use the same approach to send an event with a customized tag -onto the event bus by replacing the second argument (\fBprogress\fP) with -whatever tag is desired. However, this will not be shown on the command\-line -and will only be fired onto the event bus. -.SS Synchronous vs. Asynchronous +Salt Virt requires that virtual machine images be provided as these are not +generated on the fly. Generating these virtual machine images differs greatly +based on the underlying platform. .sp -A runner may be fired asynchronously which will immediately return control. In -this case, no output will be display to the user if \fBsalt\-run\fP is being used -from the command\-line. If used programmatically, no results will be returned. -If results are desired, they must be gathered either by firing events on the -bus from the runner and then watching for them or by some other means. +Virtual machine images can be manually created using KVM and running through +the installer, but this process is not recommended since it is very manual and +prone to errors. .sp -\fBNOTE:\fP +Virtual Machine generation applications are available for many platforms: .INDENT 0.0 +.TP +.B kiwi: (openSUSE, SLES, RHEL, CentOS) +\fI\%https://opensuse.github.io/kiwi/\fP +.TP +.B vm\-builder: +\fI\%https://wiki.debian.org/VMBuilder\fP +.sp +\fBSEE ALSO:\fP +.INDENT 7.0 .INDENT 3.5 -When running a runner in asynchronous mode, the \fB\-\-progress\fP flag will -not deliver output to the salt\-run CLI. However, progress events will -still be fired on the bus. +\fI\%url vmbuilder\-formula\fP +.UNINDENT .UNINDENT .UNINDENT .sp -In synchronous mode, which is the default, control will not be returned until -the runner has finished executing. +Once virtual machine images are available, the easiest way to make them +available to Salt Virt is to place them in the Salt file server. Just copy an +image into \fB/srv/salt\fP and it can now be used by Salt Virt. .sp -To add custom runners, put them in a directory and add it to -\fI\%runner_dirs\fP in the master configuration file. -.SS Examples +For purposes of this demo, the file name \fBcentos.img\fP will be used. +.SS Existing Virtual Machine Images .sp -Examples of runners can be found in the Salt distribution: +Many existing Linux distributions distribute virtual machine images which +can be used with Salt Virt. Please be advised that NONE OF THESE IMAGES ARE +SUPPORTED BY SALTSTACK. +.SS CentOS .sp -\fI\%salt/runners\fP +These images have been prepared for OpenNebula but should work without issue with +Salt Virt, only the raw qcow image file is needed: +\fI\%https://wiki.centos.org/Cloud/OpenNebula\fP +.SS Fedora Linux .sp -A simple runner that returns a well\-formatted list of the minions that are -responding to Salt calls could look like this: -.INDENT 0.0 -.INDENT 3.5 +Images for Fedora Linux can be found here: +\fI\%https://alt.fedoraproject.org/cloud\fP +.SS openSUSE .sp -.nf -.ft C -# Import salt modules -import salt.client - - -def up(): - \(dq\(dq\(dq - Print a list of all of the minions that are up - \(dq\(dq\(dq - client = salt.client.LocalClient(__opts__[\(dqconf_file\(dq]) - minions = client.cmd(\(dq*\(dq, \(dqtest.version\(dq, timeout=1) - for minion in sorted(minions): - print(minion) -.ft P -.fi -.UNINDENT -.UNINDENT -.SS Salt Engines +\fI\%https://download.opensuse.org/distribution/leap/15.1/jeos/openSUSE\-Leap\-15.1\-JeOS.x86_64\-15.1.0\-kvm\-and\-xen\-Current.qcow2.meta4\fP +.SS SUSE .sp -New in version 2015.8.0. - +\fI\%https://www.suse.com/products/server/jeos\fP +.SS Ubuntu Linux .sp -Salt Engines are long\-running, external system processes that leverage Salt. -.INDENT 0.0 -.IP \(bu 2 -Engines have access to Salt configuration, execution modules, and runners (\fB__opts__\fP, \fB__salt__\fP, and \fB__runners__\fP). -.IP \(bu 2 -Engines are executed in a separate process that is monitored by Salt. If a Salt engine stops, it is restarted automatically. -.IP \(bu 2 -Engines can run on the Salt master and on Salt minions. -.UNINDENT +Images for Ubuntu Linux can be found here: +\fI\%http://cloud\-images.ubuntu.com/\fP +.SS Using Salt Virt .sp -Salt engines enhance and replace the \fI\%external processes\fP functionality. -.SS Configuration +With hypervisors set up and virtual machine images ready, Salt can start +issuing cloud commands using the \fIvirt runner\fP\&. .sp -Salt engines are configured under an \fBengines\fP top\-level section in your Salt master or Salt minion configuration. Provide a list of engines and parameters under this section. +Start by running a Salt Virt hypervisor info command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -engines: - \- logstash: - host: log.my_network.com - port: 5959 - proto: tcp +salt\-run virt.host_info .ft P .fi .UNINDENT .UNINDENT .sp -New in version 3000. - +This will query the running hypervisor(s) for stats and display useful +information such as the number of CPUs and amount of memory. .sp -Multiple copies of a particular Salt engine can be configured by including the \fBengine_module\fP parameter in the engine configuration. +You can also list all VMs and their current states on all hypervisor nodes: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -engines: - \- production_logstash: - host: production_log.my_network.com - port: 5959 - proto: tcp - engine_module: logstash - \- develop_logstash: - host: develop_log.my_network.com - port: 5959 - proto: tcp - engine_module: logstash +salt\-run virt.list .ft P .fi .UNINDENT .UNINDENT .sp -Salt engines must be in the Salt path, or you can add the \fBengines_dirs\fP option in your Salt master configuration with a list of directories under which Salt attempts to find Salt engines. This option should be formatted as a list of directories to search, such as: +Now that hypervisors are available a virtual machine can be provisioned, the +\fBvirt.init\fP routine will create a new virtual machine: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -engines_dirs: - \- /home/bob/engines +salt\-run virt.init centos1 2 512 salt://centos.img .ft P .fi .UNINDENT .UNINDENT -.SS Writing an Engine -.sp -An example Salt engine, \fI\%salt/engines/test.py\fP, is available in the Salt source. To develop an engine, the only requirement is that your module implement the \fBstart()\fP function. -.SS What is YAML and How To Use It .sp -The default renderer for SLS files is the YAML renderer. -.SS What is YAML +The Salt Virt runner will now automatically select a hypervisor to deploy +the new virtual machine on. Using \fBsalt://\fP assumes that the CentOS virtual +machine image is located in the root of the \fI\%Salt File Server\fP on the master. +When images are cloned (i.e. copied locally after retrieval from the file +server), the destination directory on the hypervisor minion is determined by the +\fBvirt:images\fP config option; by default this is \fB/srv/salt\-images/\fP\&. .sp -What does YAML stand for? It\(aqs an acronym for \fIYAML Ain\(aqt Markup Language\fP\&. +When a VM is initialized using \fBvirt.init\fP, the image is copied to the +hypervisor using \fBcp.cache_file\fP and will be mounted and seeded with a minion. +Seeding includes setting pre\-authenticated keys on the new machine. A minion +will only be installed if one can not be found on the image using the default +arguments to \fBseed.apply\fP\&. .sp -\fI\%The Official YAML Website\fP defines YAML as: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -\fI\&...a human friendly data serialization\fP -\fIstandard for all programming languages.\fP +The biggest bottleneck in starting VMs is when the Salt Minion needs to be +installed. Making sure that the source VM images already have Salt +installed will GREATLY speed up virtual machine deployment. .UNINDENT .UNINDENT .sp -However, Salt uses a small subset of YAML that maps over very commonly used data -structures, like lists and dictionaries. It is the job of the YAML renderer to -take the YAML data structure and compile it into a Python data structure for use -by Salt. -.SS Defining YAML -.sp -Though YAML syntax may seem daunting and terse at first, there are only -three very simple rules to remember when writing YAML for SLS files. -.SS Rule One: Indentation -.sp -YAML uses a fixed indentation scheme to represent relationships between -data layers. Salt requires that the indentation for each level consists -of exactly two spaces. Do not use tabs. -.SS Rule Two: Colons -.sp -Python dictionaries are, of course, simply key\-value pairs. Users from other -languages may recognize this data type as hashes or associative arrays. -.sp -Dictionary keys are represented in YAML as strings terminated by a trailing -colon. Values are represented by either a string following the colon, -separated by a space: +You can also deploy an image on a particular minion by directly calling the +\fBvirt\fP execution module with an absolute image path. This can be quite handy for +testing: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_key: my_value +salt \(aqhypervisor*\(aq virt.init centos1 2 512 image=/var/lib/libvirt/images/centos.img .ft P .fi .UNINDENT .UNINDENT .sp -In Python, the above maps to: +Now that the new VM has been prepared, it can be seen via the \fBvirt.query\fP +command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqmy_key\(dq: \(dqmy_value\(dq} +salt\-run virt.query .ft P .fi .UNINDENT .UNINDENT .sp -Alternatively, a value can be associated with a key through indentation. +This command will return data about all of the hypervisors and respective +virtual machines. +.sp +Now that the new VM is booted, it should have contacted the Salt Master. A +\fBtest.ping\fP will reveal if the new VM is running. +.SS QEMU Copy on Write Support +.sp +For fast image cloning, you can use the \fI\%qcow\fP disk image format. +Pass the \fBenable_qcow\fP flag and a \fI\&.qcow2\fP image path to \fIvirt.init\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_key: - my_value +salt \(aqhypervisor*\(aq virt.init centos1 2 512 image=/var/lib/libvirt/images/centos.qcow2 enable_qcow=True start=False .ft P .fi .UNINDENT @@ -37096,3186 +35979,4410 @@ my_key: \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The above syntax is valid YAML but is uncommon in SLS files because most often, -the value for a key is not singular but instead is a \fIlist\fP of values. +Beware that attempting to boot a qcow image too quickly after cloning +can result in a race condition where libvirt may try to boot the machine +before image seeding has completed. For that reason, it is recommended to +also pass \fBstart=False\fP to \fBvirt.init\fP\&. +.sp +Also know that you \fBmust not\fP modify the original base image without +first making a copy and then \fIrebasing\fP all overlay images onto it. +See the \fBqemu\-img rebase\fP \fI\%usage docs\fP\&. .UNINDENT .UNINDENT +.SS Migrating Virtual Machines .sp -In Python, the above maps to: +Salt Virt comes with full support for virtual machine migration. Using +the libvirt state in the above formula makes migration possible. +.sp +A few things need to be available to support migration. Many operating systems +turn on firewalls when originally set up; the firewall needs to be opened up +to allow for libvirt and kvm to cross communicate and execution migration +routines. On Red Hat based hypervisors in particular, port 16514 needs to be +opened on hypervisors: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqmy_key\(dq: \(dqmy_value\(dq} +iptables \-A INPUT \-m state \-\-state NEW \-m tcp \-p tcp \-\-dport 16514 \-j ACCEPT .ft P .fi .UNINDENT .UNINDENT .sp -Dictionaries can be nested: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +More in\-depth information regarding distribution specific firewall settings can be found in: +.sp +\fI\%Opening the Firewall up for Salt\fP +.UNINDENT +.UNINDENT +.sp +Salt also needs the \fBvirt:tunnel\fP option to be turned on. This flag tells Salt +to run migrations securely via the libvirt TLS tunnel and to use port 16514. +Without \fBvirt:tunnel\fP, libvirt tries to bind to random ports when running +migrations. +.sp +To turn on \fBvirt:tunnel\fP, simply apply it to the master config file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -first_level_dict_key: - second_level_dict_key: value_in_second_level_dict +virt: + tunnel: True .ft P .fi .UNINDENT .UNINDENT .sp -And in Python: +Once the master config has been updated, restart the master and send out a call +to the minions to refresh the pillar to pick up on the change: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqfirst_level_dict_key\(dq: {\(dqsecond_level_dict_key\(dq: \(dqvalue_in_second_level_dict\(dq}} +salt \e* saltutil.refresh_modules .ft P .fi .UNINDENT .UNINDENT -.SS Rule Three: Dashes .sp -To represent lists of items, a single dash followed by a space is used. Multiple -items are a part of the same list as a function of their having the same level of indentation. +Now, migration routines can be run! To migrate a VM, simply run the Salt Virt +migrate routine: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\- list_value_one -\- list_value_two -\- list_value_three +salt\-run virt.migrate centos .ft P .fi .UNINDENT .UNINDENT +.SS VNC Consoles .sp -Lists can be the value of a key\-value pair. This is quite common in Salt: +Although not enabled by default, Salt Virt can also set up VNC consoles allowing +for remote visual consoles to be opened up. When creating a new VM using +\fBvirt.init\fP, pass the \fBenable_vnc=True\fP parameter to have a console +configured for the new VM. +.sp +The information from a \fBvirt.query\fP routine will display the VNC console port +for the specific VMs: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -my_dictionary: - \- list_value_one - \- list_value_two - \- list_value_three +centos + CPU: 2 + Memory: 524288 + State: running + Graphics: vnc \- hyper6:5900 + Disk \- vda: + Size: 2.0G + File: /srv/salt\-images/ubuntu2/system.qcow2 + File Format: qcow2 + Nic \- ac:de:48:98:08:77: + Source: br0 + Type: bridge .ft P .fi .UNINDENT .UNINDENT .sp -In Python, the above maps to: +The line \fIGraphics: vnc \- hyper6:5900\fP holds the key. First the port named, +in this case 5900, will need to be available in the hypervisor\(aqs firewall. +Once the port is open, then the console can be easily opened via vncviewer: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqmy_dictionary\(dq: [\(dqlist_value_one\(dq, \(dqlist_value_two\(dq, \(dqlist_value_three\(dq]} +vncviewer hyper6:5900 .ft P .fi .UNINDENT .UNINDENT -.SS Learning more about YAML .sp -One easy way to learn more about how YAML gets rendered into Python data structures is -to use an online YAML parser to see the Python output. +By default there is no VNC security set up on these ports, which suggests that +keeping them firewalled and mandating that SSH tunnels be used to access these +VNC interfaces. Keep in mind that activity on a VNC interface that is accessed +can be viewed by any other user that accesses that same VNC interface, and any +other user logging in can also operate with the logged in user on the virtual +machine. +.SS Conclusion .sp -Here are some excellent links for experimenting with and referencing YAML: -.INDENT 0.0 -.IP \(bu 2 -\fI\%Online YAML Parser\fP: Convert YAML -to JSON or Python data structures. -.IP \(bu 2 -\fI\%The Official YAML Specification\fP -.IP \(bu 2 -\fI\%The Wikipedia page for YAML\fP -.UNINDENT -.SS Templating +Now with Salt Virt running, new hypervisors can be seamlessly added just by +running the above states on new bare metal machines, and these machines will be +instantly available to Salt Virt. +.SS Running Salt States and Commands in Docker Containers .sp -Jinja statements and expressions are allowed by default in SLS files. See -\fI\%Understanding Jinja\fP\&. -.SS Understanding Jinja +The 2016.11.0 release of Salt introduces the ability to execute Salt States +and Salt remote execution commands directly inside of Docker containers. .sp -\fI\%Jinja\fP is the default templating language in SLS files. +This addition makes it possible to not only deploy fresh containers using +Salt States. This also allows for running containers to be audited and +modified using Salt, but without running a Salt Minion inside the container. +Some of the applications include security audits of running containers as +well as gathering operating data from containers. .sp -\fBIMPORTANT:\fP -.INDENT 0.0 -.INDENT 3.5 -\fI\%Jinja\fP supports a \fI\%secure, sandboxed template execution environment\fP that Salt -takes advantage of. Other text \fI\%Renderers\fP do not support this -functionality, so Salt highly recommends usage of \fBjinja\fP / \fBjinja|yaml\fP\&. -.UNINDENT -.UNINDENT -.SS Jinja in States +This new feature is simple and straightforward, and can be used via a running +Salt Minion, the Salt Call command, or via Salt SSH. For this tutorial we will +use the \fIsalt\-call\fP command, but like all salt commands these calls are +directly translatable to \fIsalt\fP and \fIsalt\-ssh\fP\&. +.SS Step 1 \- Install Docker .sp -Jinja is evaluated before YAML, which means it is evaluated before the States -are run. +Since setting up Docker is well covered in the Docker documentation we will +make no such effort to describe it here. Please see the Docker Installation +Documentation for installing and setting up Docker: +\fI\%https://docs.docker.com/engine/installation/\fP .sp -The most basic usage of Jinja in state files is using control structures to -wrap conditional or redundant state elements: +The Docker integration also requires that the \fIdocker\-py\fP library is installed. +This can easily be done using pip or via your system package manager: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% if grains[\(aqos\(aq] != \(aqFreeBSD\(aq %} -tcsh: - pkg: - \- installed -{% endif %} - -motd: - file.managed: - {% if grains[\(aqos\(aq] == \(aqFreeBSD\(aq %} - \- name: /etc/motd - {% elif grains[\(aqos\(aq] == \(aqDebian\(aq %} - \- name: /etc/motd.tail - {% endif %} - \- source: salt://motd +pip install docker\-py .ft P .fi .UNINDENT .UNINDENT +.SS Step 2 \- Install Salt .sp -In this example, the first \fBif\fP block will only be evaluated on minions that -aren\(aqt running FreeBSD, and the second block changes the file name based on the -\fIos\fP grain. +For this tutorial we will be using Salt Call, which is available in the +\fIsalt\-minion\fP package, please follow the +\fI\%Salt install guide\fP\&. +.SS Step 3 \- Create With Salt States .sp -Writing \fBif\-else\fP blocks can lead to very redundant state files however. In -this case, using \fI\%pillars\fP, or using a previously -defined variable might be easier: +Next some Salt States are needed, for this example a very basic state which +installs \fIvim\fP is used, but anything Salt States can do can be done here, +please see the Salt States Introduction Tutorial to learn more about Salt +States: +\fI\%https://docs.saltproject.io/en/stage/getstarted/config/\fP +.sp +For this tutorial, simply create a small state file in \fI/srv/salt/vim.sls\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set motd = [\(aq/etc/motd\(aq] %} -{% if grains[\(aqos\(aq] == \(aqDebian\(aq %} - {% set motd = [\(aq/etc/motd.tail\(aq, \(aq/var/run/motd\(aq] %} -{% endif %} - -{% for motdfile in motd %} -{{ motdfile }}: - file.managed: - \- source: salt://motd -{% endfor %} +vim: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -Using a variable set by the template, the \fI\%for loop\fP will iterate over the -list of MOTD files to update, adding a state block for each file. -.sp -The filter_by function can also be used to set variables based on grains: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +The base image you choose will need to have python 2.6 or 2.7 installed. +We are hoping to resolve this constraint in a future release. .sp -.nf -.ft C -{% set auditd = salt[\(aqgrains.filter_by\(aq]({ -\(aqRedHat\(aq: { \(aqpackage\(aq: \(aqaudit\(aq }, -\(aqDebian\(aq: { \(aqpackage\(aq: \(aqauditd\(aq }, -}) %} -.ft P -.fi +If \fIbase\fP is omitted the default image used is a minimal openSUSE +image with Python support, maintained by SUSE .UNINDENT .UNINDENT -.SS Include and Import .sp -Includes and \fI\%imports\fP can be used to share common, reusable state configuration -between state files and between files. +Next run the \fIdocker.sls_build\fP command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% from \(aqlib.sls\(aq import test %} +salt\-call \-\-local dockerng.sls_build test base=my_base_image mods=vim .ft P .fi .UNINDENT .UNINDENT .sp -This would import the \fBtest\fP template variable or macro, not the \fBtest\fP -state element, from the file \fBlib.sls\fP\&. In the case that the included file -performs checks against grains, or something else that requires context, passing -the context into the included file is required: +Now we have a fresh image called \fItest\fP to work with and vim has been +installed. +.SS Step 4 \- Running Commands Inside the Container +.sp +Salt can now run remote execution functions inside the container with another +simple \fIsalt\-call\fP command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% from \(aqlib.sls\(aq import test with context %} +salt\-call \-\-local dockerng.call test test.version +salt\-call \-\-local dockerng.call test network.interfaces +salt\-call \-\-local dockerng.call test disk.usage +salt\-call \-\-local dockerng.call test pkg.list_pkgs +salt\-call \-\-local dockerng.call test service.running httpd +salt\-call \-\-local dockerng.call test cmd.run \(aqls \-l /etc\(aq .ft P .fi .UNINDENT .UNINDENT +.SS Automatic Updates / Frozen Deployments .sp -Includes must use full paths, like so: +New in version 0.10.3.d. + .sp -spam/eggs.jinja +Salt has support for the +\fI\%Esky\fP application freezing and update +tool. This tool allows one to build a complete zipfile out of the salt scripts +and all their dependencies \- including shared objects / DLLs. +.SS Getting Started +.sp +To build frozen applications, suitable build environment will be needed for +each platform. You should probably set up a virtualenv in order to limit the +scope of Q/A. +.sp +This process does work on Windows. Directions are available at +\fI\%https://github.com/saltstack/salt\-windows\-install\fP for details on +installing Salt in Windows. Only the 32\-bit Python and dependencies have been +tested, but they have been tested on 64\-bit Windows. +.sp +Install \fBbbfreeze\fP, and then \fBesky\fP from PyPI in order to enable the +\fBbdist_esky\fP command in \fBsetup.py\fP\&. Salt itself must also be installed, in +addition to its dependencies. +.SS Building and Freezing +.sp +Once you have your tools installed and the environment configured, use +\fBsetup.py\fP to prepare the distribution files. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - {% include \(aqspam/foobar.jinja\(aq %} +python setup.py sdist +python setup.py bdist .ft P .fi .UNINDENT .UNINDENT -.SS Including Context During Include/Import .sp -By adding \fBwith context\fP to the include/import directive, the -current context can be passed to an included/imported template. +Once the distribution files are in place, Esky can be used traverse the module +tree and pack all the scripts up into a redistributable. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% import \(aqopenssl/vars.sls\(aq as ssl with context %} +python setup.py bdist_esky .ft P .fi .UNINDENT .UNINDENT -.SS Macros .sp -\fI\%Macros\fP are helpful for eliminating redundant code. Macros are most useful as -mini\-templates to repeat blocks of strings with a few parameterized variables. -Be aware that stripping whitespace from the template block, as well as -contained blocks, may be necessary to emulate a variable return from the macro. +There will be an appropriately versioned \fBsalt\-VERSION.zip\fP in \fBdist/\fP if +everything went smoothly. +.SS Windows +.sp +\fBC:\ePython27\elib\esite\-packages\ezmq\fP will need to be added to the PATH +variable. This helps bbfreeze find the zmq DLL so it can pack it up. +.SS Using the Frozen Build +.sp +Unpack the zip file in the desired install location. Scripts like +\fBsalt\-minion\fP and \fBsalt\-call\fP will be in the root of the zip file. The +associated libraries and bootstrapping will be in the directories at the same +level. (Check the \fI\%Esky\fP documentation +for more information) +.sp +To support updating your minions in the wild, put the builds on a web server +that the minions can reach. \fI\%salt.modules.saltutil.update()\fP will +trigger an update and (optionally) a restart of the minion service under the +new version. +.SS Troubleshooting +.SS A Windows minion isn\(aqt responding +.sp +The process dispatch on Windows is slower than it is on *nix. It may be +necessary to add \(aq\-t 15\(aq to salt commands to give minions plenty of time to +return. +.SS Windows and the Visual Studio Redist +.sp +The Visual C++ 2008 32\-bit redistributable will need to be installed on all +Windows minions. Esky has an option to pack the library into the zipfile, +but OpenSSL does not seem to acknowledge the new location. If a +\fBno OPENSSL_Applink\fP error appears on the console when trying to start a +frozen minion, the redistributable is not installed. +.SS Mixed Linux environments and Yum +.sp +The Yum Python module doesn\(aqt appear to be available on any of the standard +Python package mirrors. If RHEL/CentOS systems need to be supported, the frozen +build should created on that platform to support all the Linux nodes. Remember +to build the virtualenv with \fB\-\-system\-site\-packages\fP so that the \fByum\fP +module is included. +.SS Automatic (Python) module discovery +.sp +Automatic (Python) module discovery does not work with the late\-loaded scheme +that Salt uses for (Salt) modules. Any misbehaving modules will need to be +explicitly added to the \fBfreezer_includes\fP in Salt\(aqs \fBsetup.py\fP\&. Always +check the zipped application to make sure that the necessary modules were +included. +.SS ESXi Proxy Minion +.sp +New in version 2015.8.4. + +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +This tutorial assumes basic knowledge of Salt. To get up to speed, check +out the \fI\%Salt Walkthrough\fP\&. .sp -.nf -.ft C -# init.sls -{% from \(aqlib.sls\(aq import pythonpkg with context %} - -python\-virtualenv: - pkg.installed: - \- name: {{ pythonpkg(\(aqvirtualenv\(aq) }} - -python\-fabric: - pkg.installed: - \- name: {{ pythonpkg(\(aqfabric\(aq) }} -.ft P -.fi +This tutorial also assumes a basic understanding of Salt Proxy Minions. If +you\(aqre unfamiliar with Salt\(aqs Proxy Minion system, please read the +\fI\%Salt Proxy Minion\fP documentation and the +\fI\%Salt Proxy Minion End\-to\-End Example\fP +tutorial. +.sp +The third assumption that this tutorial makes is that you also have a +basic understanding of ESXi hosts. You can learn more about ESXi hosts on +\fI\%VMware\(aqs various resources\fP\&. .UNINDENT .UNINDENT +.sp +Salt\(aqs ESXi Proxy Minion allows a VMware ESXi host to be treated as an individual +Salt Minion, without installing a Salt Minion on the ESXi host. +.sp +Since an ESXi host may not necessarily run on an OS capable of hosting a Python +stack, the ESXi host can\(aqt run a regular Salt Minion directly. Therefore, Salt\(aqs +Proxy Minion functionality enables you to designate another machine to host a +proxy process that \(dqproxies\(dq communication from the Salt Master to the ESXi host. +The master does not know or care that the ESXi target is not a \(dqreal\(dq Salt Minion. +.sp +More in\-depth conceptual reading on Proxy Minions can be found in the +\fI\%Proxy Minion\fP section of Salt\(aqs documentation. +.sp +Salt\(aqs ESXi Proxy Minion was added in the 2015.8.4 release of Salt. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +Be aware that some functionality for the ESXi Proxy Minion may depend on the +type of license attached the ESXi host(s). .sp -.nf -.ft C -# lib.sls -{% macro pythonpkg(pkg) \-%} - {%\- if grains[\(aqos\(aq] == \(aqFreeBSD\(aq \-%} - py27\-{{ pkg }} - {%\- elif grains[\(aqos\(aq] == \(aqDebian\(aq \-%} - python\-{{ pkg }} - {%\- endif \-%} -{%\- endmacro %} -.ft P -.fi +For example, certain services are only available to manipulate service state +or policies with a VMware vSphere Enterprise or Enterprise Plus license, while +others are available with a Standard license. The \fBntpd\fP service is restricted +to an Enterprise Plus license, while \fBssh\fP is available via the Standard +license. +.sp +Please see the \fI\%vSphere Comparison\fP page for more information. .UNINDENT .UNINDENT +.SS Dependencies .sp -This would define a \fI\%macro\fP that would return a string of the full package name, -depending on the packaging system\(aqs naming convention. The whitespace of the -macro was eliminated, so that the macro would return a string without line -breaks, using \fI\%whitespace control\fP\&. -.SS Template Inheritance +Manipulation of the ESXi host via a Proxy Minion requires the machine running +the Proxy Minion process to have the ESXCLI package (and all of its dependencies) +and the pyVmomi Python Library to be installed. +.SS ESXi Password .sp -\fI\%Template inheritance\fP works fine from state files and files. The search path -starts at the root of the state tree or pillar. -.SS Errors +The ESXi Proxy Minion uses VMware\(aqs API to perform tasks on the host as if it was +a regular Salt Minion. In order to access the API that is already running on the +ESXi host, the ESXi host must have a username and password that is used to log +into the host. The username is usually \fBroot\fP\&. Before Salt can access the ESXi +host via VMware\(aqs API, a default password \fImust\fP be set on the host. +.SS pyVmomi .sp -Saltstack allows raising custom errors using the \fBraise\fP jinja function. +The pyVmomi Python library must be installed on the machine that is running the +proxy process. pyVmomi can be installed via pip: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ raise(\(aqCustom Error\(aq) }} +pip install pyVmomi .ft P .fi .UNINDENT .UNINDENT .sp -When rendering the template containing the above statement, a \fBTemplateError\fP -exception is raised, causing the rendering to fail with the following message: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Version 6.0 of pyVmomi has some problems with SSL error handling on certain +versions of Python. If using version 6.0 of pyVmomi, the machine that you +are running the proxy minion process from must have either Python 2.6, +Python 2.7.9, or newer. This is due to an upstream dependency in pyVmomi 6.0 +that is not supported in Python version 2.7 to 2.7.8. If the +version of Python running the proxy process is not in the supported range, you +will need to install an earlier version of pyVmomi. See \fI\%Issue #29537\fP for +more information. +.UNINDENT +.UNINDENT +.sp +Based on the note above, to install an earlier version of pyVmomi than the +version currently listed in PyPi, run the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -TemplateError: Custom Error +pip install pyVmomi==5.5.0.2014.1.1 .ft P .fi .UNINDENT .UNINDENT -.SS Filters .sp -Saltstack extends \fI\%builtin filters\fP with these custom filters: -.SS \fBstrftime\fP +The 5.5.0.2014.1.1 is a known stable version that the original ESXi Proxy Minion +was developed against. +.SS ESXCLI .sp -Converts any time related object into a time based string. It requires valid -strftime directives. An exhaustive list can be found \fI\%here\fP in the Python documentation. +Currently, about a third of the functions used for the ESXi Proxy Minion require +the ESXCLI package be installed on the machine running the Proxy Minion process. +.sp +The ESXCLI package is also referred to as the VMware vSphere CLI, or vCLI. VMware +provides vCLI package installation instructions for \fI\%vSphere 5.5\fP and +\fI\%vSphere 6.0\fP\&. +.sp +Once all of the required dependencies are in place and the vCLI package is +installed, you can check to see if you can connect to your ESXi host by running +the following command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set curtime = None | strftime() %} +esxcli \-s \-u \-p system syslog config get .ft P .fi .UNINDENT .UNINDENT .sp -Fuzzy dates require the \fI\%timelib\fP Python module is installed. +If the connection was successful, ESXCLI was successfully installed on your system. +You should see output related to the ESXi host\(aqs syslog configuration. +.SS Configuration +.sp +There are several places where various configuration values need to be set in +order for the ESXi Proxy Minion to run and connect properly. +.SS Proxy Config File +.sp +On the machine that will be running the Proxy Minion process(es), a proxy config +file must be in place. This file should be located in the \fB/etc/salt/\fP directory +and should be named \fBproxy\fP\&. If the file is not there by default, create it. +.sp +This file should contain the location of your Salt Master that the Salt Proxy +will connect to. +.sp +Example Proxy Config File: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(dq2002/12/25\(dq|strftime(\(dq%y\(dq) }} -{{ \(dq1040814000\(dq|strftime(\(dq%Y\-%m\-%d\(dq) }} -{{ datetime|strftime(\(dq%u\(dq) }} -{{ \(dqtomorrow\(dq|strftime }} +# /etc/salt/proxy + +master: .ft P .fi .UNINDENT .UNINDENT -.SS \fBsequence\fP -.sp -Ensure that parsed data is a sequence. -.SS \fByaml_encode\fP +.SS Pillar Profiles .sp -Serializes a single object into a YAML scalar with any necessary -handling for escaping special characters. This will work for any -scalar YAML data type: ints, floats, timestamps, booleans, strings, -unicode. It will \fInot\fP work for multi\-objects such as sequences or -maps. +Proxy minions get their configuration from Salt\(aqs Pillar. Every proxy must +have a stanza in Pillar and a reference in the Pillar top\-file that matches +the Proxy ID. At a minimum for communication with the ESXi host, the pillar +should look like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{%\- set bar = 7 %} -{%\- set baz = none %} -{%\- set zip = true %} -{%\- set zap = \(aqThe word of the day is \(dqsalty\(dq\(aq %} - -{%\- load_yaml as foo %} -bar: {{ bar|yaml_encode }} -baz: {{ baz|yaml_encode }} -zip: {{ zip|yaml_encode }} -zap: {{ zap|yaml_encode }} -{%\- endload %} +proxy: + proxytype: esxi + host: + username: + passwords: + \- first_password + \- second_password + \- third_password .ft P .fi .UNINDENT .UNINDENT .sp -In the above case \fB{{ bar }}\fP and \fB{{ foo.bar }}\fP should be -identical and \fB{{ baz }}\fP and \fB{{ foo.baz }}\fP should be -identical. -.SS \fByaml_dquote\fP +Some other optional settings are \fBprotocol\fP and \fBport\fP\&. These can be added +to the pillar configuration. +.SS proxytype .sp -Serializes a string into a properly\-escaped YAML double\-quoted -string. This is useful when the contents of a string are unknown -and may contain quotes or unicode that needs to be preserved. The -resulting string will be emitted with opening and closing double -quotes. +The \fBproxytype\fP key and value pair is critical, as it tells Salt which +interface to load from the \fBproxy\fP directory in Salt\(aqs install hierarchy, +or from \fB/srv/salt/_proxy\fP on the Salt Master (if you have created your +own proxy module, for example). To use this ESXi Proxy Module, set this to +\fBesxi\fP\&. +.SS host +.sp +The location, or ip/dns, of the ESXi host. Required. +.SS username +.sp +The username used to login to the ESXi host, such as \fBroot\fP\&. Required. +.SS passwords +.sp +A list of passwords to be used to try and login to the ESXi host. At least +one password in this list is required. +.sp +The proxy integration will try the passwords listed in order. It is +configured this way so you can have a regular password and the password you +may be updating for an ESXi host either via the +\fI\%vsphere.update_host_password\fP +execution module function or via the +\fI\%esxi.password_present\fP state +function. This way, after the password is changed, you should not need to +restart the proxy minion\-\-it should just pick up the new password +provided in the list. You can then change pillar at will to move that +password to the front and retire the unused ones. +.sp +Use\-case/reasoning for using a list of passwords: You are setting up an +ESXi host for the first time, and the host comes with a default password. +You know that you\(aqll be changing this password during your initial setup +from the default to a new password. If you only have one password option, +and if you have a state changing the password, any remote execution commands +or states that run after the password change will not be able to run on the +host until the password is updated in Pillar and the Proxy Minion process is +restarted. +.sp +This allows you to use any number of potential fallback passwords. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +When a password is changed on the host to one in the list of possible +passwords, the further down on the list the password is, the longer +individual commands will take to return. This is due to the nature of +pyVmomi\(aqs login system. We have to wait for the first attempt to fail +before trying the next password on the list. .sp -.nf -.ft C -{%\- set bar = \(aq\(dqThe quick brown fox . . .\(dq\(aq %} -{%\- set baz = \(aqThe word of the day is \(dqsalty\(dq.\(aq %} - -{%\- load_yaml as foo %} -bar: {{ bar|yaml_dquote }} -baz: {{ baz|yaml_dquote }} -{%\- endload %} -.ft P -.fi +This scenario is especially true, and even slower, when the proxy +minion first starts. If the correct password is not the first password +on the list, it may take up to a minute for \fBtest.version\fP to respond +with salt\(aqs version installed (Example: \fB2018.3.4\fP\&. Once the initial +authorization is complete, the responses for commands will be a little +faster. +.sp +To avoid these longer waiting periods, SaltStack recommends moving the +correct password to the top of the list and restarting the proxy minion +at your earliest convenience. .UNINDENT .UNINDENT +.SS protocol .sp -In the above case \fB{{ bar }}\fP and \fB{{ foo.bar }}\fP should be -identical and \fB{{ baz }}\fP and \fB{{ foo.baz }}\fP should be -identical. If variable contents are not guaranteed to be a string -then it is better to use \fByaml_encode\fP which handles all YAML -scalar types. -.SS \fByaml_squote\fP +If the ESXi host is not using the default protocol, set this value to an +alternate protocol. Default is \fBhttps\fP\&. For example: +.SS port .sp -Similar to the \fByaml_dquote\fP filter but with single quotes. Note -that YAML only allows special escapes inside double quotes so -\fByaml_squote\fP is not nearly as useful (viz. you likely want to -use \fByaml_encode\fP or \fByaml_dquote\fP). -.SS \fBdict_to_sls_yaml_params\fP +If the ESXi host is not using the default port, set this value to an +alternate port. Default is \fB443\fP\&. +.SS Example Configuration Files .sp -New in version 3005. - +An example of all of the basic configurations that need to be in place before +starting the Proxy Minion processes includes the Proxy Config File, Pillar +Top File, and any individual Proxy Minion Pillar files. .sp -Renders a formatted multi\-line YAML string from a Python dictionary. Each -key/value pair in the dictionary will be added as a single\-key dictionary -to a list that will then be sent to the YAML formatter. +In this example, we\(aqll assuming there are two ESXi hosts to connect to. Therefore, +we\(aqll be creating two Proxy Minion config files, one config for each ESXi host. .sp -Example: +Proxy Config File: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set thing_params = { - \(dqname\(dq: \(dqthing\(dq, - \(dqchanges\(dq: True, - \(dqwarnings\(dq: \(dqOMG! Stuff is happening!\(dq - } -%} +# /etc/salt/proxy -thing: - test.configurable_test_state: - {{ thing_params | dict_to_sls_yaml_params | indent }} +master: .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Pillar Top File: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -thing: - test.configurable_test_state: - \- name: thing - \- changes: true - \- warnings: OMG! Stuff is happening! +# /srv/pillar/top.sls + +base: + \(aqesxi\-1\(aq: + \- esxi\-1 + \(aqesxi\-2\(aq: + \- esxi\-2 .ft P .fi .UNINDENT .UNINDENT -.SS \fBto_bool\fP -.sp -New in version 2017.7.0. - -.sp -Returns the logical value of an element. .sp -Example: +Pillar Config File for the first ESXi host, esxi\-1: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqyes\(aq | to_bool }} -{{ \(aqtrue\(aq | to_bool }} -{{ 1 | to_bool }} -{{ \(aqno\(aq | to_bool }} +# /srv/pillar/esxi\-1.sls + +proxy: + proxytype: esxi + host: esxi\-1.example.com + username: \(aqroot\(aq + passwords: + \- bad\-password\-1 + \- backup\-bad\-password\-1 .ft P .fi .UNINDENT .UNINDENT .sp -Will be rendered as: +Pillar Config File for the second ESXi host, esxi\-2: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True -True -True -False +# /srv/pillar/esxi\-2.sls + +proxy: + proxytype: esxi + host: esxi\-2.example.com + username: \(aqroot\(aq + passwords: + \- bad\-password\-2 + \- backup\-bad\-password\-2 .ft P .fi .UNINDENT .UNINDENT -.SS \fBexactly_n_true\fP -.sp -New in version 2017.7.0. - -.sp -Tests that exactly N items in an iterable are \(dqtruthy\(dq (neither None, False, nor 0). +.SS Starting the Proxy Minion .sp -Example: +Once all of the correct configuration files are in place, it is time to start the +proxy processes! +.INDENT 0.0 +.IP 1. 3 +First, make sure your Salt Master is running. +.IP 2. 3 +Start the first Salt Proxy, in debug mode, by giving the Proxy Minion process +and ID that matches the config file name created in the \fI\%Configuration\fP section. +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [\(aqyes\(aq, 0, False, \(aqTrue\(aq] | exactly_n_true(2) }} +salt\-proxy \-\-proxyid=\(aqesxi\-1\(aq \-l debug .ft P .fi .UNINDENT .UNINDENT -.sp -Returns: +.INDENT 0.0 +.IP 1. 3 +Accept the \fBesxi\-1\fP Proxy Minion\(aqs key on the Salt Master: +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +# salt\-key \-L +Accepted Keys: +Denied Keys: +Unaccepted Keys: +esxi\-1 +Rejected Keys: +# +# salt\-key \-a esxi\-1 +The following keys are going to be accepted: +Unaccepted Keys: +esxi\-1 +Proceed? [n/Y] y +Key for minion esxi\-1 accepted. .ft P .fi .UNINDENT .UNINDENT -.SS \fBexactly_one_true\fP -.sp -New in version 2017.7.0. - -.sp -Tests that exactly one item in an iterable is \(dqtruthy\(dq (neither None, False, nor 0). -.sp -Example: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -{{ [\(aqyes\(aq, False, 0, None] | exactly_one_true }} -.ft P -.fi -.UNINDENT +.IP 1. 3 +Repeat for the second Salt Proxy, this time we\(aqll run the proxy process as a +daemon, as an example. .UNINDENT -.sp -Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +salt\-proxy \-\-proxyid=\(aqesxi\-2\(aq \-d .ft P .fi .UNINDENT .UNINDENT -.SS \fBquote\fP -.sp -New in version 2017.7.0. - -.sp -This text will be wrapped in quotes. -.SS \fBregex_search\fP -.sp -New in version 2017.7.0. - -.sp -Scan through string looking for a location where this regular expression -produces a match. Returns \fBNone\fP in case there were no matches found -.sp -Example: +.INDENT 0.0 +.IP 1. 3 +Accept the \fBesxi\-2\fP Proxy Minion\(aqs key on the Salt Master: +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqabcdefabcdef\(aq | regex_search(\(aqBC(.*)\(aq, ignorecase=True) }} +# salt\-key \-L +Accepted Keys: +esxi\-1 +Denied Keys: +Unaccepted Keys: +esxi\-2 +Rejected Keys: +# +# salt\-key \-a esxi\-1 +The following keys are going to be accepted: +Unaccepted Keys: +esxi\-2 +Proceed? [n/Y] y +Key for minion esxi\-1 accepted. .ft P .fi .UNINDENT .UNINDENT -.sp -Returns: +.INDENT 0.0 +.IP 1. 3 +Check and see if your Proxy Minions are responding: +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -(\(dqdefabcdef\(dq,) +# salt \(aqesxi\-*\(aq test.version +esxi\-1: + True +esxi\-3: + True .ft P .fi .UNINDENT .UNINDENT -.SS \fBregex_match\fP +.SS Executing Commands .sp -New in version 2017.7.0. - +Now that you\(aqve configured your Proxy Minions and have them responding successfully +to a \fBtest.version\fP, we can start executing commands against the ESXi hosts via Salt. .sp -If zero or more characters at the beginning of string match this regular -expression, otherwise returns \fBNone\fP\&. +It\(aqs important to understand how this particular proxy works, and there are a couple +of important pieces to be aware of in order to start running remote execution and +state commands against the ESXi host via a Proxy Minion: the +\fI\%vSphere Execution Module\fP, the \fI\%ESXi Execution Module\fP, and the \fI\%ESXi State Module\fP\&. +.SS vSphere Execution Module .sp -Example: -.INDENT 0.0 -.INDENT 3.5 +The \fI\%Salt.modules.vsphere\fP is a +standard Salt execution module that does the bulk of the work for the ESXi Proxy +Minion. If you pull up the docs for it you\(aqll see that almost every function in +the module takes credentials (\fBusername\fP and \fBpassword\fP) and a target \fBhost\fP +argument. When credentials and a host aren\(aqt passed, Salt runs commands +through \fBpyVmomi\fP or \fBESXCLI\fP against the local machine. If you wanted, +you could run functions from this module on any machine where an appropriate +version of \fBpyVmomi\fP and \fBESXCLI\fP are installed, and that machine would reach +out over the network and communicate with the ESXi host. .sp -.nf -.ft C -{{ \(aqabcdefabcdef\(aq | regex_match(\(aqBC(.*)\(aq, ignorecase=True) }} -.ft P -.fi -.UNINDENT -.UNINDENT +You\(aqll notice that most of the functions in the vSphere module require a \fBhost\fP, +\fBusername\fP, and \fBpassword\fP\&. These parameters are contained in the Pillar files and +passed through to the function via the proxy process that is already running. You don\(aqt +need to provide these parameters when you execute the commands. See the +\fI\%Running Remote Execution Commands\fP section below for an example. +.SS ESXi Execution Module .sp -Returns: +In order for the Pillar information set up in the \fI\%Configuration\fP section above to +be passed to the function call in the vSphere Execution Module, the +\fI\%salt.modules.esxi\fP execution module acts +as a \(dqshim\(dq between the vSphere execution module functions and the proxy process. +.sp +The \(dqshim\(dq takes the authentication credentials specified in the Pillar files and +passes them through to the \fBhost\fP, \fBusername\fP, \fBpassword\fP, and optional +\fBprotocol\fP and \fBport\fP options required by the vSphere Execution Module functions. +.sp +If the function takes more positional, or keyword, arguments you can append them +to the call. It\(aqs this shim that speaks to the ESXi host through the proxy, arranging +for the credentials and hostname to be pulled from the Pillar section for the ESXi +Proxy Minion. +.sp +Because of the presence of the shim, to lookup documentation for what +functions you can use to interface with the ESXi host, you\(aqll want to +look in \fI\%salt.modules.vsphere\fP +instead of \fI\%salt.modules.esxi\fP\&. +.SS Running Remote Execution Commands +.sp +To run commands from the Salt Master to execute, via the ESXi Proxy Minion, against +the ESXi host, you use the \fBesxi.cmd \fP syntax to call +functions located in the vSphere Execution Module. Both args and kwargs needed +for various vsphere execution module functions must be passed through in a kwarg\- +type manor. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -None +salt \(aqesxi\-*\(aq esxi.cmd system_info +salt \(aqexsi\-*\(aq esxi.cmd get_service_running service_name=\(aqssh\(aq .ft P .fi .UNINDENT .UNINDENT -.SS \fBregex_replace\fP -.sp -New in version 2017.7.0. - +.SS ESXi State Module .sp -Searches for a pattern and replaces with a sequence of characters. +The ESXi State Module functions similarly to other state modules. The \(dqshim\(dq provided +by the \fI\%ESXi Execution Module\fP passes the necessary \fBhost\fP, \fBusername\fP, and +\fBpassword\fP credentials through, so those options don\(aqt need to be provided in the +state. Other than that, state files are written and executed just like any other +Salt state. See the \fI\%salt.modules.esxi\fP state +for ESXi state functions. .sp -Example: +The follow state file is an example of how to configure various pieces of an ESXi host +including enabling SSH, uploading and SSH key, configuring a coredump network config, +syslog, ntp, enabling VMotion, resetting a host password, and more. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set my_text = \(aqyes, this is a TEST\(aq %} -{{ my_text | regex_replace(\(aq ([a\-z])\(aq, \(aq__\e\e1\(aq, ignorecase=True) }} +# /srv/salt/configure\-esxi.sls + +configure\-host\-ssh: + esxi.ssh_configured: + \- service_running: True + \- ssh_key_file: /etc/salt/ssh_keys/my_key.pub + \- service_policy: \(aqautomatic\(aq + \- service_restart: True + \- certificate_verify: True + +configure\-host\-coredump: + esxi.coredump_configured: + \- enabled: True + \- dump_ip: \(aqmy\-coredump\-ip.example.com\(aq + +configure\-host\-syslog: + esxi.syslog_configured: + \- syslog_configs: + loghost: ssl://localhost:5432,tcp://10.1.0.1:1514 + default\-timeout: 120 + \- firewall: True + \- reset_service: True + \- reset_syslog_config: True + \- reset_configs: loghost,default\-timeout + +configure\-host\-ntp: + esxi.ntp_configured: + \- service_running: True + \- ntp_servers: + \- 192.174.1.100 + \- 192.174.1.200 + \- service_policy: \(aqautomatic\(aq + \- service_restart: True + +configure\-vmotion: + esxi.vmotion_configured: + \- enabled: True + +configure\-host\-vsan: + esxi.vsan_configured: + \- enabled: True + \- add_disks_to_vsan: True + +configure\-host\-password: + esxi.password_present: + \- password: \(aqnew\-bad\-password\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +States are called via the ESXi Proxy Minion just as they would on a regular minion. +For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -yes,__this__is__a__TEST +salt \(aqesxi\-*\(aq state.sls configure\-esxi test=true +salt \(aqesxi\-*\(aq state.sls configure\-esxi .ft P .fi .UNINDENT .UNINDENT -.SS \fBuuid\fP -.sp -New in version 2017.7.0. - +.SS Relevant Salt Files and Resources +.INDENT 0.0 +.IP \(bu 2 +\fI\%ESXi Proxy Minion\fP +.IP \(bu 2 +\fI\%ESXi Execution Module\fP +.IP \(bu 2 +\fI\%ESXi State Module\fP +.IP \(bu 2 +\fI\%Salt Proxy Minion Docs\fP +.IP \(bu 2 +\fI\%Salt Proxy Minion End\-to\-End Example\fP +.IP \(bu 2 +\fI\%vSphere Execution Module\fP +.UNINDENT +.SS Opening the Firewall up for Salt .sp -Return a UUID. +The Salt master communicates with the minions using an AES\-encrypted ZeroMQ +connection. These communications are done over TCP ports \fB4505\fP and \fB4506\fP, +which need to be accessible on the master only. This document outlines suggested +firewall rules for allowing these incoming connections to the master. .sp -Example: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{{ \(aqrandom\(aq | uuid }} -.ft P -.fi +No firewall configuration needs to be done on Salt minions. These changes +refer to the master only. .UNINDENT .UNINDENT +.SS Fedora 18 and beyond / RHEL 7 / CentOS 7 .sp -Returns: +Starting with Fedora 18 \fI\%FirewallD\fP is the tool that is used to dynamically +manage the firewall rules on a host. It has support for IPv4/6 settings and +the separation of runtime and permanent configurations. To interact with +FirewallD use the command line client \fBfirewall\-cmd\fP\&. +.sp +\fBfirewall\-cmd example\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -3652b285\-26ad\-588e\-a5dc\-c2ee65edc804 +firewall\-cmd \-\-permanent \-\-zone= \-\-add\-port=4505\-4506/tcp .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_list\fP -.sp -New in version 2017.7.0. - .sp -Return if an object is list. +A network zone defines the security level of trust for the network. +The user should choose an appropriate zone value for their setup. +Possible values include: drop, block, public, external, dmz, work, home, internal, trusted. .sp -Example: +Don\(aqt forget to reload after you made your changes. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | is_list }} +firewall\-cmd \-\-reload .ft P .fi .UNINDENT .UNINDENT +.SS RHEL 6 / CentOS 6 .sp -Returns: +The \fBlokkit\fP command packaged with some Linux distributions makes opening +iptables firewall ports very simple via the command line. Just be careful +to not lock out access to the server by neglecting to open the ssh port. +.sp +\fBlokkit example\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +lokkit \-p 22:tcp \-p 4505:tcp \-p 4506:tcp .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_iter\fP -.sp -New in version 2017.7.0. - .sp -Return if an object is iterable. +The \fBsystem\-config\-firewall\-tui\fP command provides a text\-based interface to +modifying the firewall. .sp -Example: +\fBsystem\-config\-firewall\-tui\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | is_iter }} +system\-config\-firewall\-tui .ft P .fi .UNINDENT .UNINDENT +.SS openSUSE .sp -Returns: +Salt installs firewall rules in \fI\%/etc/sysconfig/SuSEfirewall2.d/services/salt\fP\&. +Enable with: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +SuSEfirewall2 open +SuSEfirewall2 start .ft P .fi .UNINDENT .UNINDENT -.SS \fBmin\fP .sp -New in version 2017.7.0. - -.sp -Return the minimum value from a list. +If you have an older package of Salt where the above configuration file is +not included, the \fBSuSEfirewall2\fP command makes opening iptables firewall +ports very simple via the command line. .sp -Example: +\fBSuSEfirewall example\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | min }} +SuSEfirewall2 open EXT TCP 4505 +SuSEfirewall2 open EXT TCP 4506 .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +The firewall module in YaST2 provides a text\-based interface to modifying the +firewall. +.sp +\fBYaST2\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -1 +yast2 firewall .ft P .fi .UNINDENT .UNINDENT -.SS \fBmax\fP +.SS Windows .sp -New in version 2017.7.0. - +Windows Firewall is the default component of Microsoft Windows that provides +firewalling and packet filtering. There are many 3rd party firewalls available +for Windows, some of which use rules from the Windows Firewall. If you are +experiencing problems see the vendor\(aqs specific documentation for opening the +required ports. .sp -Returns the maximum value from a list. +The Windows Firewall can be configured using the Windows Interface or from the +command line. .sp -Example: +\fBWindows Firewall (interface)\fP: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -{{ [1, 2, 3] | max }} -.ft P -.fi -.UNINDENT +.IP 1. 3 +Open the Windows Firewall Interface by typing \fBwf.msc\fP at the command +prompt or in a run dialog (\fIWindows Key + R\fP) +.IP 2. 3 +Navigate to \fBInbound Rules\fP in the console tree +.IP 3. 3 +Add a new rule by clicking \fBNew Rule...\fP in the Actions area +.IP 4. 3 +Change the Rule Type to \fBPort\fP\&. Click \fBNext\fP +.IP 5. 3 +Set the Protocol to \fBTCP\fP and specify local ports \fB4505\-4506\fP\&. Click +\fBNext\fP +.IP 6. 3 +Set the Action to \fBAllow the connection\fP\&. Click \fBNext\fP +.IP 7. 3 +Apply the rule to \fBDomain\fP, \fBPrivate\fP, and \fBPublic\fP\&. Click \fBNext\fP +.IP 8. 3 +Give the new rule a Name, ie: \fBSalt\fP\&. You may also add a description. Click +\fBFinish\fP .UNINDENT .sp -Returns: +\fBWindows Firewall (command line)\fP: +.sp +The Windows Firewall rule can be created by issuing a single command. Run the +following command from the command line or a run prompt: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -3 +netsh advfirewall firewall add rule name=\(dqSalt\(dq dir=in action=allow protocol=TCP localport=4505\-4506 .ft P .fi .UNINDENT .UNINDENT -.SS \fBavg\fP -.sp -New in version 2017.7.0. - +.SS iptables .sp -Returns the average value of the elements of a list +Different Linux distributions store their \fIiptables\fP (also known as +\fI\%netfilter\fP) rules in different places, which makes it difficult to +standardize firewall documentation. Included are some of the more +common locations, but your mileage may vary. .sp -Example: +\fBFedora / RHEL / CentOS\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | avg }} +/etc/sysconfig/iptables .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBArch Linux\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -2 +/etc/iptables/iptables.rules .ft P .fi .UNINDENT .UNINDENT -.SS \fBunion\fP .sp -New in version 2017.7.0. - +\fBDebian\fP .sp -Return the union of two lists. +Follow these instructions: \fI\%https://wiki.debian.org/iptables\fP .sp -Example: +Once you\(aqve found your firewall rules, you\(aqll need to add the below line +to allow traffic on \fBtcp/4505\fP and \fBtcp/4506\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | union([2, 3, 4]) | join(\(aq, \(aq) }} +\-A INPUT \-m state \-\-state new \-m tcp \-p tcp \-\-dport 4505:4506 \-j ACCEPT .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBUbuntu\fP +.sp +Salt installs firewall rules in \fI\%/etc/ufw/applications.d/salt.ufw\fP\&. Enable with: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -1, 2, 3, 4 +ufw allow salt .ft P .fi .UNINDENT .UNINDENT -.SS \fBintersect\fP -.sp -New in version 2017.7.0. - -.sp -Return the intersection of two lists. +.SS pf.conf .sp -Example: +The BSD\-family of operating systems uses \fI\%packet filter (pf)\fP\&. The following +example describes the addition to \fBpf.conf\fP needed to access the Salt +master. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | intersect([2, 3, 4]) | join(\(aq, \(aq) }} +pass in on $int_if proto tcp from any to $int_if port 4505:4506 .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Once this addition has been made to the \fBpf.conf\fP the rules will need to +be reloaded. This can be done using the \fBpfctl\fP command. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -2, 3 +pfctl \-vf /etc/pf.conf .ft P .fi .UNINDENT .UNINDENT -.SS \fBdifference\fP -.sp -New in version 2017.7.0. - +.SS Whitelist communication to Master .sp -Return the difference of two lists. +There are situations where you want to selectively allow Minion traffic +from specific hosts or networks into your Salt Master. The first +scenario which comes to mind is to prevent unwanted traffic to your +Master out of security concerns, but another scenario is to handle +Minion upgrades when there are backwards incompatible changes between +the installed Salt versions in your environment. .sp -Example: +Here is an example \fI\%Linux iptables\fP ruleset to +be set on the Master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | difference([2, 3, 4]) | join(\(aq, \(aq) }} +# Allow Minions from these networks +\-I INPUT \-s 10.1.2.0/24 \-p tcp \-\-dports 4505:4506 \-j ACCEPT +\-I INPUT \-s 10.1.3.0/24 \-p tcp \-\-dports 4505:4506 \-j ACCEPT +# Allow Salt to communicate with Master on the loopback interface +\-A INPUT \-i lo \-p tcp \-\-dports 4505:4506 \-j ACCEPT +# Reject everything else +\-A INPUT \-p tcp \-\-dports 4505:4506 \-j REJECT .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -1 -.ft P -.fi +The important thing to note here is that the \fBsalt\fP command +needs to communicate with the listening network socket of +\fBsalt\-master\fP on the \fIloopback\fP interface. Without this you will +see no outgoing Salt traffic from the master, even for a simple +\fBsalt \(aq*\(aq test.version\fP, because the \fBsalt\fP client never reached +the \fBsalt\-master\fP to tell it to carry out the execution. .UNINDENT .UNINDENT -.SS \fBsymmetric_difference\fP +.SS HTTP Modules .sp -New in version 2017.7.0. - +This tutorial demonstrates using the various HTTP modules available in Salt. +These modules wrap the Python \fBtornado\fP, \fBurllib2\fP, and \fBrequests\fP +libraries, extending them in a manner that is more consistent with Salt +workflows. +.SS The \fBsalt.utils.http\fP Library .sp -Return the symmetric difference of two lists. +This library forms the core of the HTTP modules. Since it is designed to be used +from the minion as an execution module, in addition to the master as a runner, +it was abstracted into this multi\-use library. This library can also be imported +by 3rd\-party programs wishing to take advantage of its extended functionality. .sp -Example: +Core functionality of the execution, state, and runner modules is derived from +this library, so common usages between them are described here. Documentation +specific to each module is described below. +.sp +This library can be imported with: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | symmetric_difference([2, 3, 4]) | join(\(aq, \(aq) }} +import salt.utils.http .ft P .fi .UNINDENT .UNINDENT +.SS Configuring Libraries .sp -Returns: +This library can make use of either \fBtornado\fP, which is required by Salt, +\fBurllib2\fP, which ships with Python, or \fBrequests\fP, which can be installed +separately. By default, \fBtornado\fP will be used. In order to switch to +\fBurllib2\fP, set the following variable: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -1, 4 +backend: urllib2 .ft P .fi .UNINDENT .UNINDENT -.SS \fBflatten\fP .sp -New in version 3005. - -.sp -Flatten a list. +In order to switch to \fBrequests\fP, set the following variable: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [3, [4, 2] ] | flatten }} -# => [3, 4, 2] +backend: requests .ft P .fi .UNINDENT .UNINDENT .sp -Flatten only the first level of a list: -.INDENT 0.0 -.INDENT 3.5 +This can be set in the master or minion configuration file, or passed as an +option directly to any \fBhttp.query()\fP functions. +.SS \fBsalt.utils.http.query()\fP .sp -.nf -.ft C -{{ [3, [4, [2]] ] | flatten(levels=1) }} -# => [3, 4, [2]] -.ft P -.fi -.UNINDENT -.UNINDENT +This function forms a basic query, but with some add\-ons not present in the +\fBtornado\fP, \fBurllib2\fP, and \fBrequests\fP libraries. Not all functionality +currently available in these libraries has been added, but can be in future +iterations. +.SS HTTPS Request Methods .sp -Preserve nulls in a list, by default \fBflatten\fP removes them. +A basic query can be performed by calling this function with no more than a +single URL: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [3, None, [4, [2]] ] | flatten(levels=1, preserve_nulls=True) }} -# => [3, None, 4, [2]] +salt.utils.http.query(\(dqhttp://example.com\(dq) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcombinations\fP -.sp -New in version 3005. - -.sp -Invokes the \fBcombinations\fP function from the \fBitertools\fP library. .sp -See the \fI\%itertools documentation\fP for more information. +By default the query will be performed with a \fBGET\fP method. The method can +be overridden with the \fBmethod\fP argument: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABCD\(dq | combinations(2) %}{{ one~two }} {% endfor %} -# => AB AC AD BC BD CD +salt.utils.http.query(\(dqhttp://example.com/delete/url\(dq, \(dqDELETE\(dq) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcombinations_with_replacement\fP -.sp -New in version 3005. - -.sp -Invokes the \fBcombinations_with_replacement\fP function from the \fBitertools\fP library. .sp -See the \fI\%itertools documentation\fP for more information. +When using the \fBPOST\fP method (and others, such as \fBPUT\fP), extra data is usually +sent as well. This data can be sent directly (would be URL encoded when necessary), +or in whatever format is required by the remote server (XML, JSON, plain text, etc). .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABC\(dq | combinations_with_replacement(2) %}{{ one~two }} {% endfor %} -# => AA AB AC BB BC CC +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, method=\(dqPOST\(dq, data=json.dumps(mydict) +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcompress\fP -.sp -New in version 3005. - -.sp -Invokes the \fBcompress\fP function from the \fBitertools\fP library. +.SS Data Formatting and Templating .sp -See the \fI\%itertools documentation\fP for more information. +Bear in mind that the data must be sent pre\-formatted; this function will not +format it for you. However, a templated file stored on the local system may be +passed through, along with variables to populate it with. To pass through only +the file (untemplated): .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for val in \(dqABCDEF\(dq | compress([1,0,1,0,1,1]) %}{{ val }} {% endfor %} -# => A C E F +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, method=\(dqPOST\(dq, data_file=\(dq/srv/salt/somefile.xml\(dq +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBpermutations\fP -.sp -New in version 3005. - -.sp -Invokes the \fBpermutations\fP function from the \fBitertools\fP library. .sp -See the \fI\%itertools documentation\fP for more information. +To pass through a file that contains jinja + yaml templating (the default): .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABCD\(dq | permutations(2) %}{{ one~two }} {% endfor %} -# => AB AC AD BA BC BD CA CB CD DA DB DC +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, + method=\(dqPOST\(dq, + data_file=\(dq/srv/salt/somefile.jinja\(dq, + data_render=True, + template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBproduct\fP -.sp -New in version 3005. - -.sp -Invokes the \fBproduct\fP function from the \fBitertools\fP library. .sp -See the \fI\%itertools documentation\fP for more information. +To pass through a file that contains mako templating: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABCD\(dq | product(\(dqxy\(dq) %}{{ one~two }} {% endfor %} -# => Ax Ay Bx By Cx Cy Dx Dy +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, + method=\(dqPOST\(dq, + data_file=\(dq/srv/salt/somefile.mako\(dq, + data_render=True, + data_renderer=\(dqmako\(dq, + template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBzip\fP -.sp -New in version 3005. - .sp -Invokes the native Python \fBzip\fP function. -.sp -The \fBzip\fP function returns a zip object, which is an iterator of tuples where -the first item in each passed iterator is paired together, and then the second -item in each passed iterator are paired together etc. -.sp -If the passed iterators have different lengths, the iterator with the least -items decides the length of the new iterator. +Because this function uses Salt\(aqs own rendering system, any Salt renderer can +be used. Because Salt\(aqs renderer requires \fB__opts__\fP to be set, an \fBopts\fP +dictionary should be passed in. If it is not, then the default \fB__opts__\fP +values for the node type (master or minion) will be used. Because this library +is intended primarily for use by minions, the default node type is \fBminion\fP\&. +However, this can be changed to \fBmaster\fP if necessary. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABCD\(dq | zip(\(dqxy\(dq) %}{{ one~two }} {% endfor %} -# => Ax By +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, + method=\(dqPOST\(dq, + data_file=\(dq/srv/salt/somefile.jinja\(dq, + data_render=True, + template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, + opts=__opts__, +) + +salt.utils.http.query( + \(dqhttp://example.com/post/url\(dq, + method=\(dqPOST\(dq, + data_file=\(dq/srv/salt/somefile.jinja\(dq, + data_render=True, + template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, + node=\(dqmaster\(dq, +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBzip_longest\fP -.sp -New in version 3005. - -.sp -Invokes the \fBzip_longest\fP function from the \fBitertools\fP library. +.SS Headers .sp -See the \fI\%itertools documentation\fP for more information. +Headers may also be passed through, either as a \fBheader_list\fP, a +\fBheader_dict\fP, or as a \fBheader_file\fP\&. As with the \fBdata_file\fP, the +\fBheader_file\fP may also be templated. Take note that because HTTP headers are +normally syntactically\-correct YAML, they will automatically be imported as an +a Python dict. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% for one, two in \(dqABCD\(dq | zip_longest(\(dqxy\(dq, fillvalue=\(dq\-\(dq) %}{{ one~two }} {% endfor %} -# => Ax By C\- D\- +salt.utils.http.query( + \(dqhttp://example.com/delete/url\(dq, + method=\(dqPOST\(dq, + header_file=\(dq/srv/salt/headers.jinja\(dq, + header_render=True, + header_renderer=\(dqjinja\(dq, + template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBmethod_call\fP -.sp -New in version 3001. - .sp -Returns a result of object\(aqs method call. +Because much of the data that would be templated between headers and data may be +the same, the \fBtemplate_dict\fP is the same for both. Correcting possible +variable name collisions is up to the user. +.SS Authentication .sp -Example #1: +The \fBquery()\fP function supports basic HTTP authentication. A username and +password may be passed in as \fBusername\fP and \fBpassword\fP, respectively. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 1, 3, 4] | method_call(\(aqindex\(aq, 1, 1, 3) }} +salt.utils.http.query(\(dqhttp://example.com\(dq, username=\(dqlarry\(dq, password=\(dq5700g3543v4r\(dq) .ft P .fi .UNINDENT .UNINDENT +.SS Cookies and Sessions .sp -Returns: +Cookies are also supported, using Python\(aqs built\-in \fBcookielib\fP\&. However, they +are turned off by default. To turn cookies on, set \fBcookies\fP to True. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -2 +salt.utils.http.query(\(dqhttp://example.com\(dq, cookies=True) .ft P .fi .UNINDENT .UNINDENT .sp -This filter can be used with the \fI\%map filter\fP to apply object methods without -using loop constructs or temporary variables. -.sp -Example #2: +By default cookies are stored in Salt\(aqs cache directory, normally +\fB/var/cache/salt\fP, as a file called \fBcookies.txt\fP\&. However, this location +may be changed with the \fBcookie_jar\fP argument: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set host_list = [\(aqweb01.example.com\(aq, \(aqdb01.example.com\(aq] %} -{% set host_list_split = [] %} -{% for item in host_list %} - {% do host_list_split.append(item.split(\(aq.\(aq, 1)) %} -{% endfor %} -{{ host_list_split }} +salt.utils.http.query( + \(dqhttp://example.com\(dq, cookies=True, cookie_jar=\(dq/path/to/cookie_jar.txt\(dq +) .ft P .fi .UNINDENT .UNINDENT .sp -Example #3: +By default, the format of the cookie jar is LWP (aka, lib\-www\-perl). This +default was chosen because it is a human\-readable text file. If desired, the +format of the cookie jar can be set to Mozilla: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ host_list|map(\(aqmethod_call\(aq, \(aqsplit\(aq, \(aq.\(aq, 1)|list }} +salt.utils.http.query( + \(dqhttp://example.com\(dq, + cookies=True, + cookie_jar=\(dq/path/to/cookie_jar.txt\(dq, + cookie_format=\(dqmozilla\(dq, +) .ft P .fi .UNINDENT .UNINDENT .sp -Return of examples #2 and #3: +Because Salt commands are normally one\-off commands that are piped together, +this library cannot normally behave as a normal browser, with session cookies +that persist across multiple HTTP requests. However, the session can be +persisted in a separate cookie jar. The default filename for this file, inside +Salt\(aqs cache directory, is \fBcookies.session.p\fP\&. This can also be changed. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[[web01, example.com], [db01, example.com]] +salt.utils.http.query( + \(dqhttp://example.com\(dq, persist_session=True, session_cookie_jar=\(dq/path/to/jar.p\(dq +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_sorted\fP -.sp -New in version 2017.7.0. - .sp -Return \fBTrue\fP if an iterable object is already sorted. +The format of this file is msgpack, which is consistent with much of the rest +of Salt\(aqs internal structure. Historically, the extension for this file is +\fB\&.p\fP\&. There are no current plans to make this configurable. +.SS Proxy .sp -Example: +If the \fBtornado\fP backend is used (\fBtornado\fP is the default), proxy +information configured in \fBproxy_host\fP, \fBproxy_port\fP, \fBproxy_username\fP, +\fBproxy_password\fP and \fBno_proxy\fP from the \fB__opts__\fP dictionary will be used. Normally +these are set in the minion configuration file. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [1, 2, 3] | is_sorted }} +proxy_host: proxy.my\-domain +proxy_port: 31337 +proxy_username: charon +proxy_password: obolus +no_proxy: [\(aq127.0.0.1\(aq, \(aqlocalhost\(aq] .ft P .fi .UNINDENT .UNINDENT -.sp -Returns: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +salt.utils.http.query(\(dqhttp://example.com\(dq, opts=__opts__, backend=\(dqtornado\(dq) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcompare_lists\fP -.sp -New in version 2017.7.0. - -.sp -Compare two lists and return a dictionary with the changes. +.SS Return Data .sp -Example: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +Return data encoding .sp -.nf -.ft C -{{ [1, 2, 3] | compare_lists([1, 2, 4]) }} -.ft P -.fi +If \fBdecode\fP is set to \fBTrue\fP, \fBquery()\fP will attempt to decode the +return data. \fBdecode_type\fP defaults to \fBauto\fP\&. Set it to a specific +encoding, \fBxml\fP, for example, to override autodetection. .UNINDENT .UNINDENT .sp -Returns: +Because Salt\(aqs http library was designed to be used with REST interfaces, +\fBquery()\fP will attempt to decode the data received from the remote server +when \fBdecode\fP is set to \fBTrue\fP\&. First it will check the \fBContent\-type\fP +header to try and find references to XML. If it does not find any, it will look +for references to JSON. If it does not find any, it will fall back to plain +text, which will not be decoded. +.sp +JSON data is translated into a dict using Python\(aqs built\-in \fBjson\fP library. +XML is translated using \fBsalt.utils.xml_util\fP, which will use Python\(aqs +built\-in XML libraries to attempt to convert the XML into a dict. In order to +force either JSON or XML decoding, the \fBdecode_type\fP may be set: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqnew\(dq: [4], \(dqold\(dq: [3]} +salt.utils.http.query(\(dqhttp://example.com\(dq, decode_type=\(dqxml\(dq) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcompare_dicts\fP -.sp -New in version 2017.7.0. - .sp -Compare two dictionaries and return a dictionary with the changes. +Once translated, the return dict from \fBquery()\fP will include a dict called +\fBdict\fP\&. .sp -Example: +If the data is not to be translated using one of these methods, decoding may be +turned off. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ {\(aqa\(aq: \(aqb\(aq} | compare_dicts({\(aqa\(aq: \(aqc\(aq}) }} +salt.utils.http.query(\(dqhttp://example.com\(dq, decode=False) .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +If decoding is turned on, and references to JSON or XML cannot be found, then +this module will default to plain text, and return the undecoded data as +\fBtext\fP (even if text is set to \fBFalse\fP; see below). +.sp +The \fBquery()\fP function can return the HTTP status code, headers, and/or text +as required. However, each must individually be turned on. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqa\(dq: {\(dqnew\(dq: \(dqc\(dq, \(dqold\(dq: \(dqb\(dq}} +salt.utils.http.query(\(dqhttp://example.com\(dq, status=True, headers=True, text=True) .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_hex\fP -.sp -New in version 2017.7.0. - .sp -Return \fBTrue\fP if the value is hexadecimal. +The return from these will be found in the return dict as \fBstatus\fP, +\fBheaders\fP and \fBtext\fP, respectively. +.SS Writing Return Data to Files .sp -Example: +It is possible to write either the return data or headers to files, as soon as +the response is received from the server, but specifying file locations via the +\fBtext_out\fP or \fBheaders_out\fP arguments. \fBtext\fP and \fBheaders\fP do not need +to be returned to the user in order to do this. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq0xabcd\(aq | is_hex }} -{{ \(aqxyzt\(aq | is_hex }} +salt.utils.http.query( + \(dqhttp://example.com\(dq, + text=False, + headers=False, + text_out=\(dq/path/to/url_download.txt\(dq, + headers_out=\(dq/path/to/headers_download.txt\(dq, +) .ft P .fi .UNINDENT .UNINDENT +.SS SSL Verification .sp -Returns: +By default, this function will verify SSL certificates. However, for testing or +debugging purposes, SSL verification can be turned off. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True -False +salt.utils.http.query(\(dqhttps://example.com\(dq, verify_ssl=False) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcontains_whitespace\fP -.sp -New in version 2017.7.0. - -.sp -Return \fBTrue\fP if a text contains whitespaces. +.SS CA Bundles .sp -Example: +The \fBrequests\fP library has its own method of detecting which CA (certificate +authority) bundle file to use. Usually this is implemented by the packager for +the specific operating system distribution that you are using. However, +\fBurllib2\fP requires a little more work under the hood. By default, Salt will +try to auto\-detect the location of this file. However, if it is not in an +expected location, or a different path needs to be specified, it may be done so +using the \fBca_bundle\fP variable. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqabcd\(aq | contains_whitespace }} -{{ \(aqab cd\(aq | contains_whitespace }} +salt.utils.http.query(\(dqhttps://example.com\(dq, ca_bundle=\(dq/path/to/ca_bundle.pem\(dq) .ft P .fi .UNINDENT .UNINDENT +.SS Updating CA Bundles .sp -Returns: +The \fBupdate_ca_bundle()\fP function can be used to update the bundle file at a +specified location. If the target location is not specified, then it will +attempt to auto\-detect the location of the bundle file. If the URL to download +the bundle from does not exist, a bundle will be downloaded from the cURL +website. +.sp +CAUTION: The \fBtarget\fP and the \fBsource\fP should always be specified! Failure +to specify the \fBtarget\fP may result in the file being written to the wrong +location on the local system. Failure to specify the \fBsource\fP may cause the +upstream URL to receive excess unnecessary traffic, and may cause a file to be +download which is hazardous or does not meet the needs of the user. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -False -True +salt.utils.http.update_ca_bundle( + target=\(dq/path/to/ca\-bundle.crt\(dq, + source=\(dqhttps://example.com/path/to/ca\-bundle.crt\(dq, + opts=__opts__, +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBsubstring_in_list\fP -.sp -New in version 2017.7.0. - -.sp -Return \fBTrue\fP if a substring is found in a list of string values. .sp -Example: +The \fBopts\fP parameter should also always be specified. If it is, then the +\fBtarget\fP and the \fBsource\fP may be specified in the relevant configuration +file (master or minion) as \fBca_bundle\fP and \fBca_bundle_url\fP, respectively. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqabcd\(aq | substring_in_list([\(aqthis\(aq, \(aqis\(aq, \(aqan abcd example\(aq]) }} +ca_bundle: /path/to/ca\-bundle.crt +ca_bundle_url: https://example.com/path/to/ca\-bundle.crt .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +If Salt is unable to auto\-detect the location of the CA bundle, it will raise +an error. +.sp +The \fBupdate_ca_bundle()\fP function can also be passed a string or a list of +strings which represent files on the local system, which should be appended (in +the specified order) to the end of the CA bundle file. This is useful in +environments where private certs need to be made available, and are not +otherwise reasonable to add to the bundle file. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +salt.utils.http.update_ca_bundle( + opts=__opts__, + merge_files=[ + \(dq/etc/ssl/private_cert_1.pem\(dq, + \(dq/etc/ssl/private_cert_2.pem\(dq, + \(dq/etc/ssl/private_cert_3.pem\(dq, + ], +) .ft P .fi .UNINDENT .UNINDENT -.SS \fBcheck_whitelist_blacklist\fP -.sp -New in version 2017.7.0. - -.sp -Check a whitelist and/or blacklist to see if the value matches it. +.SS Test Mode .sp -This filter can be used with either a whitelist or a blacklist individually, -or a whitelist and a blacklist can be passed simultaneously. +This function may be run in test mode. This mode will perform all work up until +the actual HTTP request. By default, instead of performing the request, an empty +dict will be returned. Using this function with \fBTRACE\fP logging turned on will +reveal the contents of the headers and POST data to be sent. .sp -If whitelist is used alone, value membership is checked against the -whitelist only. If the value is found, the function returns \fBTrue\fP\&. -Otherwise, it returns \fBFalse\fP\&. +Rather than returning an empty dict, an alternate \fBtest_url\fP may be passed in. +If this is detected, then test mode will replace the \fBurl\fP with the +\fBtest_url\fP, set \fBtest\fP to \fBTrue\fP in the return data, and perform the rest +of the requested operations as usual. This allows a custom, non\-destructive URL +to be used for testing when necessary. +.SS Execution Module .sp -If blacklist is used alone, value membership is checked against the -blacklist only. If the value is found, the function returns \fBFalse\fP\&. -Otherwise, it returns \fBTrue\fP\&. +The \fBhttp\fP execution module is a very thin wrapper around the +\fBsalt.utils.http\fP library. The \fBopts\fP can be passed through as well, but if +they are not specified, the minion defaults will be used as necessary. .sp -If both a whitelist and a blacklist are provided, value membership in the -blacklist will be examined first. If the value is not found in the blacklist, -then the whitelist is checked. If the value isn\(aqt found in the whitelist, -the function returns \fBFalse\fP\&. +Because passing complete data structures from the command line can be tricky at +best and dangerous (in terms of execution injection attacks) at worse, the +\fBdata_file\fP, and \fBheader_file\fP are likely to see more use here. .sp -Whitelist Example: +All methods for the library are available in the execution module, as kwargs. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} +salt myminion http.query http://example.com/restapi method=POST \e + username=\(aqlarry\(aq password=\(aq5700g3543v4r\(aq headers=True text=True \e + status=True decode_type=xml data_render=True \e + header_file=/tmp/headers.txt data_file=/tmp/data.txt \e + header_render=True cookies=True persist_session=True .ft P .fi .UNINDENT .UNINDENT +.SS Runner Module .sp -Returns: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -True -.ft P -.fi -.UNINDENT -.UNINDENT +Like the execution module, the \fBhttp\fP runner module is a very thin wrapper +around the \fBsalt.utils.http\fP library. The only significant difference is that +because runners execute on the master instead of a minion, a target is not +required, and default opts will be derived from the master config, rather than +the minion config. .sp -Blacklist Example: +All methods for the library are available in the runner module, as kwargs. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} +salt\-run http.query http://example.com/restapi method=POST \e + username=\(aqlarry\(aq password=\(aq5700g3543v4r\(aq headers=True text=True \e + status=True decode_type=xml data_render=True \e + header_file=/tmp/headers.txt data_file=/tmp/data.txt \e + header_render=True cookies=True persist_session=True .ft P .fi .UNINDENT .UNINDENT +.SS State Module +.sp +The state module is a wrapper around the runner module, which applies stateful +logic to a query. All kwargs as listed above are specified as usual in state +files, but two more kwargs are available to apply stateful logic. A required +parameter is \fBmatch\fP, which specifies a pattern to look for in the return +text. By default, this will perform a string comparison of looking for the +value of match in the return text. In Python terms this looks like: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -False +def myfunc(): + if match in html_text: + return True .ft P .fi .UNINDENT .UNINDENT -.SS \fBdate_format\fP -.sp -New in version 2017.7.0. - .sp -Converts unix timestamp into human\-readable string. +If more complex pattern matching is required, a regular expression can be used +by specifying a \fBmatch_type\fP\&. By default this is set to \fBstring\fP, but it +can be manually set to \fBpcre\fP instead. Please note that despite the name, this +will use Python\(aqs \fBre.search()\fP rather than \fBre.match()\fP\&. .sp -Example: +Therefore, the following states are valid: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ 1457456400 | date_format }} -{{ 1457456400 | date_format(\(aq%d.%m.%Y %H:%M\(aq) }} +http://example.com/restapi: + http.query: + \- match: \(aqSUCCESS\(aq + \- username: \(aqlarry\(aq + \- password: \(aq5700g3543v4r\(aq + \- data_render: True + \- header_file: /tmp/headers.txt + \- data_file: /tmp/data.txt + \- header_render: True + \- cookies: True + \- persist_session: True + +http://example.com/restapi: + http.query: + \- match_type: pcre + \- match: \(aq(?i)succe[ss|ed]\(aq + \- username: \(aqlarry\(aq + \- password: \(aq5700g3543v4r\(aq + \- data_render: True + \- header_file: /tmp/headers.txt + \- data_file: /tmp/data.txt + \- header_render: True + \- cookies: True + \- persist_session: True .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +In addition to, or instead of a match pattern, the status code for a URL can be +checked. This is done using the \fBstatus\fP argument: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -2017\-03\-08 -08.03.2017 17:00 +http://example.com/: + http.query: + \- status: 200 .ft P .fi .UNINDENT .UNINDENT -.SS \fBto_num\fP .sp -New in version 2017.7.0. - +If both are specified, both will be checked, but if only one is \fBTrue\fP and the +other is \fBFalse\fP, then \fBFalse\fP will be returned. In this case, the comments +in the return data will contain information for troubleshooting. .sp -New in version 2018.3.0: Renamed from \fBstr_to_num\fP to \fBto_num\fP\&. - +Because this is a monitoring state, it will return extra data to code that +expects it. This data will always include \fBtext\fP and \fBstatus\fP\&. Optionally, +\fBheaders\fP and \fBdict\fP may also be requested by setting the \fBheaders\fP and +\fBdecode\fP arguments to True, respectively. +.SS Using Salt at scale .sp -Converts a string to its numerical value. +The focus of this tutorial will be building a Salt infrastructure for handling +large numbers of minions. This will include tuning, topology, and best practices. .sp -Example: +For how to install the Salt Master, see the +\fI\%Salt install guide\fP\&. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +This tutorial is intended for large installations, although these same settings +won\(aqt hurt, it may not be worth the complexity to smaller installations. .sp -.nf -.ft C -{{ \(aq5\(aq | to_num }} -.ft P -.fi +When used with minions, the term \(aqmany\(aq refers to at least a thousand +and \(aqa few\(aq always means 500. +.sp +For simplicity reasons, this tutorial will default to the standard ports +used by Salt. .UNINDENT .UNINDENT +.SS The Master .sp -Returns: +The most common problems on the Salt Master are: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -5 -.ft P -.fi -.UNINDENT +.IP 1. 3 +too many minions authing at once +.IP 2. 3 +too many minions re\-authing at once +.IP 3. 3 +too many minions re\-connecting at once +.IP 4. 3 +too many minions returning at once +.IP 5. 3 +too few resources (CPU/HDD) .UNINDENT -.SS \fBto_bytes\fP .sp -New in version 2017.7.0. - -.sp -Converts string\-type object to bytes. +The first three are all \(dqthundering herd\(dq problems. To mitigate these issues +we must configure the minions to back\-off appropriately when the Master is +under heavy load. .sp -Example: -.INDENT 0.0 -.INDENT 3.5 +The fourth is caused by masters with little hardware resources in combination +with a possible bug in ZeroMQ. At least that\(aqs what it looks like till today +(\fI\%Issue 118651\fP, +\fI\%Issue 5948\fP, +\fI\%Mail thread\fP) .sp -.nf -.ft C -{{ \(aqwall of text\(aq | to_bytes }} -.ft P -.fi -.UNINDENT -.UNINDENT +To fully understand each problem, it is important to understand, how Salt works. .sp -\fBNOTE:\fP +Very briefly, the Salt Master offers two services to the minions. .INDENT 0.0 -.INDENT 3.5 -This option may have adverse effects when using the default renderer, -\fBjinja|yaml\fP\&. This is due to the fact that YAML requires proper handling -in regard to special characters. Please see the section on \fI\%YAML ASCII -support\fP in the \fI\%YAML Idiosyncrasies\fP documentation for more information. -.UNINDENT +.IP \(bu 2 +a job publisher on port 4505 +.IP \(bu 2 +an open port 4506 to receive the minions returns .UNINDENT -.SS \fBjson_encode_list\fP .sp -New in version 2017.7.0. - -.sp -New in version 2018.3.0: Renamed from \fBjson_decode_list\fP to \fBjson_encode_list\fP\&. When you encode -something you get bytes, and when you decode, you get your locale\(aqs -encoding (usually a \fBunicode\fP type). This filter was incorrectly\-named -when it was added. \fBjson_decode_list\fP will be supported until the 3003 -release. - +All minions are always connected to the publisher on port 4505 and only connect +to the open return port 4506 if necessary. On an idle Master, there will only +be connections on port 4505. +.SS Too many minions authing .sp -Deprecated since version 2018.3.3,2019.2.0: The \fI\%tojson\fP filter accomplishes what this filter was designed -to do, making this filter redundant. - +When the Minion service is first started up, it will connect to its Master\(aqs publisher +on port 4505. If too many minions are started at once, this can cause a \(dqthundering herd\(dq. +This can be avoided by not starting too many minions at once. .sp -Recursively encodes all string elements of the list to bytes. +The connection itself usually isn\(aqt the culprit, the more likely cause of master\-side +issues is the authentication that the Minion must do with the Master. If the Master +is too heavily loaded to handle the auth request it will time it out. The Minion +will then wait \fIacceptance_wait_time\fP to retry. If \fIacceptance_wait_time_max\fP is +set then the Minion will increase its wait time by the \fIacceptance_wait_time\fP each +subsequent retry until reaching \fIacceptance_wait_time_max\fP\&. +.SS Too many minions re\-authing .sp -Example: -.INDENT 0.0 -.INDENT 3.5 +This is most likely to happen in the testing phase of a Salt deployment, when +all Minion keys have already been accepted, but the framework is being tested +and parameters are frequently changed in the Salt Master\(aqs configuration +file(s). .sp -.nf -.ft C -{{ [1, 2, 3] | json_encode_list }} -.ft P -.fi -.UNINDENT -.UNINDENT +The Salt Master generates a new AES key to encrypt its publications at certain +events such as a Master restart or the removal of a Minion key. If you are +encountering this problem of too many minions re\-authing against the Master, +you will need to recalibrate your setup to reduce the rate of events like a +Master restart or Minion key removal (\fBsalt\-key \-d\fP). .sp -Returns: +When the Master generates a new AES key, the minions aren\(aqt notified of this +but will discover it on the next pub job they receive. When the Minion +receives such a job it will then re\-auth with the Master. Since Salt does +minion\-side filtering this means that all the minions will re\-auth on the next +command published on the master\-\- causing another \(dqthundering herd\(dq. This can +be avoided by setting the .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[1, 2, 3] +random_reauth_delay: 60 .ft P .fi .UNINDENT .UNINDENT -.SS \fBjson_encode_dict\fP .sp -New in version 2017.7.0. - -.sp -New in version 2018.3.0: Renamed from \fBjson_decode_dict\fP to \fBjson_encode_dict\fP\&. When you encode -something you get bytes, and when you decode, you get your locale\(aqs -encoding (usually a \fBunicode\fP type). This filter was incorrectly\-named -when it was added. \fBjson_decode_dict\fP will be supported until the 3003 -release. - -.sp -Deprecated since version 2018.3.3,2019.2.0: The \fI\%tojson\fP filter accomplishes what this filter was designed -to do, making this filter redundant. - -.sp -Recursively encodes all string items in the dictionary to bytes. +in the minions configuration file to a higher value and stagger the amount +of re\-auth attempts. Increasing this value will of course increase the time +it takes until all minions are reachable via Salt commands. +.SS Too many minions re\-connecting .sp -Example: +By default the zmq socket will re\-connect every 100ms which for some larger +installations may be too quick. This will control how quickly the TCP session is +re\-established, but has no bearing on the auth load. .sp -Assuming that \fBpillar[\(aqfoo\(aq]\fP contains \fB{u\(aqa\(aq: u\(aq\eu0414\(aq}\fP, and your locale -is \fBen_US.UTF\-8\fP: +To tune the minions sockets reconnect attempts, there are a few values in +the sample configuration file (default values) .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ pillar[\(aqfoo\(aq] | json_encode_dict }} +recon_default: 1000 +recon_max: 5000 +recon_randomize: True .ft P .fi .UNINDENT .UNINDENT +.INDENT 0.0 +.IP \(bu 2 +recon_default: the default value the socket should use, i.e. 1000. This value is in +milliseconds. (1000ms = 1 second) +.IP \(bu 2 +recon_max: the max value that the socket should use as a delay before trying to reconnect +This value is in milliseconds. (5000ms = 5 seconds) +.IP \(bu 2 +recon_randomize: enables randomization between recon_default and recon_max +.UNINDENT .sp -Returns: +To tune this values to an existing environment, a few decision have to be made. +.INDENT 0.0 +.IP 1. 3 +How long can one wait, before the minions should be online and reachable via Salt? +.IP 2. 3 +How many reconnects can the Master handle without a syn flood? +.UNINDENT +.sp +These questions can not be answered generally. Their answers depend on the +hardware and the administrators requirements. +.sp +Here is an example scenario with the goal, to have all minions reconnect +within a 60 second time\-frame on a Salt Master service restart. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqa\(dq: \(dq\exd0\ex94\(dq} +recon_default: 1000 +recon_max: 59000 +recon_randomize: True .ft P .fi .UNINDENT .UNINDENT -.SS \fBtojson\fP -.sp -New in version 2018.3.3,2019.2.0. - -.sp -Dumps a data structure to JSON. -.sp -This filter was added to provide this functionality to hosts which have a -Jinja release older than version 2.9 installed. If Jinja 2.9 or newer is -installed, then the upstream version of the filter will be used. See the -\fI\%upstream docs\fP for more information. -.SS \fBrandom_hash\fP -.sp -New in version 2017.7.0. - -.sp -New in version 2018.3.0: Renamed from \fBrand_str\fP to \fBrandom_hash\fP to more accurately describe -what the filter does. \fBrand_str\fP will be supported to ensure backwards -compatibility but please use the preferred \fBrandom_hash\fP\&. - .sp -Generates a random number between 1 and the number passed to the filter, and -then hashes it. The default hash type is the one specified by the minion\(aqs -\fI\%hash_type\fP config option, but an alternate hash type can be -passed to the filter as an argument. +Each Minion will have a randomized reconnect value between \(aqrecon_default\(aq +and \(aqrecon_default + recon_max\(aq, which in this example means between 1000ms +and 60000ms (or between 1 and 60 seconds). The generated random\-value will +be doubled after each attempt to reconnect (ZeroMQ default behavior). .sp -Example: +Lets say the generated random value is 11 seconds (or 11000ms). .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set num_range = 99999999 %} -{{ num_range | random_hash }} -{{ num_range | random_hash(\(aqsha512\(aq) }} +reconnect 1: wait 11 seconds +reconnect 2: wait 22 seconds +reconnect 3: wait 33 seconds +reconnect 4: wait 44 seconds +reconnect 5: wait 55 seconds +reconnect 6: wait time is bigger than 60 seconds (recon_default + recon_max) +reconnect 7: wait 11 seconds +reconnect 8: wait 22 seconds +reconnect 9: wait 33 seconds +reconnect x: etc. .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +With a thousand minions this will mean .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -43ec517d68b6edd3015b3edc9a11367b -d94a45acd81f8e3107d237dbc0d5d195f6a52a0d188bc0284c0763ece1eac9f9496fb6a531a296074c87b3540398dace1222b42e150e67c9301383fde3d66ae5 +1000/60 = ~16 .ft P .fi .UNINDENT .UNINDENT -.SS \fBrandom_sample\fP .sp -New in version 3005. - -.sp -Returns a given sample size from a list. The \fBseed\fP parameter can be used to -return a predictable outcome. +round about 16 connection attempts a second. These values should be altered to +values that match your environment. Keep in mind though, that it may grow over +time and that more minions might raise the problem again. +.SS Too many minions returning at once .sp -Example: +This can also happen during the testing phase, if all minions are addressed at +once with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set my_list = [\(dqone\(dq, \(dqtwo\(dq, \(dqthree\(dq, \(dqfour\(dq] %} -{{ my_list | random_sample(2) }} +$ salt * disk.usage .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +it may cause thousands of minions trying to return their data to the Salt Master +open port 4506. Also causing a flood of syn\-flood if the Master can\(aqt handle that many +returns at once. +.sp +This can be easily avoided with Salt\(aqs batch mode: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[\(dqfour\(dq, \(dqone\(dq] +$ salt * disk.usage \-b 50 .ft P .fi .UNINDENT .UNINDENT -.SS \fBrandom_shuffle\fP .sp -New in version 3005. - +This will only address 50 minions at once while looping through all addressed +minions. +.SS Too few resources .sp -Returns a shuffled copy of an input list. The \fBseed\fP parameter can be used to -return a predictable outcome. +The masters resources always have to match the environment. There is no way +to give good advise without knowing the environment the Master is supposed to +run in. But here are some general tuning tips for different situations: +.SS The Master is CPU bound .sp -Example: +In installations with large or with complex pillar files, it is possible +for the master to exhibit poor performance as a result of having to render +many pillar files at once. This exhibit itself in a number of ways, both +as high load on the master and on minions which block on waiting for their +pillar to be delivered to them. +.sp +To reduce pillar rendering times, it is possible to cache pillars on the +master. To do this, see the set of master configuration options which +are prefixed with \fIpillar_cache\fP\&. +.sp +If many pillars are encrypted using \fI\%gpg\fP renderer, it +is possible to cache GPG data. To do this, see the set of master configuration +options which are prefixed with \fIgpg_cache\fP\&. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{% set my_list = [\(dqone\(dq, \(dqtwo\(dq, \(dqthree\(dq, \(dqfour\(dq] %} -{{ my_list | random_shuffle }} -.ft P -.fi +Caching pillars or GPG data on the master may introduce security +considerations. Be certain to read caveats outlined in the master +configuration file to understand how pillar caching may affect a master\(aqs +ability to protect sensitive data! .UNINDENT .UNINDENT +.SS The Master is disk IO bound .sp -Returns: +By default, the Master saves every Minion\(aqs return for every job in its +job\-cache. The cache can then be used later, to lookup results for previous +jobs. The default directory for this is: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[\(dqfour\(dq, \(dqthree\(dq, \(dqone\(dq, \(dqtwo\(dq] +cachedir: /var/cache/salt .ft P .fi .UNINDENT .UNINDENT -.SS \fBset_dict_key_value\fP -.sp -New in version 3000. - .sp -Allows you to set a value in a nested dictionary without having to worry if all the nested keys actually exist. -Missing keys will be automatically created if they do not exist. -The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +and then in the \fB/proc\fP directory. .sp -Examples: +Each job return for every Minion is saved in a single file. Over time this +directory can grow quite large, depending on the number of published jobs. The +amount of files and directories will scale with the number of jobs published and +the retention time defined by .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - +keep_jobs_seconds: 86400 .ft P .fi .UNINDENT .UNINDENT .INDENT 0.0 -.TP -.B Example 1: -{%\- set foo = {} %} -{{ foo | set_dict_key_value(\(aqbar:baz\(aq, 42) }} -.TP -.B Example 2: -{{ {} | set_dict_key_value(\(aqbar.baz.qux\(aq, 42, delimiter=\(aq.\(aq) }} -.UNINDENT -.sp -Returns: -.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - +250 jobs/day * 2000 minions returns = 500,000 files a day .ft P .fi .UNINDENT .UNINDENT +.SS Use and External Job Cache +.sp +An external job cache allows for job storage to be placed on an external +system, such as a database. .INDENT 0.0 -.TP -.B Example 1: -{\(aqbar\(aq: {\(aqbaz\(aq: 42}} -.TP -.B Example 2: -{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 42}}} +.IP \(bu 2 +ext_job_cache: this will have the minions store their return data directly +into a returner (not sent through the Master) +.IP \(bu 2 +master_job_cache (New in \fI2014.7.0\fP): this will make the Master store the job +data using a returner (instead of the local job cache on disk). .UNINDENT -.SS \fBappend_dict_key_value\fP .sp -New in version 3000. - +If a master has many accepted keys, it may take a long time to publish a job +because the master must first determine the matching minions and deliver +that information back to the waiting client before the job can be published. .sp -Allows you to append to a list nested (deep) in a dictionary without having to worry if all the nested keys (or the list itself) actually exist. -Missing keys will automatically be created if they do not exist. -The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +To mitigate this, a key cache may be enabled. This will reduce the load +on the master to a single file open instead of thousands or tens of thousands. .sp -Examples: +This cache is updated by the maintenance process, however, which means that +minions with keys that are accepted may not be targeted by the master +for up to sixty seconds by default. +.sp +To enable the master key cache, set \fIkey_cache: \(aqsched\(aq\fP in the master +configuration file. +.SS Disable The Job Cache +.sp +The job cache is a central component of the Salt Master and many aspects of +the Salt Master will not function correctly without a running job cache. +.sp +Disabling the job cache is \fBSTRONGLY DISCOURAGED\fP and should not be done +unless the master is being used to execute routines that require no history +or reliable feedback! +.sp +The job cache can be disabled: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C - +job_cache: False .ft P .fi .UNINDENT .UNINDENT +.SS How to Convert Jinja Logic to an Execution Module +.sp +\fBNOTE:\fP .INDENT 0.0 -.TP -.B Example 1: -{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: [1, 2]}} %} -{{ foo | append_dict_key_value(\(aqbar:baz\(aq, 42) }} -.TP -.B Example 2: -{%\- set foo = {} %} -{{ foo | append_dict_key_value(\(aqbar:baz:qux\(aq, 42) }} +.INDENT 3.5 +This tutorial assumes a basic knowledge of Salt states and specifically +experience using the \fImaps.jinja\fP idiom. +.sp +This tutorial was written by a salt user who was told \(dqif your maps.jinja +is too complicated, write an execution module!\(dq. If you are experiencing +over\-complicated jinja, read on. +.UNINDENT .UNINDENT +.SS The Problem: Jinja Gone Wild .sp -Returns: +It is often said in the Salt community that \(dqJinja is not a Programming Language\(dq. +There\(aqs an even older saying known as Maslow\(aqs hammer. +It goes something like +\(dqif all you have is a hammer, everything looks like a nail\(dq. +Jinja is a reliable hammer, and so is the \fImaps.jinja\fP idiom. +Unfortunately, it can lead to code that looks like the following. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C +# storage/maps.yaml + +{% import_yaml \(aqstorage/defaults.yaml\(aq as default_settings %} +{% set storage = default_settings.storage %} +{% do storage.update(salt[\(aqgrains.filter_by\(aq]({ + \(aqDebian\(aq: { + }, + \(aqRedHat\(aq: { + } +}, merge=salt[\(aqpillar.get\(aq](\(aqstorage:lookup\(aq))) %} +{% if \(aqVirtualBox\(aq == grains.get(\(aqvirtual\(aq, None) or \(aqoracle\(aq == grains.get(\(aqvirtual\(aq, None) %} +{% do storage.update({\(aqdepot_ip\(aq: \(aq192.168.33.81\(aq, \(aqserver_ip\(aq: \(aq192.168.33.51\(aq}) %} +{% else %} +{% set colo = pillar.get(\(aqinventory\(aq, {}).get(\(aqcolo\(aq, \(aqUnknown\(aq) %} +{% set servers_list = pillar.get(\(aqstorage_servers\(aq, {}).get(colo, [storage.depot_ip, ]) %} +{% if opts.id.startswith(\(aqfoo\(aq) %} +{% set modulus = servers_list | count %} +{% set integer_id = opts.id | replace(\(aqfoo\(aq, \(aq\(aq) | int %} +{% set server_index = integer_id % modulus %} +{% else %} +{% set server_index = 0 %} +{% endif %} +{% do storage.update({\(aqserver_ip\(aq: servers_list[server_index]}) %} +{% endif %} + +{% for network, _ in salt.pillar.get(\(aqinventory:networks\(aq, {}) | dictsort %} +{% do storage.ipsets.hash_net.foo_networks.append(network) %} +{% endfor %} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B Example 1: -{\(aqbar\(aq: {\(aqbaz\(aq: [1, 2, 42]}} -.TP -.B Example 2: -{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: [42]}}} -.UNINDENT -.SS \fBextend_dict_key_value\fP -.sp -New in version 3000. - .sp -Allows you to extend a list nested (deep) in a dictionary without having to worry if all the nested keys (or the list itself) actually exist. -Missing keys will automatically be created if they do not exist. -The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +This is an example from the author\(aqs salt formulae demonstrating misuse of jinja. +Aside from being difficult to read and maintain, +accessing the logic it contains from a non\-jinja renderer +while probably possible is a significant barrier! +.SS Refactor .sp -Examples: +The first step is to reduce the maps.jinja file to something reasonable. +This gives us an idea of what the module we are writing needs to do. +There is a lot of logic around selecting a storage server ip. +Let\(aqs move that to an execution module. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C +# storage/maps.yaml + +{% import_yaml \(aqstorage/defaults.yaml\(aq as default_settings %} +{% set storage = default_settings.storage %} +{% do storage.update(salt[\(aqgrains.filter_by\(aq]({ + \(aqDebian\(aq: { + }, + \(aqRedHat\(aq: { + } +}, merge=salt[\(aqpillar.get\(aq](\(aqstorage:lookup\(aq))) %} + +{% if \(aqVirtualBox\(aq == grains.get(\(aqvirtual\(aq, None) or \(aqoracle\(aq == grains.get(\(aqvirtual\(aq, None) %} +{% do storage.update({\(aqdepot_ip\(aq: \(aq192.168.33.81\(aq}) %} +{% endif %} + +{% do storage.update({\(aqserver_ip\(aq: salt[\(aqstorage.ip\(aq]()}) %} +{% for network, _ in salt.pillar.get(\(aqinventory:networks\(aq, {}) | dictsort %} +{% do storage.ipsets.hash_net.af_networks.append(network) %} +{% endfor %} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B Example 1: -{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: [1, 2]}} %} -{{ foo | extend_dict_key_value(\(aqbar:baz\(aq, [42, 42]) }} -.TP -.B Example 2: -{{ {} | extend_dict_key_value(\(aqbar:baz:qux\(aq, [42]) }} -.UNINDENT .sp -Returns: +And then, write the module. +Note how the module encapsulates all of the logic around finding the storage server IP. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C +# _modules/storage.py +#!python + +\(dq\(dq\(dq +Functions related to storage servers. +\(dq\(dq\(dq +import re + + +def ips(): + \(dq\(dq\(dq + Provide a list of all local storage server IPs. + + CLI Example:: + + salt \e* storage.ips + \(dq\(dq\(dq + + if __grains__.get(\(dqvirtual\(dq, None) in [\(dqVirtualBox\(dq, \(dqoracle\(dq]: + return [ + \(dq192.168.33.51\(dq, + ] + + colo = __pillar__.get(\(dqinventory\(dq, {}).get(\(dqcolo\(dq, \(dqUnknown\(dq) + return __pillar__.get(\(dqstorage_servers\(dq, {}).get(colo, [\(dqunknown\(dq]) + + +def ip(): + \(dq\(dq\(dq + Select and return a local storage server IP. + + This loadbalances across storage servers by using the modulus of the client\(aqs id number. + + :maintainer: Andrew Hammond + :maturity: new + :depends: None + :platform: all + + CLI Example:: + + salt \e* storage.ip + + \(dq\(dq\(dq + + numerical_suffix = re.compile(r\(dq^.*(\ed+)$\(dq) + servers_list = ips() + + m = numerical_suffix.match(__grains__[\(dqid\(dq]) + if m: + modulus = len(servers_list) + server_number = int(m.group(1)) + server_index = server_number % modulus + else: + server_index = 0 + + return servers_list[server_index] .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.TP -.B Example 1: -{\(aqbar\(aq: {\(aqbaz\(aq: [1, 2, 42, 42]}} -.TP -.B Example 2: -{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: [42]}}} -.UNINDENT -.SS \fBupdate_dict_key_value\fP +.SS Conclusion .sp -New in version 3000. - +That was... surprisingly straight\-forward. +Now the logic is available in every renderer, instead of just Jinja. +Best of all, it can be maintained in Python, +which is a whole lot easier than Jinja. +.SS Using Apache Libcloud for declarative and procedural multi\-cloud orchestration .sp -Allows you to update a dictionary nested (deep) in another dictionary without having to worry if all the nested keys actually exist. -Missing keys will automatically be created if they do not exist. -The default delimiter for the keys is \(aq:\(aq, however, with the \fIdelimiter\fP\-parameter, a different delimiter can be specified. +New in version 2018.3.0. + .sp -Examples: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C - -.ft P -.fi +This walkthrough assumes basic knowledge of Salt and Salt States. To get up to speed, check out the +\fI\%Salt Walkthrough\fP\&. .UNINDENT .UNINDENT +.sp +Apache Libcloud is a Python library which hides differences between different cloud provider APIs and allows +you to manage different cloud resources through a unified and easy to use API. Apache Libcloud supports over +60 cloud platforms, including Amazon, Microsoft Azure, DigitalOcean, Google Cloud Platform and OpenStack. .INDENT 0.0 .TP -.B Example 1: -{%\- set foo = {\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 1}}} %} -{{ foo | update_dict_key_value(\(aqbar:baz\(aq, {\(aqquux\(aq: 3}) }} -.TP -.B Example 2: -{{ {} | update_dict_key_value(\(aqbar:baz:qux\(aq, {\(aqquux\(aq: 3}) }} +.B Execution and state modules are available for Compute, DNS, Storage and Load Balancer drivers from Apache Libcloud in +SaltStack. .UNINDENT .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C - -.ft P -.fi +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_compute\fP \- Compute \- +services such as OpenStack Nova, Amazon EC2, Microsoft Azure VMs .UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_dns\fP \- DNS as a Service \- +services such as Amazon Route 53 and Zerigo .UNINDENT -.INDENT 0.0 +.IP \(bu 2 +.INDENT 2.0 .TP -.B Example 1: -{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: 1, \(aqquux\(aq: 3}}} +.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service \- +services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 .TP -.B Example 2: -{\(aqbar\(aq: {\(aqbaz\(aq: {\(aqqux\(aq: {\(aqquux\(aq: 3}}}} +.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN \- +services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift +.UNINDENT .UNINDENT -.SS \fBmd5\fP .sp -New in version 2017.7.0. - +These modules are designed as a way of having a multi\-cloud deployment and abstracting simple differences +between platform to design a high\-availability architecture. .sp -Return the md5 digest of a string. +The Apache Libcloud functionality is available through both execution modules and Salt states. +.SS Configuring Drivers .sp -Example: +Drivers can be configured in the Salt Configuration/Minion settings. All libcloud modules expect a list of \(dqprofiles\(dq to +be configured with authentication details for each driver. +.sp +Each driver will have a string identifier, these can be found in the libcloud..types.Provider class +for each API, \fI\%https://libcloud.readthedocs.io/en/latest/supported_providers.html\fP +.sp +Some drivers require additional parameters, which are documented in the Apache Libcloud documentation. For example, +GoDaddy DNS expects \(dq\fIshopper_id\fP\(dq, which is the customer ID. These additional parameters can be added to the profile settings +and will be passed directly to the driver instantiation method. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqrandom\(aq | md5 }} +libcloud_dns: + godaddy: + driver: godaddy + shopper_id: 90425123 + key: AFDDJFGIjDFVNSDIFNASMC + secret: FG(#f8vdfgjlkm) + +libcloud_storage: + google: + driver: google_storage + key: GOOG4ASDIDFNVIdfnIVW + secret: R+qYE9hkfdhv89h4invhdfvird4Pq3an8rnK .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +You can have multiple profiles for a single driver, for example if you wanted 2 DNS profiles for Amazon Route53, +naming them \(dqroute53_prod\(dq and \(dqroute54_test\(dq would help your +administrators distinguish their purpose. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -7ddf32e17a6ac5ce04a8ecbf782ca509 +libcloud_dns: + route53_prod: + driver: route53 + key: AFDDJFGIjDFVNSDIFNASMC + secret: FG(#f8vdfgjlkm) + route53_test: + driver: route53 + key: AFDDJFGIjdfgdfgdf + secret: FG(#f8vdfgjlkm) .ft P .fi .UNINDENT .UNINDENT -.SS \fBsha256\fP +.SS Using the execution modules .sp -New in version 2017.7.0. - +Amongst over 60 clouds that Apache Libcloud supports, you can add profiles to your Salt configuration to access and control these clouds. +Each of the libcloud execution modules exposes the common API methods for controlling Compute, DNS, Load Balancers and Object Storage. +To see which functions are supported across specific clouds, see the Libcloud \fI\%supported methods\fP documentation. .sp -Return the sha256 digest of a string. +The module documentation explains each of the API methods and how to leverage them. +.INDENT 0.0 +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_compute\fP \- Compute \- +services such as OpenStack Nova, Amazon EC2, Microsoft Azure VMs +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_dns\fP \- DNS as a Service \- +services such as Amazon Route 53 and Zerigo +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service \- +services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN \- +services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift +.UNINDENT +.UNINDENT .sp -Example: +For example, listing buckets in the Google Storage platform: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqrandom\(aq | sha256 }} +$ salt\-call libcloud_storage.list_containers google + + local: + |_ + \-\-\-\-\-\-\-\-\-\- + extra: + \-\-\-\-\-\-\-\-\-\- + creation_date: + 2017\-01\-05T05:44:56.324Z + name: + anthonypjshaw .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +The Apache Libcloud storage module can be used to synchronize files between multiple storage clouds, +such as Google Storage, S3 and OpenStack Swift .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -a441b15fe9a3cf56661190a0b93b9dec7d04127288cc87250967cf3b52894d11 +salt \(aq*\(aq libcloud_storage.download_object DeploymentTools test.sh /tmp/test.sh google_storage .ft P .fi .UNINDENT .UNINDENT -.SS \fBsha512\fP +.SS Using the state modules .sp -New in version 2017.7.0. - +For each configured profile, the assets available in the API (e.g. storage objects, containers, +DNS records and load balancers) can be deployed via Salt\(aqs state system. .sp -Return the sha512 digest of a string. +The state module documentation explains the specific states that each module supports +.INDENT 0.0 +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN +.INDENT 7.0 +.IP \(bu 2 +services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift +.UNINDENT +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service +.INDENT 7.0 +.IP \(bu 2 +services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers +.UNINDENT +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B \fI\%libcloud_dns\fP \- DNS as a Service +.INDENT 7.0 +.IP \(bu 2 +services such as Amazon Route 53 and Zerigo +.UNINDENT +.UNINDENT +.UNINDENT .sp -Example: +For DNS, the state modules can be used to provide DNS resilience for multiple nameservers, for example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqrandom\(aq | sha512 }} +libcloud_dns: + godaddy: + driver: godaddy + shopper_id: 12345 + key: 2orgk34kgk34g + secret: fjgoidhjgoim + amazon: + driver: route53 + key: blah + secret: blah .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +And then in a state file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -811a90e1c8e86c7b4c0eef5b2c0bf0ec1b19c4b1b5a242e6455be93787cb473cb7bc9b0fdeb960d00d5c6881c2094dd63c5c900ce9057255e2a4e271fc25fef1 +webserver: + libcloud_dns.zone_present: + name: mywebsite.com + profile: godaddy + libcloud_dns.record_present: + name: www + zone: mywebsite.com + type: A + data: 12.34.32.3 + profile: godaddy + libcloud_dns.zone_present: + name: mywebsite.com + profile: amazon + libcloud_dns.record_present: + name: www + zone: mywebsite.com + type: A + data: 12.34.32.3 + profile: amazon .ft P .fi .UNINDENT .UNINDENT -.SS \fBbase64_encode\fP -.sp -New in version 2017.7.0. - -.sp -Encode a string as base64. .sp -Example: +This could be combined with a multi\-cloud load balancer deployment, .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqrandom\(aq | base64_encode }} +webserver: + libcloud_dns.zone_present: + \- name: mywebsite.com + \- profile: godaddy + ... + libcloud_loadbalancer.balancer_present: + \- name: web_main + \- port: 80 + \- protocol: http + \- members: + \- ip: 1.2.4.5 + port: 80 + \- ip: 2.4.5.6 + port: 80 + \- profile: google_gce + libcloud_loadbalancer.balancer_present: + \- name: web_main + \- port: 80 + \- protocol: http + \- members: + \- ip: 1.2.4.5 + port: 80 + \- ip: 2.4.5.6 + port: 80 + \- profile: amazon_elb .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Extended parameters can be passed to the specific cloud, for example you can specify the region with the Google Cloud API, because +\fIcreate_balancer\fP can accept a \fIex_region\fP argument. Adding this argument to the state will pass the additional command to the driver. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -cmFuZG9t +lb_test: + libcloud_loadbalancer.balancer_absent: + \- name: example + \- port: 80 + \- protocol: http + \- profile: google + \- ex_region: us\-east1 .ft P .fi .UNINDENT .UNINDENT -.SS \fBbase64_decode\fP +.SS Accessing custom arguments in execution modules .sp -New in version 2017.7.0. - +Some cloud providers have additional functionality that can be accessed on top of the base API, for example +the Google Cloud Engine load balancer service offers the ability to provision load balancers into a specific region. .sp -Decode a base64\-encoded string. +Looking at the \fI\%API documentation\fP, +we can see that it expects an \fIex_region\fP in the \fIcreate_balancer\fP method, so when we execute the salt command, we can add this additional parameter like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqZ2V0IHNhbHRlZA==\(aq | base64_decode }} +$ salt myminion libcloud_storage.create_balancer my_balancer 80 http profile1 ex_region=us\-east1 +$ salt myminion libcloud_storage.list_container_objects my_bucket profile1 ex_prefix=me .ft P .fi .UNINDENT .UNINDENT +.SS Accessing custom methods in Libcloud drivers .sp -Returns: +Some cloud APIs have additional methods that are prefixed with \fIex_\fP in Apache Libcloud, these methods +are part of the non\-standard API but can still +be accessed from the Salt modules for \fIlibcloud_storage\fP, \fIlibcloud_loadbalancer\fP and \fIlibcloud_dns\fP\&. +The extra methods are available via the \fIextra\fP command, which expects the name of the method as the +first argument, the profile as the second and then +accepts a list of keyword arguments to pass onto the driver method, for example, accessing permissions in Google Storage objects: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -get salted +$ salt myminion libcloud_storage.extra ex_get_permissions google container_name=my_container object_name=me.jpg \-\-out=yaml .ft P .fi .UNINDENT .UNINDENT -.SS \fBhmac\fP -.sp -New in version 2017.7.0. - -.sp -Verify a challenging hmac signature against a string / shared\-secret. Returns -a boolean value. +.SS Example profiles +.SS Google Cloud .sp -Example: +Using Service Accounts with GCE, you can provide a path to the JSON file and the project name in the parameters. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqget salted\(aq | hmac(\(aqshared secret\(aq, \(aqeBWf9bstXg+NiP5AOwppB5HMvZiYMPzEM9W5YMm/AmQ=\(aq) }} +google: + driver: gce + user_id: 234234\-compute@developer.gserviceaccount.com + key: /path/to/service_account_download.json + auth_type: SA + project: project\-name .ft P .fi .UNINDENT .UNINDENT +.SS LXC Management with Salt .sp -Returns: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -True -.ft P -.fi +This walkthrough assumes basic knowledge of Salt. To get up to speed, check +out the \fI\%Salt Walkthrough\fP\&. .UNINDENT .UNINDENT -.SS \fBhttp_query\fP +.SS Dependencies .sp -New in version 2017.7.0. - +Manipulation of LXC containers in Salt requires the minion to have an LXC +version of at least 1.0 (an alpha or beta release of LXC 1.0 is acceptable). +The following distributions are known to have new enough versions of LXC +packaged: +.INDENT 0.0 +.IP \(bu 2 +RHEL/CentOS 6 and later (via \fI\%EPEL\fP) +.IP \(bu 2 +Fedora (All non\-EOL releases) +.IP \(bu 2 +Debian 8.0 (Jessie) +.IP \(bu 2 +Ubuntu 14.04 LTS and later (LXC templates are packaged separately as +\fBlxc\-templates\fP, it is recommended to also install this package) +.IP \(bu 2 +openSUSE 13.2 and later +.UNINDENT +.SS Profiles .sp -Return the HTTP reply object from a URL. +Profiles allow for a sort of shorthand for commonly\-used +configurations to be defined in the minion config file, \fI\%grains\fP, \fI\%pillar\fP, or the master config file. The +profile is retrieved by Salt using the \fI\%config.get\fP function, which looks in those locations, in that +order. This allows for profiles to be defined centrally in the master config +file, with several options for overriding them (if necessary) on groups of +minions or individual minions. .sp -Example: +There are two types of profiles: .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{{ \(aqhttp://jsonplaceholder.typicode.com/posts/1\(aq | http_query }} -.ft P -.fi +.INDENT 0.0 +.IP \(bu 2 +One for defining the parameters used in container creation/clone. +.IP \(bu 2 +One for defining the container\(aqs network interface(s) settings. +.UNINDENT .UNINDENT .UNINDENT +.SS Container Profiles .sp -Returns: +LXC container profiles are defined underneath the +\fBlxc.container_profile\fP config option: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{ - \(aqbody\(aq: \(aq{ - \(dquserId\(dq: 1, - \(dqid\(dq: 1, - \(dqtitle\(dq: \(dqsunt aut facere repellat provident occaecati excepturi option reprehenderit\(dq, - \(dqbody\(dq: \(dqquia et suscipit\e\ensuscipit recusandae consequuntur expedita et cum\e\enreprehenderit molestiae ut ut quas totam\e\ennostrum rerum est autem sunt rem eveniet architecto\(dq - }\(aq -} +lxc.container_profile: + centos: + template: centos + backing: lvm + vgname: vg1 + lvname: lxclv + size: 10G + centos_big: + template: centos + backing: lvm + vgname: vg1 + lvname: lxclv + size: 20G .ft P .fi .UNINDENT .UNINDENT -.SS \fBtraverse\fP -.sp -New in version 2018.3.3. - .sp -Traverse a dict or list using a colon\-delimited target string. -The target \(aqfoo:bar:0\(aq will return data[\(aqfoo\(aq][\(aqbar\(aq][0] if this value exists, -and will otherwise return the provided default value. +Profiles are retrieved using the \fI\%config.get\fP +function, with the \fBrecurse\fP merge strategy. This means that a profile can be +defined at a lower level (for example, the master config file) and then parts +of it can be overridden at a higher level (for example, in pillar data). +Consider the following container profile data: .sp -Example: +\fBIn the Master config file:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ {\(aqa1\(aq: {\(aqb1\(aq: {\(aqc1\(aq: \(aqfoo\(aq}}, \(aqa2\(aq: \(aqbar\(aq} | traverse(\(aqa1:b1\(aq, \(aqdefault\(aq) }} +lxc.container_profile: + centos: + template: centos + backing: lvm + vgname: vg1 + lvname: lxclv + size: 10G .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBIn the Pillar data\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{\(dqc1\(dq: \(dqfoo\(dq} +lxc.container_profile: + centos: + size: 20G .ft P .fi .UNINDENT .UNINDENT +.sp +Any minion with the above Pillar data would have the \fBsize\fP parameter in the +\fBcentos\fP profile overridden to 20G, while those minions without the above +Pillar data would have the 10G \fBsize\fP value. This is another way of achieving +the same result as the \fBcentos_big\fP profile above, without having to define +another whole profile that differs in just one value. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{{ {\(aqa1\(aq: {\(aqb1\(aq: {\(aqc1\(aq: \(aqfoo\(aq}}, \(aqa2\(aq: \(aqbar\(aq} | traverse(\(aqa2:b2\(aq, \(aqdefault\(aq) }} -.ft P -.fi +In the 2014.7.x release cycle and earlier, container profiles are defined +under \fBlxc.profile\fP\&. This parameter will still work in version 2015.5.0, +but is deprecated and will be removed in a future release. Please note +however that the profile merging feature described above will only work +with profiles defined under \fBlxc.container_profile\fP, and only in versions +2015.5.0 and later. .UNINDENT .UNINDENT .sp -Returns: -.INDENT 0.0 -.INDENT 3.5 +Additionally, in version 2015.5.0 container profiles have been expanded to +support passing template\-specific CLI options to \fI\%lxc.create\fP\&. Below is a table describing the parameters which +can be configured in container profiles: +.TS +center; +|l|l|l|. +_ +T{ +Parameter +T} T{ +2015.5.0 and Newer +T} T{ +2014.7.x and Earlier +T} +_ +T{ +\fItemplate\fP\s-2\u1\d\s0 +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIoptions\fP\s-2\u1\d\s0 +T} T{ +Yes +T} T{ +No +T} +_ +T{ +\fIimage\fP\s-2\u1\d\s0 +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIbacking\fP +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIsnapshot\fP\s-2\u2\d\s0 +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIlvname\fP\s-2\u1\d\s0 +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIfstype\fP\s-2\u1\d\s0 +T} T{ +Yes +T} T{ +Yes +T} +_ +T{ +\fIsize\fP +T} T{ +Yes +T} T{ +Yes +T} +_ +.TE +.INDENT 0.0 +.IP 1. 3 +Parameter is only supported for container creation, and will be ignored if +the profile is used when cloning a container. +.IP 2. 3 +Parameter is only supported for container cloning, and will be ignored if +the profile is used when not cloning a container. +.UNINDENT +.SS Network Profiles +.sp +LXC network profiles are defined defined underneath the \fBlxc.network_profile\fP +config option. +By default, the module uses a DHCP based configuration and try to guess a bridge to +get connectivity. +.sp +\fBWARNING:\fP +.INDENT 0.0 +.INDENT 3.5 +on pre \fB2015.5.2\fP, you need to specify explicitly the network bridge +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 .sp .nf .ft C -\(dqdefault\(dq +lxc.network_profile: + centos: + eth0: + link: br0 + type: veth + flags: up + ubuntu: + eth0: + link: lxcbr0 + type: veth + flags: up .ft P .fi .UNINDENT .UNINDENT -.SS \fBjson_query\fP .sp -New in version 3000. - -.sp -A port of Ansible \fBjson_query\fP Jinja filter to make queries against JSON data using \fI\%JMESPath language\fP\&. -Could be used to filter \fBpillar\fP data, \fByaml\fP maps, and together with \fI\%http_query\fP\&. -Depends on the \fI\%jmespath\fP Python module. +As with container profiles, network profiles are retrieved using the +\fI\%config.get\fP function, with the \fBrecurse\fP +merge strategy. Consider the following network profile data: .sp -Examples: +\fBIn the Master config file:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example 1: {{ [1, 2, 3, 4, [5, 6]] | json_query(\(aq[]\(aq) }} - -Example 2: {{ -{\(dqmachines\(dq: [ - {\(dqname\(dq: \(dqa\(dq, \(dqstate\(dq: \(dqrunning\(dq}, - {\(dqname\(dq: \(dqb\(dq, \(dqstate\(dq: \(dqstopped\(dq}, - {\(dqname\(dq: \(dqc\(dq, \(dqstate\(dq: \(dqrunning\(dq} -]} | json_query(\(dqmachines[?state==\(aqrunning\(aq].name\(dq) }} - -Example 3: {{ -{\(dqservices\(dq: [ - {\(dqname\(dq: \(dqhttp\(dq, \(dqhost\(dq: \(dq1.2.3.4\(dq, \(dqport\(dq: 80}, - {\(dqname\(dq: \(dqsmtp\(dq, \(dqhost\(dq: \(dq1.2.3.5\(dq, \(dqport\(dq: 25}, - {\(dqname\(dq: \(dqssh\(dq, \(dqhost\(dq: \(dq1.2.3.6\(dq, \(dqport\(dq: 22}, -]} | json_query(\(dqservices[].port\(dq) }} +lxc.network_profile: + centos: + eth0: + link: br0 + type: veth + flags: up .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBIn the Pillar data\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example 1: [1, 2, 3, 4, 5, 6] - -Example 2: [\(aqa\(aq, \(aqc\(aq] - -Example 3: [80, 25, 22] +lxc.network_profile: + centos: + eth0: + link: lxcbr0 .ft P .fi .UNINDENT .UNINDENT -.SS \fBto_snake_case\fP .sp -New in version 3000. - +Any minion with the above Pillar data would use the \fBlxcbr0\fP interface as the +bridge interface for any container configured using the \fBcentos\fP network +profile, while those minions without the above Pillar data would use the +\fBbr0\fP interface for the same. .sp -Converts a string from camelCase (or CamelCase) to snake_case. +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -Example: {{ camelsWillLoveThis | to_snake_case }} -.ft P -.fi +In the 2014.7.x release cycle and earlier, network profiles are defined +under \fBlxc.nic\fP\&. This parameter will still work in version 2015.5.0, but +is deprecated and will be removed in a future release. Please note however +that the profile merging feature described above will only work with +profiles defined under \fBlxc.network_profile\fP, and only in versions +2015.5.0 and later. .UNINDENT .UNINDENT .sp -Returns: +The following are parameters which can be configured in network profiles. These +will directly correspond to a parameter in an LXC configuration file (see \fBman +5 lxc.container.conf\fP). +.INDENT 0.0 +.IP \(bu 2 +\fBtype\fP \- Corresponds to \fBlxc.network.type\fP +.IP \(bu 2 +\fBlink\fP \- Corresponds to \fBlxc.network.link\fP +.IP \(bu 2 +\fBflags\fP \- Corresponds to \fBlxc.network.flags\fP +.UNINDENT +.sp +Interface\-specific options (MAC address, IPv4/IPv6, etc.) must be passed on a +container\-by\-container basis, for instance using the \fBnic_opts\fP argument to +\fI\%lxc.create\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example: camels_will_love_this +salt myminion lxc.create container1 profile=centos network_profile=centos nic_opts=\(aq{eth0: {ipv4: 10.0.0.20/24, gateway: 10.0.0.1}}\(aq .ft P .fi .UNINDENT .UNINDENT -.SS \fBto_camelcase\fP -.sp -New in version 3000. - .sp -Converts a string from snake_case to camelCase (or UpperCamelCase if so indicated). +\fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -Example 1: {{ snake_case_for_the_win | to_camelcase }} - -Example 2: {{ snake_case_for_the_win | to_camelcase(uppercamel=True) }} -.ft P -.fi +The \fBipv4\fP, \fBipv6\fP, \fBgateway\fP, and \fBlink\fP (bridge) settings in +network profiles / nic_opts will only work if the container doesn\(aqt redefine +the network configuration (for example in +\fB/etc/sysconfig/network\-scripts/ifcfg\-\fP on RHEL/CentOS, +or \fB/etc/network/interfaces\fP on Debian/Ubuntu/etc.). Use these with +caution. The container images installed using the \fBdownload\fP template, +for instance, typically are configured for eth0 to use DHCP, which will +conflict with static IP addresses set at the container level. .UNINDENT .UNINDENT .sp -Returns: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +For LXC < 1.0.7 and DHCP support, set \fBipv4.gateway: \(aqauto\(aq\fP is your +network profile, ie.: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example 1: snakeCaseForTheWin -Example 2: SnakeCaseForTheWin +lxc.network_profile.nic: + debian: + eth0: + link: lxcbr0 + ipv4.gateway: \(aqauto\(aq .ft P .fi .UNINDENT .UNINDENT -.SS \fBhuman_to_bytes\fP +.UNINDENT +.UNINDENT +.SS Old lxc support (<1.0.7) .sp -New in version 3005. - +With saltstack \fB2015.5.2\fP and above, normally the setting is autoselected, but +before, you\(aqll need to teach your network profile to set +\fBlxc.network.ipv4.gateway\fP to \fBauto\fP when using a classic ipv4 configuration. .sp -Given a human\-readable byte string (e.g. 2G, 30MB, 64KiB), return the number of bytes. -Will return 0 if the argument has unexpected form. +Thus you\(aqll need .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example 1: {{ \(dq32GB\(dq | human_to_bytes }} - -Example 2: {{ \(dq32GB\(dq | human_to_bytes(handle_metric=True) }} - -Example 3: {{ \(dq32\(dq | human_to_bytes(default_unit=\(dqGiB\(dq) }} +lxc.network_profile.foo: + etho: + link: lxcbr0 + ipv4.gateway: auto .ft P .fi .UNINDENT .UNINDENT +.SS Tricky network setups Examples .sp -Returns: +This example covers how to make a container with both an internal ip and a +public routable ip, wired on two veth pairs. +.sp +The another interface which receives directly a public routable ip can\(aqt be on +the first interface that we reserve for private inter LXC networking. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Example 1: 34359738368 -Example 2: 32000000000 -Example 3: 34359738368 +lxc.network_profile.foo: + eth0: {gateway: null, bridge: lxcbr0} + eth1: + # replace that by your main interface + \(aqlink\(aq: \(aqbr0\(aq + \(aqmac\(aq: \(aq00:16:5b:01:24:e1\(aq + \(aqgateway\(aq: \(aq2.20.9.14\(aq + \(aqipv4\(aq: \(aq2.20.9.1\(aq .ft P .fi .UNINDENT .UNINDENT -.SS Networking Filters +.SS Creating a Container on the CLI +.SS From a Template .sp -The following networking\-related filters are supported: -.SS \fBis_ip\fP +LXC is commonly distributed with several template scripts in +/usr/share/lxc/templates. Some distros may package these separately in an +\fBlxc\-templates\fP package, so make sure to check if this is the case. .sp -New in version 2017.7.0. - +There are LXC template scripts for several different operating systems, but +some of them are designed to use tools specific to a given distribution. For +instance, the \fBubuntu\fP template uses deb_bootstrap, the \fBcentos\fP template +uses yum, etc., making these templates impractical when a container from a +different OS is desired. .sp -Return if a string is a valid IP Address. +The \fI\%lxc.create\fP function is used to create +containers using a template script. To create a CentOS container named +\fBcontainer1\fP on a CentOS minion named \fBmycentosminion\fP, using the +\fBcentos\fP LXC template, one can simply run the following command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq192.168.0.1\(aq | is_ip }} +salt mycentosminion lxc.create container1 template=centos .ft P .fi .UNINDENT .UNINDENT .sp -Additionally accepts the following options: +For these instances, there is a \fBdownload\fP template which retrieves minimal +container images for several different operating systems. To use this template, +it is necessary to provide an \fBoptions\fP parameter when creating the +container, with three values: .INDENT 0.0 -.IP \(bu 2 -global -.IP \(bu 2 -link\-local -.IP \(bu 2 -loopback -.IP \(bu 2 -multicast -.IP \(bu 2 -private -.IP \(bu 2 -public -.IP \(bu 2 -reserved -.IP \(bu 2 -site\-local -.IP \(bu 2 -unspecified +.IP 1. 3 +\fBdist\fP \- the Linux distribution (i.e. \fBubuntu\fP or \fBcentos\fP) +.IP 2. 3 +\fBrelease\fP \- the release name/version (i.e. \fBtrusty\fP or \fB6\fP) +.IP 3. 3 +\fBarch\fP \- CPU architecture (i.e. \fBamd64\fP or \fBi386\fP) .UNINDENT .sp -Example \- test if a string is a valid loopback IP address. +The \fI\%lxc.images\fP function (new in version +2015.5.0) can be used to list the available images. Alternatively, the releases +can be viewed on \fI\%http://images.linuxcontainers.org/images/\fP\&. The images are +organized in such a way that the \fBdist\fP, \fBrelease\fP, and \fBarch\fP can be +determined using the following URL format: +\fBhttp://images.linuxcontainers.org/images/dist/release/arch\fP\&. For example, +\fBhttp://images.linuxcontainers.org/images/centos/6/amd64\fP would correspond to +a \fBdist\fP of \fBcentos\fP, a \fBrelease\fP of \fB6\fP, and an \fBarch\fP of \fBamd64\fP\&. +.sp +Therefore, to use the \fBdownload\fP template to create a new 64\-bit CentOS 6 +container, the following command can be used: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq192.168.0.1\(aq | is_ip(options=\(aqloopback\(aq) }} +salt myminion lxc.create container1 template=download options=\(aq{dist: centos, release: 6, arch: amd64}\(aq .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_ipv4\fP -.sp -New in version 2017.7.0. - .sp -Returns if a string is a valid IPv4 address. Supports the same options -as \fBis_ip\fP\&. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +These command\-line options can be placed into a \fI\%container profile\fP, like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq192.168.0.1\(aq | is_ipv4 }} +lxc.container_profile.cent6: + template: download + options: + dist: centos + release: 6 + arch: amd64 .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_ipv6\fP .sp -New in version 2017.7.0. - +The \fBoptions\fP parameter is not supported in profiles for the 2014.7.x +release cycle and earlier, so it would still need to be provided on the +command\-line. +.UNINDENT +.UNINDENT +.SS Cloning an Existing Container .sp -Returns if a string is a valid IPv6 address. Supports the same options -as \fBis_ip\fP\&. +To clone a container, use the \fI\%lxc.clone\fP +function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aqfe80::\(aq | is_ipv6 }} +salt myminion lxc.clone container2 orig=container1 .ft P .fi .UNINDENT .UNINDENT -.SS \fBipaddr\fP -.sp -New in version 2017.7.0. - -.sp -From a list, returns only valid IP entries. Supports the same options -as \fBis_ip\fP\&. The list can contains also IP interfaces/networks. +.SS Using a Container Image .sp -Example: +While cloning is a good way to create new containers from a common base +container, the source container that is being cloned needs to already exist on +the minion. This makes deploying a common container across minions difficult. +For this reason, Salt\(aqs \fI\%lxc.create\fP is capable +of installing a container from a tar archive of another container\(aqs rootfs. To +create an image of a container named \fBcent6\fP, run the following command as +root: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [\(aq192.168.0.1\(aq, \(aqfoo\(aq, \(aqbar\(aq, \(aqfe80::\(aq] | ipaddr }} +tar czf cent6.tar.gz \-C /var/lib/lxc/cent6 rootfs .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Before doing this, it is recommended that the container is stopped. +.UNINDENT +.UNINDENT +.sp +The resulting tarball can then be placed alongside the files in the salt +fileserver and referenced using a \fBsalt://\fP URL. To create a container using +an image, use the \fBimage\fP parameter with \fI\%lxc.create\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[\(dq192.168.0.1\(dq, \(dqfe80::\(dq] +salt myminion lxc.create new\-cent6 image=salt://path/to/cent6.tar.gz .ft P .fi .UNINDENT .UNINDENT -.SS \fBipv4\fP -.sp -New in version 2017.7.0. - .sp -From a list, returns only valid IPv4 entries. Supports the same options -as \fBis_ip\fP\&. The list can contains also IP interfaces/networks. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +Making images of containers with LVM backing .sp -Example: +For containers with LVM backing, the rootfs is not mounted, so it is +necessary to mount it first before creating the tar archive. When a +container is created using LVM backing, an empty \fBrootfs\fP dir is handily +created within \fB/var/lib/lxc/container_name\fP, so this can be used as the +mountpoint. The location of the logical volume for the container will be +\fB/dev/vgname/lvname\fP, where \fBvgname\fP is the name of the volume group, +and \fBlvname\fP is the name of the logical volume. Therefore, assuming a +volume group of \fBvg1\fP, a logical volume of \fBlxc\-cent6\fP, and a container +name of \fBcent6\fP, the following commands can be used to create a tar +archive of the rootfs: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [\(aq192.168.0.1\(aq, \(aqfoo\(aq, \(aqbar\(aq, \(aqfe80::\(aq] | ipv4 }} +mount /dev/vg1/lxc\-cent6 /var/lib/lxc/cent6/rootfs +tar czf cent6.tar.gz \-C /var/lib/lxc/cent6 rootfs +umount /var/lib/lxc/cent6/rootfs .ft P .fi .UNINDENT .UNINDENT +.UNINDENT +.UNINDENT .sp -Returns: +\fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 +One caveat of using this method of container creation is that +\fB/etc/hosts\fP is left unmodified. This could cause confusion for some +distros if salt\-minion is later installed on the container, as the +functions that determine the hostname take \fB/etc/hosts\fP into account. .sp -.nf -.ft C -[\(dq192.168.0.1\(dq] -.ft P -.fi +Additionally, when creating an rootfs image, be sure to remove +\fB/etc/salt/minion_id\fP and make sure that \fBid\fP is not defined in +\fB/etc/salt/minion\fP, as this will cause similar issues. .UNINDENT .UNINDENT -.SS \fBipv6\fP -.sp -New in version 2017.7.0. - +.SS Initializing a New Container as a Salt Minion .sp -From a list, returns only valid IPv6 entries. Supports the same options -as \fBis_ip\fP\&. The list can contains also IP interfaces/networks. +The above examples illustrate a few ways to create containers on the CLI, but +often it is desirable to also have the new container run as a Minion. To do +this, the \fI\%lxc.init\fP function can be used. This +function will do the following: +.INDENT 0.0 +.IP 1. 3 +Create a new container +.IP 2. 3 +Optionally set password and/or DNS +.IP 3. 3 +Bootstrap the minion (using either \fI\%salt\-bootstrap\fP or a custom command) +.UNINDENT .sp -Example: +By default, the new container will be pointed at the same Salt Master as the +host machine on which the container was created. It will then request to +authenticate with the Master like any other bootstrapped Minion, at which point +it can be accepted. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [\(aq192.168.0.1\(aq, \(aqfoo\(aq, \(aqbar\(aq, \(aqfe80::\(aq] | ipv6 }} +salt myminion lxc.init test1 profile=centos +salt\-key \-a test1 .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +For even greater convenience, the \fI\%LXC runner\fP contains +a runner function of the same name (\fI\%lxc.init\fP), +which creates a keypair, seeds the new minion with it, and pre\-accepts the key, +allowing for the new Minion to be created and authorized in a single step: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[\(dqfe80::\(dq] +salt\-run lxc.init test1 host=myminion profile=centos .ft P .fi .UNINDENT .UNINDENT -.SS \fBipwrap\fP +.SS Running Commands Within a Container .sp -New in version 3006.0. - +For containers which are not running their own Minion, commands can be run +within the container in a manner similar to using (\fBcmd.run +\(aq [arguments] .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +\fBSEE ALSO:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -\(aq172.217.3.196\(aq -.ft P -.fi +\fI\%salt manpage\fP .UNINDENT .UNINDENT -.SS File filters -.SS \fBis_text_file\fP -.sp -New in version 2017.7.0. - -.sp -Return if a file is text. -.sp -Uses heuristics to guess whether the given file is text or binary, -by reading a single block of bytes from the file. -If more than 30% of the chars in the block are non\-text, or there -are NUL (\(aqx00\(aq) bytes in the block, assume this is a binary file. +.SS target .sp -Example: +The target component allows you to filter which minions should run the +following function. The default filter is a glob on the minion id. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq/etc/salt/master\(aq | is_text_file }} +salt \(aq*\(aq test.version +salt \(aq*.example.org\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Targets can be based on minion system information using the Grains system: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -True +salt \-G \(aqos:Ubuntu\(aq test.version .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_binary_file\fP -.sp -New in version 2017.7.0. - .sp -Return if a file is binary. -.sp -Detects if the file is a binary, returns bool. Returns True if the file is -a bin, False if the file is not and None if the file is not available. +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%Grains system\fP +.UNINDENT +.UNINDENT .sp -Example: +Targets can be filtered by regular expression: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq/etc/salt/master\(aq | is_binary_file }} +salt \-E \(aqvirtmach[0\-9]\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Targets can be explicitly specified in a list: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -False +salt \-L \(aqfoo,bar,baz,quo\(aq test.version .ft P .fi .UNINDENT .UNINDENT -.SS \fBis_empty_file\fP -.sp -New in version 2017.7.0. - -.sp -Return if a file is empty. .sp -Example: +Or Multiple target types can be combined in one command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq/etc/salt/master\(aq | is_empty_file }} +salt \-C \(aqG@os:Ubuntu and webser* or E@database.*\(aq test.version .ft P .fi .UNINDENT .UNINDENT +.SS function .sp -Returns: +A function is some functionality provided by a module. Salt ships with a large +collection of available functions. List all available functions on your +minions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -False +salt \(aq*\(aq sys.doc .ft P .fi .UNINDENT .UNINDENT -.SS \fBfile_hashsum\fP .sp -New in version 2017.7.0. - -.sp -Return the hashsum of a file. +Here are some examples: .sp -Example: +Show all currently available minions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq/etc/salt/master\(aq | file_hashsum }} +salt \(aq*\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Run an arbitrary shell command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -02d4ef135514934759634f10079653252c7ad594ea97bd385480c532bca0fdda +salt \(aq*\(aq cmd.run \(aquname \-a\(aq .ft P .fi .UNINDENT .UNINDENT -.SS \fBlist_files\fP -.sp -New in version 2017.7.0. - .sp -Return a recursive list of files under a specific path. +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +\fI\%the full list of modules\fP +.UNINDENT +.UNINDENT +.SS arguments .sp -Example: +Space\-delimited arguments to the function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq/etc/salt/\(aq | list_files | join(\(aq\en\(aq) }} +salt \(aq*\(aq cmd.exec_code python \(aqimport sys; print sys.version\(aq .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +Optional, keyword arguments are also supported: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/etc/salt/master -/etc/salt/proxy -/etc/salt/minion -/etc/salt/pillar/top.sls -/etc/salt/pillar/device1.sls +salt \(aq*\(aq pip.install salt timeout=5 upgrade=True .ft P .fi .UNINDENT .UNINDENT -.SS \fBpath_join\fP .sp -New in version 2017.7.0. - +They are always in the form of \fBkwarg=argument\fP\&. +.SS Multi Master Tutorial .sp -Joins absolute paths. +As of Salt 0.16.0, the ability to connect minions to multiple masters has been +made available. The multi\-master system allows for redundancy of Salt +masters and facilitates multiple points of communication out to minions. When +using a multi\-master setup, all masters are running hot, and any active master +can be used to send commands out to the minions. .sp -Example: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +If you need failover capabilities with multiple masters, there is also a +MultiMaster\-PKI setup available, that uses a different topology +\fI\%MultiMaster\-PKI with Failover Tutorial\fP +.UNINDENT +.UNINDENT .sp -.nf -.ft C -{{ \(aq/etc/salt/\(aq | path_join(\(aqpillar\(aq, \(aqdevice1.sls\(aq) }} -.ft P -.fi +In 0.16.0, the masters do not share any information, keys need to be accepted +on both masters, and shared files need to be shared manually or use tools like +the git fileserver backend to ensure that the \fI\%file_roots\fP are +kept consistent. +.sp +Beginning with Salt 2016.11.0, the \fI\%Pluggable Minion Data Cache\fP +was introduced. The minion data cache contains the Salt Mine data, minion grains, and minion +pillar information cached on the Salt Master. By default, Salt uses the \fBlocalfs\fP cache +module, but other external data stores can be used instead. +.sp +Using a pluggable minion cache modules allows for the data stored on a Salt Master about +Salt Minions to be replicated on other Salt Masters the Minion is connected to. Please see +the \fI\%Minion Data Cache\fP documentation for more information and configuration +examples. +.SS Summary of Steps +.INDENT 0.0 +.IP 1. 3 +Create a redundant master server +.IP 2. 3 +Copy primary master key to redundant master +.IP 3. 3 +Start redundant master +.IP 4. 3 +Configure minions to connect to redundant master +.IP 5. 3 +Restart minions +.IP 6. 3 +Accept keys on redundant master +.UNINDENT +.SS Prepping a Redundant Master +.sp +The first task is to prepare the redundant master. If the redundant master is +already running, stop it. There is only one requirement when preparing a +redundant master, which is that masters share the same private key. When the +first master was created, the master\(aqs identifying key pair was generated and +placed in the master\(aqs \fBpki_dir\fP\&. The default location of the master\(aqs key +pair is \fB/etc/salt/pki/master/\fP\&. Take the private key, \fBmaster.pem\fP, and +copy it to the same location on the redundant master. Do the same for the +master\(aqs public key, \fBmaster.pub\fP\&. Assuming that no minions have yet been +connected to the new redundant master, it is safe to delete any existing key +in this location and replace it. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +There is no logical limit to the number of redundant masters that can be +used. .UNINDENT .UNINDENT .sp -Returns: +Once the new key is in place, the redundant master can be safely started. +.SS Configure Minions +.sp +Since minions need to be master\-aware, the new master needs to be added to the +minion configurations. Simply update the minion configurations to list all +connected masters: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/etc/salt/pillar/device1.sls +master: + \- saltmaster1.example.com + \- saltmaster2.example.com .ft P .fi .UNINDENT .UNINDENT -.SS \fBwhich\fP -.sp -New in version 2017.7.0. - .sp -Python clone of /usr/bin/which. +Now the minion can be safely restarted. .sp -Example: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +If the ipc_mode for the minion is set to TCP (default in Windows), then +each minion in the multi\-minion setup (one per master) needs its own +tcp_pub_port and tcp_pull_port. .sp -.nf -.ft C -{{ \(aqsalt\-master\(aq | which }} -.ft P -.fi +If these settings are left as the default 4510/4511, each minion object +will receive a port 2 higher than the previous. Thus the first minion will +get 4510/4511, the second will get 4512/4513, and so on. If these port +decisions are unacceptable, you must configure tcp_pub_port and +tcp_pull_port with lists of ports for each master. The length of these +lists should match the number of masters, and there should not be overlap +in the lists. .UNINDENT .UNINDENT .sp -Returns: +Now the minions will check into the original master and also check into the new +redundant master. Both masters are first\-class and have rights to the minions. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +Minions can automatically detect failed masters and attempt to reconnect +to them quickly. To enable this functionality, set +\fImaster_alive_interval\fP in the minion config and specify a number of +seconds to poll the masters for connection status. .sp -.nf -.ft C -/usr/local/salt/virtualenv/bin/salt\-master -.ft P -.fi +If this option is not set, minions will still reconnect to failed masters +but the first command sent after a master comes back up may be lost while +the minion authenticates. .UNINDENT .UNINDENT -.SS Tests +.SS Sharing Files Between Masters .sp -Saltstack extends \fI\%builtin tests\fP with these custom tests: -.SS \fBequalto\fP +Salt does not automatically share files between multiple masters. A number of +files should be shared or sharing of these files should be strongly considered. +.SS Minion Keys .sp -Tests the equality between two values. +Minion keys can be accepted the normal way using \fBsalt\-key\fP on both +masters. Keys accepted, deleted, or rejected on one master will NOT be +automatically managed on redundant masters; this needs to be taken care of by +running salt\-key on both masters or sharing the +\fB/etc/salt/pki/master/{minions,minions_pre,minions_rejected}\fP directories +between masters. .sp -Can be used in an \fBif\fP statement directly: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +While sharing the \fB/etc/salt/pki/master\fP directory will work, it is +strongly discouraged, since allowing access to the \fBmaster.pem\fP key +outside of Salt creates a \fISERIOUS\fP security risk. +.UNINDENT +.UNINDENT +.SS File_Roots .sp -.nf -.ft C -{% if 1 is equalto(1) %} - < statements > -{% endif %} -.ft P -.fi +The \fI\%file_roots\fP contents should be kept consistent between +masters. Otherwise state runs will not always be consistent on minions since +instructions managed by one master will not agree with other masters. +.sp +The recommended way to sync these is to use a fileserver backend like gitfs or +to keep these files on shared storage. +.sp +\fBIMPORTANT:\fP +.INDENT 0.0 +.INDENT 3.5 +If using gitfs/git_pillar with the cachedir shared between masters using +\fI\%GlusterFS\fP, nfs, or another network filesystem, and the masters are +running Salt 2015.5.9 or later, it is strongly recommended not to turn off +\fI\%gitfs_global_lock\fP/\fI\%git_pillar_global_lock\fP as +doing so will cause lock files to be removed if they were created by a +different master. .UNINDENT .UNINDENT +.SS Pillar_Roots .sp -If clause evaluates to \fBTrue\fP +Pillar roots should be given the same considerations as +\fI\%file_roots\fP\&. +.SS Master Configurations .sp -or with the \fBselectattr\fP filter: +While reasons may exist to maintain separate master configurations, it is wise +to remember that each master maintains independent control over minions. +Therefore, access controls should be in sync between masters unless a valid +reason otherwise exists to keep them inconsistent. +.sp +These access control options include but are not limited to: +.INDENT 0.0 +.IP \(bu 2 +external_auth +.IP \(bu 2 +publisher_acl +.IP \(bu 2 +peer +.IP \(bu 2 +peer_run +.UNINDENT +.SS Multi\-Master\-PKI Tutorial With Failover +.sp +This tutorial will explain, how to run a salt\-environment where a single +minion can have multiple masters and fail\-over between them if its current +master fails. +.sp +The individual steps are .INDENT 0.0 +.IP \(bu 2 +setup the master(s) to sign its auth\-replies +.IP \(bu 2 +setup minion(s) to verify master\-public\-keys +.IP \(bu 2 +enable multiple masters on minion(s) +.IP \(bu 2 +enable master\-check on minion(s) +.INDENT 2.0 .INDENT 3.5 +Please note, that it is advised to have good knowledge of the salt\- +authentication and communication\-process to understand this tutorial. +All of the settings described here, go on top of the default +authentication/communication process. +.UNINDENT +.UNINDENT +.UNINDENT +.SS Motivation .sp -.nf -.ft C -{{ [{\(aqvalue\(aq: 1}, {\(aqvalue\(aq: 2} , {\(aqvalue\(aq: 3}] | selectattr(\(aqvalue\(aq, \(aqequalto\(aq, 3) | list }} -.ft P -.fi +The default behaviour of a salt\-minion is to connect to a master and accept +the masters public key. With each publication, the master sends his public\-key +for the minion to check and if this public\-key ever changes, the minion +complains and exits. Practically this means, that there can only be a single +master at any given time. +.sp +Would it not be much nicer, if the minion could have any number of masters +(1:n) and jump to the next master if its current master died because of a +network or hardware failure? +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +There is also a MultiMaster\-Tutorial with a different approach and topology +than this one, that might also suite your needs or might even be better suited +\fI\%Multi\-Master Tutorial\fP .UNINDENT .UNINDENT .sp -Returns: +It is also desirable, to add some sort of authenticity\-check to the very first +public key a minion receives from a master. Currently a minions takes the +first masters public key for granted. +.SS The Goal +.sp +Setup the master to sign the public key it sends to the minions and enable the +minions to verify this signature for authenticity. +.SS Prepping the master to sign its public key +.sp +For signing to work, both master and minion must have the signing and/or +verification settings enabled. If the master signs the public key but the +minion does not verify it, the minion will complain and exit. The same +happens, when the master does not sign but the minion tries to verify. +.sp +The easiest way to have the master sign its public key is to set .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[{\(dqvalue\(dq: 3}] +master_sign_pubkey: True .ft P .fi .UNINDENT .UNINDENT -.SS \fBmatch\fP -.sp -Tests that a string matches the regex passed as an argument. .sp -Can be used in a \fBif\fP statement directly: +After restarting the salt\-master service, the master will automatically +generate a new key\-pair .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% if \(aqa\(aq is match(\(aq[a\-b]\(aq) %} - < statements > -{% endif %} +master_sign.pem +master_sign.pub .ft P .fi .UNINDENT .UNINDENT .sp -If clause evaluates to \fBTrue\fP -.sp -or with the \fBselectattr\fP filter: +A custom name can be set for the signing key\-pair by setting .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ [{\(aqvalue\(aq: \(aqa\(aq}, {\(aqvalue\(aq: \(aqb\(aq}, {\(aqvalue\(aq: \(aqc\(aq}] | selectattr(\(aqvalue\(aq, \(aqmatch\(aq, \(aq[b\-e]\(aq) | list }} +master_sign_key_name: .ft P .fi .UNINDENT .UNINDENT .sp -Returns: +The master will then generate that key\-pair upon restart and use it for +creating the public keys signature attached to the auth\-reply. +.sp +The computation is done for every auth\-request of a minion. If many minions +auth very often, it is advised to use conf_master:\fImaster_pubkey_signature\fP +and conf_master:\fImaster_use_pubkey_signature\fP settings described below. +.sp +If multiple masters are in use and should sign their auth\-replies, the signing +key\-pair master_sign.* has to be copied to each master. Otherwise a minion +will fail to verify the masters public when connecting to a different master +than it did initially. That is because the public keys signature was created +with a different signing key\-pair. +.SS Prepping the minion to verify received public keys +.sp +The minion must have the public key (and only that one!) available to be +able to verify a signature it receives. That public key (defaults to +master_sign.pub) must be copied from the master to the minions pki\-directory. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[{\(dqvalue\(dq: \(dqb\(dq}, {\(dqvalue\(dq: \(dqc\(dq}] +/etc/salt/pki/minion/master_sign.pub .ft P .fi .UNINDENT .UNINDENT .sp -Test supports additional optional arguments: \fBignorecase\fP, \fBmultiline\fP -.SS Escape filters -.SS \fBregex_escape\fP -.sp -New in version 2017.7.0. - -.sp -Allows escaping of strings so they can be interpreted literally by another function. +\fBIMPORTANT:\fP +.INDENT 0.0 +.INDENT 3.5 +DO NOT COPY THE master_sign.pem FILE. IT MUST STAY ON THE MASTER AND +ONLY THERE! +.UNINDENT +.UNINDENT .sp -Example: +When that is done, enable the signature checking in the minions configuration .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -regex_escape = {{ \(aqhttps://example.com?foo=bar%20baz\(aq | regex_escape }} +verify_master_pubkey_sign: True .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: +and restart the minion. For the first try, the minion should be run in manual +debug mode. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -regex_escape = https\e:\e/\e/example\e.com\e?foo\e=bar\e%20baz +salt\-minion \-l debug .ft P .fi .UNINDENT .UNINDENT -.SS Set Theory Filters -.SS \fBunique\fP -.sp -New in version 2017.7.0. - -.sp -Performs set math using Jinja filters. .sp -Example: +Upon connecting to the master, the following lines should appear on the output: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -unique = {{ [\(aqfoo\(aq, \(aqfoo\(aq, \(aqbar\(aq] | unique }} +[DEBUG ] Attempting to authenticate with the Salt Master at 172.16.0.10 +[DEBUG ] Loaded minion key: /etc/salt/pki/minion/minion.pem +[DEBUG ] salt.crypt.verify_signature: Loading public key +[DEBUG ] salt.crypt.verify_signature: Verifying signature +[DEBUG ] Successfully verified signature of master public key with verification public key master_sign.pub +[INFO ] Received signed and verified master pubkey from master 172.16.0.10 +[DEBUG ] Decrypting the current master AES key .ft P .fi .UNINDENT .UNINDENT .sp -will be rendered as: +If the signature verification fails, something went wrong and it will look +like this .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -unique = [\(aqfoo\(aq, \(aqbar\(aq] +[DEBUG ] Attempting to authenticate with the Salt Master at 172.16.0.10 +[DEBUG ] Loaded minion key: /etc/salt/pki/minion/minion.pem +[DEBUG ] salt.crypt.verify_signature: Loading public key +[DEBUG ] salt.crypt.verify_signature: Verifying signature +[DEBUG ] Failed to verify signature of public key +[CRITICAL] The Salt Master server\(aqs public key did not authenticate! .ft P .fi .UNINDENT .UNINDENT -.SS Global Functions .sp -Salt Project extends \fI\%builtin global functions\fP with these custom global functions: -.SS \fBifelse\fP +In a case like this, it should be checked, that the verification pubkey +(master_sign.pub) on the minion is the same as the one on the master. .sp -Evaluate each pair of arguments up to the last one as a (matcher, value) -tuple, returning \fBvalue\fP if matched. If none match, returns the last -argument. +Once the verification is successful, the minion can be started in daemon mode +again. .sp -The \fBifelse\fP function is like a multi\-level if\-else statement. It was -inspired by CFEngine\(aqs \fBifelse\fP function which in turn was inspired by -Oracle\(aqs \fBDECODE\fP function. It must have an odd number of arguments (from -1 to N). The last argument is the default value, like the \fBelse\fP clause in -standard programming languages. Every pair of arguments before the last one -are evaluated as a pair. If the first one evaluates true then the second one -is returned, as if you had used the first one in a compound match -expression. Boolean values can also be used as the first item in a pair, as it -will be translated to a match that will always match (\(dq*\(dq) or never match -(\(dqSALT_IFELSE_MATCH_NOTHING\(dq) a target system. +For the paranoid among us, its also possible to verify the publication whenever +it is received from the master. That is, for every single auth\-attempt which +can be quite frequent. For example just the start of the minion will force the +signature to be checked 6 times for various things like auth, mine, +\fI\%highstate\fP, etc. .sp -This is essentially another way to express the \fBmatch.filter_by\fP functionality -in way that\(aqs familiar to CFEngine or Oracle users. Consider using -\fBmatch.filter_by\fP unless this function fits your workflow. +If that is desired, enable the setting .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ ifelse(\(aqfoo*\(aq, \(aqfooval\(aq, \(aqbar*\(aq, \(aqbarval\(aq, \(aqdefaultval\(aq, minion_id=\(aqbar03\(aq) }} +always_verify_signature: True .ft P .fi .UNINDENT .UNINDENT -.SS Jinja in Files +.SS Multiple Masters For A Minion .sp -\fI\%Jinja\fP can be used in the same way in managed files: +Configuring multiple masters on a minion is done by specifying two settings: +.INDENT 0.0 +.IP \(bu 2 +a list of masters addresses +.IP \(bu 2 +what type of master is defined +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# redis.sls -/etc/redis/redis.conf: - file.managed: - \- source: salt://redis.conf - \- template: jinja - \- context: - bind: 127.0.0.1 +master: + \- 172.16.0.10 + \- 172.16.0.11 + \- 172.16.0.12 .ft P .fi .UNINDENT @@ -40285,3008 +40392,3142 @@ in way that\(aqs familiar to CFEngine or Oracle users. Consider using .sp .nf .ft C -# lib.sls -{% set port = 6379 %} +master_type: failover .ft P .fi .UNINDENT .UNINDENT +.sp +This tells the minion that all the master above are available for it to +connect to. When started with this configuration, it will try the master +in the order they are defined. To randomize that order, set .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# redis.conf -{% from \(aqlib.sls\(aq import port with context %} -port {{ port }} -bind {{ bind }} +random_master: True .ft P .fi .UNINDENT .UNINDENT .sp -As an example, configuration was pulled from the file context and from an -external template file. -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Macros and variables can be shared across templates. They should not start -with one or more underscores, and should be managed by one of the -following tags: \fImacro\fP, \fIset\fP, \fIload_yaml\fP, \fIload_json\fP, \fIimport_yaml\fP and -\fIimport_json\fP\&. -.UNINDENT -.UNINDENT -.SS Escaping Jinja +The master\-list will then be shuffled before the first connection attempt. .sp -Occasionally, it may be necessary to escape Jinja syntax. There are two ways -to do this in Jinja. One is escaping individual variables or strings and the -other is to escape entire blocks. +The first master that accepts the minion, is used by the minion. If the +master does not yet know the minion, that counts as accepted and the minion +stays on that master. .sp -To escape a string commonly used in Jinja syntax such as \fB{{\fP, you can use the -following syntax: +For the minion to be able to detect if its still connected to its current +master enable the check for it .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ \(aq{{\(aq }} +master_alive_interval: .ft P .fi .UNINDENT .UNINDENT .sp -For larger blocks that contain Jinja syntax that needs to be escaped, you can use -raw blocks: +If the loss of the connection is detected, the minion will temporarily +remove the failed master from the list and try one of the other masters +defined (again shuffled if that is enabled). +.SS Testing the setup +.sp +At least two running masters are needed to test the failover setup. +.sp +Both masters should be running and the minion should be running on the command +line in debug mode .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% raw %} - some text that contains jinja characters that need to be escaped -{% endraw %} +salt\-minion \-l debug .ft P .fi .UNINDENT .UNINDENT .sp -See the \fI\%Escaping\fP section of Jinja\(aqs documentation to learn more. -.sp -A real\-word example of needing to use raw tags to escape a larger block of code -is when using \fBfile.managed\fP with the \fBcontents_pillar\fP option to manage -files that contain something like consul\-template, which shares a syntax subset -with Jinja. Raw blocks are necessary here because the Jinja in the pillar would -be rendered before the file.managed is ever called, so the Jinja syntax must be -escaped: +The minion will connect to the first master from its master list .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% raw %} -\- contents_pillar: | - job \(dqexample\-job\(dq { - - task \(dqexample\(dq { - driver = \(dqdocker\(dq - - config { - image = \(dqdocker\-registry.service.consul:5000/example\-job:{{key \(dqnomad/jobs/example\-job/version\(dq}}\(dq - -{% endraw %} +[DEBUG ] Attempting to authenticate with the Salt Master at 172.16.0.10 +[DEBUG ] Loaded minion key: /etc/salt/pki/minion/minion.pem +[DEBUG ] salt.crypt.verify_signature: Loading public key +[DEBUG ] salt.crypt.verify_signature: Verifying signature +[DEBUG ] Successfully verified signature of master public key with verification public key master_sign.pub +[INFO ] Received signed and verified master pubkey from master 172.16.0.10 +[DEBUG ] Decrypting the current master AES key .ft P .fi .UNINDENT .UNINDENT -.SS Calling Salt Functions .sp -The Jinja renderer provides a shorthand lookup syntax for the \fBsalt\fP -dictionary of \fI\%execution function\fP\&. +A test.version on the master the minion is currently connected to should be run to +test connectivity. .sp -New in version 2014.7.0. - +If successful, that master should be turned off. A firewall\-rule denying the +minions packets will also do the trick. +.sp +Depending on the configured conf_minion:\fImaster_alive_interval\fP, the minion +will notice the loss of the connection and log it to its logfile. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# The following two function calls are equivalent. -{{ salt[\(aqcmd.run\(aq](\(aqwhoami\(aq) }} -{{ salt.cmd.run(\(aqwhoami\(aq) }} +[INFO ] Connection to master 172.16.0.10 lost +[INFO ] Trying to tune in to next master from master\-list .ft P .fi .UNINDENT .UNINDENT -.SS Debugging -.sp -The \fBshow_full_context\fP function can be used to output all variables present -in the current Jinja context. .sp -New in version 2014.7.0. - +The minion will then remove the current master from the list and try connecting +to the next master .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -Context is: {{ show_full_context()|yaml(False) }} +[INFO ] Removing possibly failed master 172.16.0.10 from list of masters +[WARNING ] Master ip address changed from 172.16.0.10 to 172.16.0.11 +[DEBUG ] Attempting to authenticate with the Salt Master at 172.16.0.11 .ft P .fi .UNINDENT .UNINDENT -.SS Logs -.sp -New in version 2017.7.0. - .sp -Yes, in Salt, one is able to debug a complex Jinja template using the logs. -For example, making the call: +If everything is configured correctly, the new masters public key will be +verified successfully .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{%\- do salt.log.error(\(aqtesting jinja logging\(aq) \-%} +[DEBUG ] Loaded minion key: /etc/salt/pki/minion/minion.pem +[DEBUG ] salt.crypt.verify_signature: Loading public key +[DEBUG ] salt.crypt.verify_signature: Verifying signature +[DEBUG ] Successfully verified signature of master public key with verification public key master_sign.pub .ft P .fi .UNINDENT .UNINDENT .sp -Will insert the following message in the minion logs: +the authentication with the new master is successful .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -2017\-02\-01 01:24:40,728 [salt.module.logmod][ERROR ][3779] testing jinja logging +[INFO ] Received signed and verified master pubkey from master 172.16.0.11 +[DEBUG ] Decrypting the current master AES key +[DEBUG ] Loaded minion key: /etc/salt/pki/minion/minion.pem +[INFO ] Authentication with master successful! .ft P .fi .UNINDENT .UNINDENT -.SS Profiling .sp -New in version 3002. - +and the minion can be pinged again from its new master. +.SS Performance Tuning .sp -When working with a very large codebase, it becomes increasingly imperative to -trace inefficiencies with state and pillar render times. The \fIprofile\fP jinja -block enables the user to get finely detailed information on the most expensive -areas in the codebase. -.SS Profiling blocks +With the setup described above, the master computes a signature for every +auth\-request of a minion. With many minions and many auth\-requests, that +can chew up quite a bit of CPU\-Power. .sp -Any block of jinja code can be wrapped in a \fBprofile\fP block. The syntax for -a profile block is \fB{% profile as \(aq\(aq %}{% endprofile %}\fP, -where \fB\fP can be any string. The \fB\fP token will appear in the -log at the \fBprofile\fP level along with the render time of the block. +To avoid that, the master can use a pre\-created signature of its public\-key. +The signature is saved as a base64 encoded string which the master reads +once when starting and attaches only that string to auth\-replies. +.sp +Enabling this also gives paranoid users the possibility, to have the signing +key\-pair on a different system than the actual salt\-master and create the public +keys signature there. Probably on a system with more restrictive firewall rules, +without internet access, less users, etc. +.sp +That signature can be created with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/salt/example.sls -{%\- profile as \(aqlocal data\(aq %} - {%\- set local_data = {\(aqcounter\(aq: 0} %} - {%\- for i in range(313377) %} - {%\- do local_data.update({\(aqcounter\(aq: i}) %} - {%\- endfor %} -{%\- endprofile %} - -test: - cmd.run: - \- name: |\- - printf \(aqdata: %s\(aq \(aq{{ local_data[\(aqcounter\(aq] }}\(aq +salt\-key \-\-gen\-signature .ft P .fi .UNINDENT .UNINDENT .sp -The \fBprofile\fP block in the \fBexample.sls\fP state will emit the following log -statement: +This will create a default signature file in the master pki\-directory .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-call \-\-local \-l profile state.apply example -[...] -[PROFILE ] Time (in seconds) to render profile block \(aqlocal data\(aq: 0.9385035037994385 -[...] +/etc/salt/pki/master/master_pubkey_signature .ft P .fi .UNINDENT .UNINDENT -.SS Profiling imports .sp -Using the same logic as the \fBprofile\fP block, the \fBimport_yaml\fP, -\fBimport_json\fP, and \fBimport_text\fP blocks will emit similar statements at the -\fBprofile\fP log level. +It is a simple text\-file with the binary\-signature converted to base64. +.sp +If no signing\-pair is present yet, this will auto\-create the signing pair and +the signature file in one call .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/salt/data.sls -{%\- set values = {\(aqcounter\(aq: 0} %} -{%\- for i in range(524288) %} - {%\- do values.update({\(aqcounter\(aq: i}) %} -{%\- endfor %} - -data: {{ values[\(aqcounter\(aq] }} +salt\-key \-\-gen\-signature \-\-auto\-create .ft P .fi .UNINDENT .UNINDENT +.sp +Telling the master to use the pre\-created signature is done with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/salt/example.sls -{%\- import_yaml \(aqdata.sls\(aq as imported %} - -test: - cmd.run: - \- name: |\- - printf \(aqdata: %s\(aq \(aq{{ imported[\(aqdata\(aq] }}\(aq +master_use_pubkey_signature: True .ft P .fi .UNINDENT .UNINDENT .sp -For \fBimport_*\fP blocks, the \fBprofile\fP log statement has the following form: +That requires the file \(aqmaster_pubkey_signature\(aq to be present in the masters +pki\-directory with the correct signature. +.sp +If the signature file is named differently, its name can be set with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-call \-\-local \-l profile state.apply example -[...] -[PROFILE ] Time (in seconds) to render import_yaml \(aqdata.sls\(aq: 1.5500736236572266 -[...] +master_pubkey_signature: .ft P .fi .UNINDENT .UNINDENT -.SS Python Methods .sp -A powerful feature of jinja that is only hinted at in the official jinja -documentation is that you can use the native python methods of the -variable type. Here is the python documentation for \fI\%string methods\fP\&. +With many masters and many public\-keys (default and signing), it is advised to +use the salt\-masters hostname for the signature\-files name. Signatures can be +easily confused because they do not provide any information about the key the +signature was created from. +.sp +Verifying that everything works is done the same way as above. +.SS How the signing and verification works +.sp +The default key\-pair of the salt\-master is .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set hostname,domain = grains.id.partition(\(aq.\(aq)[::2] %}{{ hostname }} +/etc/salt/pki/master/master.pem +/etc/salt/pki/master/master.pub .ft P .fi .UNINDENT .UNINDENT +.sp +To be able to create a signature of a message (in this case a public\-key), +another key\-pair has to be added to the setup. Its default name is: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{% set strings = grains.id.split(\(aq\-\(aq) %}{{ strings[0] }} +master_sign.pem +master_sign.pub .ft P .fi .UNINDENT .UNINDENT -.SS Custom Execution Modules .sp -Custom execution modules can be used to supplement or replace complex Jinja. Many -tasks that require complex looping and logic are trivial when using Python -in a Salt execution module. Salt execution modules are easy to write and -distribute to Salt minions. +The combination of the master.* and master_sign.* key\-pairs give the +possibility of generating signatures. The signature of a given message +is unique and can be verified, if the public\-key of the signing\-key\-pair +is available to the recipient (the minion). .sp -Functions in custom execution modules are available in the Salt execution -module dictionary just like the built\-in execution modules: +The signature of the masters public\-key in master.pub is computed with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ salt[\(aqmy_custom_module.my_custom_function\(aq]() }} +master_sign.pem +master.pub +M2Crypto.EVP.sign_update() .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP \(bu 2 -\fI\%How to Convert Jinja Logic to an Execution Module\fP -.IP \(bu 2 -\fI\%Writing Execution Modules\fP -.UNINDENT -.SS Custom Jinja filters .sp -Given that all execution modules are available in the Jinja template, -one can easily define a custom module as in the previous paragraph -and use it as a Jinja filter. -However, please note that it will not be accessible through the pipe. +This results in a binary signature which is converted to base64 and attached +to the auth\-reply send to the minion. .sp -For example, instead of: +With the signing\-pairs public\-key available to the minion, the attached +signature can be verified with .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ my_variable | my_jinja_filter }} +master_sign.pub +master.pub +M2Cryptos EVP.verify_update(). .ft P .fi .UNINDENT .UNINDENT .sp -The user will need to define \fBmy_jinja_filter\fP function under an extension -module, say \fBmy_filters\fP and use as: +When running multiple masters, either the signing key\-pair has to be present +on all of them, or the master_pubkey_signature has to be pre\-computed for +each master individually (because they all have different public\-keys). .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -{{ salt.my_filters.my_jinja_filter(my_variable) }} -.ft P -.fi +DO NOT PUT THE SAME master.pub ON ALL MASTERS FOR EASE OF USE. .UNINDENT .UNINDENT +.SS Packaging External Modules for Salt +.SS External Modules Setuptools Entry\-Points Support .sp -The greatest benefit is that you are able to access thousands of existing functions, e.g.: +The salt loader was enhanced to look for external modules by looking at the +\fIsalt.loader\fP entry\-point: .INDENT 0.0 -.IP \(bu 2 -get the DNS AAAA records for a specific address using the \fI\%dnsutil\fP: -.INDENT 2.0 .INDENT 3.5 +\fI\%https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry\-points\fP +.UNINDENT +.UNINDENT .sp -.nf -.ft C -{{ salt.dnsutil.AAAA(\(aqwww.google.com\(aq) }} -.ft P -.fi +\fIpkg_resources\fP should be installed, which is normally included in setuptools. +.INDENT 0.0 +.INDENT 3.5 +\fI\%https://setuptools.readthedocs.io/en/latest/pkg_resources.html\fP .UNINDENT .UNINDENT -.IP \(bu 2 -retrieve a specific field value from a \fBRedis\fP hash: -.INDENT 2.0 +.sp +The package which has custom engines, minion modules, outputters, etc, should +require setuptools and should define the following entry points in its setup +function: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ salt.redis.hget(\(aqfoo_hash\(aq, \(aqbar_field\(aq) }} +from setuptools import setup, find_packages + +setup( + name=THE_NAME, + version=THE_VERSION, + description=THE_DESCRIPTION, + author=THE_AUTHOR_NAME, + author_email=THE_AUTHOR_EMAIL, + url=\(dq ... \(dq, + packages=find_packages(), + entry_points=\(dq\(dq\(dq + [salt.loader] + engines_dirs = .:engines_dirs + fileserver_dirs = .:fileserver_dirs + pillar_dirs = .:pillar_dirs + returner_dirs = .:returner_dirs + roster_dirs = .:roster_dirs + \(dq\(dq\(dq, +) .ft P .fi .UNINDENT .UNINDENT -.IP \(bu 2 -get the routes to \fB0.0.0.0/0\fP using the \fI\%NAPALM route\fP: -.INDENT 2.0 +.sp +The above setup script example mentions a loader module. here\(aqs an example of +how \fI/.py\fP it should look: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -{{ salt.route.show(\(aq0.0.0.0/0\(aq) }} +# \-*\- coding: utf\-8 \-*\- + +# Import python libs +import os + +PKG_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def engines_dirs(): + \(dq\(dq\(dq + yield one path per parent directory of where engines can be found + \(dq\(dq\(dq + yield os.path.join(PKG_DIR, \(dqengines_1\(dq) + yield os.path.join(PKG_DIR, \(dqengines_2\(dq) + + +def fileserver_dirs(): + \(dq\(dq\(dq + yield one path per parent directory of where fileserver modules can be found + \(dq\(dq\(dq + yield os.path.join(PKG_DIR, \(dqfileserver\(dq) + + +def pillar_dirs(): + \(dq\(dq\(dq + yield one path per parent directory of where external pillar modules can be found + \(dq\(dq\(dq + yield os.path.join(PKG_DIR, \(dqpillar\(dq) + + +def returner_dirs(): + \(dq\(dq\(dq + yield one path per parent directory of where returner modules can be found + \(dq\(dq\(dq + yield os.path.join(PKG_DIR, \(dqreturners\(dq) + + +def roster_dirs(): + \(dq\(dq\(dq + yield one path per parent directory of where roster modules can be found + \(dq\(dq\(dq + yield os.path.join(PKG_DIR, \(dqroster\(dq) .ft P .fi .UNINDENT .UNINDENT -.UNINDENT -.SS Tutorials Index -.SS Autoaccept minions from Grains +.SS Preseed Minion with Accepted Key .sp -New in version 2018.3.0. - +In some situations, it is not convenient to wait for a minion to start before +accepting its key on the master. For instance, you may want the minion to +bootstrap itself as soon as it comes online. You may also want to let your +developers provision new development machines on the fly. .sp -To automatically accept minions based on certain characteristics, e.g. the \fBuuid\fP -you can specify certain grain values on the salt master. Minions with matching grains -will have their keys automatically accepted. +\fBSEE ALSO:\fP +.INDENT 0.0 +.INDENT 3.5 +Many ways to preseed minion keys +.sp +Salt has other ways to generate and pre\-accept minion keys in addition to +the manual steps outlined below. +.sp +salt\-cloud performs these same steps automatically when new cloud VMs are +created (unless instructed not to). +.sp +salt\-api exposes an HTTP call to Salt\(aqs REST API to \fI\%generate and +download the new minion keys as a tarball\fP\&. +.UNINDENT +.UNINDENT +.sp +There is a general four step process to do this: .INDENT 0.0 .IP 1. 3 -Configure the autosign_grains_dir in the master config file: +Generate the keys on the master: .UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -autosign_grains_dir: /etc/salt/autosign_grains +root@saltmaster# salt\-key \-\-gen\-keys=[key_name] .ft P .fi .UNINDENT .UNINDENT +.sp +Pick a name for the key, such as the minion\(aqs id. .INDENT 0.0 .IP 2. 3 -Configure the grain values to be accepted +Add the public key to the accepted minion folder: .UNINDENT -.sp -Place a file named like the grain in the autosign_grains_dir and write the values that -should be accepted automatically inside that file. For example to automatically -accept minions based on their \fBuuid\fP create a file named \fB/etc/salt/autosign_grains/uuid\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -8f7d68e2\-30c5\-40c6\-b84a\-df7e978a03ee -1d3c5473\-1fbc\-479e\-b0c7\-877705a0730f +root@saltmaster# cp key_name.pub /etc/salt/pki/master/minions/[minion_id] .ft P .fi .UNINDENT .UNINDENT .sp -If already running, the master must be restarted for these config changes to take effect. -.sp -The master is now setup to accept minions with either of the two specified uuids. -Multiple values must always be written into separate lines. -Lines starting with a \fB#\fP are ignored. +It is necessary that the public key file has the same name as your minion id. +This is how Salt matches minions with their keys. Also note that the pki folder +could be in a different location, depending on your OS or if specified in the +master config file. .INDENT 0.0 .IP 3. 3 -Configure the minion to send the specific grains to the master in the minion config file: +Distribute the minion keys. +.UNINDENT +.sp +There is no single method to get the keypair to your minion. The difficulty is +finding a distribution method which is secure. For Amazon EC2 only, an AWS best +practice is to use IAM Roles to pass credentials. (See blog post, +\fI\%https://aws.amazon.com/blogs/security/using\-iam\-roles\-to\-distribute\-non\-aws\-credentials\-to\-your\-ec2\-instances/\fP ) +.INDENT 0.0 +.INDENT 3.5 +.IP "Security Warning" +.sp +Since the minion key is already accepted on the master, distributing +the private key poses a potential security risk. A malicious party +will have access to your entire state tree and other sensitive data if they +gain access to a preseeded minion key. +.UNINDENT .UNINDENT .INDENT 0.0 +.IP 4. 3 +Preseed the Minion with the keys +.UNINDENT +.sp +You will want to place the minion keys before starting the salt\-minion daemon: +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -autosign_grains: - \- uuid +/etc/salt/pki/minion/minion.pem +/etc/salt/pki/minion/minion.pub .ft P .fi .UNINDENT .UNINDENT .sp -Now you should be able to start salt\-minion and run \fBsalt\-call +Once in place, you should be able to start salt\-minion and run \fBsalt\-call state.apply\fP or any other salt commands that require master authentication. -.SS Salt as a Cloud Controller -.sp -In Salt 0.14.0, an advanced cloud control system was introduced, allowing -private cloud VMs to be managed directly with Salt. This system is generally -referred to as \fBSalt Virt\fP\&. +.SS Salt Masterless Quickstart .sp -The Salt Virt system already exists and is installed within Salt itself. This -means that besides setting up Salt, no additional salt code needs to be -deployed. +Running a masterless salt\-minion lets you use Salt\(aqs configuration management +for a single machine without calling out to a Salt master on another machine. .sp -\fBNOTE:\fP +Since the Salt minion contains such extensive functionality it can be useful +to run it standalone. A standalone minion can be used to do a number of +things: .INDENT 0.0 -.INDENT 3.5 -The \fBlibvirt\fP python module and the \fBcerttool\fP binary are required. -.UNINDENT +.IP \(bu 2 +Stand up a master server via States (Salting a Salt Master) +.IP \(bu 2 +Use salt\-call commands on a system without connectivity to a master +.IP \(bu 2 +Masterless States, run states entirely from files local to the minion .UNINDENT .sp -The main goal of Salt Virt is to facilitate a very fast and simple cloud that -can scale and is fully featured. Salt Virt comes with the ability to set up and -manage complex virtual machine networking, powerful image and disk management, -and virtual machine migration with and without shared storage. -.sp -This means that Salt Virt can be used to create a cloud from a blade center -and a SAN, but can also create a cloud out of a swarm of Linux Desktops -without a single shared storage system. Salt Virt can make clouds from -truly commodity hardware, but can also stand up the power of specialized -hardware as well. -.SS Setting up Hypervisors -.sp -The first step to set up the hypervisors involves getting the correct software -installed and setting up the hypervisor network interfaces. -.SS Installing Hypervisor Software +It is also useful for testing out state trees before deploying to a production setup. +.SS Bootstrap Salt Minion .sp -Salt Virt is made to be hypervisor agnostic but currently, the only fully -implemented hypervisor is KVM via libvirt. +The \fI\%salt\-bootstrap\fP script makes bootstrapping a server with Salt simple +for any OS with a Bourne shell: +.INDENT 0.0 +.INDENT 3.5 .sp -The required software for a hypervisor is libvirt and kvm. For advanced -features, install libguestfs or qemu\-nbd. +.nf +.ft C +curl \-L https://bootstrap.saltstack.com \-o bootstrap_salt.sh +sudo sh bootstrap_salt.sh +.ft P +.fi +.UNINDENT +.UNINDENT .sp -\fBNOTE:\fP +Before run the script, it is a good practice to verify the checksum of the downloaded +file. You can verify the checksum with SHA256 by running this command: .INDENT 0.0 .INDENT 3.5 -Libguestfs and qemu\-nbd allow for virtual machine images to be mounted -before startup and get pre\-seeded with configurations and a salt minion. +.sp +.nf +.ft C +test $(sha256sum bootstrap_salt.sh | awk \(aq{print $1}\(aq) \e + = $(curl \-sL https://bootstrap.saltproject.io/sha256 | cat \-) \e + && echo \(dqOK\(dq \e + || echo \(dqFile does not match checksum\(dq +.ft P +.fi .UNINDENT .UNINDENT .sp -This sls will set up the needed software for a hypervisor, and run the routines -to set up the libvirt pki keys. -.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Package names and setup used is Red Hat specific. Different package names -will be required for different platforms. -.UNINDENT -.UNINDENT +The previous example is the preferred method because by downloading the script +you can investigate the contents of the bootstrap script or using it again later. +Alternatively, if you want to download the bash script and run it immediately, +use: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -libvirt: - pkg.installed: [] - file.managed: - \- name: /etc/sysconfig/libvirtd - \- contents: \(aqLIBVIRTD_ARGS=\(dq\-\-listen\(dq\(aq - \- require: - \- pkg: libvirt - virt.keys: - \- require: - \- pkg: libvirt - service.running: - \- name: libvirtd - \- require: - \- pkg: libvirt - \- network: br0 - \- libvirt: libvirt - \- watch: - \- file: libvirt - -libvirt\-python: - pkg.installed: [] - -libguestfs: - pkg.installed: - \- pkgs: - \- libguestfs - \- libguestfs\-tools +curl \-L https://bootstrap.saltproject.io | sudo sh \-s \-\- .ft P .fi .UNINDENT .UNINDENT -.SS Hypervisor Network Setup +.UNINDENT +.UNINDENT .sp -The hypervisors will need to be running a network bridge to serve up network -devices for virtual machines. This formula will set up a standard bridge on -a hypervisor connecting the bridge to eth0: +See the \fI\%salt\-bootstrap\fP documentation for other one liners. When using \fI\%Vagrant\fP +to test out salt, the \fI\%Vagrant salt provisioner\fP will provision the VM for you. +.SS Telling Salt to Run Masterless +.sp +To instruct the minion to not look for a master, the \fI\%file_client\fP +configuration option needs to be set in the minion configuration file. +By default the \fI\%file_client\fP is set to \fBremote\fP so that the +minion gathers file server and pillar data from the salt master. +When setting the \fI\%file_client\fP option to \fBlocal\fP the +minion is configured to not gather this data from the master. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -eth0: - network.managed: - \- enabled: True - \- type: eth - \- bridge: br0 - -br0: - network.managed: - \- enabled: True - \- type: bridge - \- proto: dhcp - \- require: - \- network: eth0 +file_client: local .ft P .fi .UNINDENT .UNINDENT -.SS Virtual Machine Network Setup .sp -Salt Virt comes with a system to model the network interfaces used by the -deployed virtual machines. By default, a single interface is created for the -deployed virtual machine and is bridged to \fBbr0\fP\&. To get going with the -default networking setup, ensure that the bridge interface named \fBbr0\fP exists -on the hypervisor and is bridged to an active network device. +Now the salt minion will not look for a master and will assume that the local +system has all of the file and pillar resources. +.sp +Configuration which resided in the +\fI\%master configuration\fP (e.g. \fB/etc/salt/master\fP) +should be moved to the \fI\%minion configuration\fP +since the minion does not read the master configuration. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -To use more advanced networking in Salt Virt, read the \fISalt Virt -Networking\fP document: -.sp -\fI\%Salt Virt Networking\fP +When running Salt in masterless mode, do not run the salt\-minion daemon. +Otherwise, it will attempt to connect to a master and fail. The salt\-call +command stands on its own and does not need the salt\-minion daemon. .UNINDENT .UNINDENT -.SS Libvirt State +.SS Create State Tree .sp -One of the challenges of deploying a libvirt based cloud is the distribution -of libvirt certificates. These certificates allow for virtual machine -migration. Salt comes with a system used to auto deploy these certificates. -Salt manages the signing authority key and generates keys for libvirt clients -on the master, signs them with the certificate authority, and uses pillar to -distribute them. This is managed via the \fBlibvirt\fP state. Simply execute this -formula on the minion to ensure that the certificate is in place and up to -date: +Following the successful installation of a salt\-minion, the next step is to create +a state tree, which is where the SLS files that comprise the possible states of the +minion are stored. +.sp +The following example walks through the steps necessary to create a state tree that +ensures that the server has the Apache webserver installed. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The above formula includes the calls needed to set up libvirt keys. +For a complete explanation on Salt States, see the \fI\%tutorial\fP\&. .UNINDENT .UNINDENT .INDENT 0.0 +.IP 1. 3 +Create the \fBtop.sls\fP file: +.UNINDENT +.sp +\fB/srv/salt/top.sls:\fP +.INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -libvirt_keys: - virt.keys +base: + \(aq*\(aq: + \- webserver .ft P .fi .UNINDENT .UNINDENT -.SS Getting Virtual Machine Images Ready -.sp -Salt Virt requires that virtual machine images be provided as these are not -generated on the fly. Generating these virtual machine images differs greatly -based on the underlying platform. -.sp -Virtual machine images can be manually created using KVM and running through -the installer, but this process is not recommended since it is very manual and -prone to errors. -.sp -Virtual Machine generation applications are available for many platforms: .INDENT 0.0 -.TP -.B kiwi: (openSUSE, SLES, RHEL, CentOS) -\fI\%https://opensuse.github.io/kiwi/\fP -.TP -.B vm\-builder: -\fI\%https://wiki.debian.org/VMBuilder\fP -.sp -\fBSEE ALSO:\fP -.INDENT 7.0 -.INDENT 3.5 -\fI\%url vmbuilder\-formula\fP -.UNINDENT -.UNINDENT +.IP 2. 3 +Create the webserver state tree: .UNINDENT .sp -Once virtual machine images are available, the easiest way to make them -available to Salt Virt is to place them in the Salt file server. Just copy an -image into \fB/srv/salt\fP and it can now be used by Salt Virt. -.sp -For purposes of this demo, the file name \fBcentos.img\fP will be used. -.SS Existing Virtual Machine Images -.sp -Many existing Linux distributions distribute virtual machine images which -can be used with Salt Virt. Please be advised that NONE OF THESE IMAGES ARE -SUPPORTED BY SALTSTACK. -.SS CentOS -.sp -These images have been prepared for OpenNebula but should work without issue with -Salt Virt, only the raw qcow image file is needed: -\fI\%https://wiki.centos.org/Cloud/OpenNebula\fP -.SS Fedora Linux -.sp -Images for Fedora Linux can be found here: -\fI\%https://alt.fedoraproject.org/cloud\fP -.SS openSUSE -.sp -\fI\%https://download.opensuse.org/distribution/leap/15.1/jeos/openSUSE\-Leap\-15.1\-JeOS.x86_64\-15.1.0\-kvm\-and\-xen\-Current.qcow2.meta4\fP -.SS SUSE -.sp -\fI\%https://www.suse.com/products/server/jeos\fP -.SS Ubuntu Linux -.sp -Images for Ubuntu Linux can be found here: -\fI\%http://cloud\-images.ubuntu.com/\fP -.SS Using Salt Virt -.sp -With hypervisors set up and virtual machine images ready, Salt can start -issuing cloud commands using the \fIvirt runner\fP\&. -.sp -Start by running a Salt Virt hypervisor info command: +\fB/srv/salt/webserver.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run virt.host_info +apache: # ID declaration + pkg: # state declaration + \- installed # function declaration .ft P .fi .UNINDENT .UNINDENT .sp -This will query the running hypervisor(s) for stats and display useful -information such as the number of CPUs and amount of memory. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The apache package has different names on different platforms, for +instance on Debian/Ubuntu it is apache2, on Fedora/RHEL it is httpd +and on Arch it is apache +.UNINDENT +.UNINDENT .sp -You can also list all VMs and their current states on all hypervisor nodes: +The only thing left is to provision our minion using \fBsalt\-call\fP\&. +.SS Salt\-call +.sp +The salt\-call command is used to run remote execution functions locally on a +minion instead of executing them from the master. Normally the salt\-call +command checks into the master to retrieve file server and pillar data, but +when running standalone salt\-call needs to be instructed to not check the +master for this data: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run virt.list +salt\-call \-\-local state.apply .ft P .fi .UNINDENT .UNINDENT .sp -Now that hypervisors are available a virtual machine can be provisioned, the -\fBvirt.init\fP routine will create a new virtual machine: +The \fB\-\-local\fP flag tells the salt\-minion to look for the state tree in the +local file system and not to contact a Salt Master for instructions. +.sp +To provide verbose output, use \fB\-l debug\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run virt.init centos1 2 512 salt://centos.img +salt\-call \-\-local state.apply \-l debug .ft P .fi .UNINDENT .UNINDENT .sp -The Salt Virt runner will now automatically select a hypervisor to deploy -the new virtual machine on. Using \fBsalt://\fP assumes that the CentOS virtual -machine image is located in the root of the \fI\%Salt File Server\fP on the master. -When images are cloned (i.e. copied locally after retrieval from the file -server), the destination directory on the hypervisor minion is determined by the -\fBvirt:images\fP config option; by default this is \fB/srv/salt\-images/\fP\&. +The minion first examines the \fBtop.sls\fP file and determines that it is a part +of the group matched by \fB*\fP glob and that the \fBwebserver\fP SLS should be applied. .sp -When a VM is initialized using \fBvirt.init\fP, the image is copied to the -hypervisor using \fBcp.cache_file\fP and will be mounted and seeded with a minion. -Seeding includes setting pre\-authenticated keys on the new machine. A minion -will only be installed if one can not be found on the image using the default -arguments to \fBseed.apply\fP\&. +It then examines the \fBwebserver.sls\fP file and finds the \fBapache\fP state, which +installs the Apache package. .sp -\fBNOTE:\fP +The minion should now have Apache installed, and the next step is to begin +learning how to write \fI\%more complex states\fP\&. +.SS running salt as normal user tutorial +.sp +\fBBefore continuing\fP make sure you have a working Salt installation by +following the instructions in the +\fI\%Salt install guide\fP\&. .INDENT 0.0 .INDENT 3.5 -The biggest bottleneck in starting VMs is when the Salt Minion needs to be -installed. Making sure that the source VM images already have Salt -installed will GREATLY speed up virtual machine deployment. +.IP "Stuck?" +.sp +The Salt Project community can help offer advice and help troubleshoot +technical issues as you\(aqre learning about Salt. One of the best places to +talk to the community is on the +\fI\%Salt Project Slack workspace\fP\&. .UNINDENT .UNINDENT +.SS Running Salt functions as non root user .sp -You can also deploy an image on a particular minion by directly calling the -\fBvirt\fP execution module with an absolute image path. This can be quite handy for -testing: +If you don\(aqt want to run salt cloud as root or even install it you can +configure it to have a virtual root in your working directory. +.sp +The salt system uses the \fBsalt.syspath\fP module to find the variables +.sp +If you run the salt\-build, it will generated in: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aqhypervisor*\(aq virt.init centos1 2 512 image=/var/lib/libvirt/images/centos.img +\&./build/lib.linux\-x86_64\-2.7/salt/_syspaths.py .ft P .fi .UNINDENT .UNINDENT .sp -Now that the new VM has been prepared, it can be seen via the \fBvirt.query\fP -command: +To generate it, run the command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run virt.query +python setup.py build .ft P .fi .UNINDENT .UNINDENT .sp -This command will return data about all of the hypervisors and respective -virtual machines. -.sp -Now that the new VM is booted, it should have contacted the Salt Master. A -\fBtest.ping\fP will reveal if the new VM is running. -.SS QEMU Copy on Write Support -.sp -For fast image cloning, you can use the \fI\%qcow\fP disk image format. -Pass the \fBenable_qcow\fP flag and a \fI\&.qcow2\fP image path to \fIvirt.init\fP: +Copy the generated module into your salt directory .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aqhypervisor*\(aq virt.init centos1 2 512 image=/var/lib/libvirt/images/centos.qcow2 enable_qcow=True start=False +cp ./build/lib.linux\-x86_64\-2.7/salt/_syspaths.py salt/_syspaths.py .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Beware that attempting to boot a qcow image too quickly after cloning -can result in a race condition where libvirt may try to boot the machine -before image seeding has completed. For that reason, it is recommended to -also pass \fBstart=False\fP to \fBvirt.init\fP\&. -.sp -Also know that you \fBmust not\fP modify the original base image without -first making a copy and then \fIrebasing\fP all overlay images onto it. -See the \fBqemu\-img rebase\fP \fI\%usage docs\fP\&. -.UNINDENT -.UNINDENT -.SS Migrating Virtual Machines -.sp -Salt Virt comes with full support for virtual machine migration. Using -the libvirt state in the above formula makes migration possible. -.sp -A few things need to be available to support migration. Many operating systems -turn on firewalls when originally set up; the firewall needs to be opened up -to allow for libvirt and kvm to cross communicate and execution migration -routines. On Red Hat based hypervisors in particular, port 16514 needs to be -opened on hypervisors: -.INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -iptables \-A INPUT \-m state \-\-state NEW \-m tcp \-p tcp \-\-dport 16514 \-j ACCEPT -.ft P -.fi -.UNINDENT -.UNINDENT -.sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -More in\-depth information regarding distribution specific firewall settings can be found in: -.sp -\fI\%Opening the Firewall up for Salt\fP -.UNINDENT -.UNINDENT -.sp -Salt also needs the \fBvirt:tunnel\fP option to be turned on. This flag tells Salt -to run migrations securely via the libvirt TLS tunnel and to use port 16514. -Without \fBvirt:tunnel\fP, libvirt tries to bind to random ports when running -migrations. -.sp -To turn on \fBvirt:tunnel\fP, simply apply it to the master config file: +Edit it to include needed variables and your new paths .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -virt: - tunnel: True +# you need to edit this +_your_current_dir_ = ... +ROOT_DIR = _your_current_dir_ + \(dq/salt/root\(dq + +# you need to edit this +_location_of_source_code_ = ... +INSTALL_DIR = _location_of_source_code_ + +CONFIG_DIR = ROOT_DIR + \(dq/etc/salt\(dq +CACHE_DIR = ROOT_DIR + \(dq/var/cache/salt\(dq +SOCK_DIR = ROOT_DIR + \(dq/var/run/salt\(dq +SRV_ROOT_DIR = ROOT_DIR + \(dq/srv\(dq +BASE_FILE_ROOTS_DIR = ROOT_DIR + \(dq/srv/salt\(dq +BASE_PILLAR_ROOTS_DIR = ROOT_DIR + \(dq/srv/pillar\(dq +BASE_MASTER_ROOTS_DIR = ROOT_DIR + \(dq/srv/salt\-master\(dq +LOGS_DIR = ROOT_DIR + \(dq/var/log/salt\(dq +PIDFILE_DIR = ROOT_DIR + \(dq/var/run\(dq +CLOUD_DIR = INSTALL_DIR + \(dq/cloud\(dq +BOOTSTRAP = CLOUD_DIR + \(dq/deploy/bootstrap\-salt.sh\(dq .ft P .fi .UNINDENT .UNINDENT .sp -Once the master config has been updated, restart the master and send out a call -to the minions to refresh the pillar to pick up on the change: +Create the directory structure .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \e* saltutil.refresh_modules +mkdir \-p root/etc/salt root/var/cache/run root/run/salt root/srv +root/srv/salt root/srv/pillar root/srv/salt\-master root/var/log/salt root/var/run .ft P .fi .UNINDENT .UNINDENT .sp -Now, migration routines can be run! To migrate a VM, simply run the Salt Virt -migrate routine: +Populate the configuration files: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run virt.migrate centos +cp \-r conf/* root/etc/salt/ .ft P .fi .UNINDENT .UNINDENT -.SS VNC Consoles .sp -Although not enabled by default, Salt Virt can also set up VNC consoles allowing -for remote visual consoles to be opened up. When creating a new VM using -\fBvirt.init\fP, pass the \fBenable_vnc=True\fP parameter to have a console -configured for the new VM. -.sp -The information from a \fBvirt.query\fP routine will display the VNC console port -for the specific VMs: +Edit your \fBroot/etc/salt/master\fP configuration that is used by salt\-cloud: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -centos - CPU: 2 - Memory: 524288 - State: running - Graphics: vnc \- hyper6:5900 - Disk \- vda: - Size: 2.0G - File: /srv/salt\-images/ubuntu2/system.qcow2 - File Format: qcow2 - Nic \- ac:de:48:98:08:77: - Source: br0 - Type: bridge +user: *your user name* .ft P .fi .UNINDENT .UNINDENT .sp -The line \fIGraphics: vnc \- hyper6:5900\fP holds the key. First the port named, -in this case 5900, will need to be available in the hypervisor\(aqs firewall. -Once the port is open, then the console can be easily opened via vncviewer: +Run like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -vncviewer hyper6:5900 +PYTHONPATH=\(gapwd\(ga scripts/salt\-cloud .ft P .fi .UNINDENT .UNINDENT +.SS Salt Bootstrap .sp -By default there is no VNC security set up on these ports, which suggests that -keeping them firewalled and mandating that SSH tunnels be used to access these -VNC interfaces. Keep in mind that activity on a VNC interface that is accessed -can be viewed by any other user that accesses that same VNC interface, and any -other user logging in can also operate with the logged in user on the virtual -machine. -.SS Conclusion -.sp -Now with Salt Virt running, new hypervisors can be seamlessly added just by -running the above states on new bare metal machines, and these machines will be -instantly available to Salt Virt. -.SS Running Salt States and Commands in Docker Containers +The Salt Bootstrap Script allows a user to install the Salt Minion or Master +on a variety of system distributions and versions. .sp -The 2016.11.0 release of Salt introduces the ability to execute Salt States -and Salt remote execution commands directly inside of Docker containers. +The Salt Bootstrap Script is a shell script is known as \fBbootstrap\-salt.sh\fP\&. +It runs through a series of checks to determine the operating system type and +version. It then installs the Salt binaries using the appropriate methods. .sp -This addition makes it possible to not only deploy fresh containers using -Salt States. This also allows for running containers to be audited and -modified using Salt, but without running a Salt Minion inside the container. -Some of the applications include security audits of running containers as -well as gathering operating data from containers. +The Salt Bootstrap Script installs the minimum number of packages required to +run Salt. This means that in the event you run the bootstrap to install via +package, Git will not be installed. Installing the minimum number of packages +helps ensure the script stays as lightweight as possible, assuming the user +will install any other required packages after the Salt binaries are present +on the system. .sp -This new feature is simple and straightforward, and can be used via a running -Salt Minion, the Salt Call command, or via Salt SSH. For this tutorial we will -use the \fIsalt\-call\fP command, but like all salt commands these calls are -directly translatable to \fIsalt\fP and \fIsalt\-ssh\fP\&. -.SS Step 1 \- Install Docker +The Salt Bootstrap Script is maintained in a separate repo from Salt, complete +with its own issues, pull requests, contributing guidelines, release protocol, +etc. .sp -Since setting up Docker is well covered in the Docker documentation we will -make no such effort to describe it here. Please see the Docker Installation -Documentation for installing and setting up Docker: -\fI\%https://docs.docker.com/engine/installation/\fP +To learn more, please see the Salt Bootstrap repo links: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Salt Bootstrap repo\fP +.IP \(bu 2 +\fI\%README\fP: includes supported operating systems, example usage, and more. +.IP \(bu 2 +\fI\%Contributing Guidelines\fP +.IP \(bu 2 +\fI\%Release Process\fP +.UNINDENT .sp -The Docker integration also requires that the \fIdocker\-py\fP library is installed. -This can easily be done using pip or via your system package manager: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -pip install docker\-py -.ft P -.fi +The Salt Bootstrap script can be found in the Salt repo under the +\fBsalt/cloud/deploy/bootstrap\-salt.sh\fP path. Any changes to this file +will be overwritten! Bug fixes and feature additions must be submitted +via the \fI\%Salt Bootstrap repo\fP\&. Please see the Salt Bootstrap Script\(aqs +\fI\%Release Process\fP for more information. .UNINDENT .UNINDENT -.SS Step 2 \- Install Salt -.sp -For this tutorial we will be using Salt Call, which is available in the -\fIsalt\-minion\fP package, please follow the -\fI\%Salt install guide\fP\&. -.SS Step 3 \- Create With Salt States -.sp -Next some Salt States are needed, for this example a very basic state which -installs \fIvim\fP is used, but anything Salt States can do can be done here, -please see the Salt States Introduction Tutorial to learn more about Salt -States: -\fI\%https://docs.saltproject.io/en/stage/getstarted/config/\fP +.SS Standalone Minion .sp -For this tutorial, simply create a small state file in \fI/srv/salt/vim.sls\fP: +Since the Salt minion contains such extensive functionality it can be useful +to run it standalone. A standalone minion can be used to do a number of +things: .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -vim: - pkg.installed -.ft P -.fi -.UNINDENT +.IP \(bu 2 +Use salt\-call commands on a system without connectivity to a master +.IP \(bu 2 +Masterless States, run states entirely from files local to the minion .UNINDENT .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The base image you choose will need to have python 2.6 or 2.7 installed. -We are hoping to resolve this constraint in a future release. +When running Salt in masterless mode, it is not required to run the +salt\-minion daemon. By default the salt\-minion daemon will attempt to +connect to a master and fail. The salt\-call command stands on its own +and does not need the salt\-minion daemon. .sp -If \fIbase\fP is omitted the default image used is a minimal openSUSE -image with Python support, maintained by SUSE +As of version 2016.11.0 you can have a running minion (with engines and +beacons) without a master connection. If you wish to run the salt\-minion +daemon you will need to set the \fI\%master_type\fP configuration +setting to be set to \(aqdisable\(aq. .UNINDENT .UNINDENT +.SS Minion Configuration .sp -Next run the \fIdocker.sls_build\fP command: +Throughout this document there are several references to setting different +options to configure a masterless Minion. Salt Minions are easy to configure +via a configuration file that is located, by default, in \fB/etc/salt/minion\fP\&. +Note, however, that on FreeBSD systems, the minion configuration file is located +in \fB/usr/local/etc/salt/minion\fP\&. +.sp +You can learn more about minion configuration options in the +\fI\%Configuring the Salt Minion\fP docs. +.SS Telling Salt Call to Run Masterless +.sp +The salt\-call command is used to run module functions locally on a minion +instead of executing them from the master. Normally the salt\-call command +checks into the master to retrieve file server and pillar data, but when +running standalone salt\-call needs to be instructed to not check the master for +this data. To instruct the minion to not look for a master when running +salt\-call the \fI\%file_client\fP configuration option needs to be set. +By default the \fI\%file_client\fP is set to \fBremote\fP so that the +minion knows that file server and pillar data are to be gathered from the +master. When setting the \fI\%file_client\fP option to \fBlocal\fP the +minion is configured to not gather this data from the master. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-call \-\-local dockerng.sls_build test base=my_base_image mods=vim +file_client: local .ft P .fi .UNINDENT .UNINDENT .sp -Now we have a fresh image called \fItest\fP to work with and vim has been -installed. -.SS Step 4 \- Running Commands Inside the Container +Now the salt\-call command will not look for a master and will assume that the +local system has all of the file and pillar resources. +.SS Running States Masterless .sp -Salt can now run remote execution functions inside the container with another -simple \fIsalt\-call\fP command: +The state system can be easily run without a Salt master, with all needed files +local to the minion. To do this the minion configuration file needs to be set +up to know how to return file_roots information like the master. The file_roots +setting defaults to /srv/salt for the base environment just like on the master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-call \-\-local dockerng.call test test.version -salt\-call \-\-local dockerng.call test network.interfaces -salt\-call \-\-local dockerng.call test disk.usage -salt\-call \-\-local dockerng.call test pkg.list_pkgs -salt\-call \-\-local dockerng.call test service.running httpd -salt\-call \-\-local dockerng.call test cmd.run \(aqls \-l /etc\(aq +file_roots: + base: + \- /srv/salt .ft P .fi .UNINDENT .UNINDENT -.SS Automatic Updates / Frozen Deployments -.sp -New in version 0.10.3.d. - -.sp -Salt has support for the -\fI\%Esky\fP application freezing and update -tool. This tool allows one to build a complete zipfile out of the salt scripts -and all their dependencies \- including shared objects / DLLs. -.SS Getting Started .sp -To build frozen applications, suitable build environment will be needed for -each platform. You should probably set up a virtualenv in order to limit the -scope of Q/A. +Now set up the Salt State Tree, top file, and SLS modules in the same way that +they would be set up on a master. Now, with the \fI\%file_client\fP +option set to \fBlocal\fP and an available state tree then calls to functions in +the state module will use the information in the file_roots on the minion +instead of checking in with the master. .sp -This process does work on Windows. Directions are available at -\fI\%https://github.com/saltstack/salt\-windows\-install\fP for details on -installing Salt in Windows. Only the 32\-bit Python and dependencies have been -tested, but they have been tested on 64\-bit Windows. +Remember that when creating a state tree on a minion there are no syntax or +path changes needed, SLS modules written to be used from a master do not need +to be modified in any way to work with a minion. .sp -Install \fBbbfreeze\fP, and then \fBesky\fP from PyPI in order to enable the -\fBbdist_esky\fP command in \fBsetup.py\fP\&. Salt itself must also be installed, in -addition to its dependencies. -.SS Building and Freezing +This makes it easy to \(dqscript\(dq deployments with Salt states without having to +set up a master, and allows for these SLS modules to be easily moved into a +Salt master as the deployment grows. .sp -Once you have your tools installed and the environment configured, use -\fBsetup.py\fP to prepare the distribution files. +The declared state can now be executed with: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -python setup.py sdist -python setup.py bdist +salt\-call state.apply .ft P .fi .UNINDENT .UNINDENT .sp -Once the distribution files are in place, Esky can be used traverse the module -tree and pack all the scripts up into a redistributable. +Or the salt\-call command can be executed with the \fB\-\-local\fP flag, this makes +it unnecessary to change the configuration file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -python setup.py bdist_esky +salt\-call state.apply \-\-local .ft P .fi .UNINDENT .UNINDENT +.SS External Pillars .sp -There will be an appropriately versioned \fBsalt\-VERSION.zip\fP in \fBdist/\fP if -everything went smoothly. -.SS Windows -.sp -\fBC:\ePython27\elib\esite\-packages\ezmq\fP will need to be added to the PATH -variable. This helps bbfreeze find the zmq DLL so it can pack it up. -.SS Using the Frozen Build -.sp -Unpack the zip file in the desired install location. Scripts like -\fBsalt\-minion\fP and \fBsalt\-call\fP will be in the root of the zip file. The -associated libraries and bootstrapping will be in the directories at the same -level. (Check the \fI\%Esky\fP documentation -for more information) -.sp -To support updating your minions in the wild, put the builds on a web server -that the minions can reach. \fI\%salt.modules.saltutil.update()\fP will -trigger an update and (optionally) a restart of the minion service under the -new version. -.SS Troubleshooting -.SS A Windows minion isn\(aqt responding -.sp -The process dispatch on Windows is slower than it is on *nix. It may be -necessary to add \(aq\-t 15\(aq to salt commands to give minions plenty of time to -return. -.SS Windows and the Visual Studio Redist -.sp -The Visual C++ 2008 32\-bit redistributable will need to be installed on all -Windows minions. Esky has an option to pack the library into the zipfile, -but OpenSSL does not seem to acknowledge the new location. If a -\fBno OPENSSL_Applink\fP error appears on the console when trying to start a -frozen minion, the redistributable is not installed. -.SS Mixed Linux environments and Yum +\fI\%External pillars\fP are supported when running in masterless mode. +.SS How Do I Use Salt States? .sp -The Yum Python module doesn\(aqt appear to be available on any of the standard -Python package mirrors. If RHEL/CentOS systems need to be supported, the frozen -build should created on that platform to support all the Linux nodes. Remember -to build the virtualenv with \fB\-\-system\-site\-packages\fP so that the \fByum\fP -module is included. -.SS Automatic (Python) module discovery +Simplicity, Simplicity, Simplicity .sp -Automatic (Python) module discovery does not work with the late\-loaded scheme -that Salt uses for (Salt) modules. Any misbehaving modules will need to be -explicitly added to the \fBfreezer_includes\fP in Salt\(aqs \fBsetup.py\fP\&. Always -check the zipped application to make sure that the necessary modules were -included. -.SS ESXi Proxy Minion +Many of the most powerful and useful engineering solutions are founded on +simple principles. Salt States strive to do just that: K.I.S.S. (Keep It +Stupidly Simple) .sp -New in version 2015.8.4. - +The core of the Salt State system is the SLS, or \fBS\fPtructured \fBL\fPayered \fBS\fPtate. +The SLS is a representation of the state in which +a system should be in, and is set up to contain this data in a simple format. +This is often called configuration management. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -This tutorial assumes basic knowledge of Salt. To get up to speed, check -out the \fI\%Salt Walkthrough\fP\&. -.sp -This tutorial also assumes a basic understanding of Salt Proxy Minions. If -you\(aqre unfamiliar with Salt\(aqs Proxy Minion system, please read the -\fI\%Salt Proxy Minion\fP documentation and the -\fI\%Salt Proxy Minion End\-to\-End Example\fP -tutorial. -.sp -The third assumption that this tutorial makes is that you also have a -basic understanding of ESXi hosts. You can learn more about ESXi hosts on -\fI\%VMware\(aqs various resources\fP\&. +This is just the beginning of using states, make sure to read up on pillar +\fI\%Pillar\fP next. .UNINDENT .UNINDENT +.SS It is All Just Data .sp -Salt\(aqs ESXi Proxy Minion allows a VMware ESXi host to be treated as an individual -Salt Minion, without installing a Salt Minion on the ESXi host. +Before delving into the particulars, it will help to understand that the SLS +file is just a data structure under the hood. While understanding that the SLS +is just a data structure isn\(aqt critical for understanding and making use of +Salt States, it should help bolster knowledge of where the real power is. .sp -Since an ESXi host may not necessarily run on an OS capable of hosting a Python -stack, the ESXi host can\(aqt run a regular Salt Minion directly. Therefore, Salt\(aqs -Proxy Minion functionality enables you to designate another machine to host a -proxy process that \(dqproxies\(dq communication from the Salt Master to the ESXi host. -The master does not know or care that the ESXi target is not a \(dqreal\(dq Salt Minion. +SLS files are therefore, in reality, just dictionaries, lists, strings, and +numbers. By using this approach Salt can be much more flexible. As one writes +more state files, it becomes clearer exactly what is being written. The result +is a system that is easy to understand, yet grows with the needs of the admin +or developer. +.SS The Top File .sp -More in\-depth conceptual reading on Proxy Minions can be found in the -\fI\%Proxy Minion\fP section of Salt\(aqs documentation. +The example SLS files in the below sections can be assigned to hosts using a +file called \fBtop.sls\fP\&. This file is described in\-depth \fI\%here\fP\&. +.SS Default Data \- YAML .sp -Salt\(aqs ESXi Proxy Minion was added in the 2015.8.4 release of Salt. +By default Salt represents the SLS data in what is one of the simplest +serialization formats available \- \fI\%YAML\fP\&. +.sp +A typical SLS file will often look like this in YAML: .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Be aware that some functionality for the ESXi Proxy Minion may depend on the -type of license attached the ESXi host(s). -.sp -For example, certain services are only available to manipulate service state -or policies with a VMware vSphere Enterprise or Enterprise Plus license, while -others are available with a Standard license. The \fBntpd\fP service is restricted -to an Enterprise Plus license, while \fBssh\fP is available via the Standard -license. +These demos use some generic service and package names, different +distributions often use different names for packages and services. For +instance \fIapache\fP should be replaced with \fIhttpd\fP on a Red Hat system. +Salt uses the name of the init script, systemd name, upstart name etc. +based on what the underlying service management for the platform. To +get a list of the available service names on a platform execute the +service.get_all salt function. .sp -Please see the \fI\%vSphere Comparison\fP page for more information. +Information on how to make states work with multiple distributions +is later in the tutorial. .UNINDENT .UNINDENT -.SS Dependencies -.sp -Manipulation of the ESXi host via a Proxy Minion requires the machine running -the Proxy Minion process to have the ESXCLI package (and all of its dependencies) -and the pyVmomi Python Library to be installed. -.SS ESXi Password -.sp -The ESXi Proxy Minion uses VMware\(aqs API to perform tasks on the host as if it was -a regular Salt Minion. In order to access the API that is already running on the -ESXi host, the ESXi host must have a username and password that is used to log -into the host. The username is usually \fBroot\fP\&. Before Salt can access the ESXi -host via VMware\(aqs API, a default password \fImust\fP be set on the host. -.SS pyVmomi -.sp -The pyVmomi Python library must be installed on the machine that is running the -proxy process. pyVmomi can be installed via pip: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pip install pyVmomi +apache: + pkg.installed: [] + service.running: + \- require: + \- pkg: apache .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -Version 6.0 of pyVmomi has some problems with SSL error handling on certain -versions of Python. If using version 6.0 of pyVmomi, the machine that you -are running the proxy minion process from must have either Python 2.6, -Python 2.7.9, or newer. This is due to an upstream dependency in pyVmomi 6.0 -that is not supported in Python version 2.7 to 2.7.8. If the -version of Python running the proxy process is not in the supported range, you -will need to install an earlier version of pyVmomi. See \fI\%Issue #29537\fP for -more information. -.UNINDENT -.UNINDENT +This SLS data will ensure that the package named apache is installed, and +that the apache service is running. The components can be explained in a +simple way. .sp -Based on the note above, to install an earlier version of pyVmomi than the -version currently listed in PyPi, run the following: +The first line is the ID for a set of data, and it is called the ID +Declaration. This ID sets the name of the thing that needs to be manipulated. +.sp +The second and third lines contain the state module function to be run, in the +format \fB.\fP\&. The \fBpkg.installed\fP state module +function ensures that a software package is installed via the system\(aqs native +package manager. The \fBservice.running\fP state module function ensures that a +given system daemon is running. +.sp +Finally, on line four, is the word \fBrequire\fP\&. This is called a Requisite +Statement, and it makes sure that the Apache service is only started after +a successful installation of the apache package. +.SS Adding Configs and Users +.sp +When setting up a service like an Apache web server, many more components may +need to be added. The Apache configuration file will most likely be managed, +and a user and group may need to be set up. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pip install pyVmomi==5.5.0.2014.1.1 +apache: + pkg.installed: [] + service.running: + \- watch: + \- pkg: apache + \- file: /etc/httpd/conf/httpd.conf + \- user: apache + user.present: + \- uid: 87 + \- gid: 87 + \- home: /var/www/html + \- shell: /bin/nologin + \- require: + \- group: apache + group.present: + \- gid: 87 + \- require: + \- pkg: apache + +/etc/httpd/conf/httpd.conf: + file.managed: + \- source: salt://apache/httpd.conf + \- user: root + \- group: root + \- mode: 644 .ft P .fi .UNINDENT .UNINDENT .sp -The 5.5.0.2014.1.1 is a known stable version that the original ESXi Proxy Minion -was developed against. -.SS ESXCLI +This SLS data greatly extends the first example, and includes a config file, +a user, a group and new requisite statement: \fBwatch\fP\&. .sp -Currently, about a third of the functions used for the ESXi Proxy Minion require -the ESXCLI package be installed on the machine running the Proxy Minion process. +Adding more states is easy, since the new user and group states are under +the Apache ID, the user and group will be the Apache user and group. The +\fBrequire\fP statements will make sure that the user will only be made after +the group, and that the group will be made only after the Apache package is +installed. .sp -The ESXCLI package is also referred to as the VMware vSphere CLI, or vCLI. VMware -provides vCLI package installation instructions for \fI\%vSphere 5.5\fP and -\fI\%vSphere 6.0\fP\&. +Next, the \fBrequire\fP statement under service was changed to watch, and is +now watching 3 states instead of just one. The watch statement does the same +thing as require, making sure that the other states run before running the +state with a watch, but it adds an extra component. The \fBwatch\fP statement +will run the state\(aqs watcher function for any changes to the watched states. +So if the package was updated, the config file changed, or the user +uid modified, then the service state\(aqs watcher will be run. The service +state\(aqs watcher just restarts the service, so in this case, a change in the +config file will also trigger a restart of the respective service. +.SS Moving Beyond a Single SLS .sp -Once all of the required dependencies are in place and the vCLI package is -installed, you can check to see if you can connect to your ESXi host by running -the following command: +When setting up Salt States in a scalable manner, more than one SLS will need +to be used. The above examples were in a single SLS file, but two or more +SLS files can be combined to build out a State Tree. The above example also +references a file with a strange source \- \fBsalt://apache/httpd.conf\fP\&. That +file will need to be available as well. +.sp +The SLS files are laid out in a directory structure on the Salt master; an +SLS is just a file and files to download are just files. +.sp +The Apache example would be laid out in the root of the Salt file server like +this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -esxcli \-s \-u \-p system syslog config get +apache/init.sls +apache/httpd.conf .ft P .fi .UNINDENT .UNINDENT .sp -If the connection was successful, ESXCLI was successfully installed on your system. -You should see output related to the ESXi host\(aqs syslog configuration. -.SS Configuration +So the httpd.conf is just a file in the apache directory, and is referenced +directly. +.INDENT 0.0 +.INDENT 3.5 +.IP "Do not use dots in SLS file names or their directories" .sp -There are several places where various configuration values need to be set in -order for the ESXi Proxy Minion to run and connect properly. -.SS Proxy Config File +The initial implementation of \fI\%top.sls\fP and +\fI\%Include declaration\fP followed the python import model where a slash +is represented as a period. This means that a SLS file with a period in +the name ( besides the suffix period) can not be referenced. For example, +webserver_1.0.sls is not referenceable because webserver_1.0 would refer +to the directory/file webserver_1/0.sls .sp -On the machine that will be running the Proxy Minion process(es), a proxy config -file must be in place. This file should be located in the \fB/etc/salt/\fP directory -and should be named \fBproxy\fP\&. If the file is not there by default, create it. +The same applies for any subdirectories, this is especially \(aqtricky\(aq when +git repos are created. Another command that typically can\(aqt render its +output is \fB\(gastate.show_sls\(ga\fP of a file in a path that contains a dot. +.UNINDENT +.UNINDENT .sp -This file should contain the location of your Salt Master that the Salt Proxy -will connect to. +But when using more than one single SLS file, more components can be added to +the toolkit. Consider this SSH example: .sp -Example Proxy Config File: +\fBssh/init.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /etc/salt/proxy +openssh\-client: + pkg.installed -master: +/etc/ssh/ssh_config: + file.managed: + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/ssh_config + \- require: + \- pkg: openssh\-client .ft P .fi .UNINDENT .UNINDENT -.SS Pillar Profiles .sp -Proxy minions get their configuration from Salt\(aqs Pillar. Every proxy must -have a stanza in Pillar and a reference in the Pillar top\-file that matches -the Proxy ID. At a minimum for communication with the ESXi host, the pillar -should look like this: +\fBssh/server.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -proxy: - proxytype: esxi - host: - username: - passwords: - \- first_password - \- second_password - \- third_password +include: + \- ssh + +openssh\-server: + pkg.installed + +sshd: + service.running: + \- require: + \- pkg: openssh\-client + \- pkg: openssh\-server + \- file: /etc/ssh/banner + \- file: /etc/ssh/sshd_config + +/etc/ssh/sshd_config: + file.managed: + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/sshd_config + \- require: + \- pkg: openssh\-server + +/etc/ssh/banner: + file: + \- managed + \- user: root + \- group: root + \- mode: 644 + \- source: salt://ssh/banner + \- require: + \- pkg: openssh\-server .ft P .fi .UNINDENT .UNINDENT .sp -Some other optional settings are \fBprotocol\fP and \fBport\fP\&. These can be added -to the pillar configuration. -.SS proxytype -.sp -The \fBproxytype\fP key and value pair is critical, as it tells Salt which -interface to load from the \fBproxy\fP directory in Salt\(aqs install hierarchy, -or from \fB/srv/salt/_proxy\fP on the Salt Master (if you have created your -own proxy module, for example). To use this ESXi Proxy Module, set this to -\fBesxi\fP\&. -.SS host -.sp -The location, or ip/dns, of the ESXi host. Required. -.SS username -.sp -The username used to login to the ESXi host, such as \fBroot\fP\&. Required. -.SS passwords -.sp -A list of passwords to be used to try and login to the ESXi host. At least -one password in this list is required. -.sp -The proxy integration will try the passwords listed in order. It is -configured this way so you can have a regular password and the password you -may be updating for an ESXi host either via the -\fI\%vsphere.update_host_password\fP -execution module function or via the -\fI\%esxi.password_present\fP state -function. This way, after the password is changed, you should not need to -restart the proxy minion\-\-it should just pick up the new password -provided in the list. You can then change pillar at will to move that -password to the front and retire the unused ones. -.sp -Use\-case/reasoning for using a list of passwords: You are setting up an -ESXi host for the first time, and the host comes with a default password. -You know that you\(aqll be changing this password during your initial setup -from the default to a new password. If you only have one password option, -and if you have a state changing the password, any remote execution commands -or states that run after the password change will not be able to run on the -host until the password is updated in Pillar and the Proxy Minion process is -restarted. -.sp -This allows you to use any number of potential fallback passwords. -.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -When a password is changed on the host to one in the list of possible -passwords, the further down on the list the password is, the longer -individual commands will take to return. This is due to the nature of -pyVmomi\(aqs login system. We have to wait for the first attempt to fail -before trying the next password on the list. +Notice that we use two similar ways of denoting that a file +is managed by Salt. In the \fI/etc/ssh/sshd_config\fP state section above, +we use the \fIfile.managed\fP state declaration whereas with the +\fI/etc/ssh/banner\fP state section, we use the \fIfile\fP state declaration +and add a \fImanaged\fP attribute to that state declaration. Both ways +produce an identical result; the first way \-\- using \fIfile.managed\fP \-\- +is merely a shortcut. +.UNINDENT +.UNINDENT .sp -This scenario is especially true, and even slower, when the proxy -minion first starts. If the correct password is not the first password -on the list, it may take up to a minute for \fBtest.version\fP to respond -with salt\(aqs version installed (Example: \fB2018.3.4\fP\&. Once the initial -authorization is complete, the responses for commands will be a little -faster. +Now our State Tree looks like this: +.INDENT 0.0 +.INDENT 3.5 .sp -To avoid these longer waiting periods, SaltStack recommends moving the -correct password to the top of the list and restarting the proxy minion -at your earliest convenience. +.nf +.ft C +apache/init.sls +apache/httpd.conf +ssh/init.sls +ssh/server.sls +ssh/banner +ssh/ssh_config +ssh/sshd_config +.ft P +.fi .UNINDENT .UNINDENT -.SS protocol .sp -If the ESXi host is not using the default protocol, set this value to an -alternate protocol. Default is \fBhttps\fP\&. For example: -.SS port +This example now introduces the \fBinclude\fP statement. The include statement +includes another SLS file so that components found in it can be required, +watched or as will soon be demonstrated \- extended. .sp -If the ESXi host is not using the default port, set this value to an -alternate port. Default is \fB443\fP\&. -.SS Example Configuration Files +The include statement allows for states to be cross linked. When an SLS +has an include statement it is literally extended to include the contents of +the included SLS files. .sp -An example of all of the basic configurations that need to be in place before -starting the Proxy Minion processes includes the Proxy Config File, Pillar -Top File, and any individual Proxy Minion Pillar files. +Note that some of the SLS files are called init.sls, while others are not. More +info on what this means can be found in the \fI\%States Tutorial\fP\&. +.SS Extending Included SLS Data .sp -In this example, we\(aqll assuming there are two ESXi hosts to connect to. Therefore, -we\(aqll be creating two Proxy Minion config files, one config for each ESXi host. +Sometimes SLS data needs to be extended. Perhaps the apache service needs to +watch additional resources, or under certain circumstances a different file +needs to be placed. .sp -Proxy Config File: +In these examples, the first will add a custom banner to ssh and the second will +add more watchers to apache to include mod_python. +.sp +\fBssh/custom\-server.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /etc/salt/proxy +include: + \- ssh.server -master: +extend: + /etc/ssh/banner: + file: + \- source: salt://ssh/custom\-banner .ft P .fi .UNINDENT .UNINDENT .sp -Pillar Top File: +\fBpython/mod_python.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/pillar/top.sls +include: + \- apache -base: - \(aqesxi\-1\(aq: - \- esxi\-1 - \(aqesxi\-2\(aq: - \- esxi\-2 +extend: + apache: + service: + \- watch: + \- pkg: mod_python + +mod_python: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -Pillar Config File for the first ESXi host, esxi\-1: +The \fBcustom\-server.sls\fP file uses the extend statement to overwrite where the +banner is being downloaded from, and therefore changing what file is being used +to configure the banner. +.sp +In the new mod_python SLS the mod_python package is added, but more importantly +the apache service was extended to also watch the mod_python package. +.INDENT 0.0 +.INDENT 3.5 +.IP "Using extend with require or watch" +.sp +The \fBextend\fP statement works differently for \fBrequire\fP or \fBwatch\fP\&. +It appends to, rather than replacing the requisite component. +.UNINDENT +.UNINDENT +.SS Understanding the Render System +.sp +Since SLS data is simply that (data), it does not need to be represented +with YAML. Salt defaults to YAML because it is very straightforward and easy +to learn and use. But the SLS files can be rendered from almost any imaginable +medium, so long as a renderer module is provided. +.sp +The default rendering system is the \fBjinja|yaml\fP renderer. The +\fBjinja|yaml\fP renderer will first pass the template through the \fI\%Jinja2\fP +templating system, and then through the YAML parser. The benefit here is that +full programming constructs are available when creating SLS files. +.sp +Other renderers available are \fByaml_mako\fP and \fByaml_wempy\fP which each use +the \fI\%Mako\fP or \fI\%Wempy\fP templating system respectively rather than the jinja +templating system, and more notably, the pure Python or \fBpy\fP, \fBpydsl\fP & +\fBpyobjects\fP renderers. +The \fBpy\fP renderer allows for SLS files to be written in pure Python, +allowing for the utmost level of flexibility and power when preparing SLS +data; while the \fI\%pydsl\fP renderer +provides a flexible, domain\-specific language for authoring SLS data in Python; +and the \fI\%pyobjects\fP renderer +gives you a \fI\%\(dqPythonic\(dq\fP interface to building state data. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The templating engines described above aren\(aqt just available in SLS files. +They can also be used in \fI\%file.managed\fP +states, making file management much more dynamic and flexible. Some +examples for using templates in managed files can be found in the +documentation for the \fI\%file state\fP, as well as the +\fI\%MooseFS example\fP below. +.UNINDENT +.UNINDENT +.SS Getting to Know the Default \- jinja|yaml +.sp +The default renderer \- \fBjinja|yaml\fP, allows for use of the jinja +templating system. A guide to the Jinja templating system can be found here: +\fI\%https://jinja.palletsprojects.com/en/2.11.x/\fP +.sp +When working with renderers a few very useful bits of data are passed in. In +the case of templating engine based renderers, three critical components are +available, \fBsalt\fP, \fBgrains\fP, and \fBpillar\fP\&. The \fBsalt\fP object allows for +any Salt function to be called from within the template, and \fBgrains\fP allows +for the Grains to be accessed from within the template. A few examples: +.sp +\fBapache/init.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/pillar/esxi\-1.sls +apache: + pkg.installed: + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq%} + \- name: httpd + {% endif %} + service.running: + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq%} + \- name: httpd + {% endif %} + \- watch: + \- pkg: apache + \- file: /etc/httpd/conf/httpd.conf + \- user: apache + user.present: + \- uid: 87 + \- gid: 87 + \- home: /var/www/html + \- shell: /bin/nologin + \- require: + \- group: apache + group.present: + \- gid: 87 + \- require: + \- pkg: apache -proxy: - proxytype: esxi - host: esxi\-1.example.com - username: \(aqroot\(aq - passwords: - \- bad\-password\-1 - \- backup\-bad\-password\-1 +/etc/httpd/conf/httpd.conf: + file.managed: + \- source: salt://apache/httpd.conf + \- user: root + \- group: root + \- mode: 644 .ft P .fi .UNINDENT .UNINDENT .sp -Pillar Config File for the second ESXi host, esxi\-2: +This example is simple. If the \fBos\fP grain states that the operating system is +Red Hat, then the name of the Apache package and service needs to be httpd. +.sp +A more aggressive way to use Jinja can be found here, in a module to set up +a MooseFS distributed filesystem chunkserver: +.sp +\fBmoosefs/chunk.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/pillar/esxi\-2.sls +include: + \- moosefs -proxy: - proxytype: esxi - host: esxi\-2.example.com - username: \(aqroot\(aq - passwords: - \- bad\-password\-2 - \- backup\-bad\-password\-2 +{% for mnt in salt[\(aqcmd.run\(aq](\(aqls /dev/data/moose*\(aq).split() %} +/mnt/moose{{ mnt[\-1] }}: + mount.mounted: + \- device: {{ mnt }} + \- fstype: xfs + \- mkmnt: True + file.directory: + \- user: mfs + \- group: mfs + \- require: + \- user: mfs + \- group: mfs +{% endfor %} + +/etc/mfshdd.cfg: + file.managed: + \- source: salt://moosefs/mfshdd.cfg + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- require: + \- pkg: mfs\-chunkserver + +/etc/mfschunkserver.cfg: + file.managed: + \- source: salt://moosefs/mfschunkserver.cfg + \- user: root + \- group: root + \- mode: 644 + \- template: jinja + \- require: + \- pkg: mfs\-chunkserver + +mfs\-chunkserver: + pkg.installed: [] +mfschunkserver: + service.running: + \- require: +{% for mnt in salt[\(aqcmd.run\(aq](\(aqls /dev/data/moose*\(aq) %} + \- mount: /mnt/moose{{ mnt[\-1] }} + \- file: /mnt/moose{{ mnt[\-1] }} +{% endfor %} + \- file: /etc/mfschunkserver.cfg + \- file: /etc/mfshdd.cfg + \- file: /var/lib/mfs .ft P .fi .UNINDENT .UNINDENT -.SS Starting the Proxy Minion .sp -Once all of the correct configuration files are in place, it is time to start the -proxy processes! -.INDENT 0.0 -.IP 1. 3 -First, make sure your Salt Master is running. -.IP 2. 3 -Start the first Salt Proxy, in debug mode, by giving the Proxy Minion process -and ID that matches the config file name created in the \fI\%Configuration\fP section. -.UNINDENT +This example shows much more of the available power of Jinja. +Multiple for loops are used to dynamically detect available hard drives +and set them up to be mounted, and the \fBsalt\fP object is used multiple +times to call shell commands to gather data. +.SS Introducing the Python, PyDSL, and the Pyobjects Renderers +.sp +Sometimes the chosen default renderer might not have enough logical power to +accomplish the needed task. When this happens, the Python renderer can be +used. Normally a YAML renderer should be used for the majority of SLS files, +but an SLS file set to use another renderer can be easily added to the tree. +.sp +This example shows a very basic Python SLS file: +.sp +\fBpython/django.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-proxy \-\-proxyid=\(aqesxi\-1\(aq \-l debug +#!py + + +def run(): + \(dq\(dq\(dq + Install the django package + \(dq\(dq\(dq + return {\(dqinclude\(dq: [\(dqpython\(dq], \(dqdjango\(dq: {\(dqpkg\(dq: [\(dqinstalled\(dq]}} .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP 1. 3 -Accept the \fBesxi\-1\fP Proxy Minion\(aqs key on the Salt Master: -.UNINDENT +.sp +This is a very simple example; the first line has an SLS shebang that +tells Salt to not use the default renderer, but to use the \fBpy\fP renderer. +Then the run function is defined, the return value from the run function +must be a Salt friendly data structure, or better known as a Salt +\fI\%HighState data structure\fP\&. +.sp +Alternatively, using the \fI\%pydsl\fP +renderer, the above example can be written more succinctly as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-key \-L -Accepted Keys: -Denied Keys: -Unaccepted Keys: -esxi\-1 -Rejected Keys: -# -# salt\-key \-a esxi\-1 -The following keys are going to be accepted: -Unaccepted Keys: -esxi\-1 -Proceed? [n/Y] y -Key for minion esxi\-1 accepted. +#!pydsl + +include(\(dqpython\(dq, delayed=True) +state(\(dqdjango\(dq).pkg.installed() .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP 1. 3 -Repeat for the second Salt Proxy, this time we\(aqll run the proxy process as a -daemon, as an example. -.UNINDENT +.sp +The \fI\%pyobjects\fP renderer +provides an \fI\%\(dqPythonic\(dq\fP object based approach for building the state data. +The above example could be written as: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-proxy \-\-proxyid=\(aqesxi\-2\(aq \-d +#!pyobjects + +include(\(dqpython\(dq) +Pkg.installed(\(dqdjango\(dq) .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP 1. 3 -Accept the \fBesxi\-2\fP Proxy Minion\(aqs key on the Salt Master: -.UNINDENT +.sp +These Python examples would look like this if they were written in YAML: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt\-key \-L -Accepted Keys: -esxi\-1 -Denied Keys: -Unaccepted Keys: -esxi\-2 -Rejected Keys: -# -# salt\-key \-a esxi\-1 -The following keys are going to be accepted: -Unaccepted Keys: -esxi\-2 -Proceed? [n/Y] y -Key for minion esxi\-1 accepted. +include: + \- python + +django: + pkg.installed .ft P .fi .UNINDENT .UNINDENT -.INDENT 0.0 -.IP 1. 3 -Check and see if your Proxy Minions are responding: -.UNINDENT +.sp +This example clearly illustrates that; one, using the YAML renderer by default +is a wise decision and two, unbridled power can be obtained where needed by +using a pure Python SLS. +.SS Running and Debugging Salt States +.sp +Once the rules in an SLS are ready, they should be tested to ensure they +work properly. To invoke these rules, simply execute +\fBsalt \(aq*\(aq state.apply\fP on the command line. If you get back only +hostnames with a \fB:\fP after, but no return, chances are there is a problem with +one or more of the sls files. On the minion, use the \fBsalt\-call\fP command to +examine the output for errors: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# salt \(aqesxi\-*\(aq test.version -esxi\-1: - True -esxi\-3: - True +salt\-call state.apply \-l debug .ft P .fi .UNINDENT .UNINDENT -.SS Executing Commands -.sp -Now that you\(aqve configured your Proxy Minions and have them responding successfully -to a \fBtest.version\fP, we can start executing commands against the ESXi hosts via Salt. .sp -It\(aqs important to understand how this particular proxy works, and there are a couple -of important pieces to be aware of in order to start running remote execution and -state commands against the ESXi host via a Proxy Minion: the -\fI\%vSphere Execution Module\fP, the \fI\%ESXi Execution Module\fP, and the \fI\%ESXi State Module\fP\&. -.SS vSphere Execution Module +This should help troubleshoot the issue. The minion can also be started in the +foreground in debug mode by running \fBsalt\-minion \-l debug\fP\&. +.SS Next Reading .sp -The \fI\%Salt.modules.vsphere\fP is a -standard Salt execution module that does the bulk of the work for the ESXi Proxy -Minion. If you pull up the docs for it you\(aqll see that almost every function in -the module takes credentials (\fBusername\fP and \fBpassword\fP) and a target \fBhost\fP -argument. When credentials and a host aren\(aqt passed, Salt runs commands -through \fBpyVmomi\fP or \fBESXCLI\fP against the local machine. If you wanted, -you could run functions from this module on any machine where an appropriate -version of \fBpyVmomi\fP and \fBESXCLI\fP are installed, and that machine would reach -out over the network and communicate with the ESXi host. +With an understanding of states, the next recommendation is to become familiar +with Salt\(aqs pillar interface: +.INDENT 0.0 +.INDENT 3.5 +\fI\%Pillar Walkthrough\fP +.UNINDENT +.UNINDENT +.SS States tutorial, part 1 \- Basic Usage .sp -You\(aqll notice that most of the functions in the vSphere module require a \fBhost\fP, -\fBusername\fP, and \fBpassword\fP\&. These parameters are contained in the Pillar files and -passed through to the function via the proxy process that is already running. You don\(aqt -need to provide these parameters when you execute the commands. See the -\fI\%Running Remote Execution Commands\fP section below for an example. -.SS ESXi Execution Module +The purpose of this tutorial is to demonstrate how quickly you can configure a +system to be managed by Salt States. For detailed information about the state +system please refer to the full \fI\%states reference\fP\&. .sp -In order for the Pillar information set up in the \fI\%Configuration\fP section above to -be passed to the function call in the vSphere Execution Module, the -\fI\%salt.modules.esxi\fP execution module acts -as a \(dqshim\(dq between the vSphere execution module functions and the proxy process. +This tutorial will walk you through using Salt to configure a minion to run the +Apache HTTP server and to ensure the server is running. .sp -The \(dqshim\(dq takes the authentication credentials specified in the Pillar files and -passes them through to the \fBhost\fP, \fBusername\fP, \fBpassword\fP, and optional -\fBprotocol\fP and \fBport\fP options required by the vSphere Execution Module functions. +\fBBefore continuing\fP make sure you have a working Salt installation by +following the instructions in the +\fI\%Salt install guide\fP\&. +.INDENT 0.0 +.INDENT 3.5 +.IP "Stuck?" .sp -If the function takes more positional, or keyword, arguments you can append them -to the call. It\(aqs this shim that speaks to the ESXi host through the proxy, arranging -for the credentials and hostname to be pulled from the Pillar section for the ESXi -Proxy Minion. +The Salt Project community can help offer advice and help troubleshoot +technical issues as you\(aqre learning about Salt. One of the best places to +talk to the community is on the +\fI\%Salt Project Slack workspace\fP\&. +.UNINDENT +.UNINDENT +.SS Setting up the Salt State Tree .sp -Because of the presence of the shim, to lookup documentation for what -functions you can use to interface with the ESXi host, you\(aqll want to -look in \fI\%salt.modules.vsphere\fP -instead of \fI\%salt.modules.esxi\fP\&. -.SS Running Remote Execution Commands +States are stored in text files on the master and transferred to the minions on +demand via the master\(aqs File Server. The collection of state files make up the +\fBState Tree\fP\&. .sp -To run commands from the Salt Master to execute, via the ESXi Proxy Minion, against -the ESXi host, you use the \fBesxi.cmd \fP syntax to call -functions located in the vSphere Execution Module. Both args and kwargs needed -for various vsphere execution module functions must be passed through in a kwarg\- -type manor. For example: +To start using a central state system in Salt, the Salt File Server must first +be set up. Edit the master config file (\fI\%file_roots\fP) and +uncomment the following lines: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aqesxi\-*\(aq esxi.cmd system_info -salt \(aqexsi\-*\(aq esxi.cmd get_service_running service_name=\(aqssh\(aq +file_roots: + base: + \- /srv/salt .ft P .fi .UNINDENT .UNINDENT -.SS ESXi State Module .sp -The ESXi State Module functions similarly to other state modules. The \(dqshim\(dq provided -by the \fI\%ESXi Execution Module\fP passes the necessary \fBhost\fP, \fBusername\fP, and -\fBpassword\fP credentials through, so those options don\(aqt need to be provided in the -state. Other than that, state files are written and executed just like any other -Salt state. See the \fI\%salt.modules.esxi\fP state -for ESXi state functions. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +If you are deploying on FreeBSD via ports, the \fBfile_roots\fP path defaults +to \fB/usr/local/etc/salt/states\fP\&. +.UNINDENT +.UNINDENT .sp -The follow state file is an example of how to configure various pieces of an ESXi host -including enabling SSH, uploading and SSH key, configuring a coredump network config, -syslog, ntp, enabling VMotion, resetting a host password, and more. +Restart the Salt master in order to pick up this change: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# /srv/salt/configure\-esxi.sls - -configure\-host\-ssh: - esxi.ssh_configured: - \- service_running: True - \- ssh_key_file: /etc/salt/ssh_keys/my_key.pub - \- service_policy: \(aqautomatic\(aq - \- service_restart: True - \- certificate_verify: True - -configure\-host\-coredump: - esxi.coredump_configured: - \- enabled: True - \- dump_ip: \(aqmy\-coredump\-ip.example.com\(aq - -configure\-host\-syslog: - esxi.syslog_configured: - \- syslog_configs: - loghost: ssl://localhost:5432,tcp://10.1.0.1:1514 - default\-timeout: 120 - \- firewall: True - \- reset_service: True - \- reset_syslog_config: True - \- reset_configs: loghost,default\-timeout - -configure\-host\-ntp: - esxi.ntp_configured: - \- service_running: True - \- ntp_servers: - \- 192.174.1.100 - \- 192.174.1.200 - \- service_policy: \(aqautomatic\(aq - \- service_restart: True - -configure\-vmotion: - esxi.vmotion_configured: - \- enabled: True - -configure\-host\-vsan: - esxi.vsan_configured: - \- enabled: True - \- add_disks_to_vsan: True - -configure\-host\-password: - esxi.password_present: - \- password: \(aqnew\-bad\-password\(aq +pkill salt\-master +salt\-master \-d .ft P .fi .UNINDENT .UNINDENT +.SS Preparing the Top File .sp -States are called via the ESXi Proxy Minion just as they would on a regular minion. -For example: +On the master, in the directory uncommented in the previous step, +(\fB/srv/salt\fP by default), create a new file called +\fI\%top.sls\fP and add the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aqesxi\-*\(aq state.sls configure\-esxi test=true -salt \(aqesxi\-*\(aq state.sls configure\-esxi +base: + \(aq*\(aq: + \- webserver .ft P .fi .UNINDENT .UNINDENT -.SS Relevant Salt Files and Resources -.INDENT 0.0 -.IP \(bu 2 -\fI\%ESXi Proxy Minion\fP -.IP \(bu 2 -\fI\%ESXi Execution Module\fP -.IP \(bu 2 -\fI\%ESXi State Module\fP -.IP \(bu 2 -\fI\%Salt Proxy Minion Docs\fP -.IP \(bu 2 -\fI\%Salt Proxy Minion End\-to\-End Example\fP -.IP \(bu 2 -\fI\%vSphere Execution Module\fP -.UNINDENT -.SS Opening the Firewall up for Salt -.sp -The Salt master communicates with the minions using an AES\-encrypted ZeroMQ -connection. These communications are done over TCP ports \fB4505\fP and \fB4506\fP, -which need to be accessible on the master only. This document outlines suggested -firewall rules for allowing these incoming connections to the master. .sp -\fBNOTE:\fP +The \fI\%top file\fP is separated into environments (discussed +later). The default environment is \fBbase\fP\&. Under the \fBbase\fP environment a +collection of minion matches is defined; for now simply specify all hosts +(\fB*\fP). .INDENT 0.0 .INDENT 3.5 -No firewall configuration needs to be done on Salt minions. These changes -refer to the master only. -.UNINDENT -.UNINDENT -.SS Fedora 18 and beyond / RHEL 7 / CentOS 7 -.sp -Starting with Fedora 18 \fI\%FirewallD\fP is the tool that is used to dynamically -manage the firewall rules on a host. It has support for IPv4/6 settings and -the separation of runtime and permanent configurations. To interact with -FirewallD use the command line client \fBfirewall\-cmd\fP\&. +.IP "Targeting minions" .sp -\fBfirewall\-cmd example\fP: +The expressions can use any of the targeting mechanisms used by Salt — +minions can be matched by glob, PCRE regular expression, or by \fI\%grains\fP\&. For example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -firewall\-cmd \-\-permanent \-\-zone= \-\-add\-port=4505\-4506/tcp +base: + \(aqos:Fedora\(aq: + \- match: grain + \- webserver .ft P .fi .UNINDENT .UNINDENT +.UNINDENT +.UNINDENT +.SS Create an \fBsls\fP file .sp -A network zone defines the security level of trust for the network. -The user should choose an appropriate zone value for their setup. -Possible values include: drop, block, public, external, dmz, work, home, internal, trusted. -.sp -Don\(aqt forget to reload after you made your changes. +In the same directory as the \fI\%top file\fP, create a file +named \fBwebserver.sls\fP, containing the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -firewall\-cmd \-\-reload +apache: # ID declaration + pkg: # state declaration + \- installed # function declaration .ft P .fi .UNINDENT .UNINDENT -.SS RHEL 6 / CentOS 6 .sp -The \fBlokkit\fP command packaged with some Linux distributions makes opening -iptables firewall ports very simple via the command line. Just be careful -to not lock out access to the server by neglecting to open the ssh port. +The first line, called the \fI\%ID declaration\fP, is an arbitrary identifier. +In this case it defines the name of the package to be installed. .sp -\fBlokkit example\fP: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -lokkit \-p 22:tcp \-p 4505:tcp \-p 4506:tcp -.ft P -.fi +The package name for the Apache httpd web server may differ depending on +OS or distro — for example, on Fedora it is \fBhttpd\fP but on +Debian/Ubuntu it is \fBapache2\fP\&. .UNINDENT .UNINDENT .sp -The \fBsystem\-config\-firewall\-tui\fP command provides a text\-based interface to -modifying the firewall. +The second line, called the \fI\%State declaration\fP, defines which of the Salt +States we are using. In this example, we are using the \fI\%pkg state\fP to ensure that a given package is installed. .sp -\fBsystem\-config\-firewall\-tui\fP: +The third line, called the \fI\%Function declaration\fP, defines which function +in the \fI\%pkg state\fP module to call. .INDENT 0.0 .INDENT 3.5 +.IP "Renderers" .sp -.nf -.ft C -system\-config\-firewall\-tui -.ft P -.fi +States \fBsls\fP files can be written in many formats. Salt requires only +a simple data structure and is not concerned with how that data structure +is built. Templating languages and \fI\%DSLs\fP are a dime\-a\-dozen and everyone +has a favorite. +.sp +Building the expected data structure is the job of Salt \fI\%Renderers\fP +and they are dead\-simple to write. +.sp +In this tutorial we will be using YAML in Jinja2 templates, which is the +default format. The default can be changed by editing +\fI\%renderer\fP in the master configuration file. .UNINDENT .UNINDENT -.SS openSUSE +.SS Install the package .sp -Salt installs firewall rules in \fI\%/etc/sysconfig/SuSEfirewall2.d/services/salt\fP\&. -Enable with: +Next, let\(aqs run the state we created. Open a terminal on the master and run: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -SuSEfirewall2 open -SuSEfirewall2 start +salt \(aq*\(aq state.apply .ft P .fi .UNINDENT .UNINDENT .sp -If you have an older package of Salt where the above configuration file is -not included, the \fBSuSEfirewall2\fP command makes opening iptables firewall -ports very simple via the command line. +Our master is instructing all targeted minions to run \fBstate.apply\fP\&. When this function is executed without any SLS +targets, a minion will download the \fI\%top file\fP and attempt to +match the expressions within it. When the minion does match an expression the +modules listed for it will be downloaded, compiled, and executed. .sp -\fBSuSEfirewall example\fP: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -SuSEfirewall2 open EXT TCP 4505 -SuSEfirewall2 open EXT TCP 4506 -.ft P -.fi +This action is referred to as a \(dqhighstate\(dq, and can be run using the +\fI\%state.highstate\fP function. +However, to make the usage easier to understand (\(dqhighstate\(dq is not +necessarily an intuitive name), a \fI\%state.apply\fP function was added in version 2015.5.0, which +when invoked without any SLS names will trigger a highstate. +\fI\%state.highstate\fP still exists and +can be used, but the documentation (as can be seen above) has been updated +to reference \fI\%state.apply\fP, so keep +the following in mind as you read the documentation: +.INDENT 0.0 +.IP \(bu 2 +\fI\%state.apply\fP invoked without any +SLS names will run \fI\%state.highstate\fP +.IP \(bu 2 +\fI\%state.apply\fP invoked with SLS names +will run \fI\%state.sls\fP +.UNINDENT .UNINDENT .UNINDENT .sp -The firewall module in YaST2 provides a text\-based interface to modifying the -firewall. +Once completed, the minion will report back with a summary of all actions taken +and all changes made. .sp -\fBYaST2\fP: +\fBWARNING:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -yast2 firewall -.ft P -.fi +If you have created \fI\%custom grain modules\fP, they will +not be available in the top file until after the first \fI\%highstate\fP\&. To make custom grains available on a minion\(aqs first +\fI\%highstate\fP, it is recommended to use \fI\%this +example\fP to ensure that the custom grains are synced +when the minion starts. .UNINDENT .UNINDENT -.SS Windows -.sp -Windows Firewall is the default component of Microsoft Windows that provides -firewalling and packet filtering. There are many 3rd party firewalls available -for Windows, some of which use rules from the Windows Firewall. If you are -experiencing problems see the vendor\(aqs specific documentation for opening the -required ports. -.sp -The Windows Firewall can be configured using the Windows Interface or from the -command line. +.INDENT 0.0 +.INDENT 3.5 +.IP "SLS File Namespace" .sp -\fBWindows Firewall (interface)\fP: +Note that in the \fI\%example\fP above, the SLS file +\fBwebserver.sls\fP was referred to simply as \fBwebserver\fP\&. The namespace +for SLS files when referenced in \fI\%top.sls\fP or an \fI\%Include declaration\fP +follows a few simple rules: .INDENT 0.0 .IP 1. 3 -Open the Windows Firewall Interface by typing \fBwf.msc\fP at the command -prompt or in a run dialog (\fIWindows Key + R\fP) +The \fB\&.sls\fP is discarded (i.e. \fBwebserver.sls\fP becomes +\fBwebserver\fP). .IP 2. 3 -Navigate to \fBInbound Rules\fP in the console tree +.INDENT 3.0 +.TP +.B Subdirectories can be used for better organization. +.INDENT 7.0 +.IP a. 3 +Each subdirectory under the configured file_roots (default: +\fB/srv/salt/\fP) is represented with a dot (following the Python +import model) in Salt states and on the command line. +\fBwebserver/dev.sls\fP on the filesystem is referred to as +\fBwebserver.dev\fP in Salt +.IP b. 3 +Because slashes are represented as dots, SLS files can not contain +dots in the name (other than the dot for the SLS suffix). The SLS +file \fBwebserver_1.0.sls\fP can not be matched, and \fBwebserver_1.0\fP +would match the directory/file \fBwebserver_1/0.sls\fP +.UNINDENT +.UNINDENT .IP 3. 3 -Add a new rule by clicking \fBNew Rule...\fP in the Actions area +A file called \fBinit.sls\fP in a subdirectory is referred to by the path +of the directory. So, \fBwebserver/init.sls\fP is referred to as +\fBwebserver\fP\&. .IP 4. 3 -Change the Rule Type to \fBPort\fP\&. Click \fBNext\fP -.IP 5. 3 -Set the Protocol to \fBTCP\fP and specify local ports \fB4505\-4506\fP\&. Click -\fBNext\fP -.IP 6. 3 -Set the Action to \fBAllow the connection\fP\&. Click \fBNext\fP -.IP 7. 3 -Apply the rule to \fBDomain\fP, \fBPrivate\fP, and \fBPublic\fP\&. Click \fBNext\fP -.IP 8. 3 -Give the new rule a Name, ie: \fBSalt\fP\&. You may also add a description. Click -\fBFinish\fP +If both \fBwebserver.sls\fP and \fBwebserver/init.sls\fP happen to exist, +\fBwebserver/init.sls\fP will be ignored and \fBwebserver.sls\fP will be the +file referred to as \fBwebserver\fP\&. .UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.IP "Troubleshooting Salt" .sp -\fBWindows Firewall (command line)\fP: -.sp -The Windows Firewall rule can be created by issuing a single command. Run the -following command from the command line or a run prompt: +If the expected output isn\(aqt seen, the following tips can help to +narrow down the problem. .INDENT 0.0 +.TP +.B Turn up logging +Salt can be quite chatty when you change the logging setting to +\fBdebug\fP: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -netsh advfirewall firewall add rule name=\(dqSalt\(dq dir=in action=allow protocol=TCP localport=4505\-4506 +salt\-minion \-l debug .ft P .fi .UNINDENT .UNINDENT -.SS iptables -.sp -Different Linux distributions store their \fIiptables\fP (also known as -\fI\%netfilter\fP) rules in different places, which makes it difficult to -standardize firewall documentation. Included are some of the more -common locations, but your mileage may vary. -.sp -\fBFedora / RHEL / CentOS\fP: -.INDENT 0.0 +.TP +.B Run the minion in the foreground +By not starting the minion in daemon mode (\fI\%\-d\fP) +one can view any output from the minion as it works: +.INDENT 7.0 .INDENT 3.5 .sp .nf .ft C -/etc/sysconfig/iptables +salt\-minion .ft P .fi .UNINDENT .UNINDENT +.UNINDENT .sp -\fBArch Linux\fP: +Increase the default timeout value when running \fBsalt\fP\&. For +example, to change the default timeout to 60 seconds: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -/etc/iptables/iptables.rules +salt \-t 60 .ft P .fi .UNINDENT .UNINDENT .sp -\fBDebian\fP -.sp -Follow these instructions: \fI\%https://wiki.debian.org/iptables\fP -.sp -Once you\(aqve found your firewall rules, you\(aqll need to add the below line -to allow traffic on \fBtcp/4505\fP and \fBtcp/4506\fP: +For best results, combine all three: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -\-A INPUT \-m state \-\-state new \-m tcp \-p tcp \-\-dport 4505:4506 \-j ACCEPT +salt\-minion \-l debug # On the minion +salt \(aq*\(aq state.apply \-t 60 # On the master .ft P .fi .UNINDENT .UNINDENT +.UNINDENT +.UNINDENT +.SS Next steps .sp -\fBUbuntu\fP +This tutorial focused on getting a simple Salt States configuration working. +\fI\%Part 2\fP will build on this example to cover more advanced +\fBsls\fP syntax and will explore more of the states that ship with Salt. +.SS States tutorial, part 2 \- More Complex States, Requisites .sp -Salt installs firewall rules in \fI\%/etc/ufw/applications.d/salt.ufw\fP\&. Enable with: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -ufw allow salt -.ft P -.fi +This tutorial builds on topics covered in \fI\%part 1\fP\&. It is +recommended that you begin there. .UNINDENT .UNINDENT -.SS pf.conf .sp -The BSD\-family of operating systems uses \fI\%packet filter (pf)\fP\&. The following -example describes the addition to \fBpf.conf\fP needed to access the Salt -master. +In the \fI\%last part\fP of the Salt States tutorial we covered the +basics of installing a package. We will now modify our \fBwebserver.sls\fP file +to have requirements, and use even more Salt States. +.SS Call multiple States +.sp +You can specify multiple \fI\%State declaration\fP under an +\fI\%ID declaration\fP\&. For example, a quick modification to our +\fBwebserver.sls\fP to also start Apache if it is not running: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -pass in on $int_if proto tcp from any to $int_if port 4505:4506 +apache: + pkg.installed: [] + service.running: + \- require: + \- pkg: apache .ft P .fi .UNINDENT .UNINDENT .sp -Once this addition has been made to the \fBpf.conf\fP the rules will need to -be reloaded. This can be done using the \fBpfctl\fP command. +Try stopping Apache before running \fI\%state.apply\fP once again and observe the output. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -pfctl \-vf /etc/pf.conf -.ft P -.fi +For those running RedhatOS derivatives (Centos, AWS), you will want to specify the +service name to be httpd. More on state service here, \fI\%service state\fP\&. With the example above, just add \(dq\- name: httpd\(dq +above the require line and with the same spacing. .UNINDENT .UNINDENT -.SS Whitelist communication to Master -.sp -There are situations where you want to selectively allow Minion traffic -from specific hosts or networks into your Salt Master. The first -scenario which comes to mind is to prevent unwanted traffic to your -Master out of security concerns, but another scenario is to handle -Minion upgrades when there are backwards incompatible changes between -the installed Salt versions in your environment. +.SS Require other states .sp -Here is an example \fI\%Linux iptables\fP ruleset to -be set on the Master: +We now have a working installation of Apache so let\(aqs add an HTML file to +customize our website. It isn\(aqt exactly useful to have a website without a +webserver so we don\(aqt want Salt to install our HTML file until Apache is +installed and running. Include the following at the bottom of your +\fBwebserver/init.sls\fP file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# Allow Minions from these networks -\-I INPUT \-s 10.1.2.0/24 \-p tcp \-\-dports 4505:4506 \-j ACCEPT -\-I INPUT \-s 10.1.3.0/24 \-p tcp \-\-dports 4505:4506 \-j ACCEPT -# Allow Salt to communicate with Master on the loopback interface -\-A INPUT \-i lo \-p tcp \-\-dports 4505:4506 \-j ACCEPT -# Reject everything else -\-A INPUT \-p tcp \-\-dports 4505:4506 \-j REJECT +apache: + pkg.installed: [] + service.running: + \- require: + \- pkg: apache + +/var/www/index.html: # ID declaration + file: # state declaration + \- managed # function + \- source: salt://webserver/index.html # function arg + \- require: # requisite declaration + \- pkg: apache # requisite reference .ft P .fi .UNINDENT .UNINDENT .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -The important thing to note here is that the \fBsalt\fP command -needs to communicate with the listening network socket of -\fBsalt\-master\fP on the \fIloopback\fP interface. Without this you will -see no outgoing Salt traffic from the master, even for a simple -\fBsalt \(aq*\(aq test.version\fP, because the \fBsalt\fP client never reached -the \fBsalt\-master\fP to tell it to carry out the execution. -.UNINDENT -.UNINDENT -.SS HTTP Modules +\fBline 7\fP is the \fI\%ID declaration\fP\&. In this example it is the location we +want to install our custom HTML file. (\fBNote:\fP the default location that +Apache serves may differ from the above on your OS or distro. \fB/srv/www\fP +could also be a likely place to look.) .sp -This tutorial demonstrates using the various HTTP modules available in Salt. -These modules wrap the Python \fBtornado\fP, \fBurllib2\fP, and \fBrequests\fP -libraries, extending them in a manner that is more consistent with Salt -workflows. -.SS The \fBsalt.utils.http\fP Library +\fBLine 8\fP the \fI\%State declaration\fP\&. This example uses the Salt \fI\%file +state\fP\&. .sp -This library forms the core of the HTTP modules. Since it is designed to be used -from the minion as an execution module, in addition to the master as a runner, -it was abstracted into this multi\-use library. This library can also be imported -by 3rd\-party programs wishing to take advantage of its extended functionality. +\fBLine 9\fP is the \fI\%Function declaration\fP\&. The \fI\%managed function\fP will download a file from the master and install it +in the location specified. .sp -Core functionality of the execution, state, and runner modules is derived from -this library, so common usages between them are described here. Documentation -specific to each module is described below. +\fBLine 10\fP is a \fI\%Function arg declaration\fP which, in this example, passes +the \fBsource\fP argument to the \fI\%managed function\fP\&. .sp -This library can be imported with: +\fBLine 11\fP is a \fI\%Requisite declaration\fP\&. +.sp +\fBLine 12\fP is a \fI\%Requisite reference\fP which refers to a state and an ID. +In this example, it is referring to the \fBID declaration\fP from our example in +\fI\%part 1\fP\&. This declaration tells Salt not to install the HTML +file until Apache is installed. +.sp +Next, create the \fBindex.html\fP file and save it in the \fBwebserver\fP +directory: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -import salt.utils.http + + + Salt rocks + +

This file brought to you by Salt

+ + .ft P .fi .UNINDENT .UNINDENT -.SS Configuring Libraries .sp -This library can make use of either \fBtornado\fP, which is required by Salt, -\fBurllib2\fP, which ships with Python, or \fBrequests\fP, which can be installed -separately. By default, \fBtornado\fP will be used. In order to switch to -\fBurllib2\fP, set the following variable: +Last, call \fI\%state.apply\fP again and the minion +will fetch and execute the \fI\%highstate\fP as well as our +HTML file from the master using Salt\(aqs File Server: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -backend: urllib2 +salt \(aq*\(aq state.apply .ft P .fi .UNINDENT .UNINDENT .sp -In order to switch to \fBrequests\fP, set the following variable: +Verify that Apache is now serving your custom HTML. +.INDENT 0.0 +.INDENT 3.5 +.IP "\fBrequire\fP vs. \fBwatch\fP" +.sp +There are two \fI\%Requisite declaration\fP, “require”, and “watch”. Not +every state supports “watch”. The \fI\%service state\fP does support “watch” and will restart a service +based on the watch condition. +.sp +For example, if you use Salt to install an Apache virtual host +configuration file and want to restart Apache whenever that file is changed +you could modify our Apache example from earlier as follows: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -backend: requests +/etc/httpd/extra/httpd\-vhosts.conf: + file.managed: + \- source: salt://webserver/httpd\-vhosts.conf + +apache: + pkg.installed: [] + service.running: + \- watch: + \- file: /etc/httpd/extra/httpd\-vhosts.conf + \- require: + \- pkg: apache .ft P .fi .UNINDENT .UNINDENT .sp -This can be set in the master or minion configuration file, or passed as an -option directly to any \fBhttp.query()\fP functions. -.SS \fBsalt.utils.http.query()\fP +If the pkg and service names differ on your OS or distro of choice you can +specify each one separately using a \fI\%Name declaration\fP which explained +in \fI\%Part 3\fP\&. +.UNINDENT +.UNINDENT +.SS Next steps .sp -This function forms a basic query, but with some add\-ons not present in the -\fBtornado\fP, \fBurllib2\fP, and \fBrequests\fP libraries. Not all functionality -currently available in these libraries has been added, but can be in future -iterations. -.SS HTTPS Request Methods +In \fI\%part 3\fP we will discuss how to use includes, extends, and +templating to make a more complete State Tree configuration. +.SS States tutorial, part 3 \- Templating, Includes, Extends .sp -A basic query can be performed by calling this function with no more than a -single URL: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -salt.utils.http.query(\(dqhttp://example.com\(dq) -.ft P -.fi +This tutorial builds on topics covered in \fI\%part 1\fP and +\fI\%part 2\fP\&. It is recommended that you begin there. .UNINDENT .UNINDENT .sp -By default the query will be performed with a \fBGET\fP method. The method can -be overridden with the \fBmethod\fP argument: +This part of the tutorial will cover more advanced templating and +configuration techniques for \fBsls\fP files. +.SS Templating SLS modules +.sp +SLS modules may require programming logic or inline execution. This is +accomplished with module templating. The default module templating system used +is \fI\%Jinja2\fP and may be configured by changing the \fI\%renderer\fP +value in the master config. +.sp +All states are passed through a templating system when they are initially read. +To make use of the templating system, simply add some templating markup. +An example of an sls module with templating markup may look like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com/delete/url\(dq, \(dqDELETE\(dq) +{% for usr in [\(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq] %} +{{ usr }}: + user.present +{% endfor %} .ft P .fi .UNINDENT .UNINDENT .sp -When using the \fBPOST\fP method (and others, such as \fBPUT\fP), extra data is usually -sent as well. This data can be sent directly (would be URL encoded when necessary), -or in whatever format is required by the remote server (XML, JSON, plain text, etc). +This templated sls file once generated will look like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, method=\(dqPOST\(dq, data=json.dumps(mydict) -) +moe: + user.present +larry: + user.present +curly: + user.present .ft P .fi .UNINDENT .UNINDENT -.SS Data Formatting and Templating .sp -Bear in mind that the data must be sent pre\-formatted; this function will not -format it for you. However, a templated file stored on the local system may be -passed through, along with variables to populate it with. To pass through only -the file (untemplated): +Here\(aqs a more complex example: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, method=\(dqPOST\(dq, data_file=\(dq/srv/salt/somefile.xml\(dq -) +# Comments in yaml start with a hash symbol. +# Since jinja rendering occurs before yaml parsing, if you want to include jinja +# in the comments you may need to escape them using \(aqjinja\(aq comments to prevent +# jinja from trying to render something which is not well\-defined jinja. +# e.g. +# {# iterate over the Three Stooges using a {% for %}..{% endfor %} loop +# with the iterator variable {{ usr }} becoming the state ID. #} +{% for usr in \(aqmoe\(aq,\(aqlarry\(aq,\(aqcurly\(aq %} +{{ usr }}: + group: + \- present + user: + \- present + \- gid_from_name: True + \- require: + \- group: {{ usr }} +{% endfor %} .ft P .fi .UNINDENT .UNINDENT +.SS Using Grains in SLS modules .sp -To pass through a file that contains jinja + yaml templating (the default): +Often times a state will need to behave differently on different systems. +\fI\%Salt grains\fP objects are made available in the template +context. The \fIgrains\fP can be used from within sls modules: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, - method=\(dqPOST\(dq, - data_file=\(dq/srv/salt/somefile.jinja\(dq, - data_render=True, - template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, -) +apache: + pkg.installed: + {% if grains[\(aqos\(aq] == \(aqRedHat\(aq %} + \- name: httpd + {% elif grains[\(aqos\(aq] == \(aqUbuntu\(aq %} + \- name: apache2 + {% endif %} .ft P .fi .UNINDENT .UNINDENT +.SS Using Environment Variables in SLS modules .sp -To pass through a file that contains mako templating: +You can use \fBsalt[\(aqenviron.get\(aq](\(aqVARNAME\(aq)\fP to use an environment +variable in a Salt state. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, - method=\(dqPOST\(dq, - data_file=\(dq/srv/salt/somefile.mako\(dq, - data_render=True, - data_renderer=\(dqmako\(dq, - template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, -) +MYENVVAR=\(dqworld\(dq salt\-call state.template test.sls .ft P .fi .UNINDENT .UNINDENT -.sp -Because this function uses Salt\(aqs own rendering system, any Salt renderer can -be used. Because Salt\(aqs renderer requires \fB__opts__\fP to be set, an \fBopts\fP -dictionary should be passed in. If it is not, then the default \fB__opts__\fP -values for the node type (master or minion) will be used. Because this library -is intended primarily for use by minions, the default node type is \fBminion\fP\&. -However, this can be changed to \fBmaster\fP if necessary. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, - method=\(dqPOST\(dq, - data_file=\(dq/srv/salt/somefile.jinja\(dq, - data_render=True, - template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, - opts=__opts__, -) - -salt.utils.http.query( - \(dqhttp://example.com/post/url\(dq, - method=\(dqPOST\(dq, - data_file=\(dq/srv/salt/somefile.jinja\(dq, - data_render=True, - template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, - node=\(dqmaster\(dq, -) +Create a file with contents from an environment variable: + file.managed: + \- name: /tmp/hello + \- contents: {{ salt[\(aqenviron.get\(aq](\(aqMYENVVAR\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.SS Headers .sp -Headers may also be passed through, either as a \fBheader_list\fP, a -\fBheader_dict\fP, or as a \fBheader_file\fP\&. As with the \fBdata_file\fP, the -\fBheader_file\fP may also be templated. Take note that because HTTP headers are -normally syntactically\-correct YAML, they will automatically be imported as an -a Python dict. +Error checking: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com/delete/url\(dq, - method=\(dqPOST\(dq, - header_file=\(dq/srv/salt/headers.jinja\(dq, - header_render=True, - header_renderer=\(dqjinja\(dq, - template_dict={\(dqkey1\(dq: \(dqvalue1\(dq, \(dqkey2\(dq: \(dqvalue2\(dq}, -) +{% set myenvvar = salt[\(aqenviron.get\(aq](\(aqMYENVVAR\(aq) %} +{% if myenvvar %} + +Create a file with contents from an environment variable: + file.managed: + \- name: /tmp/hello + \- contents: {{ salt[\(aqenviron.get\(aq](\(aqMYENVVAR\(aq) }} + +{% else %} + +Fail \- no environment passed in: + test.fail_without_changes + +{% endif %} .ft P .fi .UNINDENT .UNINDENT +.SS Calling Salt modules from templates .sp -Because much of the data that would be templated between headers and data may be -the same, the \fBtemplate_dict\fP is the same for both. Correcting possible -variable name collisions is up to the user. -.SS Authentication +All of the Salt modules loaded by the minion are available within the +templating system. This allows data to be gathered in real time on the target +system. It also allows for shell commands to be run easily from within the sls +modules. .sp -The \fBquery()\fP function supports basic HTTP authentication. A username and -password may be passed in as \fBusername\fP and \fBpassword\fP, respectively. +The Salt module functions are also made available in the template context as +\fBsalt:\fP +.sp +The following example illustrates calling the \fBgroup_to_gid\fP function in the +\fBfile\fP execution module with a single positional argument called +\fBsome_group_that_exists\fP\&. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, username=\(dqlarry\(dq, password=\(dq5700g3543v4r\(dq) +moe: + user.present: + \- gid: {{ salt[\(aqfile.group_to_gid\(aq](\(aqsome_group_that_exists\(aq) }} .ft P .fi .UNINDENT .UNINDENT -.SS Cookies and Sessions .sp -Cookies are also supported, using Python\(aqs built\-in \fBcookielib\fP\&. However, they -are turned off by default. To turn cookies on, set \fBcookies\fP to True. +One way to think about this might be that the \fBgid\fP key is being assigned +a value equivalent to the following python pseudo\-code: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, cookies=True) +import salt.modules.file + +file.group_to_gid(\(dqsome_group_that_exists\(dq) .ft P .fi .UNINDENT .UNINDENT .sp -By default cookies are stored in Salt\(aqs cache directory, normally -\fB/var/cache/salt\fP, as a file called \fBcookies.txt\fP\&. However, this location -may be changed with the \fBcookie_jar\fP argument: +Note that for the above example to work, \fBsome_group_that_exists\fP must exist +before the state file is processed by the templating engine. +.sp +Below is an example that uses the \fBnetwork.hw_addr\fP function to retrieve the +MAC address for eth0: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com\(dq, cookies=True, cookie_jar=\(dq/path/to/cookie_jar.txt\(dq -) +salt[\(dqnetwork.hw_addr\(dq](\(dqeth0\(dq) .ft P .fi .UNINDENT .UNINDENT .sp -By default, the format of the cookie jar is LWP (aka, lib\-www\-perl). This -default was chosen because it is a human\-readable text file. If desired, the -format of the cookie jar can be set to Mozilla: +To examine the possible arguments to each execution module function, +one can examine the \fI\%module reference documentation\fP: +.SS Advanced SLS module syntax +.sp +Lastly, we will cover some incredibly useful techniques for more complex State +trees. +.SS Include declaration +.sp +A previous example showed how to spread a Salt tree across several files. +Similarly, \fI\%Requisites and Other Global State Arguments\fP span multiple files by +using an \fI\%Include declaration\fP\&. For example: +.sp +\fBpython/python\-libs.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com\(dq, - cookies=True, - cookie_jar=\(dq/path/to/cookie_jar.txt\(dq, - cookie_format=\(dqmozilla\(dq, -) +python\-dateutil: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -Because Salt commands are normally one\-off commands that are piped together, -this library cannot normally behave as a normal browser, with session cookies -that persist across multiple HTTP requests. However, the session can be -persisted in a separate cookie jar. The default filename for this file, inside -Salt\(aqs cache directory, is \fBcookies.session.p\fP\&. This can also be changed. +\fBpython/django.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com\(dq, persist_session=True, session_cookie_jar=\(dq/path/to/jar.p\(dq -) +include: + \- python.python\-libs + +django: + pkg.installed: + \- require: + \- pkg: python\-dateutil .ft P .fi .UNINDENT .UNINDENT +.SS Extend declaration .sp -The format of this file is msgpack, which is consistent with much of the rest -of Salt\(aqs internal structure. Historically, the extension for this file is -\fB\&.p\fP\&. There are no current plans to make this configurable. -.SS Proxy +You can modify previous declarations by using an \fI\%Extend declaration\fP\&. For +example the following modifies the Apache tree to also restart Apache when the +vhosts file is changed: .sp -If the \fBtornado\fP backend is used (\fBtornado\fP is the default), proxy -information configured in \fBproxy_host\fP, \fBproxy_port\fP, \fBproxy_username\fP, -\fBproxy_password\fP and \fBno_proxy\fP from the \fB__opts__\fP dictionary will be used. Normally -these are set in the minion configuration file. +\fBapache/apache.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -proxy_host: proxy.my\-domain -proxy_port: 31337 -proxy_username: charon -proxy_password: obolus -no_proxy: [\(aq127.0.0.1\(aq, \(aqlocalhost\(aq] +apache: + pkg.installed .ft P .fi .UNINDENT .UNINDENT +.sp +\fBapache/mywebsite.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, opts=__opts__, backend=\(dqtornado\(dq) +include: + \- apache.apache + +extend: + apache: + service: + \- running + \- watch: + \- file: /etc/httpd/extra/httpd\-vhosts.conf + +/etc/httpd/extra/httpd\-vhosts.conf: + file.managed: + \- source: salt://apache/httpd\-vhosts.conf .ft P .fi .UNINDENT .UNINDENT -.SS Return Data -.sp -\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Return data encoding +.IP "Using extend with require or watch" .sp -If \fBdecode\fP is set to \fBTrue\fP, \fBquery()\fP will attempt to decode the -return data. \fBdecode_type\fP defaults to \fBauto\fP\&. Set it to a specific -encoding, \fBxml\fP, for example, to override autodetection. +The \fBextend\fP statement works differently for \fBrequire\fP or \fBwatch\fP\&. +It appends to, rather than replacing the requisite component. .UNINDENT .UNINDENT +.SS Name declaration .sp -Because Salt\(aqs http library was designed to be used with REST interfaces, -\fBquery()\fP will attempt to decode the data received from the remote server -when \fBdecode\fP is set to \fBTrue\fP\&. First it will check the \fBContent\-type\fP -header to try and find references to XML. If it does not find any, it will look -for references to JSON. If it does not find any, it will fall back to plain -text, which will not be decoded. +You can override the \fI\%ID declaration\fP by using a \fI\%Name declaration\fP\&. +For example, the previous example is a bit more maintainable if rewritten as +follows: .sp -JSON data is translated into a dict using Python\(aqs built\-in \fBjson\fP library. -XML is translated using \fBsalt.utils.xml_util\fP, which will use Python\(aqs -built\-in XML libraries to attempt to convert the XML into a dict. In order to -force either JSON or XML decoding, the \fBdecode_type\fP may be set: +\fBapache/mywebsite.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, decode_type=\(dqxml\(dq) +include: + \- apache.apache + +extend: + apache: + service: + \- running + \- watch: + \- file: mywebsite + +mywebsite: + file.managed: + \- name: /etc/httpd/extra/httpd\-vhosts.conf + \- source: salt://apache/httpd\-vhosts.conf .ft P .fi .UNINDENT .UNINDENT +.SS Names declaration .sp -Once translated, the return dict from \fBquery()\fP will include a dict called -\fBdict\fP\&. -.sp -If the data is not to be translated using one of these methods, decoding may be -turned off. +Even more powerful is using a \fI\%Names declaration\fP to override the +\fI\%ID declaration\fP for multiple states at once. This often can remove the +need for looping in a template. For example, the first example in this tutorial +can be rewritten without the loop: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, decode=False) +stooges: + user.present: + \- names: + \- moe + \- larry + \- curly .ft P .fi .UNINDENT .UNINDENT +.SS Next steps .sp -If decoding is turned on, and references to JSON or XML cannot be found, then -this module will default to plain text, and return the undecoded data as -\fBtext\fP (even if text is set to \fBFalse\fP; see below). +In \fI\%part 4\fP we will discuss how to use salt\(aqs +\fI\%file_roots\fP to set up a workflow in which states can be +\(dqpromoted\(dq from dev, to QA, to production. +.SS States tutorial, part 4 .sp -The \fBquery()\fP function can return the HTTP status code, headers, and/or text -as required. However, each must individually be turned on. +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -salt.utils.http.query(\(dqhttp://example.com\(dq, status=True, headers=True, text=True) -.ft P -.fi +This tutorial builds on topics covered in \fI\%part 1\fP, +\fI\%part 2\fP, and \fI\%part 3\fP\&. +It is recommended that you begin there. .UNINDENT .UNINDENT .sp -The return from these will be found in the return dict as \fBstatus\fP, -\fBheaders\fP and \fBtext\fP, respectively. -.SS Writing Return Data to Files +This part of the tutorial will show how to use salt\(aqs \fI\%file_roots\fP +to set up a workflow in which states can be \(dqpromoted\(dq from dev, to QA, to +production. +.SS Salt fileserver path inheritance .sp -It is possible to write either the return data or headers to files, as soon as -the response is received from the server, but specifying file locations via the -\fBtext_out\fP or \fBheaders_out\fP arguments. \fBtext\fP and \fBheaders\fP do not need -to be returned to the user in order to do this. +Salt\(aqs fileserver allows for more than one root directory per environment, like +in the below example, which uses both a local directory and a secondary +location shared to the salt master via NFS: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query( - \(dqhttp://example.com\(dq, - text=False, - headers=False, - text_out=\(dq/path/to/url_download.txt\(dq, - headers_out=\(dq/path/to/headers_download.txt\(dq, -) +# In the master config file (/etc/salt/master) +file_roots: + base: + \- /srv/salt + \- /mnt/salt\-nfs/base .ft P .fi .UNINDENT .UNINDENT -.SS SSL Verification .sp -By default, this function will verify SSL certificates. However, for testing or -debugging purposes, SSL verification can be turned off. +Salt\(aqs fileserver collapses the list of root directories into a single virtual +environment containing all files from each root. If the same file exists at the +same relative path in more than one root, then the top\-most match \(dqwins\(dq. For +example, if \fB/srv/salt/foo.txt\fP and \fB/mnt/salt\-nfs/base/foo.txt\fP both +exist, then \fBsalt://foo.txt\fP will point to \fB/srv/salt/foo.txt\fP\&. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +When using multiple fileserver backends, the order in which they are listed +in the \fI\%fileserver_backend\fP parameter also matters. If both +\fBroots\fP and \fBgit\fP backends contain a file with the same relative path, +and \fBroots\fP appears before \fBgit\fP in the +\fI\%fileserver_backend\fP list, then the file in \fBroots\fP will +\(dqwin\(dq, and the file in gitfs will be ignored. +.sp +A more thorough explanation of how Salt\(aqs modular fileserver works can be +found \fI\%here\fP\&. We recommend reading this. +.UNINDENT +.UNINDENT +.SS Environment configuration +.sp +Configure a multiple\-environment setup like so: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttps://example.com\(dq, verify_ssl=False) +file_roots: + base: + \- /srv/salt/prod + qa: + \- /srv/salt/qa + \- /srv/salt/prod + dev: + \- /srv/salt/dev + \- /srv/salt/qa + \- /srv/salt/prod .ft P .fi .UNINDENT .UNINDENT -.SS CA Bundles .sp -The \fBrequests\fP library has its own method of detecting which CA (certificate -authority) bundle file to use. Usually this is implemented by the packager for -the specific operating system distribution that you are using. However, -\fBurllib2\fP requires a little more work under the hood. By default, Salt will -try to auto\-detect the location of this file. However, if it is not in an -expected location, or a different path needs to be specified, it may be done so -using the \fBca_bundle\fP variable. +Given the path inheritance described above, files within \fB/srv/salt/prod\fP +would be available in all environments. Files within \fB/srv/salt/qa\fP would be +available in both \fBqa\fP, and \fBdev\fP\&. Finally, the files within +\fB/srv/salt/dev\fP would only be available within the \fBdev\fP environment. +.sp +Based on the order in which the roots are defined, new files/states can be +placed within \fB/srv/salt/dev\fP, and pushed out to the dev hosts for testing. +.sp +Those files/states can then be moved to the same relative path within +\fB/srv/salt/qa\fP, and they are now available only in the \fBdev\fP and \fBqa\fP +environments, allowing them to be pushed to QA hosts and tested. +.sp +Finally, if moved to the same relative path within \fB/srv/salt/prod\fP, the +files are now available in all three environments. +.SS Requesting files from specific fileserver environments +.sp +See \fI\%here\fP for documentation on how to request +files from specific environments. +.SS Practical Example +.sp +As an example, consider a simple website, installed to \fB/var/www/foobarcom\fP\&. +Below is a top.sls that can be used to deploy the website: +.sp +\fB/srv/salt/prod/top.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.query(\(dqhttps://example.com\(dq, ca_bundle=\(dq/path/to/ca_bundle.pem\(dq) +base: + \(aqweb*prod*\(aq: + \- webserver.foobarcom +qa: + \(aqweb*qa*\(aq: + \- webserver.foobarcom +dev: + \(aqweb*dev*\(aq: + \- webserver.foobarcom .ft P .fi .UNINDENT .UNINDENT -.SS Updating CA Bundles .sp -The \fBupdate_ca_bundle()\fP function can be used to update the bundle file at a -specified location. If the target location is not specified, then it will -attempt to auto\-detect the location of the bundle file. If the URL to download -the bundle from does not exist, a bundle will be downloaded from the cURL -website. +Using pillar, roles can be assigned to the hosts: .sp -CAUTION: The \fBtarget\fP and the \fBsource\fP should always be specified! Failure -to specify the \fBtarget\fP may result in the file being written to the wrong -location on the local system. Failure to specify the \fBsource\fP may cause the -upstream URL to receive excess unnecessary traffic, and may cause a file to be -download which is hazardous or does not meet the needs of the user. +\fB/srv/pillar/top.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.update_ca_bundle( - target=\(dq/path/to/ca\-bundle.crt\(dq, - source=\(dqhttps://example.com/path/to/ca\-bundle.crt\(dq, - opts=__opts__, -) +base: + \(aqweb*prod*\(aq: + \- webserver.prod + \(aqweb*qa*\(aq: + \- webserver.qa + \(aqweb*dev*\(aq: + \- webserver.dev .ft P .fi .UNINDENT .UNINDENT .sp -The \fBopts\fP parameter should also always be specified. If it is, then the -\fBtarget\fP and the \fBsource\fP may be specified in the relevant configuration -file (master or minion) as \fBca_bundle\fP and \fBca_bundle_url\fP, respectively. +\fB/srv/pillar/webserver/prod.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -ca_bundle: /path/to/ca\-bundle.crt -ca_bundle_url: https://example.com/path/to/ca\-bundle.crt +webserver_role: prod .ft P .fi .UNINDENT .UNINDENT .sp -If Salt is unable to auto\-detect the location of the CA bundle, it will raise -an error. -.sp -The \fBupdate_ca_bundle()\fP function can also be passed a string or a list of -strings which represent files on the local system, which should be appended (in -the specified order) to the end of the CA bundle file. This is useful in -environments where private certs need to be made available, and are not -otherwise reasonable to add to the bundle file. +\fB/srv/pillar/webserver/qa.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt.utils.http.update_ca_bundle( - opts=__opts__, - merge_files=[ - \(dq/etc/ssl/private_cert_1.pem\(dq, - \(dq/etc/ssl/private_cert_2.pem\(dq, - \(dq/etc/ssl/private_cert_3.pem\(dq, - ], -) +webserver_role: qa .ft P .fi .UNINDENT .UNINDENT -.SS Test Mode -.sp -This function may be run in test mode. This mode will perform all work up until -the actual HTTP request. By default, instead of performing the request, an empty -dict will be returned. Using this function with \fBTRACE\fP logging turned on will -reveal the contents of the headers and POST data to be sent. -.sp -Rather than returning an empty dict, an alternate \fBtest_url\fP may be passed in. -If this is detected, then test mode will replace the \fBurl\fP with the -\fBtest_url\fP, set \fBtest\fP to \fBTrue\fP in the return data, and perform the rest -of the requested operations as usual. This allows a custom, non\-destructive URL -to be used for testing when necessary. -.SS Execution Module -.sp -The \fBhttp\fP execution module is a very thin wrapper around the -\fBsalt.utils.http\fP library. The \fBopts\fP can be passed through as well, but if -they are not specified, the minion defaults will be used as necessary. -.sp -Because passing complete data structures from the command line can be tricky at -best and dangerous (in terms of execution injection attacks) at worse, the -\fBdata_file\fP, and \fBheader_file\fP are likely to see more use here. .sp -All methods for the library are available in the execution module, as kwargs. +\fB/srv/pillar/webserver/dev.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion http.query http://example.com/restapi method=POST \e - username=\(aqlarry\(aq password=\(aq5700g3543v4r\(aq headers=True text=True \e - status=True decode_type=xml data_render=True \e - header_file=/tmp/headers.txt data_file=/tmp/data.txt \e - header_render=True cookies=True persist_session=True +webserver_role: dev .ft P .fi .UNINDENT .UNINDENT -.SS Runner Module .sp -Like the execution module, the \fBhttp\fP runner module is a very thin wrapper -around the \fBsalt.utils.http\fP library. The only significant difference is that -because runners execute on the master instead of a minion, a target is not -required, and default opts will be derived from the master config, rather than -the minion config. +And finally, the SLS to deploy the website: .sp -All methods for the library are available in the runner module, as kwargs. +\fB/srv/salt/prod/webserver/foobarcom.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt\-run http.query http://example.com/restapi method=POST \e - username=\(aqlarry\(aq password=\(aq5700g3543v4r\(aq headers=True text=True \e - status=True decode_type=xml data_render=True \e - header_file=/tmp/headers.txt data_file=/tmp/data.txt \e - header_render=True cookies=True persist_session=True +{% if pillar.get(\(aqwebserver_role\(aq, \(aq\(aq) %} +/var/www/foobarcom: + file.recurse: + \- source: salt://webserver/src/foobarcom + \- env: {{ pillar[\(aqwebserver_role\(aq] }} + \- user: www + \- group: www + \- dir_mode: 755 + \- file_mode: 644 +{% endif %} .ft P .fi .UNINDENT .UNINDENT -.SS State Module .sp -The state module is a wrapper around the runner module, which applies stateful -logic to a query. All kwargs as listed above are specified as usual in state -files, but two more kwargs are available to apply stateful logic. A required -parameter is \fBmatch\fP, which specifies a pattern to look for in the return -text. By default, this will perform a string comparison of looking for the -value of match in the return text. In Python terms this looks like: +Given the above SLS, the source for the website should initially be placed in +\fB/srv/salt/dev/webserver/src/foobarcom\fP\&. +.sp +First, let\(aqs deploy to dev. Given the configuration in the top file, this can +be done using \fI\%state.apply\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -def myfunc(): - if match in html_text: - return True +salt \-\-pillar \(aqwebserver_role:dev\(aq state.apply .ft P .fi .UNINDENT .UNINDENT .sp -If more complex pattern matching is required, a regular expression can be used -by specifying a \fBmatch_type\fP\&. By default this is set to \fBstring\fP, but it -can be manually set to \fBpcre\fP instead. Please note that despite the name, this -will use Python\(aqs \fBre.search()\fP rather than \fBre.match()\fP\&. -.sp -Therefore, the following states are valid: +However, in the event that it is not desirable to apply all states configured +in the top file (which could be likely in more complex setups), it is possible +to apply just the states for the \fBfoobarcom\fP website, by invoking +\fI\%state.apply\fP with the desired SLS target +as an argument: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -http://example.com/restapi: - http.query: - \- match: \(aqSUCCESS\(aq - \- username: \(aqlarry\(aq - \- password: \(aq5700g3543v4r\(aq - \- data_render: True - \- header_file: /tmp/headers.txt - \- data_file: /tmp/data.txt - \- header_render: True - \- cookies: True - \- persist_session: True - -http://example.com/restapi: - http.query: - \- match_type: pcre - \- match: \(aq(?i)succe[ss|ed]\(aq - \- username: \(aqlarry\(aq - \- password: \(aq5700g3543v4r\(aq - \- data_render: True - \- header_file: /tmp/headers.txt - \- data_file: /tmp/data.txt - \- header_render: True - \- cookies: True - \- persist_session: True +salt \-\-pillar \(aqwebserver_role:dev\(aq state.apply webserver.foobarcom .ft P .fi .UNINDENT .UNINDENT .sp -In addition to, or instead of a match pattern, the status code for a URL can be -checked. This is done using the \fBstatus\fP argument: +Once the site has been tested in dev, then the files can be moved from +\fB/srv/salt/dev/webserver/src/foobarcom\fP to +\fB/srv/salt/qa/webserver/src/foobarcom\fP, and deployed using the following: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -http://example.com/: - http.query: - \- status: 200 +salt \-\-pillar \(aqwebserver_role:qa\(aq state.apply webserver.foobarcom .ft P .fi .UNINDENT .UNINDENT .sp -If both are specified, both will be checked, but if only one is \fBTrue\fP and the -other is \fBFalse\fP, then \fBFalse\fP will be returned. In this case, the comments -in the return data will contain information for troubleshooting. -.sp -Because this is a monitoring state, it will return extra data to code that -expects it. This data will always include \fBtext\fP and \fBstatus\fP\&. Optionally, -\fBheaders\fP and \fBdict\fP may also be requested by setting the \fBheaders\fP and -\fBdecode\fP arguments to True, respectively. -.SS Using Salt at scale -.sp -The focus of this tutorial will be building a Salt infrastructure for handling -large numbers of minions. This will include tuning, topology, and best practices. -.sp -For how to install the Salt Master, see the -\fI\%Salt install guide\fP\&. -.sp -\fBNOTE:\fP +Finally, once the site has been tested in qa, then the files can be moved from +\fB/srv/salt/qa/webserver/src/foobarcom\fP to +\fB/srv/salt/prod/webserver/src/foobarcom\fP, and deployed using the following: .INDENT 0.0 .INDENT 3.5 -This tutorial is intended for large installations, although these same settings -won\(aqt hurt, it may not be worth the complexity to smaller installations. -.sp -When used with minions, the term \(aqmany\(aq refers to at least a thousand -and \(aqa few\(aq always means 500. .sp -For simplicity reasons, this tutorial will default to the standard ports -used by Salt. +.nf +.ft C +salt \-\-pillar \(aqwebserver_role:prod\(aq state.apply webserver.foobarcom +.ft P +.fi .UNINDENT .UNINDENT -.SS The Master .sp -The most common problems on the Salt Master are: -.INDENT 0.0 -.IP 1. 3 -too many minions authing at once -.IP 2. 3 -too many minions re\-authing at once -.IP 3. 3 -too many minions re\-connecting at once -.IP 4. 3 -too many minions returning at once -.IP 5. 3 -too few resources (CPU/HDD) -.UNINDENT +Thanks to Salt\(aqs fileserver inheritance, even though the files have been moved +to within \fB/srv/salt/prod\fP, they are still available from the same +\fBsalt://\fP URI in both the qa and dev environments. +.SS Continue Learning .sp -The first three are all \(dqthundering herd\(dq problems. To mitigate these issues -we must configure the minions to back\-off appropriately when the Master is -under heavy load. +The best way to continue learning about Salt States is to read through the +\fI\%reference documentation\fP and to look through examples +of existing state trees. Many pre\-configured state trees +can be found on GitHub in the \fI\%saltstack\-formulas\fP collection of repositories. .sp -The fourth is caused by masters with little hardware resources in combination -with a possible bug in ZeroMQ. At least that\(aqs what it looks like till today -(\fI\%Issue 118651\fP, -\fI\%Issue 5948\fP, -\fI\%Mail thread\fP) +If you have any questions, suggestions, or just want to chat with other people +who are using Salt, we have a very active community and we\(aqd love to hear from +you. One of the best places to talk to the community is on the +\fI\%Salt Project Slack workspace\fP\&. .sp -To fully understand each problem, it is important to understand, how Salt works. +In addition, by continuing to the \fI\%Orchestrate Runner\fP docs, +you can learn about the powerful orchestration of which Salt is capable. +.SS States Tutorial, Part 5 \- Orchestration with Salt .sp -Very briefly, the Salt Master offers two services to the minions. +This was moved to \fI\%Orchestrate Runner\fP\&. +.SS Syslog\-ng usage +.SS Overview +.sp +Syslog_ng state module is for generating syslog\-ng +configurations. You can do the following things: .INDENT 0.0 .IP \(bu 2 -a job publisher on port 4505 +generate syslog\-ng configuration from YAML, .IP \(bu 2 -an open port 4506 to receive the minions returns +use non\-YAML configuration, +.IP \(bu 2 +start, stop or reload syslog\-ng. .UNINDENT .sp -All minions are always connected to the publisher on port 4505 and only connect -to the open return port 4506 if necessary. On an idle Master, there will only -be connections on port 4505. -.SS Too many minions authing -.sp -When the Minion service is first started up, it will connect to its Master\(aqs publisher -on port 4505. If too many minions are started at once, this can cause a \(dqthundering herd\(dq. -This can be avoided by not starting too many minions at once. -.sp -The connection itself usually isn\(aqt the culprit, the more likely cause of master\-side -issues is the authentication that the Minion must do with the Master. If the Master -is too heavily loaded to handle the auth request it will time it out. The Minion -will then wait \fIacceptance_wait_time\fP to retry. If \fIacceptance_wait_time_max\fP is -set then the Minion will increase its wait time by the \fIacceptance_wait_time\fP each -subsequent retry until reaching \fIacceptance_wait_time_max\fP\&. -.SS Too many minions re\-authing -.sp -This is most likely to happen in the testing phase of a Salt deployment, when -all Minion keys have already been accepted, but the framework is being tested -and parameters are frequently changed in the Salt Master\(aqs configuration -file(s). +There is also an execution module, which can check the syntax of the +configuration, get the version and other information about syslog\-ng. +.SS Configuration .sp -The Salt Master generates a new AES key to encrypt its publications at certain -events such as a Master restart or the removal of a Minion key. If you are -encountering this problem of too many minions re\-authing against the Master, -you will need to recalibrate your setup to reduce the rate of events like a -Master restart or Minion key removal (\fBsalt\-key \-d\fP). +Users can create syslog\-ng configuration statements with the +\fI\%syslog_ng.config\fP function. It requires +a \fIname\fP and a \fIconfig\fP parameter. The \fIname\fP parameter determines the name of +the generated statement and the \fIconfig\fP parameter holds a parsed YAML structure. .sp -When the Master generates a new AES key, the minions aren\(aqt notified of this -but will discover it on the next pub job they receive. When the Minion -receives such a job it will then re\-auth with the Master. Since Salt does -minion\-side filtering this means that all the minions will re\-auth on the next -command published on the master\-\- causing another \(dqthundering herd\(dq. This can -be avoided by setting the +A statement can be declared in the following forms (both are equivalent): .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -random_reauth_delay: 60 +source.s_localhost: + syslog_ng.config: + \- config: + \- tcp: + \- ip: \(dq127.0.0.1\(dq + \- port: 1233 .ft P .fi .UNINDENT .UNINDENT -.sp -in the minions configuration file to a higher value and stagger the amount -of re\-auth attempts. Increasing this value will of course increase the time -it takes until all minions are reachable via Salt commands. -.SS Too many minions re\-connecting -.sp -By default the zmq socket will re\-connect every 100ms which for some larger -installations may be too quick. This will control how quickly the TCP session is -re\-established, but has no bearing on the auth load. -.sp -To tune the minions sockets reconnect attempts, there are a few values in -the sample configuration file (default values) .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -recon_default: 1000 -recon_max: 5000 -recon_randomize: True +s_localhost: + syslog_ng.config: + \- config: + source: + \- tcp: + \- ip: \(dq127.0.0.1\(dq + \- port: 1233 .ft P .fi .UNINDENT .UNINDENT +.sp +The first one is called short form, because it needs less typing. Users can use lists +and dictionaries to specify their configuration. The format is quite self describing and +there are more examples [at the end](#examples) of this document. +.SS Quotation .INDENT 0.0 +.TP +.B The quotation can be tricky sometimes but here are some rules to follow: +.INDENT 7.0 .IP \(bu 2 -recon_default: the default value the socket should use, i.e. 1000. This value is in -milliseconds. (1000ms = 1 second) -.IP \(bu 2 -recon_max: the max value that the socket should use as a delay before trying to reconnect -This value is in milliseconds. (5000ms = 5 seconds) +when a string meant to be \fB\(dqstring\(dq\fP in the generated configuration, it should be like \fB\(aq\(dqstring\(dq\(aq\fP in the YAML document .IP \(bu 2 -recon_randomize: enables randomization between recon_default and recon_max +similarly, users should write \fB\(dq\(aqstring\(aq\(dq\fP to get \fB\(aqstring\(aq\fP in the generated configuration .UNINDENT +.UNINDENT +.SS Full example .sp -To tune this values to an existing environment, a few decision have to be made. +The following configuration is an example, how a complete syslog\-ng configuration looks like: .INDENT 0.0 -.IP 1. 3 -How long can one wait, before the minions should be online and reachable via Salt? -.IP 2. 3 -How many reconnects can the Master handle without a syn flood? +.INDENT 3.5 +.sp +.nf +.ft C +# Set the location of the configuration file +set_location: + module.run: + \- name: syslog_ng.set_config_file + \- m_name: \(dq/home/tibi/install/syslog\-ng/etc/syslog\-ng.conf\(dq + +# The syslog\-ng and syslog\-ng\-ctl binaries are here. You needn\(aqt use +# this method if these binaries can be found in a directory in your PATH. +set_bin_path: + module.run: + \- name: syslog_ng.set_binary_path + \- m_name: \(dq/home/tibi/install/syslog\-ng/sbin\(dq + +# Writes the first lines into the config file, also erases its previous +# content +write_version: + module.run: + \- name: syslog_ng.write_version + \- m_name: \(dq3.6\(dq + +# There is a shorter form to set the above variables +set_variables: + module.run: + \- name: syslog_ng.set_parameters + \- version: \(dq3.6\(dq + \- binary_path: \(dq/home/tibi/install/syslog\-ng/sbin\(dq + \- config_file: \(dq/home/tibi/install/syslog\-ng/etc/syslog\-ng.conf\(dq + + +# Some global options +options.global_options: + syslog_ng.config: + \- config: + \- time_reap: 30 + \- mark_freq: 10 + \- keep_hostname: \(dqyes\(dq + +source.s_localhost: + syslog_ng.config: + \- config: + \- tcp: + \- ip: \(dq127.0.0.1\(dq + \- port: 1233 + +destination.d_log_server: + syslog_ng.config: + \- config: + \- tcp: + \- \(dq127.0.0.1\(dq + \- port: 1234 + +log.l_log_to_central_server: + syslog_ng.config: + \- config: + \- source: s_localhost + \- destination: d_log_server + +some_comment: + module.run: + \- name: syslog_ng.write_config + \- config: | + # Multi line + # comment + +# Another mode to use comments or existing configuration snippets +config.other_comment_form: + syslog_ng.config: + \- config: | + # Multi line + # comment +.ft P +.fi +.UNINDENT .UNINDENT .sp -These questions can not be answered generally. Their answers depend on the -hardware and the administrators requirements. +The \fI\%syslog_ng.reloaded\fP function can generate syslog\-ng configuration from YAML. If the statement (source, destination, parser, +etc.) has a name, this function uses the id as the name, otherwise (log +statement) its purpose is like a mandatory comment. .sp -Here is an example scenario with the goal, to have all minions reconnect -within a 60 second time\-frame on a Salt Master service restart. +After execution this example the syslog_ng state will generate this +file: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -recon_default: 1000 -recon_max: 59000 -recon_randomize: True +#Generated by Salt on 2014\-08\-18 00:11:11 +@version: 3.6 + +options { + time_reap( + 30 + ); + mark_freq( + 10 + ); + keep_hostname( + yes + ); +}; + + +source s_localhost { + tcp( + ip( + 127.0.0.1 + ), + port( + 1233 + ) + ); +}; + + +destination d_log_server { + tcp( + 127.0.0.1, + port( + 1234 + ) + ); +}; + + +log { + source( + s_localhost + ); + destination( + d_log_server + ); +}; + + +# Multi line +# comment + + +# Multi line +# comment .ft P .fi .UNINDENT .UNINDENT .sp -Each Minion will have a randomized reconnect value between \(aqrecon_default\(aq -and \(aqrecon_default + recon_max\(aq, which in this example means between 1000ms -and 60000ms (or between 1 and 60 seconds). The generated random\-value will -be doubled after each attempt to reconnect (ZeroMQ default behavior). +Users can include arbitrary texts in the generated configuration with +using the \fBconfig\fP statement (see the example above). +.SS Syslog_ng module functions .sp -Lets say the generated random value is 11 seconds (or 11000ms). +You can use \fI\%syslog_ng.set_binary_path\fP +to set the directory which contains the +syslog\-ng and syslog\-ng\-ctl binaries. If this directory is in your PATH, +you don\(aqt need to use this function. There is also a \fI\%syslog_ng.set_config_file\fP +function to set the location of the configuration file. +.SS Examples +.SS Simple source .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -reconnect 1: wait 11 seconds -reconnect 2: wait 22 seconds -reconnect 3: wait 33 seconds -reconnect 4: wait 44 seconds -reconnect 5: wait 55 seconds -reconnect 6: wait time is bigger than 60 seconds (recon_default + recon_max) -reconnect 7: wait 11 seconds -reconnect 8: wait 22 seconds -reconnect 9: wait 33 seconds -reconnect x: etc. +source s_tail { + file( + \(dq/var/log/apache/access.log\(dq, + follow_freq(1), + flags(no\-parse, validate\-utf8) + ); +}; .ft P .fi .UNINDENT .UNINDENT -.sp -With a thousand minions this will mean .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -1000/60 = ~16 +s_tail: + # Salt will call the source function of syslog_ng module + syslog_ng.config: + \- config: + source: + \- file: + \- file: \(aq\(aq\(dq/var/log/apache/access.log\(dq\(aq\(aq + \- follow_freq : 1 + \- flags: + \- no\-parse + \- validate\-utf8 .ft P .fi .UNINDENT .UNINDENT .sp -round about 16 connection attempts a second. These values should be altered to -values that match your environment. Keep in mind though, that it may grow over -time and that more minions might raise the problem again. -.SS Too many minions returning at once -.sp -This can also happen during the testing phase, if all minions are addressed at -once with +OR .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt * disk.usage +s_tail: + syslog_ng.config: + \- config: + source: + \- file: + \- \(aq\(aq\(dq/var/log/apache/access.log\(dq\(aq\(aq + \- follow_freq : 1 + \- flags: + \- no\-parse + \- validate\-utf8 .ft P .fi .UNINDENT .UNINDENT .sp -it may cause thousands of minions trying to return their data to the Salt Master -open port 4506. Also causing a flood of syn\-flood if the Master can\(aqt handle that many -returns at once. -.sp -This can be easily avoided with Salt\(aqs batch mode: +OR .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt * disk.usage \-b 50 +source.s_tail: + syslog_ng.config: + \- config: + \- file: + \- \(aq\(aq\(dq/var/log/apache/access.log\(dq\(aq\(aq + \- follow_freq : 1 + \- flags: + \- no\-parse + \- validate\-utf8 .ft P .fi .UNINDENT .UNINDENT -.sp -This will only address 50 minions at once while looping through all addressed -minions. -.SS Too few resources -.sp -The masters resources always have to match the environment. There is no way -to give good advise without knowing the environment the Master is supposed to -run in. But here are some general tuning tips for different situations: -.SS The Master is CPU bound -.sp -In installations with large or with complex pillar files, it is possible -for the master to exhibit poor performance as a result of having to render -many pillar files at once. This exhibit itself in a number of ways, both -as high load on the master and on minions which block on waiting for their -pillar to be delivered to them. -.sp -To reduce pillar rendering times, it is possible to cache pillars on the -master. To do this, see the set of master configuration options which -are prefixed with \fIpillar_cache\fP\&. -.sp -If many pillars are encrypted using \fI\%gpg\fP renderer, it -is possible to cache GPG data. To do this, see the set of master configuration -options which are prefixed with \fIgpg_cache\fP\&. -.sp -\fBNOTE:\fP +.SS Complex source .INDENT 0.0 .INDENT 3.5 -Caching pillars or GPG data on the master may introduce security -considerations. Be certain to read caveats outlined in the master -configuration file to understand how pillar caching may affect a master\(aqs -ability to protect sensitive data! +.sp +.nf +.ft C +source s_gsoc2014 { + tcp( + ip(\(dq0.0.0.0\(dq), + port(1234), + flags(no\-parse) + ); +}; +.ft P +.fi .UNINDENT .UNINDENT -.SS The Master is disk IO bound -.sp -By default, the Master saves every Minion\(aqs return for every job in its -job\-cache. The cache can then be used later, to lookup results for previous -jobs. The default directory for this is: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -cachedir: /var/cache/salt +s_gsoc2014: + syslog_ng.config: + \- config: + source: + \- tcp: + \- ip: 0.0.0.0 + \- port: 1234 + \- flags: no\-parse .ft P .fi .UNINDENT .UNINDENT -.sp -and then in the \fB/proc\fP directory. -.sp -Each job return for every Minion is saved in a single file. Over time this -directory can grow quite large, depending on the number of published jobs. The -amount of files and directories will scale with the number of jobs published and -the retention time defined by +.SS Filter .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -keep_jobs_seconds: 86400 +filter f_json { + match( + \(dq@json:\(dq + ); +}; .ft P .fi .UNINDENT @@ -43296,11095 +43537,11469 @@ keep_jobs_seconds: 86400 .sp .nf .ft C -250 jobs/day * 2000 minions returns = 500,000 files a day +f_json: + syslog_ng.config: + \- config: + filter: + \- match: + \- \(aq\(aq\(dq@json:\(dq\(aq\(aq .ft P .fi .UNINDENT .UNINDENT -.SS Use and External Job Cache -.sp -An external job cache allows for job storage to be placed on an external -system, such as a database. +.SS Template .INDENT 0.0 -.IP \(bu 2 -ext_job_cache: this will have the minions store their return data directly -into a returner (not sent through the Master) -.IP \(bu 2 -master_job_cache (New in \fI2014.7.0\fP): this will make the Master store the job -data using a returner (instead of the local job cache on disk). -.UNINDENT -.sp -If a master has many accepted keys, it may take a long time to publish a job -because the master must first determine the matching minions and deliver -that information back to the waiting client before the job can be published. -.sp -To mitigate this, a key cache may be enabled. This will reduce the load -on the master to a single file open instead of thousands or tens of thousands. -.sp -This cache is updated by the maintenance process, however, which means that -minions with keys that are accepted may not be targeted by the master -for up to sixty seconds by default. -.sp -To enable the master key cache, set \fIkey_cache: \(aqsched\(aq\fP in the master -configuration file. -.SS Disable The Job Cache -.sp -The job cache is a central component of the Salt Master and many aspects of -the Salt Master will not function correctly without a running job cache. -.sp -Disabling the job cache is \fBSTRONGLY DISCOURAGED\fP and should not be done -unless the master is being used to execute routines that require no history -or reliable feedback! +.INDENT 3.5 .sp -The job cache can be disabled: +.nf +.ft C +template t_demo_filetemplate { + template( + \(dq$ISODATE $HOST $MSG \(dq + ); + template_escape( + no + ); +}; +.ft P +.fi +.UNINDENT +.UNINDENT .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -job_cache: False +t_demo_filetemplate: + syslog_ng.config: + \-config: + template: + \- template: + \- \(aq\(dq$ISODATE $HOST $MSG\en\(dq\(aq + \- template_escape: + \- \(dqno\(dq .ft P .fi .UNINDENT .UNINDENT -.SS How to Convert Jinja Logic to an Execution Module -.sp -\fBNOTE:\fP +.SS Rewrite .INDENT 0.0 .INDENT 3.5 -This tutorial assumes a basic knowledge of Salt states and specifically -experience using the \fImaps.jinja\fP idiom. .sp -This tutorial was written by a salt user who was told \(dqif your maps.jinja -is too complicated, write an execution module!\(dq. If you are experiencing -over\-complicated jinja, read on. +.nf +.ft C +rewrite r_set_message_to_MESSAGE { + set( + \(dq${.json.message}\(dq, + value(\(dq$MESSAGE\(dq) + ); +}; +.ft P +.fi .UNINDENT .UNINDENT -.SS The Problem: Jinja Gone Wild -.sp -It is often said in the Salt community that \(dqJinja is not a Programming Language\(dq. -There\(aqs an even older saying known as Maslow\(aqs hammer. -It goes something like -\(dqif all you have is a hammer, everything looks like a nail\(dq. -Jinja is a reliable hammer, and so is the \fImaps.jinja\fP idiom. -Unfortunately, it can lead to code that looks like the following. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# storage/maps.yaml - -{% import_yaml \(aqstorage/defaults.yaml\(aq as default_settings %} -{% set storage = default_settings.storage %} -{% do storage.update(salt[\(aqgrains.filter_by\(aq]({ - \(aqDebian\(aq: { - }, - \(aqRedHat\(aq: { - } -}, merge=salt[\(aqpillar.get\(aq](\(aqstorage:lookup\(aq))) %} - -{% if \(aqVirtualBox\(aq == grains.get(\(aqvirtual\(aq, None) or \(aqoracle\(aq == grains.get(\(aqvirtual\(aq, None) %} -{% do storage.update({\(aqdepot_ip\(aq: \(aq192.168.33.81\(aq, \(aqserver_ip\(aq: \(aq192.168.33.51\(aq}) %} -{% else %} -{% set colo = pillar.get(\(aqinventory\(aq, {}).get(\(aqcolo\(aq, \(aqUnknown\(aq) %} -{% set servers_list = pillar.get(\(aqstorage_servers\(aq, {}).get(colo, [storage.depot_ip, ]) %} -{% if opts.id.startswith(\(aqfoo\(aq) %} -{% set modulus = servers_list | count %} -{% set integer_id = opts.id | replace(\(aqfoo\(aq, \(aq\(aq) | int %} -{% set server_index = integer_id % modulus %} -{% else %} -{% set server_index = 0 %} -{% endif %} -{% do storage.update({\(aqserver_ip\(aq: servers_list[server_index]}) %} -{% endif %} - -{% for network, _ in salt.pillar.get(\(aqinventory:networks\(aq, {}) | dictsort %} -{% do storage.ipsets.hash_net.foo_networks.append(network) %} -{% endfor %} +r_set_message_to_MESSAGE: + syslog_ng.config: + \- config: + rewrite: + \- set: + \- \(aq\(dq${.json.message}\(dq\(aq + \- value : \(aq\(dq$MESSAGE\(dq\(aq .ft P .fi .UNINDENT .UNINDENT -.sp -This is an example from the author\(aqs salt formulae demonstrating misuse of jinja. -Aside from being difficult to read and maintain, -accessing the logic it contains from a non\-jinja renderer -while probably possible is a significant barrier! -.SS Refactor -.sp -The first step is to reduce the maps.jinja file to something reasonable. -This gives us an idea of what the module we are writing needs to do. -There is a lot of logic around selecting a storage server ip. -Let\(aqs move that to an execution module. +.SS Global options .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# storage/maps.yaml - -{% import_yaml \(aqstorage/defaults.yaml\(aq as default_settings %} -{% set storage = default_settings.storage %} -{% do storage.update(salt[\(aqgrains.filter_by\(aq]({ - \(aqDebian\(aq: { - }, - \(aqRedHat\(aq: { - } -}, merge=salt[\(aqpillar.get\(aq](\(aqstorage:lookup\(aq))) %} - -{% if \(aqVirtualBox\(aq == grains.get(\(aqvirtual\(aq, None) or \(aqoracle\(aq == grains.get(\(aqvirtual\(aq, None) %} -{% do storage.update({\(aqdepot_ip\(aq: \(aq192.168.33.81\(aq}) %} -{% endif %} - -{% do storage.update({\(aqserver_ip\(aq: salt[\(aqstorage.ip\(aq]()}) %} - -{% for network, _ in salt.pillar.get(\(aqinventory:networks\(aq, {}) | dictsort %} -{% do storage.ipsets.hash_net.af_networks.append(network) %} -{% endfor %} +options { + time_reap(30); + mark_freq(10); + keep_hostname(yes); +}; .ft P .fi .UNINDENT .UNINDENT -.sp -And then, write the module. -Note how the module encapsulates all of the logic around finding the storage server IP. .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -# _modules/storage.py -#!python - -\(dq\(dq\(dq -Functions related to storage servers. -\(dq\(dq\(dq - -import re - - -def ips(): - \(dq\(dq\(dq - Provide a list of all local storage server IPs. - - CLI Example:: - - salt \e* storage.ips - \(dq\(dq\(dq - - if __grains__.get(\(dqvirtual\(dq, None) in [\(dqVirtualBox\(dq, \(dqoracle\(dq]: - return [ - \(dq192.168.33.51\(dq, - ] - - colo = __pillar__.get(\(dqinventory\(dq, {}).get(\(dqcolo\(dq, \(dqUnknown\(dq) - return __pillar__.get(\(dqstorage_servers\(dq, {}).get(colo, [\(dqunknown\(dq]) - - -def ip(): - \(dq\(dq\(dq - Select and return a local storage server IP. - - This loadbalances across storage servers by using the modulus of the client\(aqs id number. - - :maintainer: Andrew Hammond - :maturity: new - :depends: None - :platform: all - - CLI Example:: - - salt \e* storage.ip - - \(dq\(dq\(dq - - numerical_suffix = re.compile(r\(dq^.*(\ed+)$\(dq) - servers_list = ips() - - m = numerical_suffix.match(__grains__[\(dqid\(dq]) - if m: - modulus = len(servers_list) - server_number = int(m.group(1)) - server_index = server_number % modulus - else: - server_index = 0 - - return servers_list[server_index] +global_options: + syslog_ng.config: + \- config: + options: + \- time_reap: 30 + \- mark_freq: 10 + \- keep_hostname: \(dqyes\(dq .ft P .fi .UNINDENT .UNINDENT -.SS Conclusion -.sp -That was... surprisingly straight\-forward. -Now the logic is available in every renderer, instead of just Jinja. -Best of all, it can be maintained in Python, -which is a whole lot easier than Jinja. -.SS Using Apache Libcloud for declarative and procedural multi\-cloud orchestration +.SS Log +.INDENT 0.0 +.INDENT 3.5 .sp -New in version 2018.3.0. +.nf +.ft C +log { + source(s_gsoc2014); + junction { + channel { + filter(f_json); + parser(p_json); + rewrite(r_set_json_tag); + rewrite(r_set_message_to_MESSAGE); + destination { + file( + \(dq/tmp/json\-input.log\(dq, + template(t_gsoc2014) + ); + }; + flags(final); + }; + channel { + filter(f_not_json); + parser { + syslog\-parser( -.sp -\fBNOTE:\fP + ); + }; + rewrite(r_set_syslog_tag); + flags(final); + }; + }; + destination { + file( + \(dq/tmp/all.log\(dq, + template(t_gsoc2014) + ); + }; +}; +.ft P +.fi +.UNINDENT +.UNINDENT .INDENT 0.0 .INDENT 3.5 -This walkthrough assumes basic knowledge of Salt and Salt States. To get up to speed, check out the -\fI\%Salt Walkthrough\fP\&. +.sp +.nf +.ft C +l_gsoc2014: + syslog_ng.config: + \- config: + log: + \- source: s_gsoc2014 + \- junction: + \- channel: + \- filter: f_json + \- parser: p_json + \- rewrite: r_set_json_tag + \- rewrite: r_set_message_to_MESSAGE + \- destination: + \- file: + \- \(aq\(dq/tmp/json\-input.log\(dq\(aq + \- template: t_gsoc2014 + \- flags: final + \- channel: + \- filter: f_not_json + \- parser: + \- syslog\-parser: [] + \- rewrite: r_set_syslog_tag + \- flags: final + \- destination: + \- file: + \- \(dq/tmp/all.log\(dq + \- template: t_gsoc2014 +.ft P +.fi .UNINDENT .UNINDENT +.SS Salt in 10 Minutes .sp -Apache Libcloud is a Python library which hides differences between different cloud provider APIs and allows -you to manage different cloud resources through a unified and easy to use API. Apache Libcloud supports over -60 cloud platforms, including Amazon, Microsoft Azure, DigitalOcean, Google Cloud Platform and OpenStack. +\fBNOTE:\fP .INDENT 0.0 -.TP -.B Execution and state modules are available for Compute, DNS, Storage and Load Balancer drivers from Apache Libcloud in -SaltStack. -.UNINDENT +.INDENT 3.5 +Welcome to Salt Project! I am excited that you are interested in Salt and +starting down the path to better infrastructure management. I developed +(and am continuing to develop) Salt with the goal of making the best +software available to manage computers of almost any kind. I hope you enjoy +working with Salt and that the software can solve your real world needs! .INDENT 0.0 .IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_compute\fP \- Compute \- -services such as OpenStack Nova, Amazon EC2, Microsoft Azure VMs -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_dns\fP \- DNS as a Service \- -services such as Amazon Route 53 and Zerigo -.UNINDENT +Thomas S Hatch .IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service \- -services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers +Salt Project creator and Chief Developer of Salt Project .UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN \- -services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift .UNINDENT .UNINDENT +.SS Getting Started +.SS What is Salt? .sp -These modules are designed as a way of having a multi\-cloud deployment and abstracting simple differences -between platform to design a high\-availability architecture. +Salt is a different approach to infrastructure management, founded on +the idea that high\-speed communication with large numbers of systems can open +up new capabilities. This approach makes Salt a powerful multitasking system +that can solve many specific problems in an infrastructure. .sp -The Apache Libcloud functionality is available through both execution modules and Salt states. -.SS Configuring Drivers +The backbone of Salt is the remote execution engine, which creates a high\-speed, +secure and bi\-directional communication net for groups of systems. On top of this +communication system, Salt provides an extremely fast, flexible, and easy\-to\-use +configuration management system called \fBSalt States\fP\&. +.SS Installing Salt .sp -Drivers can be configured in the Salt Configuration/Minion settings. All libcloud modules expect a list of \(dqprofiles\(dq to -be configured with authentication details for each driver. +SaltStack has been made to be very easy to install and get started. The +\fI\%Salt install guide\fP +provides instructions for all supported platforms. +.SS Starting Salt .sp -Each driver will have a string identifier, these can be found in the libcloud..types.Provider class -for each API, \fI\%https://libcloud.readthedocs.io/en/latest/supported_providers.html\fP +Salt functions on a master/minion topology. A master server acts as a +central control bus for the clients, which are called \fBminions\fP\&. The minions +connect back to the master. +.SS Setting Up the Salt Master .sp -Some drivers require additional parameters, which are documented in the Apache Libcloud documentation. For example, -GoDaddy DNS expects \(dq\fIshopper_id\fP\(dq, which is the customer ID. These additional parameters can be added to the profile settings -and will be passed directly to the driver instantiation method. +Turning on the Salt Master is easy \-\- just turn it on! The default configuration +is suitable for the vast majority of installations. The Salt Master can be +controlled by the local Linux/Unix service manager: +.sp +On Systemd based platforms (newer Debian, openSUSE, Fedora): .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -libcloud_dns: - godaddy: - driver: godaddy - shopper_id: 90425123 - key: AFDDJFGIjDFVNSDIFNASMC - secret: FG(#f8vdfgjlkm) - -libcloud_storage: - google: - driver: google_storage - key: GOOG4ASDIDFNVIdfnIVW - secret: R+qYE9hkfdhv89h4invhdfvird4Pq3an8rnK +systemctl start salt\-master .ft P .fi .UNINDENT .UNINDENT .sp -You can have multiple profiles for a single driver, for example if you wanted 2 DNS profiles for Amazon Route53, -naming them \(dqroute53_prod\(dq and \(dqroute54_test\(dq would help your -administrators distinguish their purpose. +On Upstart based systems (Ubuntu, older Fedora/RHEL): .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -libcloud_dns: - route53_prod: - driver: route53 - key: AFDDJFGIjDFVNSDIFNASMC - secret: FG(#f8vdfgjlkm) - route53_test: - driver: route53 - key: AFDDJFGIjdfgdfgdf - secret: FG(#f8vdfgjlkm) +service salt\-master start .ft P .fi .UNINDENT .UNINDENT -.SS Using the execution modules -.sp -Amongst over 60 clouds that Apache Libcloud supports, you can add profiles to your Salt configuration to access and control these clouds. -Each of the libcloud execution modules exposes the common API methods for controlling Compute, DNS, Load Balancers and Object Storage. -To see which functions are supported across specific clouds, see the Libcloud \fI\%supported methods\fP documentation. .sp -The module documentation explains each of the API methods and how to leverage them. +On SysV Init systems (Gentoo, older Debian etc.): .INDENT 0.0 -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_compute\fP \- Compute \- -services such as OpenStack Nova, Amazon EC2, Microsoft Azure VMs -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_dns\fP \- DNS as a Service \- -services such as Amazon Route 53 and Zerigo -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service \- -services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN \- -services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift +.INDENT 3.5 +.sp +.nf +.ft C +/etc/init.d/salt\-master start +.ft P +.fi .UNINDENT .UNINDENT .sp -For example, listing buckets in the Google Storage platform: +Alternatively, the Master can be started directly on the command\-line: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt\-call libcloud_storage.list_containers google - - local: - |_ - \-\-\-\-\-\-\-\-\-\- - extra: - \-\-\-\-\-\-\-\-\-\- - creation_date: - 2017\-01\-05T05:44:56.324Z - name: - anthonypjshaw +salt\-master \-d .ft P .fi .UNINDENT .UNINDENT .sp -The Apache Libcloud storage module can be used to synchronize files between multiple storage clouds, -such as Google Storage, S3 and OpenStack Swift +The Salt Master can also be started in the foreground in debug mode, thus +greatly increasing the command output: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt \(aq*\(aq libcloud_storage.download_object DeploymentTools test.sh /tmp/test.sh google_storage +salt\-master \-l debug .ft P .fi .UNINDENT .UNINDENT -.SS Using the state modules .sp -For each configured profile, the assets available in the API (e.g. storage objects, containers, -DNS records and load balancers) can be deployed via Salt\(aqs state system. +The Salt Master needs to bind to two TCP network ports on the system. These ports +are \fB4505\fP and \fB4506\fP\&. For more in depth information on firewalling these ports, +the firewall tutorial is available \fI\%here\fP\&. +.SS Finding the Salt Master .sp -The state module documentation explains the specific states that each module supports +When a minion starts, by default it searches for a system that resolves to the \fBsalt\fP hostname on the network. +If found, the minion initiates the handshake and key authentication process with the Salt master. +This means that the easiest configuration approach is to set internal DNS to resolve the name \fBsalt\fP back to the Salt Master IP. +.sp +Otherwise, the minion configuration file will need to be edited so that the +configuration option \fBmaster\fP points to the DNS name or the IP of the Salt Master: +.sp +\fBNOTE:\fP .INDENT 0.0 -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_storage\fP \- Cloud Object Storage and CDN -.INDENT 7.0 -.IP \(bu 2 -services such as Amazon S3 and Rackspace CloudFiles, OpenStack Swift -.UNINDENT -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_loadbalancer\fP \- Load Balancers as a Service -.INDENT 7.0 -.IP \(bu 2 -services such as Amazon Elastic Load Balancer and GoGrid LoadBalancers -.UNINDENT -.UNINDENT -.IP \(bu 2 -.INDENT 2.0 -.TP -.B \fI\%libcloud_dns\fP \- DNS as a Service -.INDENT 7.0 -.IP \(bu 2 -services such as Amazon Route 53 and Zerigo -.UNINDENT +.INDENT 3.5 +The default location of the configuration files is \fB/etc/salt\fP\&. Most +platforms adhere to this convention, but platforms such as FreeBSD and +Microsoft Windows place this file in different locations. .UNINDENT .UNINDENT .sp -For DNS, the state modules can be used to provide DNS resilience for multiple nameservers, for example: +\fB/etc/salt/minion:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -libcloud_dns: - godaddy: - driver: godaddy - shopper_id: 12345 - key: 2orgk34kgk34g - secret: fjgoidhjgoim - amazon: - driver: route53 - key: blah - secret: blah +master: saltmaster.example.com .ft P .fi .UNINDENT .UNINDENT +.SS Setting up a Salt Minion .sp -And then in a state file: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +The Salt Minion can operate with or without a Salt Master. This walk\-through +assumes that the minion will be connected to the master, for information on +how to run a master\-less minion please see the master\-less quick\-start guide: .sp -.nf -.ft C -webserver: - libcloud_dns.zone_present: - name: mywebsite.com - profile: godaddy - libcloud_dns.record_present: - name: www - zone: mywebsite.com - type: A - data: 12.34.32.3 - profile: godaddy - libcloud_dns.zone_present: - name: mywebsite.com - profile: amazon - libcloud_dns.record_present: - name: www - zone: mywebsite.com - type: A - data: 12.34.32.3 - profile: amazon -.ft P -.fi +\fI\%Masterless Minion Quickstart\fP .UNINDENT .UNINDENT .sp -This could be combined with a multi\-cloud load balancer deployment, +Now that the master can be found, start the minion in the same way as the +master; with the platform init system or via the command line directly: +.sp +As a daemon: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -webserver: - libcloud_dns.zone_present: - \- name: mywebsite.com - \- profile: godaddy - ... - libcloud_loadbalancer.balancer_present: - \- name: web_main - \- port: 80 - \- protocol: http - \- members: - \- ip: 1.2.4.5 - port: 80 - \- ip: 2.4.5.6 - port: 80 - \- profile: google_gce - libcloud_loadbalancer.balancer_present: - \- name: web_main - \- port: 80 - \- protocol: http - \- members: - \- ip: 1.2.4.5 - port: 80 - \- ip: 2.4.5.6 - port: 80 - \- profile: amazon_elb +salt\-minion \-d .ft P .fi .UNINDENT .UNINDENT .sp -Extended parameters can be passed to the specific cloud, for example you can specify the region with the Google Cloud API, because -\fIcreate_balancer\fP can accept a \fIex_region\fP argument. Adding this argument to the state will pass the additional command to the driver. +In the foreground in debug mode: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lb_test: - libcloud_loadbalancer.balancer_absent: - \- name: example - \- port: 80 - \- protocol: http - \- profile: google - \- ex_region: us\-east1 +salt\-minion \-l debug .ft P .fi .UNINDENT .UNINDENT -.SS Accessing custom arguments in execution modules .sp -Some cloud providers have additional functionality that can be accessed on top of the base API, for example -the Google Cloud Engine load balancer service offers the ability to provision load balancers into a specific region. +When the minion is started, it will generate an \fBid\fP value, unless it has +been generated on a previous run and cached (in \fB/etc/salt/minion_id\fP by +default). This is the name by which the minion will attempt +to authenticate to the master. The following steps are attempted, in order to +try to find a value that is not \fBlocalhost\fP: +.INDENT 0.0 +.IP 1. 3 +The Python function \fBsocket.getfqdn()\fP is run +.IP 2. 3 +\fB/etc/hostname\fP is checked (non\-Windows only) +.IP 3. 3 +\fB/etc/hosts\fP (\fB%WINDIR%\esystem32\edrivers\eetc\ehosts\fP on Windows hosts) is +checked for hostnames that map to anything within \fB127.0.0.0/8\fP\&. +.UNINDENT .sp -Looking at the \fI\%API documentation\fP, -we can see that it expects an \fIex_region\fP in the \fIcreate_balancer\fP method, so when we execute the salt command, we can add this additional parameter like this: +If none of the above are able to produce an id which is not \fBlocalhost\fP, then +a sorted list of IP addresses on the minion (excluding any within +\fB127.0.0.0/8\fP) is inspected. The first publicly\-routable IP address is +used, if there is one. Otherwise, the first privately\-routable IP address is +used. +.sp +If all else fails, then \fBlocalhost\fP is used as a fallback. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +Overriding the \fBid\fP .sp -.nf -.ft C -$ salt myminion libcloud_storage.create_balancer my_balancer 80 http profile1 ex_region=us\-east1 -$ salt myminion libcloud_storage.list_container_objects my_bucket profile1 ex_prefix=me -.ft P -.fi +The minion id can be manually specified using the \fI\%id\fP +parameter in the minion config file. If this configuration value is +specified, it will override all other sources for the \fBid\fP\&. .UNINDENT .UNINDENT -.SS Accessing custom methods in Libcloud drivers .sp -Some cloud APIs have additional methods that are prefixed with \fIex_\fP in Apache Libcloud, these methods -are part of the non\-standard API but can still -be accessed from the Salt modules for \fIlibcloud_storage\fP, \fIlibcloud_loadbalancer\fP and \fIlibcloud_dns\fP\&. -The extra methods are available via the \fIextra\fP command, which expects the name of the method as the -first argument, the profile as the second and then -accepts a list of keyword arguments to pass onto the driver method, for example, accessing permissions in Google Storage objects: +Now that the minion is started, it will generate cryptographic keys and attempt +to connect to the master. The next step is to venture back to the master server +and accept the new minion\(aqs public key. +.SS Using salt\-key +.sp +Salt authenticates minions using public\-key encryption and authentication. For +a minion to start accepting commands from the master, the minion keys need to be +accepted by the master. +.sp +The \fBsalt\-key\fP command is used to manage all of the keys on the +master. To list the keys that are on the master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ salt myminion libcloud_storage.extra ex_get_permissions google container_name=my_container object_name=me.jpg \-\-out=yaml +salt\-key \-L .ft P .fi .UNINDENT .UNINDENT -.SS Example profiles -.SS Google Cloud .sp -Using Service Accounts with GCE, you can provide a path to the JSON file and the project name in the parameters. +The keys that have been rejected, accepted, and pending acceptance are listed. +The easiest way to accept the minion key is to accept all pending keys: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -google: - driver: gce - user_id: 234234\-compute@developer.gserviceaccount.com - key: /path/to/service_account_download.json - auth_type: SA - project: project\-name +salt\-key \-A .ft P .fi .UNINDENT .UNINDENT -.SS LXC Management with Salt .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -This walkthrough assumes basic knowledge of Salt. To get up to speed, check -out the \fI\%Salt Walkthrough\fP\&. -.UNINDENT -.UNINDENT -.SS Dependencies -.sp -Manipulation of LXC containers in Salt requires the minion to have an LXC -version of at least 1.0 (an alpha or beta release of LXC 1.0 is acceptable). -The following distributions are known to have new enough versions of LXC -packaged: -.INDENT 0.0 -.IP \(bu 2 -RHEL/CentOS 6 and later (via \fI\%EPEL\fP) -.IP \(bu 2 -Fedora (All non\-EOL releases) -.IP \(bu 2 -Debian 8.0 (Jessie) -.IP \(bu 2 -Ubuntu 14.04 LTS and later (LXC templates are packaged separately as -\fBlxc\-templates\fP, it is recommended to also install this package) -.IP \(bu 2 -openSUSE 13.2 and later -.UNINDENT -.SS Profiles -.sp -Profiles allow for a sort of shorthand for commonly\-used -configurations to be defined in the minion config file, \fI\%grains\fP, \fI\%pillar\fP, or the master config file. The -profile is retrieved by Salt using the \fI\%config.get\fP function, which looks in those locations, in that -order. This allows for profiles to be defined centrally in the master config -file, with several options for overriding them (if necessary) on groups of -minions or individual minions. +Keys should be verified! Print the master key fingerprint by running \fBsalt\-key \-F master\fP +on the Salt master. Copy the \fBmaster.pub\fP fingerprint from the Local Keys section, +and then set this value as the \fI\%master_finger\fP in the minion configuration +file. Restart the Salt minion. .sp -There are two types of profiles: -.INDENT 0.0 -.INDENT 3.5 -.INDENT 0.0 -.IP \(bu 2 -One for defining the parameters used in container creation/clone. -.IP \(bu 2 -One for defining the container\(aqs network interface(s) settings. -.UNINDENT -.UNINDENT -.UNINDENT -.SS Container Profiles +On the master, run \fBsalt\-key \-f minion\-id\fP to print the fingerprint of the +minion\(aqs public key that was received by the master. On the minion, run +\fBsalt\-call key.finger \-\-local\fP to print the fingerprint of the minion key. .sp -LXC container profiles are defined underneath the -\fBlxc.container_profile\fP config option: +On the master: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.container_profile: - centos: - template: centos - backing: lvm - vgname: vg1 - lvname: lxclv - size: 10G - centos_big: - template: centos - backing: lvm - vgname: vg1 - lvname: lxclv - size: 20G +# salt\-key \-f foo.domain.com +Unaccepted Keys: +foo.domain.com: 39:f9:e4:8a:aa:74:8d:52:1a:ec:92:03:82:09:c8:f9 .ft P .fi .UNINDENT .UNINDENT .sp -Profiles are retrieved using the \fI\%config.get\fP -function, with the \fBrecurse\fP merge strategy. This means that a profile can be -defined at a lower level (for example, the master config file) and then parts -of it can be overridden at a higher level (for example, in pillar data). -Consider the following container profile data: -.sp -\fBIn the Master config file:\fP +On the minion: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.container_profile: - centos: - template: centos - backing: lvm - vgname: vg1 - lvname: lxclv - size: 10G +# salt\-call key.finger \-\-local +local: + 39:f9:e4:8a:aa:74:8d:52:1a:ec:92:03:82:09:c8:f9 .ft P .fi .UNINDENT .UNINDENT .sp -\fBIn the Pillar data\fP +If they match, approve the key with \fBsalt\-key \-a foo.domain.com\fP\&. +.UNINDENT +.UNINDENT +.SS Sending the First Commands +.sp +Now that the minion is connected to the master and authenticated, the master +can start to command the minion. +.sp +Salt commands allow for a vast set of functions to be executed and for +specific minions and groups of minions to be targeted for execution. +.sp +The \fBsalt\fP command is comprised of command options, target specification, +the function to execute, and arguments to the function. +.sp +A simple command to +start with looks like this: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.container_profile: - centos: - size: 20G +salt \(aq*\(aq test.version .ft P .fi .UNINDENT .UNINDENT .sp -Any minion with the above Pillar data would have the \fBsize\fP parameter in the -\fBcentos\fP profile overridden to 20G, while those minions without the above -Pillar data would have the 10G \fBsize\fP value. This is another way of achieving -the same result as the \fBcentos_big\fP profile above, without having to define -another whole profile that differs in just one value. +The \fB*\fP is the target, which specifies all minions. +.sp +\fBtest.version\fP tells the minion to run the \fI\%test.version\fP function. +.sp +In the case of \fBtest.version\fP, \fBtest\fP refers to a \fI\%execution module\fP\&. \fBversion\fP refers to the \fI\%version\fP function contained in the aforementioned \fBtest\fP +module. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -In the 2014.7.x release cycle and earlier, container profiles are defined -under \fBlxc.profile\fP\&. This parameter will still work in version 2015.5.0, -but is deprecated and will be removed in a future release. Please note -however that the profile merging feature described above will only work -with profiles defined under \fBlxc.container_profile\fP, and only in versions -2015.5.0 and later. -.UNINDENT +Execution modules are the workhorses of Salt. They do the work on the +system to perform various tasks, such as manipulating files and restarting +services. .UNINDENT -.sp -Additionally, in version 2015.5.0 container profiles have been expanded to -support passing template\-specific CLI options to \fI\%lxc.create\fP\&. Below is a table describing the parameters which -can be configured in container profiles: -.TS -center; -|l|l|l|. -_ -T{ -Parameter -T} T{ -2015.5.0 and Newer -T} T{ -2014.7.x and Earlier -T} -_ -T{ -\fItemplate\fP\s-2\u1\d\s0 -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIoptions\fP\s-2\u1\d\s0 -T} T{ -Yes -T} T{ -No -T} -_ -T{ -\fIimage\fP\s-2\u1\d\s0 -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIbacking\fP -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIsnapshot\fP\s-2\u2\d\s0 -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIlvname\fP\s-2\u1\d\s0 -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIfstype\fP\s-2\u1\d\s0 -T} T{ -Yes -T} T{ -Yes -T} -_ -T{ -\fIsize\fP -T} T{ -Yes -T} T{ -Yes -T} -_ -.TE -.INDENT 0.0 -.IP 1. 3 -Parameter is only supported for container creation, and will be ignored if -the profile is used when cloning a container. -.IP 2. 3 -Parameter is only supported for container cloning, and will be ignored if -the profile is used when not cloning a container. .UNINDENT -.SS Network Profiles .sp -LXC network profiles are defined defined underneath the \fBlxc.network_profile\fP -config option. -By default, the module uses a DHCP based configuration and try to guess a bridge to -get connectivity. +The result of running this command will be the master instructing all of the +minions to execute \fI\%test.version\fP in parallel +and return the result. Using \fI\%test.version\fP +is a good way of confirming that a minion is connected, and reaffirm to the user +the salt version(s) they have installed on the minions. .sp -\fBWARNING:\fP +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -on pre \fB2015.5.2\fP, you need to specify explicitly the network bridge +Each minion registers itself with a unique minion ID. This ID defaults to +the minion\(aqs hostname, but can be explicitly defined in the minion config as +well by using the \fI\%id\fP parameter. .UNINDENT .UNINDENT +.sp +Of course, there are hundreds of other modules that can be called just as +\fBtest.version\fP can. For example, the following would return disk usage on all +targeted minions: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.network_profile: - centos: - eth0: - link: br0 - type: veth - flags: up - ubuntu: - eth0: - link: lxcbr0 - type: veth - flags: up +salt \(aq*\(aq disk.usage .ft P .fi .UNINDENT .UNINDENT +.SS Getting to Know the Functions .sp -As with container profiles, network profiles are retrieved using the -\fI\%config.get\fP function, with the \fBrecurse\fP -merge strategy. Consider the following network profile data: -.sp -\fBIn the Master config file:\fP +Salt comes with a vast library of functions available for execution, and Salt +functions are self\-documenting. To see what functions are available on the +minions execute the \fBsys.doc\fP function: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.network_profile: - centos: - eth0: - link: br0 - type: veth - flags: up +salt \(aq*\(aq sys.doc .ft P .fi .UNINDENT .UNINDENT .sp -\fBIn the Pillar data\fP +This will display a very large list of available functions and documentation on +them. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -.sp -.nf -.ft C -lxc.network_profile: - centos: - eth0: - link: lxcbr0 -.ft P -.fi +Module documentation is also available \fI\%on the web\fP\&. .UNINDENT .UNINDENT .sp -Any minion with the above Pillar data would use the \fBlxcbr0\fP interface as the -bridge interface for any container configured using the \fBcentos\fP network -profile, while those minions without the above Pillar data would use the -\fBbr0\fP interface for the same. +These functions cover everything from shelling out to package management to +manipulating database servers. They comprise a powerful system management API +which is the backbone to Salt configuration management and many other aspects +of Salt. .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -In the 2014.7.x release cycle and earlier, network profiles are defined -under \fBlxc.nic\fP\&. This parameter will still work in version 2015.5.0, but -is deprecated and will be removed in a future release. Please note however -that the profile merging feature described above will only work with -profiles defined under \fBlxc.network_profile\fP, and only in versions -2015.5.0 and later. -.UNINDENT +Salt comes with many plugin systems. The functions that are available via +the \fBsalt\fP command are called \fI\%Execution Modules\fP\&. .UNINDENT -.sp -The following are parameters which can be configured in network profiles. These -will directly correspond to a parameter in an LXC configuration file (see \fBman -5 lxc.container.conf\fP). -.INDENT 0.0 -.IP \(bu 2 -\fBtype\fP \- Corresponds to \fBlxc.network.type\fP -.IP \(bu 2 -\fBlink\fP \- Corresponds to \fBlxc.network.link\fP -.IP \(bu 2 -\fBflags\fP \- Corresponds to \fBlxc.network.flags\fP .UNINDENT +.SS Helpful Functions to Know .sp -Interface\-specific options (MAC address, IPv4/IPv6, etc.) must be passed on a -container\-by\-container basis, for instance using the \fBnic_opts\fP argument to -\fI\%lxc.create\fP: +The \fI\%cmd\fP module contains +functions to shell out on minions, such as \fI\%cmd.run\fP and \fI\%cmd.run_all\fP: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion lxc.create container1 profile=centos network_profile=centos nic_opts=\(aq{eth0: {ipv4: 10.0.0.20/24, gateway: 10.0.0.1}}\(aq +salt \(aq*\(aq cmd.run \(aqls \-l /etc\(aq .ft P .fi .UNINDENT .UNINDENT .sp -\fBWARNING:\fP +The \fBpkg\fP functions automatically map local system package managers to the +same salt functions. This means that \fBpkg.install\fP will install packages via +\fByum\fP on Red Hat based systems, \fBapt\fP on Debian systems, etc.: .INDENT 0.0 .INDENT 3.5 -The \fBipv4\fP, \fBipv6\fP, \fBgateway\fP, and \fBlink\fP (bridge) settings in -network profiles / nic_opts will only work if the container doesn\(aqt redefine -the network configuration (for example in -\fB/etc/sysconfig/network\-scripts/ifcfg\-\fP on RHEL/CentOS, -or \fB/etc/network/interfaces\fP on Debian/Ubuntu/etc.). Use these with -caution. The container images installed using the \fBdownload\fP template, -for instance, typically are configured for eth0 to use DHCP, which will -conflict with static IP addresses set at the container level. +.sp +.nf +.ft C +salt \(aq*\(aq pkg.install vim +.ft P +.fi .UNINDENT .UNINDENT .sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -For LXC < 1.0.7 and DHCP support, set \fBipv4.gateway: \(aqauto\(aq\fP is your -network profile, ie.: +Some custom Linux spins and derivatives of other distributions are not properly +detected by Salt. If the above command returns an error message saying that +\fBpkg.install\fP is not available, then you may need to override the pkg +provider. This process is explained \fI\%here\fP\&. +.UNINDENT +.UNINDENT +.sp +The \fI\%network.interfaces\fP function will +list all interfaces on a minion, along with their IP addresses, netmasks, MAC +addresses, etc: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.network_profile.nic: - debian: - eth0: - link: lxcbr0 - ipv4.gateway: \(aqauto\(aq +salt \(aq*\(aq network.interfaces .ft P .fi .UNINDENT .UNINDENT -.UNINDENT -.UNINDENT -.SS Old lxc support (<1.0.7) -.sp -With saltstack \fB2015.5.2\fP and above, normally the setting is autoselected, but -before, you\(aqll need to teach your network profile to set -\fBlxc.network.ipv4.gateway\fP to \fBauto\fP when using a classic ipv4 configuration. +.SS Changing the Output Format .sp -Thus you\(aqll need +The default output format used for most Salt commands is called the \fBnested\fP +outputter, but there are several other outputters that can be used to change +the way the output is displayed. For instance, the \fBpprint\fP outputter can be +used to display the return data using Python\(aqs \fBpprint\fP module: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.network_profile.foo: - etho: - link: lxcbr0 - ipv4.gateway: auto +root@saltmaster:~# salt myminion grains.item pythonpath \-\-out=pprint +{\(aqmyminion\(aq: {\(aqpythonpath\(aq: [\(aq/usr/lib64/python2.7\(aq, + \(aq/usr/lib/python2.7/plat\-linux2\(aq, + \(aq/usr/lib64/python2.7/lib\-tk\(aq, + \(aq/usr/lib/python2.7/lib\-tk\(aq, + \(aq/usr/lib/python2.7/site\-packages\(aq, + \(aq/usr/lib/python2.7/site\-packages/gst\-0.10\(aq, + \(aq/usr/lib/python2.7/site\-packages/gtk\-2.0\(aq]}} .ft P .fi .UNINDENT .UNINDENT -.SS Tricky network setups Examples .sp -This example covers how to make a container with both an internal ip and a -public routable ip, wired on two veth pairs. +The full list of Salt outputters, as well as example output, can be found +\fI\%here\fP\&. +.SS \fBsalt\-call\fP .sp -The another interface which receives directly a public routable ip can\(aqt be on -the first interface that we reserve for private inter LXC networking. +The examples so far have described running commands from the Master using the +\fBsalt\fP command, but when troubleshooting it can be more beneficial to login +to the minion directly and use \fBsalt\-call\fP\&. +.sp +Doing so allows you to see the minion log messages specific to the command you +are running (which are \fInot\fP part of the return data you see when running the +command from the Master using \fBsalt\fP), making it unnecessary to tail the +minion log. More information on \fBsalt\-call\fP and how to use it can be found +\fI\%here\fP\&. +.SS Grains +.sp +Salt uses a system called \fI\%Grains\fP to build up +static data about minions. This data includes information about the operating +system that is running, CPU architecture and much more. The grains system is +used throughout Salt to deliver platform data to many components and to users. +.sp +Grains can also be statically set, this makes it easy to assign values to +minions for grouping and managing. +.sp +A common practice is to assign grains to minions to specify what the role or +roles a minion might be. These static grains can be set in the minion +configuration file or via the \fI\%grains.setval\fP +function. +.SS Targeting +.sp +Salt allows for minions to be targeted based on a wide range of criteria. The +default targeting system uses globular expressions to match minions, hence if +there are minions named \fBlarry1\fP, \fBlarry2\fP, \fBcurly1\fP, and \fBcurly2\fP, a +glob of \fBlarry*\fP will match \fBlarry1\fP and \fBlarry2\fP, and a glob of \fB*1\fP +will match \fBlarry1\fP and \fBcurly1\fP\&. +.sp +Many other targeting systems can be used other than globs, these systems +include: +.INDENT 0.0 +.TP +.B Regular Expressions +Target using PCRE\-compliant regular expressions +.TP +.B Grains +Target based on grains data: +\fI\%Targeting with Grains\fP +.TP +.B Pillar +Target based on pillar data: +\fI\%Targeting with Pillar\fP +.TP +.B IP +Target based on IP address/subnet/range +.TP +.B Compound +Create logic to target based on multiple targets: +\fI\%Targeting with Compound\fP +.TP +.B Nodegroup +Target with nodegroups: +\fI\%Targeting with Nodegroup\fP +.UNINDENT +.sp +The concepts of targets are used on the command line with Salt, but also +function in many other areas as well, including the state system and the +systems used for ACLs and user permissions. +.SS Passing in Arguments +.sp +Many of the functions available accept arguments which can be passed in on +the command line: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.network_profile.foo: - eth0: {gateway: null, bridge: lxcbr0} - eth1: - # replace that by your main interface - \(aqlink\(aq: \(aqbr0\(aq - \(aqmac\(aq: \(aq00:16:5b:01:24:e1\(aq - \(aqgateway\(aq: \(aq2.20.9.14\(aq - \(aqipv4\(aq: \(aq2.20.9.1\(aq +salt \(aq*\(aq pkg.install vim .ft P .fi .UNINDENT .UNINDENT -.SS Creating a Container on the CLI -.SS From a Template -.sp -LXC is commonly distributed with several template scripts in -/usr/share/lxc/templates. Some distros may package these separately in an -\fBlxc\-templates\fP package, so make sure to check if this is the case. -.sp -There are LXC template scripts for several different operating systems, but -some of them are designed to use tools specific to a given distribution. For -instance, the \fBubuntu\fP template uses deb_bootstrap, the \fBcentos\fP template -uses yum, etc., making these templates impractical when a container from a -different OS is desired. .sp -The \fI\%lxc.create\fP function is used to create -containers using a template script. To create a CentOS container named -\fBcontainer1\fP on a CentOS minion named \fBmycentosminion\fP, using the -\fBcentos\fP LXC template, one can simply run the following command: +This example passes the argument \fBvim\fP to the pkg.install function. Since +many functions can accept more complex input than just a string, the arguments +are parsed through YAML, allowing for more complex data to be sent on the +command line: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt mycentosminion lxc.create container1 template=centos +salt \(aq*\(aq test.echo \(aqfoo: bar\(aq .ft P .fi .UNINDENT .UNINDENT .sp -For these instances, there is a \fBdownload\fP template which retrieves minimal -container images for several different operating systems. To use this template, -it is necessary to provide an \fBoptions\fP parameter when creating the -container, with three values: +In this case Salt translates the string \(aqfoo: bar\(aq into the dictionary +\(dq{\(aqfoo\(aq: \(aqbar\(aq}\(dq +.sp +\fBNOTE:\fP .INDENT 0.0 -.IP 1. 3 -\fBdist\fP \- the Linux distribution (i.e. \fBubuntu\fP or \fBcentos\fP) -.IP 2. 3 -\fBrelease\fP \- the release name/version (i.e. \fBtrusty\fP or \fB6\fP) -.IP 3. 3 -\fBarch\fP \- CPU architecture (i.e. \fBamd64\fP or \fBi386\fP) +.INDENT 3.5 +Any line that contains a newline will not be parsed by YAML. +.UNINDENT .UNINDENT +.SS Salt States .sp -The \fI\%lxc.images\fP function (new in version -2015.5.0) can be used to list the available images. Alternatively, the releases -can be viewed on \fI\%http://images.linuxcontainers.org/images/\fP\&. The images are -organized in such a way that the \fBdist\fP, \fBrelease\fP, and \fBarch\fP can be -determined using the following URL format: -\fBhttp://images.linuxcontainers.org/images/dist/release/arch\fP\&. For example, -\fBhttp://images.linuxcontainers.org/images/centos/6/amd64\fP would correspond to -a \fBdist\fP of \fBcentos\fP, a \fBrelease\fP of \fB6\fP, and an \fBarch\fP of \fBamd64\fP\&. +Now that the basics are covered the time has come to evaluate \fBStates\fP\&. Salt +\fBStates\fP, or the \fBState System\fP is the component of Salt made for +configuration management. .sp -Therefore, to use the \fBdownload\fP template to create a new 64\-bit CentOS 6 -container, the following command can be used: +The state system is already available with a basic Salt setup, no additional +configuration is required. States can be set up immediately. +.sp +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +Before diving into the state system, a brief overview of how states are +constructed will make many of the concepts clearer. Salt states are based +on data modeling and build on a low level data structure that is used to +execute each state function. Then more logical layers are built on top of +each other. .sp -.nf -.ft C -salt myminion lxc.create container1 template=download options=\(aq{dist: centos, release: 6, arch: amd64}\(aq -.ft P -.fi +The high layers of the state system which this tutorial will +cover consists of everything that needs to be known to use states, the two +high layers covered here are the \fIsls\fP layer and the highest layer +\fIhighstate\fP\&. +.sp +Understanding the layers of data management in the State System will help with +understanding states, but they never need to be used. Just as understanding +how a compiler functions assists when learning a programming language, +understanding what is going on under the hood of a configuration management +system will also prove to be a valuable asset. .UNINDENT .UNINDENT +.SS The First SLS Formula .sp -\fBNOTE:\fP -.INDENT 0.0 -.INDENT 3.5 -These command\-line options can be placed into a \fI\%container profile\fP, like so: +The state system is built on SLS (SaLt State) formulas. These formulas are built out in +files on Salt\(aqs file server. To make a very basic SLS formula open up a file +under /srv/salt named vim.sls. The following state ensures that vim is installed +on a system to which that state has been applied. +.sp +\fB/srv/salt/vim.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -lxc.container_profile.cent6: - template: download - options: - dist: centos - release: 6 - arch: amd64 +vim: + pkg.installed .ft P .fi .UNINDENT .UNINDENT .sp -The \fBoptions\fP parameter is not supported in profiles for the 2014.7.x -release cycle and earlier, so it would still need to be provided on the -command\-line. -.UNINDENT -.UNINDENT -.SS Cloning an Existing Container -.sp -To clone a container, use the \fI\%lxc.clone\fP -function: +Now install vim on the minions by calling the SLS directly: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion lxc.clone container2 orig=container1 +salt \(aq*\(aq state.apply vim .ft P .fi .UNINDENT .UNINDENT -.SS Using a Container Image .sp -While cloning is a good way to create new containers from a common base -container, the source container that is being cloned needs to already exist on -the minion. This makes deploying a common container across minions difficult. -For this reason, Salt\(aqs \fI\%lxc.create\fP is capable -of installing a container from a tar archive of another container\(aqs rootfs. To -create an image of a container named \fBcent6\fP, run the following command as -root: +This command will invoke the state system and run the \fBvim\fP SLS. +.sp +Now, to beef up the vim SLS formula, a \fBvimrc\fP can be added: +.sp +\fB/srv/salt/vim.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -tar czf cent6.tar.gz \-C /var/lib/lxc/cent6 rootfs +vim: + pkg.installed: [] + +/etc/vimrc: + file.managed: + \- source: salt://vimrc + \- mode: 644 + \- user: root + \- group: root .ft P .fi .UNINDENT .UNINDENT .sp +Now the desired \fBvimrc\fP needs to be copied into the Salt file server to +\fB/srv/salt/vimrc\fP\&. In Salt, everything is a file, so no path redirection needs +to be accounted for. The \fBvimrc\fP file is placed right next to the \fBvim.sls\fP file. +The same command as above can be executed to all the vim SLS formulas and now +include managing the file. +.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Before doing this, it is recommended that the container is stopped. +Salt does not need to be restarted/reloaded or have the master manipulated +in any way when changing SLS formulas. They are instantly available. .UNINDENT .UNINDENT +.SS Adding Some Depth .sp -The resulting tarball can then be placed alongside the files in the salt -fileserver and referenced using a \fBsalt://\fP URL. To create a container using -an image, use the \fBimage\fP parameter with \fI\%lxc.create\fP: +Obviously maintaining SLS formulas right in a single directory at the root of +the file server will not scale out to reasonably sized deployments. This is +why more depth is required. Start by making an nginx formula a better way, +make an nginx subdirectory and add an init.sls file: +.sp +\fB/srv/salt/nginx/init.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion lxc.create new\-cent6 image=salt://path/to/cent6.tar.gz +nginx: + pkg.installed: [] + service.running: + \- require: + \- pkg: nginx .ft P .fi .UNINDENT .UNINDENT .sp +A few concepts are introduced in this SLS formula. +.sp +First is the service statement which ensures that the \fBnginx\fP service is running. +.sp +Of course, the nginx service can\(aqt be started unless the package is installed \-\- +hence the \fBrequire\fP statement which sets up a dependency between the two. +.sp +The \fBrequire\fP statement makes sure that the required component is executed before +and that it results in success. +.sp \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -Making images of containers with LVM backing +The \fIrequire\fP option belongs to a family of options called \fIrequisites\fP\&. +Requisites are a powerful component of Salt States, for more information +on how requisites work and what is available see: +\fI\%Requisites\fP .sp -For containers with LVM backing, the rootfs is not mounted, so it is -necessary to mount it first before creating the tar archive. When a -container is created using LVM backing, an empty \fBrootfs\fP dir is handily -created within \fB/var/lib/lxc/container_name\fP, so this can be used as the -mountpoint. The location of the logical volume for the container will be -\fB/dev/vgname/lvname\fP, where \fBvgname\fP is the name of the volume group, -and \fBlvname\fP is the name of the logical volume. Therefore, assuming a -volume group of \fBvg1\fP, a logical volume of \fBlxc\-cent6\fP, and a container -name of \fBcent6\fP, the following commands can be used to create a tar -archive of the rootfs: +Also evaluation ordering is available in Salt as well: +\fI\%Ordering States\fP +.UNINDENT +.UNINDENT +.sp +This new sls formula has a special name \-\- \fBinit.sls\fP\&. When an SLS formula is +named \fBinit.sls\fP it inherits the name of the directory path that contains it. +This formula can be referenced via the following command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -mount /dev/vg1/lxc\-cent6 /var/lib/lxc/cent6/rootfs -tar czf cent6.tar.gz \-C /var/lib/lxc/cent6 rootfs -umount /var/lib/lxc/cent6/rootfs +salt \(aq*\(aq state.apply nginx .ft P .fi .UNINDENT .UNINDENT -.UNINDENT -.UNINDENT .sp -\fBWARNING:\fP +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -One caveat of using this method of container creation is that -\fB/etc/hosts\fP is left unmodified. This could cause confusion for some -distros if salt\-minion is later installed on the container, as the -functions that determine the hostname take \fB/etc/hosts\fP into account. -.sp -Additionally, when creating an rootfs image, be sure to remove -\fB/etc/salt/minion_id\fP and make sure that \fBid\fP is not defined in -\fB/etc/salt/minion\fP, as this will cause similar issues. +\fI\%state.apply\fP is just another remote +execution function, just like \fI\%test.version\fP +or \fI\%disk.usage\fP\&. It simply takes the +name of an SLS file as an argument. .UNINDENT .UNINDENT -.SS Initializing a New Container as a Salt Minion .sp -The above examples illustrate a few ways to create containers on the CLI, but -often it is desirable to also have the new container run as a Minion. To do -this, the \fI\%lxc.init\fP function can be used. This -function will do the following: -.INDENT 0.0 -.IP 1. 3 -Create a new container -.IP 2. 3 -Optionally set password and/or DNS -.IP 3. 3 -Bootstrap the minion (using either \fI\%salt\-bootstrap\fP or a custom command) -.UNINDENT +Now that subdirectories can be used, the \fBvim.sls\fP formula can be cleaned up. +To make things more flexible, move the \fBvim.sls\fP and vimrc into a new subdirectory +called \fBedit\fP and change the \fBvim.sls\fP file to reflect the change: .sp -By default, the new container will be pointed at the same Salt Master as the -host machine on which the container was created. It will then request to -authenticate with the Master like any other bootstrapped Minion, at which point -it can be accepted. +\fB/srv/salt/edit/vim.sls:\fP .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -salt myminion lxc.init test1 profile=centos -salt\-key \-a test1 +vim: + pkg.installed + +/etc/vimrc: + file.managed: + \- source: salt://edit/vimrc + \- mode: 644 + \- user: root + \- group: root .ft P .fi .UNINDENT .UNINDENT .sp -For even greater convenience, the \fI\%LXC runner\fP contains -a runner function of the same name (\fI\%lxc.init\fP), -which creates a keypair, seeds the new minion with it, and pre\-accepts the key, -allowing for the new Minion to be created and authorized in a single step: +Only the source path to the vimrc file has changed. Now the formula is +referenced as \fBedit.vim\fP because it resides in the edit subdirectory. +Now the edit subdirectory can contain formulas for emacs, nano, joe or any other +editor that may need to be deployed. +.SS Next Reading +.sp +Two walk\-throughs are specifically recommended at this point. First, a deeper +run through States, followed by an explanation of Pillar. .INDENT 0.0 -.INDENT 3.5 +.IP 1. 3 +\fI\%Starting States\fP +.IP 2. 3 +\fI\%Pillar Walkthrough\fP +.UNINDENT .sp -.nf -.ft C -salt\-run lxc.init test1 host=myminion profile=centos -.ft P -.fi +An understanding of Pillar is extremely helpful in using States. +.SS Getting Deeper Into States +.sp +Two more in\-depth States tutorials exist, which delve much more deeply into States +functionality. +.INDENT 0.0 +.IP 1. 3 +\fI\%How Do I Use Salt States?\fP, covers much +more to get off the ground with States. +.IP 2. 3 +The \fI\%States Tutorial\fP also provides a +fantastic introduction. .UNINDENT +.sp +These tutorials include much more in\-depth information including templating +SLS formulas etc. +.SS So Much More! +.sp +This concludes the initial Salt walk\-through, but there are many more things still +to learn! These documents will cover important core aspects of Salt: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Pillar\fP +.IP \(bu 2 +\fI\%Job Management\fP .UNINDENT -.SS Running Commands Within a Container .sp -For containers which are not running their own Minion, commands can be run -within the container in a manner similar to using (\fBcmd.run -