-
Notifications
You must be signed in to change notification settings - Fork 46
/
Copy pathbackupninja-backup-pipeline.groovy
225 lines (220 loc) · 12.6 KB
/
backupninja-backup-pipeline.groovy
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
def backupSaltMasterAndMaas = (env.getProperty('BACKUP_SALTMASTER_AND_MAAS') ?: true).toBoolean()
def backupDogtag = (env.getProperty('BACKUP_DOGTAG') ?: true).toBoolean()
def backupKeystone = (env.getProperty('BACKUP_KEYSTONE_CREDENTIAL_KEYS') ?: true).toBoolean()
def saltMasterTargetMatcher = "I@backupninja:client and I@salt:master"
def dogtagTagetMatcher = "I@backupninja:client and I@dogtag:server"
def keystoneTargetMatcher = "I@backupninja:client and I@keystone:server"
logBackupSuccess = []
logBackupFailure = []
def checkBackupninjaLog(output, backupName='', terminateOnFailure=true) {
def common = new com.mirantis.mk.Common()
def outputPattern = java.util.regex.Pattern.compile("\\d+")
def outputMatcher = outputPattern.matcher(output)
if (outputMatcher.find()) {
try {
result = outputMatcher.getAt([0, 1, 2, 3])
if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
common.successMsg("[${backupName}] - Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
logBackupSuccess.add(backupName)
} else {
common.errorMsg("[${backupName}] - Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
logBackupFailure.add(backupName)
}
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("[${backupName}] - Backupninja log parsing failed.")
logBackupFailure.add(backupName)
}
}
}
timeout(time: 12, unit: 'HOURS') {
node() {
def saltMasterBackupNode = ''
def dogtagBackupNode = ''
def keystoneBackupNode = ''
def backupServer = ''
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Verify pillar for backups') {
if (backupSaltMasterAndMaas) {
try {
def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data').get('return')[0].values()[0]
if (masterPillar.isEmpty()) {
throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
}
def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data').get('return')[0].values()[0]
if (minionPillar.isEmpty()) {
throw new Exception("Problem with salt-minion pillar on I@salt:master node.")
}
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
throw e
}
}
if (backupDogtag) {
def barbicanBackendPresent = salt.getPillar(pepperEnv, "I@salt:master", "_param:barbican_backend").get('return')[0].values()[0]
if (barbicanBackendPresent == 'dogtag') {
try {
def dogtagPillar = salt.getPillar(pepperEnv, "I@dogtag:server", "dogtag:server").get('return')[0].values()[0]
if (dogtagPillar.isEmpty()) {
throw new Exception("Problem with dogtag pillar on I@dogtag:server node.")
}
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Looks like dogtag pillar is not defined. Fix your pillar or disable dogtag backup by setting the BACKUP_DOGTAG parameter to False if you're using different barbican backend.")
throw e
}
} else {
backupDogtag = false
common.warningMsg('Backup for Dogtag is enabled, but service itself is not present. Skipping...')
}
}
}
stage('Check backup location') {
if (backupSaltMasterAndMaas) {
try {
saltMasterBackupNode = salt.getMinionsSorted(pepperEnv, saltMasterTargetMatcher)[0]
salt.minionsReachable(pepperEnv, "I@salt:master", saltMasterBackupNode)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar on Salt master node or the minion is not reachable")
currentBuild.result = "FAILURE"
throw e
}
def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
if (!maasNodes.isEmpty()) {
def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
if (! postgresqlMajorVersion) {
common.errorMsg("Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar?")
if (askConfirmation) {
input message: "Confirm to proceed anyway."
}
} else {
def postgresqlClientPackages = "postgresql-client-${postgresqlMajorVersion}"
try {
if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': saltMasterBackupNode, 'packageName': postgresqlClientPackages, 'output': false])) {
if (askConfirmation) {
input message: "Do you want to install ${postgresqlClientPackages} package on targeted nodes: ${saltMasterBackupNode}? It's required to make backup. Click to confirm."
} else {
common.infoMsg("Package ${postgresqlClientPackages} will be installed. It's required to make backup.")
}
// update also common fake package
salt.runSaltProcessStep(pepperEnv, saltMasterBackupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackages}"])
}
} catch (Exception e) {
common.errorMsg("Unable to determine status of ${postgresqlClientPackages} packages on target nodes: ${saltMasterBackupNode}.")
if (askConfirmation) {
input message: "Do you want to continue? Click to confirm"
}
}
}
}
}
if (backupDogtag) {
try {
dogtagBackupNode = salt.getMinionsSorted(pepperEnv, dogtagTagetMatcher)[0]
salt.minionsReachable(pepperEnv, "I@salt:master", dogtagBackupNode)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and dogtag:server pillars defined or the minion is not reachable")
currentBuild.result = "FAILURE"
throw e
}
}
try {
backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
currentBuild.result = "FAILURE"
throw e
}
if (backupKeystone) {
try {
keystoneBackupNode = salt.getMinionsSorted(pepperEnv, keystoneTargetMatcher)[0]
salt.minionsReachable(pepperEnv, "I@salt:master", keystoneBackupNode)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and keystone:server pillars defined or the minion is not reachable")
currentBuild.result = "FAILURE"
throw e
}
}
try {
backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
currentBuild.result = "FAILURE"
throw e
}
}
stage('Prepare for backup') {
if (backupSaltMasterAndMaas) {
salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
salt.enforceState(['saltId': pepperEnv, 'target': saltMasterTargetMatcher, 'state': 'backupninja'])
def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:master:initial_data:source'))
def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:minion:initial_data:source'))
// TODO: Remove ssh-keyscan once we have openssh meta for backupninja implemented
[backupServer, backupMasterSource, backupMinionSource].unique().each {
salt.cmdRun(pepperEnv, saltMasterBackupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
}
def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
if (!maasNodes.isEmpty()) {
common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && ' +
'getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt &&' +
'getfacl -pR /etc/maas/ > /etc/maas/file_permissions.txt || true')
}
}
if (backupDogtag) {
salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
salt.enforceState(['saltId': pepperEnv, 'target': dogtagTagetMatcher, 'state': 'backupninja'])
}
if (backupKeystone) {
salt.enforceState(['saltId': pepperEnv, 'target': keystoneTargetMatcher, 'state': 'backupninja'])
salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
}
}
stage('Backup') {
if (backupSaltMasterAndMaas) {
def output = salt.getReturnValues(salt.cmdRun(pepperEnv, saltMasterBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
checkBackupninjaLog(output, "Salt Master/MAAS")
}
if (backupDogtag) {
def output = salt.getReturnValues(salt.cmdRun(pepperEnv, dogtagBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
checkBackupninjaLog(output, "Dogtag")
}
if (backupKeystone) {
def output = salt.getReturnValues(salt.cmdRun(pepperEnv, keystoneBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
checkBackupninjaLog(output, "Keystone")
}
}
stage('Results') {
if (logBackupSuccess.size() > 0) {
common.infoMsg("Following backups finished successfully: ${logBackupSuccess.join(",")}")
}
if (logBackupFailure.size() > 0) {
common.errorMsg("Following backups has failed: ${logBackupFailure.join(",")}. Make sure to check the logs.")
currentBuild.result = "FAILURE"
}
}
}
}