-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy path.gitlab-ci.yml
201 lines (181 loc) · 7.02 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
stages:
- test # Deploy to a DEV system for testing
- staging # Deploy to IRC/NLE (manually)
- production # Deploy to LIVE (manually)
- check # Run checks across all environments
variables:
# Our servers old git version does not support fetching, see https://stackoverflow.com/questions/56663096/gitlab-runner-doesnt-work-on-a-specific-project
GIT_STRATEGY: clone
# Shared Docker-group-writable storage paths for different environments:
DEV_STORAGE_PATH: /mnt/nfs/data/npld-access-stack
DLS_STORAGE_PATH: /home/axsadmin/service-storage
# -------------------------------------------------------------------------------------
# Deployment (run only on push events, not on schedules)
# -------------------------------------------------------------------------------------
# Job templates re-used across stages, so each job only specifies what's different
.deploy:
rules:
# Only run on push events, i.e. do not run during regularly scheduled tasks:
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
variables:
LOCKS_AUTH: $LOCKS_AUTH_DEV
STORAGE_PATH_SHARED: ${DLS_STORAGE_PATH}
STORAGE_PATH_ACL: ${STORAGE_PATH_SHARED}/wayback_excludes_update/ldukwa/acl
STORAGE_PATH_PROMETHEUS: ${STORAGE_PATH_SHARED}/prometheus-data
PYWB_IMAGE: ukwa/ukwa-pywb:2.6.9-LDA-3.2
EXTRA_CONFIG:
before_script:
# These credentials are only valid during the time the job runs, so deployment must wait for images to be fetched.
- echo $CI_DEPENDENCY_PROXY_PASSWORD | docker login --username $CI_DEPENDENCY_PROXY_USER --password-stdin $CI_DEPENDENCY_PROXY_SERVER
script:
# Make sure the access control lists are there:
- ./clone-or-pull-acls.sh
# Deploy the stack:
- ./deploy-rrwb.sh
# -------------------------------------------------------------------------------------
# Test Stage deployment in DEV environment
# -------------------------------------------------------------------------------------
Deploy DEV:
extends: .deploy
stage: test
environment:
name: dev
url: http://dev1.n45.wa.bl.uk:8209/
variables:
STORAGE_PATH_SHARED: ${DEV_STORAGE_PATH}
DLS_ACCESS_SERVER: http://staffaccess.dl.bl.uk
EXTRA_CONFIG: -c docker-compose.dev.yml -c docker-compose.irc.yml
UKWA_INDEX: http://cdxserver:8080/tc?url={url}&closest={closest}&sort=closest&filter=!statuscode:429&filter=!mimetype:warc/revisit
UKWA_ARCHIVE: webhdfs://warc-server:8000/by-filename/
#UKWA_INDEX: http://cdx.api.wa.bl.uk/data-heritrix?url={url}&closest={closest}&sort=closest&filter=!statuscode:429&filter=!mimetype:warc/revisit
#UKWA_ARCHIVE: webhdfs://warc-server.api.wa.bl.uk/by-filename/
tags:
- dev
# -------------------------------------------------------------------------------------
# Staging deployment in IRC environment
# -------------------------------------------------------------------------------------
Deploy IRC:
extends: .deploy
stage: staging
when: manual
needs:
# Explicitly state that this can only be done after DEV
- Deploy DEV
environment:
name: irc-bsp
url: https://blstaff-alpha.ldls.org.uk/
variables:
DLS_ACCESS_SERVER: http://192.168.217.72
# Point to internal/local test systems:
EXTRA_CONFIG: -c docker-compose.irc.yml
UKWA_INDEX: http://cdxserver:8080/tc?url={url}&closest={closest}&sort=closest&filter=!statuscode:429&filter=!mimetype:warc/revisit
UKWA_ARCHIVE: webhdfs://warc-server:8000/by-filename/
tags:
- irc-bsp-npld.dom.test
# -------------------------------------------------------------------------------------
# Production deployment in LIVE environment
# -------------------------------------------------------------------------------------
Deploy LIVE-BSP:
extends: .deploy
stage: production
when: manual # Only run manually
needs:
# Only allow if IRC has been deployed
- Deploy IRC
environment:
name: live-bsp
url: https://blstaff-beta.ldls.org.uk/
variables:
DLS_ACCESS_SERVER: http://192.168.218.72
# Proxied via DLS-BSP-WB02/192.168.218.86
UKWA_INDEX: http://192.168.218.86/wa/cdx?url={url}&closest={closest}&sort=closest&filter=!statuscode:429&filter=!mimetype:warc/revisit
UKWA_ARCHIVE: webhdfs://192.168.218.86
tags:
- dls-bsp-npld.dom.live
Deploy LIVE-LON:
extends: .deploy
stage: production
when: manual # Only run manually
needs:
# Only allow if IRC has been deployed
- Deploy IRC
environment:
name: live-lon
url: https://blstaff-beta.ldls.org.uk/
variables:
DLS_ACCESS_SERVER: http://192.168.219.72
# Proxied via DLS-LON-WB02/192.168.219.86
UKWA_INDEX: http://192.168.219.86/wa/cdx?url={url}&closest={closest}&sort=closest&filter=!statuscode:429&filter=!mimetype:warc/revisit
UKWA_ARCHIVE: webhdfs://192.168.219.86
tags:
- dls-lon-npld.dom.live
# -------------------------------------------------------------------------------------
# Updates, checks and metrics
# To be run as scheduled tasks, not on code changes
# -------------------------------------------------------------------------------------
#
# Template job, to iterate over all deployments:
#
.environments:
stage: check
when: manual
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
when: always
parallel:
matrix:
- ENV: dev
RUNNER: dev
STORAGE_PATH_SHARED: ${DEV_STORAGE_PATH}
PUSH_GATEWAY: http://monitor-pushgateway.dapi.wa.bl.uk
- ENV: irc-bsp
RUNNER: irc-bsp-npld.dom.test
STORAGE_PATH_SHARED: ${DLS_STORAGE_PATH}
PUSH_GATEWAY: http://monitor-pushgateway.api.wa.bl.uk
- ENV: live-bsp
RUNNER: dls-bsp-npld.dom.live
STORAGE_PATH_SHARED: ${DLS_STORAGE_PATH}
PUSH_GATEWAY: http://monitor-pushgateway.api.wa.bl.uk
- ENV: live-lon
RUNNER: dls-lon-npld.dom.live
STORAGE_PATH_SHARED: ${DLS_STORAGE_PATH}
PUSH_GATEWAY: http://monitor-pushgateway.api.wa.bl.uk
tags:
- $RUNNER
# Don't specify 'environment' as that is intended to indicate deployment.
variables:
GIT_STRATEGY: none # Avoid trashing the checked-out directory (referenced by the deployment):
#
# Update ACLs:
#
Update ACLs:
extends: .environments
script:
- ./clone-or-pull-acls.sh
#
# This job gets the Prometheus metrics as a local file and stores it as an artefact:
#
Get Metrics 1/2:
extends: .environments
script:
- curl -o metrics.${ENV}.prom 'http://127.0.0.1:3903/metrics'
# Can't use the Federation endpoint like this as it uses timestamps:
#- curl -o metrics.${ENV}.prom 'http://127.0.0.1:3990/federate?match\[\]=\{job!=""\}'
artifacts:
paths:
- metrics.$ENV.prom
expire_in: 1 day
#
# This picks up the artifacts and pushes them to the appropriate Push Gateway:
#
Get Metrics 2/2:
extends: .environments
needs:
- job: Get Metrics 1/2
artifacts: true
script:
# Send the metrics to the Push Gateway: https://github.com/prometheus/pushgateway#command-line
- curl --data-binary @metrics.${ENV}.prom ${PUSH_GATEWAY}/metrics/job/access-rrwb/instance/${ENV}
tags:
# Run this on the BETA Swarm:
- beta