Skip to content

Commit 08c975e

Browse files
cc3630cc3630
committed
fix: virtual host url
1 parent 5d6a83f commit 08c975e

File tree

7 files changed

+202
-40
lines changed

7 files changed

+202
-40
lines changed

‎README.md

+15-4
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
# 使用文件备份
2727
MONGO_URI=mongodb://username:password@192.168.1.20 BACKUP_PATH=./backup FILE_PREFIX=tmp python docker/backup.py
2828
# 启用 s3 备份
29-
MONGO_URI=mongodb://username:password@192.168.1.20 BACKUP_PATH=./backup FILE_PREFIX=tmp S3_ENABLE=1 S3_EP="https://minio-api.36node.com" S3_ACCESS_KEY="xxxx" S3_ACCESS_SECRET="xxxx" S3_BUCKET="test" S3_PREFIX="prefix" python docker/backup.py
29+
MONGO_URI=mongodb://username:password@192.168.1.20 BACKUP_PATH=./backup FILE_PREFIX=tmp S3_ENABLE=true S3_EP="https://minio-api.36node.com" S3_ACCESS_KEY="xxxx" S3_ACCESS_SECRET="xxxx" S3_BUCKET="test" S3_PREFIX="prefix" python docker/backup.py
3030

3131
# 使用文件恢复
3232
MONGO_URI=mongodb://username:password@192.168.1.21 BACKUP_PATH=./backup python docker/restore.py
@@ -116,14 +116,21 @@ kubectl -n mongodb-backup get pod
116116
kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
117117
```
118118

119-
#### 存储
119+
### 存储
120120

121121
支持 挂载磁盘 或 PVC，容器内的挂载路径默认为 `/backup`
122122

123123
- 本地磁盘，指定 nodeSelector 及 hostPath
124124
- PVC，指定 existingClaim
125125
- comming feature 支持 storage_class
126126

127+
### 关于 S3 endpoint 的说明
128+
129+
S3 支持使用虚拟域名作为endpoint，即可以将 region 或者 bucket 放入域名中使用，不同的 S3 配置会略有不同，需进行测试。其他配置，可参考[boto3文档](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html)
130+
131+
- 36node 自建 minio，使用默认配置即可
132+
- xxx.aliyuncs.com，需要设置 S3_EP_VIRTUAL 为 true
133+
127134
## 环境变量说明
128135

129136
### backup
@@ -135,16 +142,18 @@ kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
135142
- MONGO_EXCLUDE_COLLECTIONS: 选填，忽略的集合名称，支持多个，例如 test1,test2，若不为空，则需保证 MONGO_DB 也存在，且若 MONGO_COLLECTION 不为空，则忽略该参数
136143
- BACKUP_PWD: 选填，加密密码，备份文件可用 zip 加密
137144

138-
- S3_ENABLE: 选填，是否启用 S3 存储备份，不为空值即认为启用，例如 1 视为启用 S3
145+
- S3_ENABLE: 选填，是否启用 S3 存储备份，true 表示启用
139146
- S3_EP: 选填，S3 url，例如 https://minio-api.36node.com
147+
- S3_EP_VIRTUAL: 选填，是否启用虚拟 host url，true 表示启用
140148
- S3_ACCESS_KEY: 选填，S3 access key
141149
- S3_ACCESS_SECRET: 选填，S3 access secret
150+
- S3_REGION: 选填，地区名
142151
- S3_BUCKET: 选填，要存储的桶名
143152
- S3_PREFIX: 选填，要存储的前缀
144153

145154
### restore
146155

147-
- RESTORE_FROM_S3: 选填，是否从 S3 中进行恢复，不为空值即认为启用，例如 1 视为从 S3 中进行恢复
156+
- S3_ENABLE: 选填，是否从 S3 中进行恢复，true 表示启用
148157

149158
同 backup 的变量
150159

@@ -154,8 +163,10 @@ kubectl -n mongodb-backup exec -it restore-xxx-xxx -- python3 /app/restore.py
154163
- BACKUP_PWD
155164

156165
- S3_EP
166+
- S3_EP_VIRTUAL
157167
- S3_ACCESS_KEY
158168
- S3_ACCESS_SECRET
169+
- S3_REGION
159170
- S3_BUCKET
160171
- S3_PREFIX
161172

‎docker/backup.py

+54-11
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import subprocess
66
from urllib.parse import urlparse
77
import boto3
8+
from botocore.client import Config
89

910
# 数据库清理备份脚本
1011
# 1. 按要求备份数据，并保存到指定路径
@@ -25,11 +26,13 @@
2526
"MONGO_COLLECTION",
2627
"MONGO_EXCLUDE_COLLECTIONS",
2728
"BACKUP_PWD",
28-
29+
# S3 CONFIG
2930
"S3_ENABLE",
3031
"S3_EP",
32+
"S3_EP_VIRTUAL",
3133
"S3_ACCESS_KEY",
3234
"S3_ACCESS_SECRET",
35+
"S3_REGION",
3336
"S3_BUCKET",
3437
"S3_PREFIX",
3538
]
@@ -46,25 +49,45 @@ def check_var(key):
4649
return False
4750

4851

52+
def check_bool(key):
53+
if os.environ[key].lower() == "true":
54+
return True
55+
return False
56+
57+
4958
# 必填
5059
uri = os.environ["MONGO_URI"]
5160

5261
# 选填
53-
file_prefix = os.environ["FILE_PREFIX"] if check_var("FILE_PREFIX") else DEFAULT_FILE_PREFIX
54-
backup_path = os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
55-
backup_save_nums = int(os.environ["BACKUP_SAVE_NUMS"]) if check_var("BACKUP_SAVE_NUMS") else DEFAULT_BACKUP_SAVE_NUMS
62+
file_prefix = (
63+
os.environ["FILE_PREFIX"] if check_var("FILE_PREFIX") else DEFAULT_FILE_PREFIX
64+
)
65+
backup_path = (
66+
os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
67+
)
68+
backup_save_nums = (
69+
int(os.environ["BACKUP_SAVE_NUMS"])
70+
if check_var("BACKUP_SAVE_NUMS")
71+
else DEFAULT_BACKUP_SAVE_NUMS
72+
)
5673
collection = os.environ["MONGO_COLLECTION"] if check_var("MONGO_COLLECTION") else None
5774
excludeCollections = (
58-
os.environ["MONGO_EXCLUDE_COLLECTIONS"].split(",") if check_var("MONGO_EXCLUDE_COLLECTIONS") else None
75+
os.environ["MONGO_EXCLUDE_COLLECTIONS"].split(",")
76+
if check_var("MONGO_EXCLUDE_COLLECTIONS")
77+
else None
5978
)
6079
backup_pwd = os.environ["BACKUP_PWD"] if check_var("BACKUP_PWD") else None
6180

62-
s3_enable = True if check_var("S3_ENABLE") else False
81+
s3_enable = check_bool("S3_ENABLE") if check_var("S3_ENABLE") else False
6382
s3_ep = os.environ["S3_EP"] if check_var("S3_EP") else None
83+
s3_ep_virtual = check_bool("S3_EP_VIRTUAL") if check_var("S3_EP_VIRTUAL") else False
6484
s3_access_key = os.environ["S3_ACCESS_KEY"] if check_var("S3_ACCESS_KEY") else None
65-
s3_access_secret = os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
85+
s3_access_secret = (
86+
os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
87+
)
6688
s3_bucket = os.environ["S3_BUCKET"] if check_var("S3_BUCKET") else None
6789
s3_prefix = os.environ["S3_PREFIX"] if check_var("S3_PREFIX") else DEFAULT_S3_PREFIX
90+
s3_region = os.environ["S3_REGION"] if check_var("S3_REGION") else None
6891

6992
# 计算当前日期，按照 年月日时分 格式
7093
date = (datetime.utcnow() + timedelta(hours=8)).strftime("%Y%m%d%H%M%S")
@@ -149,30 +172,46 @@ def backup_file(prefix):
149172
subprocess.call(f"zip -e {source}.crypt -P {backup_pwd} {source}", shell=True)
150173
os.remove(source)
151174

175+
152176
def upload_s3(prefix):
177+
# config
178+
config_s3 = {}
179+
if s3_ep_virtual:
180+
config_s3["addressing_style"] = "virtual"
181+
182+
if s3_region:
183+
config = Config(s3=config_s3, region_name=s3_region)
184+
else:
185+
config = Config(s3=config_s3)
186+
153187
client = boto3.client(
154188
"s3",
155189
endpoint_url=s3_ep,
156190
aws_access_key_id=s3_access_key,
157191
aws_secret_access_key=s3_access_secret,
192+
config=config,
158193
)
159194

160195
# 上传备份文件
161-
file_name = f"{prefix}{date}.tar.gz.crypt" if backup_pwd else f"{prefix}{date}.tar.gz"
196+
file_name = (
197+
f"{prefix}{date}.tar.gz.crypt" if backup_pwd else f"{prefix}{date}.tar.gz"
198+
)
162199
upload_path = f"{s3_prefix}/{file_name}" if s3_prefix else file_name
163200
client.upload_file(f"{backup_path}/{file_name}", s3_bucket, upload_path)
164201

165202
# 清理 S3 上的多余备份文件
166203
list_prefix = f"{s3_prefix}/" if s3_prefix else ""
167-
resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter='/')
204+
resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter="/")
168205
if "Contents" in resp:
169206
objects = resp["Contents"]
170207

171208
file_prefix = f"{list_prefix}{prefix}"
172209
# 构造正则表达式
173210
regex_pattern = f"^{re.escape(file_prefix)}(\\d{{14}})\\.tar\\.gz(\\.crypt)?$"
174211
compiled_regex = re.compile(regex_pattern)
175-
keys = [object["Key"] for object in objects if compiled_regex.match(object["Key"])]
212+
keys = [
213+
object["Key"] for object in objects if compiled_regex.match(object["Key"])
214+
]
176215

177216
# 根据文件名排序（这使得最新的备份文件位于列表的末尾）
178217
keys.sort()
@@ -182,9 +221,13 @@ def upload_s3(prefix):
182221

183222
# 删除旧的备份文件
184223
if keys_to_remove:
185-
client.delete_objects(Bucket=s3_bucket, Delete={"Objects": [{"Key": key} for key in keys_to_remove]})
224+
client.delete_objects(
225+
Bucket=s3_bucket,
226+
Delete={"Objects": [{"Key": key} for key in keys_to_remove]},
227+
)
186228
print(f"Deleted s3 old backup files: {keys_to_remove}")
187229

230+
188231
try:
189232
if not os.path.exists(backup_path):
190233
os.makedirs(backup_path)

‎docker/restore.py

+44-13
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from datetime import datetime, timedelta
55
import subprocess
66
import boto3
7+
from botocore.client import Config
78

89
# 数据库还原
910

@@ -16,13 +17,15 @@
1617
"FILE_PREFIX",
1718
"BACKUP_PATH",
1819
"BACKUP_PWD",
19-
20-
"RESTORE_FROM_S3",
20+
# S3 CONFIG
21+
"S3_ENABLE",
2122
"S3_EP",
23+
"S3_EP_VIRTUAL",
2224
"S3_ACCESS_KEY",
2325
"S3_ACCESS_SECRET",
2426
"S3_BUCKET",
2527
"S3_PREFIX",
28+
"S3_REGION",
2629
]
2730

2831
for key in must_inputs:
@@ -37,19 +40,31 @@ def check_var(key):
3740
return False
3841

3942

43+
def check_bool(key):
44+
if os.environ[key].lower() == "true":
45+
return True
46+
return False
47+
48+
4049
# 必填
4150
uri = os.environ["MONGO_URI"]
4251

4352
# 选填
44-
backup_path = os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
53+
backup_path = (
54+
os.environ["BACKUP_PATH"] if check_var("BACKUP_PATH") else DEFAULT_BACKUP_PATH
55+
)
4556
backup_pwd = os.environ["BACKUP_PWD"] if check_var("BACKUP_PWD") else None
4657

47-
restore_from_s3 = True if check_var("RESTORE_FROM_S3") else False
58+
s3_enable = check_bool("S3_ENABLE") if check_var("S3_ENABLE") else False
4859
s3_ep = os.environ["S3_EP"] if check_var("S3_EP") else None
60+
s3_ep_virtual = check_bool("S3_EP_VIRTUAL") if check_var("S3_EP_VIRTUAL") else False
4961
s3_access_key = os.environ["S3_ACCESS_KEY"] if check_var("S3_ACCESS_KEY") else None
50-
s3_access_secret = os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
62+
s3_access_secret = (
63+
os.environ["S3_ACCESS_SECRET"] if check_var("S3_ACCESS_SECRET") else None
64+
)
5165
s3_bucket = os.environ["S3_BUCKET"] if check_var("S3_BUCKET") else None
5266
s3_prefix = os.environ["S3_PREFIX"] if check_var("S3_PREFIX") else DEFAULT_S3_PREFIX
67+
s3_region = os.environ["S3_REGION"] if check_var("S3_REGION") else None
5368

5469
date = (datetime.utcnow() + timedelta(hours=8)).strftime("%Y%m%d%H%M%S")
5570

@@ -65,26 +80,29 @@ def get_files(compiled_regex):
6580
matched_files.sort(reverse=True)
6681
return matched_files
6782

83+
6884
def get_keys_from_s3(client, compiled_regex):
6985
list_prefix = f"{s3_prefix}/" if s3_prefix else ""
70-
resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter='/')
86+
resp = client.list_objects_v2(Bucket=s3_bucket, Prefix=list_prefix, Delimiter="/")
7187
if "Contents" in resp:
7288
objects = resp["Contents"]
7389

74-
keys = [object["Key"] for object in objects if compiled_regex.match(object["Key"])]
90+
keys = [
91+
object["Key"] for object in objects if compiled_regex.match(object["Key"])
92+
]
7593

7694
if keys:
7795
# 根据文件名排序（这使得最新的备份文件位于列表的末尾）
7896
keys.sort(reverse=True)
7997
return keys
8098
raise Exception("没有可用的备份")
8199

100+
82101
def download_file(client, key):
83102
file_name = os.path.basename(key)
84103
save_path = f"/tmp/{file_name}"
85104
client.download_file(s3_bucket, key, save_path)
86105
return save_path
87-
88106

89107

90108
def restore_file(file_path):
@@ -99,7 +117,9 @@ def restore_file(file_path):
99117
# 先解密
100118
unzip_path = os.path.dirname(file_path)
101119
# -o 覆盖已有文件，-j 不保留文件夹
102-
subprocess.call(f"unzip -P {backup_pwd} -oj {file_path} -d {unzip_path}", shell=True)
120+
subprocess.call(
121+
f"unzip -P {backup_pwd} -oj {file_path} -d {unzip_path}", shell=True
122+
)
103123

104124
# 恢复数据
105125
cmd = f'mongorestore --uri="{uri}" --gzip --archive={restore_path}'
@@ -117,31 +137,42 @@ def restore_file(file_path):
117137

118138
client = None
119139
# 1. 获取备份文件列表
120-
if restore_from_s3:
140+
if s3_enable:
141+
# s3 config
142+
config_s3 = {}
143+
if s3_ep_virtual:
144+
config_s3["addressing_style"] = "virtual"
145+
146+
if s3_region:
147+
config = Config(s3=config_s3, region_name=s3_region)
148+
else:
149+
config = Config(s3=config_s3)
150+
121151
client = boto3.client(
122152
"s3",
123153
endpoint_url=s3_ep,
124154
aws_access_key_id=s3_access_key,
125155
aws_secret_access_key=s3_access_secret,
156+
config=config,
126157
)
127158
backup_files = get_keys_from_s3(client, compiled_regex)
128159
else:
129-
backup_files = get_files(compiled_regex)
160+
backup_files = get_files(compiled_regex)
130161

131162
input_cmd = "请选择要还原的备份文件：\n"
132163
for index, item in enumerate(backup_files):
133164
input_cmd += f"{index + 1}. {item}\n"
134165
value = input(input_cmd)
135166

136-
if restore_from_s3:
167+
if s3_enable:
137168
# 需要下载文件
138169
file = download_file(client, backup_files[int(value) - 1])
139170
else:
140171
file = f"{backup_path}/{backup_files[int(value) - 1]}"
141172

142173
restore_file(file)
143174

144-
if restore_from_s3:
175+
if s3_enable:
145176
# 删除临时文件
146177
os.remove(file)
147178

‎helm-chart/templates/backup.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,19 @@ spec:
3535
value: "true"
3636
- name: "S3_EP"
3737
value: {{ index .Values.storage.s3 "endpoint" | quote }}
38+
- name: "S3_EP_VIRTUAL"
39+
value: {{ index .Values.storage.s3 "endpoint_virtual" | quote }}
3840
- name: "S3_ACCESS_KEY"
3941
value: {{ index .Values.storage.s3 "access_key" | quote }}
4042
- name: "S3_ACCESS_SECRET"
4143
value: {{ index .Values.storage.s3 "access_secret" | quote }}
44+
- name: "S3_REGION"
45+
value: {{ index .Values.storage.s3 "region" | quote }}
4246
- name: "S3_BUCKET"
4347
value: {{ index .Values.storage.s3 "bucket" | quote }}
4448
- name: "S3_PREFIX"
4549
value: {{ index .Values.storage.s3 "prefix" | quote }}
50+
4651
{{- end }}
4752
{{- range $key, $value := .Values.backup.env }}
4853
- name: "{{ $key }}"

‎helm-chart/templates/restore.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,14 @@ spec:
3535
value: "true"
3636
- name: "S3_EP"
3737
value: {{ index .Values.storage.s3 "endpoint" | quote }}
38+
- name: "S3_EP_VIRTUAL"
39+
value: {{ index .Values.storage.s3 "endpoint_virtual" | quote }}
3840
- name: "S3_ACCESS_KEY"
3941
value: {{ index .Values.storage.s3 "access_key" | quote }}
4042
- name: "S3_ACCESS_SECRET"
4143
value: {{ index .Values.storage.s3 "access_secret" | quote }}
44+
- name: "S3_REGION"
45+
value: {{ index .Values.storage.s3 "region" | quote }}
4246
- name: "S3_BUCKET"
4347
value: {{ index .Values.storage.s3 "bucket" | quote }}
4448
- name: "S3_PREFIX"

0 commit comments

Comments
 (0)