-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
167 lines (146 loc) · 5.25 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
import os
import telegram
from telegram.constants import ParseMode
import datetime as dt
import traceback
import custom_logging
import asyncio
import helpers
import docker
import loader
import workflows
# Constants
curr_path = os.path.dirname(__file__)
# Default backup settings
default_config = {
"min_backup_interval": 0,
"ghost_backup_keep_days": float("inf"), # mapped to -1
"backup_keep_num": float("inf"), # mapped to -1
"warn_large_backup_mb": 1024,
"backup_by_default": True,
"container_paths": {},
"archive_dir_path": curr_path,
"telegram_chat_id": None,
"telegram_bot_token": None,
}
async def main(df, bot):
total_prune_size = 0
total_backup_size = 0
# Conduct disk usage check and send start message on telegram
free_percent, free_str = helpers.check_available_space(backup_dir_path)
message = (
f"Starting backup script...\n"
f"Remaining disk space: {free_percent}%, {free_str}"
)
logger.info(message)
if bot:
await bot.send_message(
text=message,
chat_id=config["telegram_chat_id"],
parse_mode=ParseMode.HTML,
)
# Get docker client
docker_client = None
try:
docker_client = docker.from_env().api
logger.info("Docker client loaded!")
except:
logger.error("Cannot access docker service!")
logger.error(traceback.format_exc())
# If docker client is available, update information of all tracked instances
if docker_client:
try:
df = workflows.update_instances(
df.copy(),
docker_client,
config["container_paths"],
config["backup_by_default"]
)
except:
logger.error(
"Error while updating instances information! Stopping other actions for safety reasons..."
)
logger.error(traceback.format_exc())
df = None
# Prune backups of instances that are marked as deleted
try:
df, prune_size = workflows.prune_ghost_backups(
df.copy(), backup_dir_path, config["ghost_backup_keep_days"]
)
total_prune_size += prune_size
except:
logger.warning("Error while pruning ghost backups!")
logger.warning(traceback.format_exc())
# If docker client is available, backup tracked instances (that are not marked as deleted)
if docker_client:
# Find the instances that require backup, prune extra backups for each instance if necessary, then create backups
try:
df, prune_size, total_backup_size = (
workflows.prune_extra_and_create_backups(
df.copy(),
docker_client,
backup_dir_path,
config["min_backup_interval"],
config["backup_keep_num"],
config["warn_large_backup_mb"]
)
)
total_prune_size += prune_size
except:
logger.error("Error either while pruning extra or making backups!")
logger.error(traceback.format_exc())
# Save instance information to disk
df.to_csv(instance_info_path, index=False)
# Compile alert messages
error_message = []
if len(logLevelCountHandler.level_messages["CRITICAL"]) > 0:
error_message.append(
(
f"Critical error(s) encountered!\n"
f"{helpers.format_messages(logLevelCountHandler.level_messages['CRITICAL'])}"
)
)
if len(logLevelCountHandler.level_messages["ERROR"]) > 0:
error_message.append(
(
f"Error(s) encountered!\n"
f"{helpers.format_messages(logLevelCountHandler.level_messages['ERROR'])}"
)
)
if len(logLevelCountHandler.level_messages["WARNING"]) > 0:
error_message.append(
(
f"Warning(s) encountered!\n"
f"{helpers.format_messages(logLevelCountHandler.level_messages['WARNING'])}"
)
)
error_message = "\n\n".join(error_message)
# print(error_message)
# Send completion message on telegram
free_percent, free_str = helpers.check_available_space(backup_dir_path)
final_str = (
f"{error_message}\n\n"
f"Pruned: {helpers.convert_bytes_to_readable(total_prune_size)}, Created: {helpers.convert_bytes_to_readable(total_backup_size)}\n"
f"Remaining disk space: {free_percent}%, {free_str}"
)
if bot:
await bot.send_message(
text=final_str,
chat_id=config["telegram_chat_id"],
parse_mode=ParseMode.HTML,
)
async def main_bot_wrapped(df, bot):
async with bot:
await main(df, bot)
if __name__ == "__main__":
config, instance_info_path, df, backup_dir_path = loader.load_and_parse_args(
default_config, curr_path
)
logger, logLevelCountHandler = custom_logging.get_logger(__name__)
if config["telegram_chat_id"] and config["telegram_bot_token"]:
bot = telegram.Bot(config["telegram_bot_token"])
logger.info("Connected to Telegram!")
asyncio.run(main_bot_wrapped(df, bot))
else:
asyncio.run(main(df, None))
logger.info("Gracefully exiting...")