+ /bin/clickhouse-backup server 2024/04/01 12:16:12.131786 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:12.132927 error clickhouse connection ping: tcp://localhost:9000 return error: dial tcp [::1]:9000: connect: connection refused logger=clickhouse 2024/04/01 12:16:12.132951 error dial tcp [::1]:9000: connect: connection refused logger=server.Run 2024/04/01 12:16:17.134803 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:17.135314 error clickhouse connection ping: tcp://localhost:9000 return error: dial tcp [::1]:9000: connect: connection refused logger=clickhouse 2024/04/01 12:16:17.135336 error dial tcp [::1]:9000: connect: connection refused logger=server.Run 2024/04/01 12:16:22.137021 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:22.138350 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:16:22.138438 info Create integration tables logger=server 2024/04/01 12:16:22.138467 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:22.139458 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:16:22.140234 info SELECT value FROM `system`.`build_options` where name='VERSION_INTEGER' logger=clickhouse 2024/04/01 12:16:22.142739 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:16:22.145536 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:16:22.149412 info SELECT engine FROM system.databases WHERE name = 'system' logger=clickhouse 2024/04/01 12:16:22.150569 info DROP TABLE IF EXISTS `system`.`backup_actions` NO DELAY logger=clickhouse 2024/04/01 12:16:22.151405 info CREATE TABLE system.backup_actions (command String, start DateTime, finish DateTime, status String, error String) ENGINE=URL('http://127.0.0.1:7171/backup/actions', JSONEachRow) SETTINGS input_format_skip_unknown_fields=1 logger=clickhouse 2024/04/01 12:16:22.637015 info SELECT engine FROM system.databases WHERE name = 'system' logger=clickhouse 2024/04/01 12:16:22.638979 info DROP TABLE IF EXISTS `system`.`backup_list` NO DELAY logger=clickhouse 2024/04/01 12:16:22.639976 info CREATE TABLE system.backup_list (name String, created DateTime, size Int64, location String, required String, desc String) ENGINE=URL('http://127.0.0.1:7171/backup/list', JSONEachRow) SETTINGS input_format_skip_unknown_fields=1 logger=clickhouse 2024/04/01 12:16:22.687314 info clickhouse connection closed logger=clickhouse 2024/04/01 12:16:22.687812 info Starting API server on 0.0.0.0:7171 logger=server.Run 2024/04/01 12:16:22.689537 info Update backup metrics start (onlyLocal=false) logger=server 2024/04/01 12:16:22.689629 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:22.689637 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:22.690706 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:16:22.690722 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:16:22.690752 info SELECT value FROM `system`.`build_options` where name='VERSION_INTEGER' logger=clickhouse 2024/04/01 12:16:22.690729 info SELECT value FROM `system`.`build_options` where name='VERSION_INTEGER' logger=clickhouse 2024/04/01 12:16:22.692831 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:16:22.692851 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:16:22.695691 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:16:22.695838 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:16:22.699251 error ResumeOperationsAfterRestart return error: open /var/lib/clickhouse/backup: no such file or directory logger=server.Run 2024/04/01 12:16:22.699785 info clickhouse connection closed logger=clickhouse 2024/04/01 12:16:22.699846 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:16:22.701285 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:16:22.701344 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:16:22.703589 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:16:22.704757 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:16:22.706873 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:16:22.713927 debug /tmp/.clickhouse-backup-metadata.cache.S3 not found, load 0 elements logger=s3 2024/04/01 12:16:22.716232 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C22715AE313138, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:16:22.716716 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:16:22.716818 info clickhouse connection closed logger=clickhouse 2024/04/01 12:16:22.716850 info Update backup metrics finish LastBackupCreateLocal= LastBackupCreateRemote= LastBackupSizeLocal=0 LastBackupSizeRemote=0 LastBackupUpload= NumberBackupsLocal=0 NumberBackupsRemote=0 duration=27ms logger=server 2024/04/01 12:16:27.852887 debug API call GET /metrics logger=server 2024/04/01 12:16:51.938846 debug API call GET /metrics logger=server 2024/04/01 12:17:16.987579 debug API call GET /metrics logger=server 2024/04/01 12:17:46.987626 debug API call GET /metrics logger=server 2024/04/01 12:18:16.993411 debug API call GET /metrics logger=server 2024/04/01 12:18:46.988806 debug API call GET /metrics logger=server 2024/04/01 12:19:16.987959 debug API call GET /metrics logger=server 2024/04/01 12:19:46.988608 debug API call GET /metrics logger=server 2024/04/01 12:19:53.119047 info API call POST /backup/watch logger=server 2024/04/01 12:19:53.120323 debug api.status.Start -> status.commands[0] == {ActionRowStatus:{Command:watch --watch-interval="8h" --full-interval="24h" Status:in progress Start:2024-04-01 12:19:53 Finish: Error:} Ctx:context.Background.WithCancel Cancel:0x4a7ba0} logger=status 2024/04/01 12:19:53.120755 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:19:53.122097 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:19:53.122146 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:53.125334 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:53.126369 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:53.128269 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:53.134713 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:19:53.136846 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C22746AC361F67, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:19:53.137038 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:19:53.137199 info clickhouse connection closed logger=clickhouse 2024/04/01 12:19:53.137218 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:53.140081 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:53.141160 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:19:53.141902 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:19:53.142632 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:53.145390 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:53.146500 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:19:53.147194 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:19:53.147247 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:19:53.149158 info SELECT name, engine FROM system.databases WHERE NOT match(name,'^(system|INFORMATION_SCHEMA|information_schema|_temporary_and_external_tables)$') logger=clickhouse 2024/04/01 12:19:53.151509 info SHOW CREATE DATABASE `_internal_ddl_health_check_` logger=clickhouse 2024/04/01 12:19:53.152380 info SHOW CREATE DATABASE `alert` logger=clickhouse 2024/04/01 12:19:53.153101 info SHOW CREATE DATABASE `default` logger=clickhouse 2024/04/01 12:19:53.153830 info SHOW CREATE DATABASE `distributed_1` logger=clickhouse 2024/04/01 12:19:53.154482 info SHOW CREATE DATABASE `dtsmart` logger=clickhouse 2024/04/01 12:19:53.155118 info SHOW CREATE DATABASE `local_1` logger=clickhouse 2024/04/01 12:19:53.155837 info SELECT name, count(*) as is_present FROM system.settings WHERE name IN (?, ?) GROUP BY name with args [show_table_uuid_in_table_create_query_if_not_nil display_secrets_in_show_and_select] logger=clickhouse 2024/04/01 12:19:53.159548 info SELECT name FROM system.databases WHERE engine IN ('MySQL','PostgreSQL','MaterializedPostgreSQL') logger=clickhouse 2024/04/01 12:19:53.160791 info SELECT countIf(name='data_path') is_data_path_present, countIf(name='data_paths') is_data_paths_present, countIf(name='uuid') is_uuid_present, countIf(name='create_table_query') is_create_table_query_present, countIf(name='total_bytes') is_total_bytes_present FROM system.columns WHERE database='system' AND table='tables' logger=clickhouse 2024/04/01 12:19:53.164135 info SELECT database, name, engine , data_paths , uuid , create_table_query , coalesce(total_bytes, 0) AS total_bytes FROM system.tables WHERE is_temporary = 0 ORDER BY total_bytes DESC SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1 logger=clickhouse 2024/04/01 12:19:53.191318 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:19:53.194269 info SELECT count() as cnt FROM system.columns WHERE database='system' AND table='functions' AND name='create_query' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:53.196730 info SELECT name, create_query FROM system.functions WHERE create_query!='' logger=clickhouse 2024/04/01 12:19:53.201620 info SELECT value FROM `system`.`build_options` where name='VERSION_INTEGER' logger=clickhouse 2024/04/01 12:19:53.203122 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:19:53.244213 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:19:53.274862 info BACKUP TABLE `local_1`.`system_metrics`, TABLE `local_1`.`ddd`, TABLE `local_1`.`metrics_summary`, TABLE `local_1`.`arms_frontend_log`, TABLE `local_1`.`system_attribute_meta`, TABLE `local_1`.`easydo_collect_task_monitor`, TABLE `local_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`dt_chart`, TABLE `distributed_1`.`dt_collect_model`, TABLE `distributed_1`.`dt_collect_model_task`, TABLE `distributed_1`.`dt_dashboard`, TABLE `distributed_1`.`dt_data_index`, TABLE `distributed_1`.`dt_host`, TABLE `distributed_1`.`dt_inspect_record`, TABLE `distributed_1`.`dt_inspect_template`, TABLE `distributed_1`.`dt_resource`, TABLE `distributed_1`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`easydo_collect_task_monitor`, TABLE `distributed_1`.`metrics_summary`, TABLE `distributed_1`.`system_attribute_meta`, TABLE `distributed_1`.`system_metrics`, TABLE `dtsmart`.`dt_alert`, TABLE `dtsmart`.`dt_arms_app`, TABLE `dtsmart`.`dt_arms_service_group`, TABLE `dtsmart`.`dt_arms_service_group_app`, TABLE `dtsmart`.`dt_chart`, TABLE `dtsmart`.`dt_collect_model`, TABLE `dtsmart`.`dt_collect_model_task`, TABLE `dtsmart`.`dt_dashboard`, TABLE `dtsmart`.`dt_data_index`, TABLE `dtsmart`.`dt_host`, TABLE `dtsmart`.`dt_inspect_record`, TABLE `dtsmart`.`dt_inspect_template`, TABLE `dtsmart`.`dt_resource`, TABLE `dtsmart`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`dt_arms_app`, TABLE `distributed_1`.`dt_arms_service_group`, TABLE `distributed_1`.`dt_arms_service_group_app`, TABLE `distributed_1`.`dt_alert`, TABLE `distributed_1`.`ddd`, TABLE `distributed_1`.`arms_frontend_log` TO Disk(?,?) with args [backups_s3 shard0-full-20240401121953] logger=clickhouse 2024/04/01 12:19:53.978072 info SELECT sum(total_bytes) AS backup_data_size FROM system.tables WHERE concat(database,'.',name) IN ('local_1.system_metrics', 'local_1.ddd', 'local_1.metrics_summary', 'local_1.arms_frontend_log', 'local_1.system_attribute_meta', 'local_1.easydo_collect_task_monitor', 'local_1.easydo_agent_monitor', 'distributed_1.dt_chart', 'distributed_1.dt_collect_model', 'distributed_1.dt_collect_model_task', 'distributed_1.dt_dashboard', 'distributed_1.dt_data_index', 'distributed_1.dt_host', 'distributed_1.dt_inspect_record', 'distributed_1.dt_inspect_template', 'distributed_1.dt_resource', 'distributed_1.dt_resource_optimization_suggest', 'distributed_1.easydo_agent_monitor', 'distributed_1.easydo_collect_task_monitor', 'distributed_1.metrics_summary', 'distributed_1.system_attribute_meta', 'distributed_1.system_metrics', 'dtsmart.dt_alert', 'dtsmart.dt_arms_app', 'dtsmart.dt_arms_service_group', 'dtsmart.dt_arms_service_group_app', 'dtsmart.dt_chart', 'dtsmart.dt_collect_model', 'dtsmart.dt_collect_model_task', 'dtsmart.dt_dashboard', 'dtsmart.dt_data_index', 'dtsmart.dt_host', 'dtsmart.dt_inspect_record', 'dtsmart.dt_inspect_template', 'dtsmart.dt_resource', 'dtsmart.dt_resource_optimization_suggest', 'distributed_1.dt_arms_app', 'distributed_1.dt_arms_service_group', 'distributed_1.dt_arms_service_group_app', 'distributed_1.dt_alert', 'distributed_1.ddd', 'distributed_1.arms_frontend_log') logger=clickhouse 2024/04/01 12:19:53.981360 debug calculate parts list from embedded backup disk backup=shard0-full-20240401121953 logger=backuper operation=create 2024/04/01 12:19:53.985877 info SELECT value FROM `system`.`build_options` where name='VERSION_DESCRIBE' logger=clickhouse 2024/04/01 12:19:53.987463 info done backup=shard0-full-20240401121953 duration=841ms logger=backuper operation=create_embedded 2024/04/01 12:19:53.987549 info clickhouse connection closed logger=clickhouse 2024/04/01 12:19:53.987594 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:19:53.989212 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:19:53.989267 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:19:53.992391 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:19:53.996151 info SELECT max(toInt64(bytes_on_disk * 1.02)) AS max_file_size FROM system.parts logger=clickhouse 2024/04/01 12:19:53.998831 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:54.001064 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:54.002139 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:19:54.004036 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:19:54.007970 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:19:54.009635 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C22746E04614DF, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:19:54.009801 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:19:54.010273 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/arms_frontend_log: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010367 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/ddd: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010482 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010580 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010698 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010811 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.010930 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011030 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011144 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011239 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011350 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011446 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011551 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011674 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011772 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011872 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.011978 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/easydo_agent_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012072 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/easydo_collect_task_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012165 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/metrics_summary: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012266 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/system_attribute_meta: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012457 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/distributed_1/system_metrics: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012604 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012714 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012809 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.012911 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013020 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013204 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013321 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013415 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013517 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013611 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013719 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013812 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013904 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.013999 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/dtsmart/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:19:54.015101 warn /var/lib/clickhouse/backup/shard0-full-20240401121953/upload.state empty, will continue from scratch error: open /var/lib/clickhouse/backup/shard0-full-20240401121953/upload.state: no such file or directory logger=resumable 2024/04/01 12:19:54.015145 debug prepare table concurrent semaphore with concurrency=2 len(tablesForUpload)=42 backup=shard0-full-20240401121953 operation=upload 2024/04/01 12:19:54.015264 debug start local_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:19:54.015264 debug start dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:19:54.015425 debug finish dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:19:54.015549 debug start upload 6 files to shard0-full-20240401121953/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:19:54.015609 debug start upload 7 files to shard0-full-20240401121953/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:20:16.988130 debug API call GET /metrics logger=server 2024/04/01 12:20:46.988231 debug API call GET /metrics logger=server 2024/04/01 12:21:16.988233 debug API call GET /metrics logger=server 2024/04/01 12:21:24.036322 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2275BD63202AD, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:21:24.036373 debug start upload 3 files to shard0-full-20240401121953/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.036322 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2275BD631FDFB, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:21:24.036420 debug start dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.036514 debug finish dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.036857 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.036878 debug start dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.036896 debug finish dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.036926 debug start dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.036953 debug finish dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.037211 debug start local_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=4 logger=uploadTableData 2024/04/01 12:21:24.037280 debug start local_1.ddd with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:21:24.037485 debug start upload 7 files to shard0-full-20240401121953/shadow/local_1/ddd/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.037597 debug start upload 6 files to shard0-full-20240401121953/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.037636 debug start upload 7 files to shard0-full-20240401121953/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.037854 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.037917 debug start local_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:21:24.037951 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.038005 debug start upload 3 files to shard0-full-20240401121953/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.038059 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.038097 debug start upload 11 files to shard0-full-20240401121953/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.038112 debug start upload 3 files to shard0-full-20240401121953/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.038342 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.038558 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.038601 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.038622 debug start dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.038644 debug finish dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.038687 debug start dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.038709 debug finish dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.039018 debug start local_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:21:24.039065 debug start local_1.system_metrics with concurrency=2 len(table.Parts[...])=2 logger=uploadTableData 2024/04/01 12:21:24.039310 debug start upload 6 files to shard0-full-20240401121953/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.039326 debug start upload 7 files to shard0-full-20240401121953/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.039323 debug start upload 6 files to shard0-full-20240401121953/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.039453 debug start upload 8 files to shard0-full-20240401121953/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.039859 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.039896 debug start upload 4 files to shard0-full-20240401121953/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.039900 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.040047 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.040094 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.040711 debug start local_1.metrics_summary with concurrency=2 len(table.Parts[...])=4 logger=uploadTableData 2024/04/01 12:21:24.040948 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.041024 debug start dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.041054 debug finish dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.041174 debug start upload 6 files to shard0-full-20240401121953/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.041254 debug start upload 7 files to shard0-full-20240401121953/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.041841 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.041876 debug start upload 3 files to shard0-full-20240401121953/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.041887 debug start dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.041908 debug finish dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.042146 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.042194 debug start upload 3 files to shard0-full-20240401121953/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:21:24.042276 debug start dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.042289 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.042298 debug finish dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.042430 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:21:24.042513 debug start dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.042537 debug finish dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.042601 debug start dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.042629 debug finish dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.042827 debug start dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.042847 debug finish dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.042909 debug start dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.042930 debug finish dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043064 debug start dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043086 debug finish dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043185 debug start distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043206 debug finish distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043339 debug start distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043358 debug finish distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043460 debug start distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043480 debug finish distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043601 debug start distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043620 debug finish distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043721 debug start distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043742 debug finish distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.043887 debug start distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.043906 debug finish distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044059 debug start distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044082 debug finish distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044174 debug start distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044215 debug finish distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044337 debug start distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044358 debug finish distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044471 debug start distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044492 debug finish distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044609 debug start distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044630 debug finish distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044750 debug start distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044770 debug finish distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.044905 debug start distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.044926 debug finish distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.045021 debug start distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.045041 debug finish distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.045203 debug start distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.045224 debug finish distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.045714 debug start distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.045740 debug finish distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.045759 debug start distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.045780 debug finish distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.046073 debug start distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.046087 debug start distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.046129 debug finish distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.046096 debug finish distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.046479 debug start distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.046500 debug finish distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.046529 debug start distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:24.046554 debug finish distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:24.046943 info clickhouse connection closed logger=clickhouse 2024/04/01 12:21:24.046981 error metrics.ExecuteWithMetrics(create_remote) return error: one of upload table go-routine return error: can't embeeded upload metadata: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2275BD6320050, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=metrics 2024/04/01 12:21:24.047089 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:21:24.048456 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:21:24.048499 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:21:24.053452 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:21:24.059703 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:24.066826 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:24.068230 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:24.070752 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:24.076464 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:21:24.078253 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C2275BD8C50C11, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:21:24.078430 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:21:24.078547 debug b.skipIfTheSameRemoteBackupPresent return skip=false logger=RemoveBackupLocal 2024/04/01 12:21:24.078593 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:21:24.081928 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:24.088201 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:24.089515 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:21:24.092406 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:21:24.100019 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/.backup) 2024/04/01 12:21:24.106121 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/checksums.txt) 2024/04/01 12:21:24.109315 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/columns.txt) 2024/04/01 12:21:24.112719 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/count.txt) 2024/04/01 12:21:24.115366 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.bin) 2024/04/01 12:21:24.118721 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.cmrk3) 2024/04/01 12:21:24.121166 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/default_compression_codec.txt) 2024/04/01 12:21:24.123535 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/metadata_version.txt) 2024/04/01 12:21:24.126195 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/minmax__at_date_.idx) 2024/04/01 12:21:24.132774 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/partition.dat) 2024/04/01 12:21:24.135079 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/primary.cidx) 2024/04/01 12:21:24.137497 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/checksums.txt) 2024/04/01 12:21:24.139847 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/columns.txt) 2024/04/01 12:21:24.142277 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/count.txt) 2024/04/01 12:21:24.144611 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.bin) 2024/04/01 12:21:24.147025 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.cmrk3) 2024/04/01 12:21:24.149719 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/primary.cidx) 2024/04/01 12:21:24.152564 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:21:24.155557 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:21:24.158141 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:21:24.160577 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:21:24.162899 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:21:24.165486 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:21:24.168208 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:21:24.170863 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:21:24.173632 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:21:24.176294 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:21:24.179149 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:21:24.181912 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:21:24.184436 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:21:24.187018 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:21:24.189844 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_19_19_0/checksums.txt) 2024/04/01 12:21:24.192322 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_19_19_0/data.bin) 2024/04/01 12:21:24.194768 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_19_19_0/primary.cidx) 2024/04/01 12:21:24.197204 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:21:24.200280 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:21:24.202862 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:21:24.208825 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:21:24.232953 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:21:24.236432 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:21:24.239278 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:21:24.241783 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:21:24.244908 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:21:24.248002 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:21:24.250469 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:21:24.252847 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:21:24.255415 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:21:24.257605 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:21:24.260180 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:21:24.262140 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:21:24.264504 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:21:24.266856 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:21:24.269803 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:21:24.272334 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:21:24.274660 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:21:24.276845 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:21:24.279070 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:21:24.281481 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:21:24.283782 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:21:24.286073 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:21:24.288752 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:21:24.290994 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:21:24.293357 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_19_19_0/checksums.txt) 2024/04/01 12:21:24.295486 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_19_19_0/data.bin) 2024/04/01 12:21:24.297777 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_19_19_0/primary.cidx) 2024/04/01 12:21:24.300877 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/checksums.txt) 2024/04/01 12:21:24.303138 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/columns.txt) 2024/04/01 12:21:24.305356 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/count.txt) 2024/04/01 12:21:24.307773 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.bin) 2024/04/01 12:21:24.309991 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.cmrk3) 2024/04/01 12:21:24.312824 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/primary.cidx) 2024/04/01 12:21:24.315493 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/checksums.txt) 2024/04/01 12:21:24.318218 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/count.txt) 2024/04/01 12:21:24.320794 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.bin) 2024/04/01 12:21:24.323785 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.cmrk3) 2024/04/01 12:21:24.343002 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/primary.cidx) 2024/04/01 12:21:24.346416 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/checksums.txt) 2024/04/01 12:21:24.349077 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.bin) 2024/04/01 12:21:24.351016 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.cmrk3) 2024/04/01 12:21:24.353294 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/primary.cidx) 2024/04/01 12:21:24.355780 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/checksums.txt) 2024/04/01 12:21:24.357902 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/columns.txt) 2024/04/01 12:21:24.360168 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/count.txt) 2024/04/01 12:21:24.362422 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/data.bin) 2024/04/01 12:21:24.364679 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/data.cmrk3) 2024/04/01 12:21:24.366919 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/metadata_version.txt) 2024/04/01 12:21:24.369379 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/primary.cidx) 2024/04/01 12:21:24.371636 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/checksums.txt) 2024/04/01 12:21:24.373991 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/count.txt) 2024/04/01 12:21:24.376284 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/data.bin) 2024/04/01 12:21:24.388521 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/data.cmrk3) 2024/04/01 12:21:24.391330 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/primary.cidx) 2024/04/01 12:21:24.394048 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/arms_frontend_log.sql) 2024/04/01 12:21:24.396333 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/ddd.sql) 2024/04/01 12:21:24.398551 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_alert.sql) 2024/04/01 12:21:24.444808 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_arms_app.sql) 2024/04/01 12:21:24.447571 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_arms_service_group.sql) 2024/04/01 12:21:24.450986 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_arms_service_group_app.sql) 2024/04/01 12:21:24.453737 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_chart.sql) 2024/04/01 12:21:24.455907 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_collect_model.sql) 2024/04/01 12:21:24.458069 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_collect_model_task.sql) 2024/04/01 12:21:24.460197 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_dashboard.sql) 2024/04/01 12:21:24.463153 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_data_index.sql) 2024/04/01 12:21:24.465333 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_host.sql) 2024/04/01 12:21:24.467615 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_inspect_record.sql) 2024/04/01 12:21:24.470035 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_inspect_template.sql) 2024/04/01 12:21:24.472331 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_resource.sql) 2024/04/01 12:21:24.474309 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/dt_resource_optimization_suggest.sql) 2024/04/01 12:21:24.476443 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/easydo_agent_monitor.sql) 2024/04/01 12:21:24.478518 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/easydo_collect_task_monitor.sql) 2024/04/01 12:21:24.480803 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/metrics_summary.sql) 2024/04/01 12:21:24.483049 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/system_attribute_meta.sql) 2024/04/01 12:21:24.485458 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/distributed_1/system_metrics.sql) 2024/04/01 12:21:24.487839 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_alert.sql) 2024/04/01 12:21:24.490656 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_arms_app.sql) 2024/04/01 12:21:24.492902 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_arms_service_group.sql) 2024/04/01 12:21:24.495168 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_arms_service_group_app.sql) 2024/04/01 12:21:24.497563 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_chart.sql) 2024/04/01 12:21:24.499746 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_collect_model.sql) 2024/04/01 12:21:24.501936 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_collect_model_task.sql) 2024/04/01 12:21:24.504085 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_dashboard.sql) 2024/04/01 12:21:24.506253 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_data_index.sql) 2024/04/01 12:21:24.508584 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_host.sql) 2024/04/01 12:21:24.510809 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_inspect_record.sql) 2024/04/01 12:21:24.513823 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_inspect_template.sql) 2024/04/01 12:21:24.543191 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_resource.sql) 2024/04/01 12:21:24.546244 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/dtsmart/dt_resource_optimization_suggest.sql) 2024/04/01 12:21:24.548629 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/arms_frontend_log.sql) 2024/04/01 12:21:24.551096 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/ddd.sql) 2024/04/01 12:21:24.553373 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/easydo_agent_monitor.sql) 2024/04/01 12:21:24.555894 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/easydo_collect_task_monitor.sql) 2024/04/01 12:21:24.558245 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/metrics_summary.sql) 2024/04/01 12:21:24.560034 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/system_attribute_meta.sql) 2024/04/01 12:21:24.562156 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953/metadata/local_1/system_metrics.sql) 2024/04/01 12:21:24.564774 debug remove '/var/lib/clickhouse/backup/shard0-full-20240401121953' logger=RemoveBackupLocal 2024/04/01 12:21:24.564838 debug remove '/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401121953' logger=RemoveBackupLocal 2024/04/01 12:21:24.573634 info done backup=shard0-full-20240401121953 duration=527ms location=local logger=RemoveBackupLocal operation=delete 2024/04/01 12:21:24.573769 info clickhouse connection closed logger=clickhouse 2024/04/01 12:21:24.573807 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:21:24.574768 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:21:24.575738 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:24.578459 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:24.579639 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:21:24.580300 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:21:24.580334 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:21:24.582055 info SELECT name, engine FROM system.databases WHERE NOT match(name,'^(system|INFORMATION_SCHEMA|information_schema|_temporary_and_external_tables)$') logger=clickhouse 2024/04/01 12:21:24.584065 info SHOW CREATE DATABASE `_internal_ddl_health_check_` logger=clickhouse 2024/04/01 12:21:24.585011 info SHOW CREATE DATABASE `alert` logger=clickhouse 2024/04/01 12:21:24.585805 info SHOW CREATE DATABASE `default` logger=clickhouse 2024/04/01 12:21:24.586635 info SHOW CREATE DATABASE `distributed_1` logger=clickhouse 2024/04/01 12:21:24.587404 info SHOW CREATE DATABASE `dtsmart` logger=clickhouse 2024/04/01 12:21:24.588120 info SHOW CREATE DATABASE `local_1` logger=clickhouse 2024/04/01 12:21:24.588885 info SELECT name, count(*) as is_present FROM system.settings WHERE name IN (?, ?) GROUP BY name with args [show_table_uuid_in_table_create_query_if_not_nil display_secrets_in_show_and_select] logger=clickhouse 2024/04/01 12:21:24.591709 info SELECT name FROM system.databases WHERE engine IN ('MySQL','PostgreSQL','MaterializedPostgreSQL') logger=clickhouse 2024/04/01 12:21:24.592881 info SELECT countIf(name='data_path') is_data_path_present, countIf(name='data_paths') is_data_paths_present, countIf(name='uuid') is_uuid_present, countIf(name='create_table_query') is_create_table_query_present, countIf(name='total_bytes') is_total_bytes_present FROM system.columns WHERE database='system' AND table='tables' logger=clickhouse 2024/04/01 12:21:24.595593 info SELECT database, name, engine , data_paths , uuid , create_table_query , coalesce(total_bytes, 0) AS total_bytes FROM system.tables WHERE is_temporary = 0 ORDER BY total_bytes DESC SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1 logger=clickhouse 2024/04/01 12:21:24.618956 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:21:24.621643 info SELECT count() as cnt FROM system.columns WHERE database='system' AND table='functions' AND name='create_query' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:24.623711 info SELECT name, create_query FROM system.functions WHERE create_query!='' logger=clickhouse 2024/04/01 12:21:24.626621 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:21:24.629215 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:21:24.632411 info BACKUP TABLE `local_1`.`system_metrics`, TABLE `local_1`.`ddd`, TABLE `local_1`.`metrics_summary`, TABLE `local_1`.`arms_frontend_log`, TABLE `local_1`.`system_attribute_meta`, TABLE `local_1`.`easydo_collect_task_monitor`, TABLE `local_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`dt_chart`, TABLE `distributed_1`.`dt_collect_model`, TABLE `distributed_1`.`dt_collect_model_task`, TABLE `distributed_1`.`dt_dashboard`, TABLE `distributed_1`.`dt_data_index`, TABLE `distributed_1`.`dt_host`, TABLE `distributed_1`.`dt_inspect_record`, TABLE `distributed_1`.`dt_inspect_template`, TABLE `distributed_1`.`dt_resource`, TABLE `distributed_1`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`easydo_collect_task_monitor`, TABLE `distributed_1`.`metrics_summary`, TABLE `distributed_1`.`system_attribute_meta`, TABLE `distributed_1`.`system_metrics`, TABLE `dtsmart`.`dt_alert`, TABLE `dtsmart`.`dt_arms_app`, TABLE `dtsmart`.`dt_arms_service_group`, TABLE `dtsmart`.`dt_arms_service_group_app`, TABLE `dtsmart`.`dt_chart`, TABLE `dtsmart`.`dt_collect_model`, TABLE `dtsmart`.`dt_collect_model_task`, TABLE `dtsmart`.`dt_dashboard`, TABLE `dtsmart`.`dt_data_index`, TABLE `dtsmart`.`dt_host`, TABLE `dtsmart`.`dt_inspect_record`, TABLE `dtsmart`.`dt_inspect_template`, TABLE `dtsmart`.`dt_resource`, TABLE `dtsmart`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`dt_arms_app`, TABLE `distributed_1`.`dt_arms_service_group`, TABLE `distributed_1`.`dt_arms_service_group_app`, TABLE `distributed_1`.`dt_alert`, TABLE `distributed_1`.`ddd`, TABLE `distributed_1`.`arms_frontend_log` TO Disk(?,?) with args [backups_s3 shard0-full-20240401122124] logger=clickhouse 2024/04/01 12:21:25.175301 info SELECT sum(total_bytes) AS backup_data_size FROM system.tables WHERE concat(database,'.',name) IN ('local_1.system_metrics', 'local_1.ddd', 'local_1.metrics_summary', 'local_1.arms_frontend_log', 'local_1.system_attribute_meta', 'local_1.easydo_collect_task_monitor', 'local_1.easydo_agent_monitor', 'distributed_1.dt_chart', 'distributed_1.dt_collect_model', 'distributed_1.dt_collect_model_task', 'distributed_1.dt_dashboard', 'distributed_1.dt_data_index', 'distributed_1.dt_host', 'distributed_1.dt_inspect_record', 'distributed_1.dt_inspect_template', 'distributed_1.dt_resource', 'distributed_1.dt_resource_optimization_suggest', 'distributed_1.easydo_agent_monitor', 'distributed_1.easydo_collect_task_monitor', 'distributed_1.metrics_summary', 'distributed_1.system_attribute_meta', 'distributed_1.system_metrics', 'dtsmart.dt_alert', 'dtsmart.dt_arms_app', 'dtsmart.dt_arms_service_group', 'dtsmart.dt_arms_service_group_app', 'dtsmart.dt_chart', 'dtsmart.dt_collect_model', 'dtsmart.dt_collect_model_task', 'dtsmart.dt_dashboard', 'dtsmart.dt_data_index', 'dtsmart.dt_host', 'dtsmart.dt_inspect_record', 'dtsmart.dt_inspect_template', 'dtsmart.dt_resource', 'dtsmart.dt_resource_optimization_suggest', 'distributed_1.dt_arms_app', 'distributed_1.dt_arms_service_group', 'distributed_1.dt_arms_service_group_app', 'distributed_1.dt_alert', 'distributed_1.ddd', 'distributed_1.arms_frontend_log') logger=clickhouse 2024/04/01 12:21:25.178694 debug calculate parts list from embedded backup disk backup=shard0-full-20240401122124 logger=backuper operation=create 2024/04/01 12:21:25.184083 info SELECT value FROM `system`.`build_options` where name='VERSION_DESCRIBE' logger=clickhouse 2024/04/01 12:21:25.185960 info done backup=shard0-full-20240401122124 duration=606ms logger=backuper operation=create_embedded 2024/04/01 12:21:25.186048 info clickhouse connection closed logger=clickhouse 2024/04/01 12:21:25.186140 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:21:25.186857 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:21:25.186898 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:21:25.190349 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:21:25.194471 info SELECT max(toInt64(bytes_on_disk * 1.02)) AS max_file_size FROM system.parts logger=clickhouse 2024/04/01 12:21:25.196874 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:25.198828 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:25.199802 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:21:25.201731 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:21:25.206167 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:21:25.207713 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C2275C1C16E331, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:21:25.207854 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:21:25.208323 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/arms_frontend_log: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208416 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/ddd: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208502 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208588 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208700 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208791 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208876 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.208960 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209096 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209196 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209276 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209355 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209442 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209545 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209630 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209725 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209808 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/easydo_agent_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.209903 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/easydo_collect_task_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210002 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/metrics_summary: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210088 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/system_attribute_meta: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210281 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/distributed_1/system_metrics: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210425 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210518 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210626 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210721 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210801 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.210916 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211005 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211084 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211205 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211309 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211395 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211476 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211562 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.211655 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/dtsmart/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:21:25.212780 warn /var/lib/clickhouse/backup/shard0-full-20240401122124/upload.state empty, will continue from scratch error: open /var/lib/clickhouse/backup/shard0-full-20240401122124/upload.state: no such file or directory logger=resumable 2024/04/01 12:21:25.212829 debug prepare table concurrent semaphore with concurrency=2 len(tablesForUpload)=42 backup=shard0-full-20240401122124 operation=upload 2024/04/01 12:21:25.212892 debug start local_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:21:25.212967 debug start dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:21:25.213002 debug finish dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:21:25.213111 debug start upload 6 files to shard0-full-20240401122124/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:25.213458 debug start upload 7 files to shard0-full-20240401122124/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:21:46.988009 debug API call GET /metrics logger=server 2024/04/01 12:22:16.988621 debug API call GET /metrics logger=server 2024/04/01 12:22:46.987882 debug API call GET /metrics logger=server 2024/04/01 12:22:55.235757 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C22771120DB908, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:22:55.235817 debug start upload 3 files to shard0-full-20240401122124/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.235852 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C22771120DB6F7, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:22:55.235869 debug start dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.235930 debug finish dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.236237 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.236287 debug start dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.236319 debug finish dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.236296 debug start dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.236385 debug finish dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.236626 debug start local_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:22:55.236771 debug start upload 7 files to shard0-full-20240401122124/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.236786 debug start local_1.ddd with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:22:55.236932 debug start upload 7 files to shard0-full-20240401122124/shadow/local_1/ddd/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.237202 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.237259 debug start local_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:22:55.237279 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.237353 debug start dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.237370 debug finish dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.237838 debug start upload 11 files to shard0-full-20240401122124/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.238097 debug start dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.238125 debug finish dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.238228 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.238272 debug start local_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:22:55.238510 debug start local_1.system_metrics with concurrency=2 len(table.Parts[...])=4 logger=uploadTableData 2024/04/01 12:22:55.238576 debug start upload 6 files to shard0-full-20240401122124/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.238641 debug start upload 7 files to shard0-full-20240401122124/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.238868 debug start upload 6 files to shard0-full-20240401122124/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.238908 debug start upload 8 files to shard0-full-20240401122124/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.238928 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.238963 debug start upload 4 files to shard0-full-20240401122124/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.239052 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239194 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239248 debug start upload 6 files to shard0-full-20240401122124/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.239285 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239358 debug start local_1.metrics_summary with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:22:55.239446 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239474 debug start upload 7 files to shard0-full-20240401122124/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.239479 debug start upload 5 files to shard0-full-20240401122124/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:22:55.239561 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239788 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239854 debug start dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.239870 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:22:55.239875 debug finish dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.239942 debug start dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.239959 debug finish dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.240292 debug start dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.240325 debug finish dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.240294 debug start dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.240416 debug finish dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.240617 debug start dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.240641 debug finish dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.241564 debug start dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.241645 debug start dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.241681 debug finish dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.241683 debug finish dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242250 debug start dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242273 debug finish dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242364 debug start distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242387 debug finish distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242607 debug start distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242620 debug start distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242628 debug finish distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242635 debug finish distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242912 debug start distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242933 debug finish distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.242965 debug start distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.242986 debug finish distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243264 debug start distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243291 debug finish distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243345 debug start distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243366 debug finish distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243548 debug start distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243572 debug finish distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243689 debug start distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243709 debug finish distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243833 debug start distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243855 debug finish distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.243950 debug start distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.243969 debug finish distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244090 debug start distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244113 debug finish distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244211 debug start distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244233 debug finish distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244435 debug start distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244459 debug finish distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244469 debug start distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244485 debug finish distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244732 debug start distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244753 debug finish distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244755 debug start distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.244866 debug finish distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.244984 debug start distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.245006 debug finish distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.245112 debug start distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.245133 debug finish distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.245243 debug start distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.245263 debug finish distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.245381 debug start distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:55.245406 debug finish distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:55.245759 info clickhouse connection closed logger=clickhouse 2024/04/01 12:22:55.245798 error metrics.ExecuteWithMetrics(create_remote) return error: one of upload table go-routine return error: can't embeeded upload metadata: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C22771120DB54A, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=metrics 2024/04/01 12:22:55.245871 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:22:55.246743 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:22:55.246799 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:22:55.250433 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:22:55.257865 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:55.260075 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:22:55.261093 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:55.263226 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:22:55.268761 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:22:55.271038 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C227711446B744, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:22:55.271206 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:22:55.271315 debug b.skipIfTheSameRemoteBackupPresent return skip=false logger=RemoveBackupLocal 2024/04/01 12:22:55.271428 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/.backup) 2024/04/01 12:22:55.286366 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/checksums.txt) 2024/04/01 12:22:55.290797 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/columns.txt) 2024/04/01 12:22:55.294370 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/count.txt) 2024/04/01 12:22:55.297439 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.bin) 2024/04/01 12:22:55.300980 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.cmrk3) 2024/04/01 12:22:55.303750 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/default_compression_codec.txt) 2024/04/01 12:22:55.306642 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/metadata_version.txt) 2024/04/01 12:22:55.309073 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/minmax__at_date_.idx) 2024/04/01 12:22:55.311285 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/partition.dat) 2024/04/01 12:22:55.313651 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/primary.cidx) 2024/04/01 12:22:55.316227 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/checksums.txt) 2024/04/01 12:22:55.318709 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/columns.txt) 2024/04/01 12:22:55.320850 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/count.txt) 2024/04/01 12:22:55.323025 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.bin) 2024/04/01 12:22:55.325873 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.cmrk3) 2024/04/01 12:22:55.328092 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/primary.cidx) 2024/04/01 12:22:55.330647 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:22:55.332951 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:22:55.335263 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:22:55.337478 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:22:55.339872 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:22:55.342239 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:22:55.344694 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:22:55.346726 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:22:55.348763 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:22:55.351093 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:22:55.353887 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:22:55.355859 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:22:55.358035 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:22:55.359972 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:22:55.362340 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:22:55.364578 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:22:55.366748 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:22:55.369102 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:22:55.371113 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:22:55.373156 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:22:55.375201 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:22:55.378146 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:22:55.380735 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:22:55.383244 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:22:55.386168 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:22:55.388615 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:22:55.390941 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/checksums.txt) 2024/04/01 12:22:55.393551 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/columns.txt) 2024/04/01 12:22:55.396372 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/count.txt) 2024/04/01 12:22:55.399197 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.bin) 2024/04/01 12:22:55.402354 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.cmrk3) 2024/04/01 12:22:55.404948 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/primary.cidx) 2024/04/01 12:22:55.407344 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/checksums.txt) 2024/04/01 12:22:55.409458 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/count.txt) 2024/04/01 12:22:55.411897 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.bin) 2024/04/01 12:22:55.414288 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.cmrk3) 2024/04/01 12:22:55.416785 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/primary.cidx) 2024/04/01 12:22:55.419137 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/checksums.txt) 2024/04/01 12:22:55.445063 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.bin) 2024/04/01 12:22:55.448102 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.cmrk3) 2024/04/01 12:22:55.450720 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/primary.cidx) 2024/04/01 12:22:55.453471 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/checksums.txt) 2024/04/01 12:22:55.456415 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/columns.txt) 2024/04/01 12:22:55.458888 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/count.txt) 2024/04/01 12:22:55.461189 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/data.bin) 2024/04/01 12:22:55.463322 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/data.cmrk3) 2024/04/01 12:22:55.465633 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/metadata_version.txt) 2024/04/01 12:22:55.467879 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_218_41/primary.cidx) 2024/04/01 12:22:55.470035 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/checksums.txt) 2024/04/01 12:22:55.472179 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/count.txt) 2024/04/01 12:22:55.474773 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/data.bin) 2024/04/01 12:22:55.478047 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/data.cmrk3) 2024/04/01 12:22:55.480334 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_219_219_0/primary.cidx) 2024/04/01 12:22:55.482579 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_220_220_0/checksums.txt) 2024/04/01 12:22:55.484922 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_220_220_0/count.txt) 2024/04/01 12:22:55.487140 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_220_220_0/data.bin) 2024/04/01 12:22:55.489852 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_220_220_0/data.cmrk3) 2024/04/01 12:22:55.492109 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_220_220_0/primary.cidx) 2024/04/01 12:22:55.494428 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_221_221_0/checksums.txt) 2024/04/01 12:22:55.496765 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_221_221_0/data.bin) 2024/04/01 12:22:55.499881 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_221_221_0/data.cmrk3) 2024/04/01 12:22:55.502138 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_221_221_0/primary.cidx) 2024/04/01 12:22:55.504541 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/arms_frontend_log.sql) 2024/04/01 12:22:55.506786 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/ddd.sql) 2024/04/01 12:22:55.509074 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_alert.sql) 2024/04/01 12:22:55.511418 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_arms_app.sql) 2024/04/01 12:22:55.513759 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_arms_service_group.sql) 2024/04/01 12:22:55.516286 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_arms_service_group_app.sql) 2024/04/01 12:22:55.544561 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_chart.sql) 2024/04/01 12:22:55.548726 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_collect_model.sql) 2024/04/01 12:22:55.551280 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_collect_model_task.sql) 2024/04/01 12:22:55.554400 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_dashboard.sql) 2024/04/01 12:22:55.557049 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_data_index.sql) 2024/04/01 12:22:55.559915 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_host.sql) 2024/04/01 12:22:55.562708 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_inspect_record.sql) 2024/04/01 12:22:55.565309 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_inspect_template.sql) 2024/04/01 12:22:55.568050 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_resource.sql) 2024/04/01 12:22:55.570229 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/dt_resource_optimization_suggest.sql) 2024/04/01 12:22:55.572404 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/easydo_agent_monitor.sql) 2024/04/01 12:22:55.574992 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/easydo_collect_task_monitor.sql) 2024/04/01 12:22:55.577275 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/metrics_summary.sql) 2024/04/01 12:22:55.579746 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/system_attribute_meta.sql) 2024/04/01 12:22:55.582024 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/distributed_1/system_metrics.sql) 2024/04/01 12:22:55.584835 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_alert.sql) 2024/04/01 12:22:55.587082 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_arms_app.sql) 2024/04/01 12:22:55.589123 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_arms_service_group.sql) 2024/04/01 12:22:55.591227 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_arms_service_group_app.sql) 2024/04/01 12:22:55.593440 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_chart.sql) 2024/04/01 12:22:55.595815 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_collect_model.sql) 2024/04/01 12:22:55.597969 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_collect_model_task.sql) 2024/04/01 12:22:55.599916 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_dashboard.sql) 2024/04/01 12:22:55.602049 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_data_index.sql) 2024/04/01 12:22:55.604062 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_host.sql) 2024/04/01 12:22:55.608237 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_inspect_record.sql) 2024/04/01 12:22:55.610383 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_inspect_template.sql) 2024/04/01 12:22:55.612385 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_resource.sql) 2024/04/01 12:22:55.614366 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/dtsmart/dt_resource_optimization_suggest.sql) 2024/04/01 12:22:55.616404 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/arms_frontend_log.sql) 2024/04/01 12:22:55.618340 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/ddd.sql) 2024/04/01 12:22:55.644759 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/easydo_agent_monitor.sql) 2024/04/01 12:22:55.647201 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/easydo_collect_task_monitor.sql) 2024/04/01 12:22:55.649490 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/metrics_summary.sql) 2024/04/01 12:22:55.651801 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/system_attribute_meta.sql) 2024/04/01 12:22:55.654014 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124/metadata/local_1/system_metrics.sql) 2024/04/01 12:22:55.656414 debug remove '/var/lib/clickhouse/backup/shard0-full-20240401122124' logger=RemoveBackupLocal 2024/04/01 12:22:55.656450 debug remove '/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122124' logger=RemoveBackupLocal 2024/04/01 12:22:55.661964 info done backup=shard0-full-20240401122124 duration=416ms location=local logger=RemoveBackupLocal operation=delete 2024/04/01 12:22:55.662070 info clickhouse connection closed logger=clickhouse 2024/04/01 12:22:55.662098 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:22:55.662812 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:22:55.663694 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:55.666328 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:22:55.667531 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:22:55.668637 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:22:55.668714 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:22:55.670558 info SELECT name, engine FROM system.databases WHERE NOT match(name,'^(system|INFORMATION_SCHEMA|information_schema|_temporary_and_external_tables)$') logger=clickhouse 2024/04/01 12:22:55.672771 info SHOW CREATE DATABASE `_internal_ddl_health_check_` logger=clickhouse 2024/04/01 12:22:55.673725 info SHOW CREATE DATABASE `alert` logger=clickhouse 2024/04/01 12:22:55.674500 info SHOW CREATE DATABASE `default` logger=clickhouse 2024/04/01 12:22:55.675270 info SHOW CREATE DATABASE `distributed_1` logger=clickhouse 2024/04/01 12:22:55.676080 info SHOW CREATE DATABASE `dtsmart` logger=clickhouse 2024/04/01 12:22:55.676795 info SHOW CREATE DATABASE `local_1` logger=clickhouse 2024/04/01 12:22:55.677551 info SELECT name, count(*) as is_present FROM system.settings WHERE name IN (?, ?) GROUP BY name with args [show_table_uuid_in_table_create_query_if_not_nil display_secrets_in_show_and_select] logger=clickhouse 2024/04/01 12:22:55.680985 info SELECT name FROM system.databases WHERE engine IN ('MySQL','PostgreSQL','MaterializedPostgreSQL') logger=clickhouse 2024/04/01 12:22:55.682256 info SELECT countIf(name='data_path') is_data_path_present, countIf(name='data_paths') is_data_paths_present, countIf(name='uuid') is_uuid_present, countIf(name='create_table_query') is_create_table_query_present, countIf(name='total_bytes') is_total_bytes_present FROM system.columns WHERE database='system' AND table='tables' logger=clickhouse 2024/04/01 12:22:55.685524 info SELECT database, name, engine , data_paths , uuid , create_table_query , coalesce(total_bytes, 0) AS total_bytes FROM system.tables WHERE is_temporary = 0 ORDER BY total_bytes DESC SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1 logger=clickhouse 2024/04/01 12:22:55.709861 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:22:55.712391 info SELECT count() as cnt FROM system.columns WHERE database='system' AND table='functions' AND name='create_query' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:55.714570 info SELECT name, create_query FROM system.functions WHERE create_query!='' logger=clickhouse 2024/04/01 12:22:55.718229 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:22:55.720937 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:22:55.724722 info BACKUP TABLE `local_1`.`system_metrics`, TABLE `local_1`.`ddd`, TABLE `local_1`.`arms_frontend_log`, TABLE `local_1`.`metrics_summary`, TABLE `local_1`.`system_attribute_meta`, TABLE `local_1`.`easydo_collect_task_monitor`, TABLE `local_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`dt_chart`, TABLE `distributed_1`.`dt_collect_model`, TABLE `distributed_1`.`dt_collect_model_task`, TABLE `distributed_1`.`dt_dashboard`, TABLE `distributed_1`.`dt_data_index`, TABLE `distributed_1`.`dt_host`, TABLE `distributed_1`.`dt_inspect_record`, TABLE `distributed_1`.`dt_inspect_template`, TABLE `distributed_1`.`dt_resource`, TABLE `distributed_1`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`easydo_collect_task_monitor`, TABLE `distributed_1`.`metrics_summary`, TABLE `distributed_1`.`system_attribute_meta`, TABLE `distributed_1`.`system_metrics`, TABLE `dtsmart`.`dt_alert`, TABLE `dtsmart`.`dt_arms_app`, TABLE `dtsmart`.`dt_arms_service_group`, TABLE `dtsmart`.`dt_arms_service_group_app`, TABLE `dtsmart`.`dt_chart`, TABLE `dtsmart`.`dt_collect_model`, TABLE `dtsmart`.`dt_collect_model_task`, TABLE `dtsmart`.`dt_dashboard`, TABLE `dtsmart`.`dt_data_index`, TABLE `dtsmart`.`dt_host`, TABLE `dtsmart`.`dt_inspect_record`, TABLE `dtsmart`.`dt_inspect_template`, TABLE `dtsmart`.`dt_resource`, TABLE `dtsmart`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`dt_arms_app`, TABLE `distributed_1`.`dt_arms_service_group`, TABLE `distributed_1`.`dt_arms_service_group_app`, TABLE `distributed_1`.`dt_alert`, TABLE `distributed_1`.`ddd`, TABLE `distributed_1`.`arms_frontend_log` TO Disk(?,?) with args [backups_s3 shard0-full-20240401122255] logger=clickhouse 2024/04/01 12:22:56.365603 info SELECT sum(total_bytes) AS backup_data_size FROM system.tables WHERE concat(database,'.',name) IN ('local_1.system_metrics', 'local_1.ddd', 'local_1.arms_frontend_log', 'local_1.metrics_summary', 'local_1.system_attribute_meta', 'local_1.easydo_collect_task_monitor', 'local_1.easydo_agent_monitor', 'distributed_1.dt_chart', 'distributed_1.dt_collect_model', 'distributed_1.dt_collect_model_task', 'distributed_1.dt_dashboard', 'distributed_1.dt_data_index', 'distributed_1.dt_host', 'distributed_1.dt_inspect_record', 'distributed_1.dt_inspect_template', 'distributed_1.dt_resource', 'distributed_1.dt_resource_optimization_suggest', 'distributed_1.easydo_agent_monitor', 'distributed_1.easydo_collect_task_monitor', 'distributed_1.metrics_summary', 'distributed_1.system_attribute_meta', 'distributed_1.system_metrics', 'dtsmart.dt_alert', 'dtsmart.dt_arms_app', 'dtsmart.dt_arms_service_group', 'dtsmart.dt_arms_service_group_app', 'dtsmart.dt_chart', 'dtsmart.dt_collect_model', 'dtsmart.dt_collect_model_task', 'dtsmart.dt_dashboard', 'dtsmart.dt_data_index', 'dtsmart.dt_host', 'dtsmart.dt_inspect_record', 'dtsmart.dt_inspect_template', 'dtsmart.dt_resource', 'dtsmart.dt_resource_optimization_suggest', 'distributed_1.dt_arms_app', 'distributed_1.dt_arms_service_group', 'distributed_1.dt_arms_service_group_app', 'distributed_1.dt_alert', 'distributed_1.ddd', 'distributed_1.arms_frontend_log') logger=clickhouse 2024/04/01 12:22:56.368515 debug calculate parts list from embedded backup disk backup=shard0-full-20240401122255 logger=backuper operation=create 2024/04/01 12:22:56.374292 info SELECT value FROM `system`.`build_options` where name='VERSION_DESCRIBE' logger=clickhouse 2024/04/01 12:22:56.375968 info done backup=shard0-full-20240401122255 duration=708ms logger=backuper operation=create_embedded 2024/04/01 12:22:56.376061 info clickhouse connection closed logger=clickhouse 2024/04/01 12:22:56.376118 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:22:56.379408 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:22:56.379452 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:22:56.382096 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:22:56.385412 info SELECT max(toInt64(bytes_on_disk * 1.02)) AS max_file_size FROM system.parts logger=clickhouse 2024/04/01 12:22:56.387792 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:56.389618 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:22:56.390959 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:22:56.392858 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:22:56.397959 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:22:56.399183 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C227715789F338, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:22:56.399313 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:22:56.399793 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/arms_frontend_log: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.399891 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/ddd: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.399995 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400133 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400234 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400320 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400438 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400524 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400615 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400714 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400811 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400898 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.400982 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401111 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401208 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401297 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401395 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/easydo_agent_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401482 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/easydo_collect_task_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401569 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/metrics_summary: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401684 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/system_attribute_meta: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.401884 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/distributed_1/system_metrics: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402018 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402108 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402196 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402284 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402380 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.402471 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403010 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403106 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403194 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403286 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403387 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403480 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403572 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.403690 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/dtsmart/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:22:56.404806 warn /var/lib/clickhouse/backup/shard0-full-20240401122255/upload.state empty, will continue from scratch error: open /var/lib/clickhouse/backup/shard0-full-20240401122255/upload.state: no such file or directory logger=resumable 2024/04/01 12:22:56.404841 debug prepare table concurrent semaphore with concurrency=2 len(tablesForUpload)=42 backup=shard0-full-20240401122255 operation=upload 2024/04/01 12:22:56.404901 debug start local_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:22:56.404935 debug start dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:22:56.404967 debug finish dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:22:56.405402 debug start upload 7 files to shard0-full-20240401122255/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:22:56.405383 debug start upload 6 files to shard0-full-20240401122255/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:23:16.988307 debug API call GET /metrics logger=server 2024/04/01 12:23:46.988634 debug API call GET /metrics logger=server 2024/04/01 12:24:16.988062 debug API call GET /metrics logger=server 2024/04/01 12:24:26.428054 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C227864D9621B0, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:24:26.428133 debug start upload 3 files to shard0-full-20240401122255/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.428263 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , canceled, context canceled logger=uploadTableData 2024/04/01 12:24:26.428597 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.428731 debug start dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.428785 debug finish dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.428738 debug start dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.428858 debug finish dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.429098 debug start dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.429123 debug finish dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.429384 debug start local_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:24:26.429421 debug start local_1.ddd with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:24:26.429542 debug start upload 7 files to shard0-full-20240401122255/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.429575 debug start upload 7 files to shard0-full-20240401122255/shadow/local_1/ddd/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.429942 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.429944 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.429995 debug start local_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:24:26.430034 debug start dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.430058 debug finish dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.430316 debug start upload 6 files to shard0-full-20240401122255/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.430344 debug start upload 11 files to shard0-full-20240401122255/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.430400 debug start dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.430418 debug finish dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.430659 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.430693 debug start upload 4 files to shard0-full-20240401122255/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.430770 debug start local_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=4 logger=uploadTableData 2024/04/01 12:24:26.430827 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.430960 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.431008 debug start local_1.system_metrics with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:24:26.431073 debug start upload 6 files to shard0-full-20240401122255/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.431105 debug start upload 8 files to shard0-full-20240401122255/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.431549 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.431574 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.431594 debug start upload 4 files to shard0-full-20240401122255/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.431609 debug start upload 7 files to shard0-full-20240401122255/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.431604 debug start local_1.metrics_summary with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:24:26.431960 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.431981 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.432000 debug start upload 7 files to shard0-full-20240401122255/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.432016 debug start upload 3 files to shard0-full-20240401122255/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:24:26.432273 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.432351 debug start dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.432382 debug finish dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.432591 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:24:26.432674 debug start dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.432695 debug finish dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.432935 debug start dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.432959 debug finish dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.433096 debug start dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.433119 debug finish dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.433431 debug start dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.433457 debug finish dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.433466 debug start dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.433491 debug finish dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.433860 debug start dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.433880 debug finish dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.433942 debug start dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.433967 debug finish dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.434459 debug start distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.434502 debug finish distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.434473 debug start distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.434621 debug finish distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.434851 debug start distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.434871 debug finish distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.435066 debug start distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.435419 debug finish distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.435611 debug start distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.435634 debug finish distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.436567 debug start distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.436603 debug finish distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.436572 debug start distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.436720 debug finish distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.436893 debug start distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.436918 debug finish distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437158 debug start distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437178 debug finish distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437234 debug start distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437265 debug finish distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437418 debug start distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437438 debug finish distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437593 debug start distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437618 debug finish distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437706 debug start distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437726 debug finish distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437960 debug start distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.437996 debug finish distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.437966 debug start distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.438081 debug finish distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.438353 debug start distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.438380 debug start distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.438404 debug finish distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.438384 debug finish distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.438728 debug start distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.438744 debug start distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.438755 debug finish distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.438762 debug finish distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.438997 debug start distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.439015 debug start distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:26.439040 debug finish distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.439020 debug finish distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:26.439458 info clickhouse connection closed logger=clickhouse 2024/04/01 12:24:26.439491 error metrics.ExecuteWithMetrics(create_remote) return error: one of upload table go-routine return error: one of uploadTableData go-routine return error: can't upload: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C227864D9621B0, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=metrics 2024/04/01 12:24:26.439574 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:24:26.440463 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:24:26.440523 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:24:26.444147 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:24:26.448437 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:26.450503 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:24:26.451446 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:26.453446 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:24:26.457639 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:24:26.459974 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C227864F88B218, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:24:26.460340 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:24:26.460455 debug b.skipIfTheSameRemoteBackupPresent return skip=false logger=RemoveBackupLocal 2024/04/01 12:24:26.460575 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/.backup) 2024/04/01 12:24:26.474453 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/checksums.txt) 2024/04/01 12:24:26.477135 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/columns.txt) 2024/04/01 12:24:26.479812 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/count.txt) 2024/04/01 12:24:26.482145 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.bin) 2024/04/01 12:24:26.484589 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.cmrk3) 2024/04/01 12:24:26.488686 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/default_compression_codec.txt) 2024/04/01 12:24:26.491480 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/metadata_version.txt) 2024/04/01 12:24:26.493932 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/minmax__at_date_.idx) 2024/04/01 12:24:26.496605 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/partition.dat) 2024/04/01 12:24:26.498988 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/primary.cidx) 2024/04/01 12:24:26.501487 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/checksums.txt) 2024/04/01 12:24:26.503949 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/count.txt) 2024/04/01 12:24:26.506342 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/data.bin) 2024/04/01 12:24:26.508793 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/data.cmrk3) 2024/04/01 12:24:26.511947 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/primary.cidx) 2024/04/01 12:24:26.514503 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/checksums.txt) 2024/04/01 12:24:26.516923 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/data.bin) 2024/04/01 12:24:26.519772 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/data.cmrk3) 2024/04/01 12:24:26.522059 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/primary.cidx) 2024/04/01 12:24:26.524545 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/checksums.txt) 2024/04/01 12:24:26.527285 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/columns.txt) 2024/04/01 12:24:26.529837 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/count.txt) 2024/04/01 12:24:26.532155 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.bin) 2024/04/01 12:24:26.534618 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.cmrk3) 2024/04/01 12:24:26.536820 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/primary.cidx) 2024/04/01 12:24:26.539017 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:24:26.542947 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:24:26.545221 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:24:26.547480 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:24:26.550148 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:24:26.552347 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:24:26.555114 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:24:26.558034 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:24:26.560214 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:24:26.562608 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:24:26.564938 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:24:26.567322 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:24:26.569710 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:24:26.571946 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:24:26.574021 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:24:26.576124 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:24:26.578247 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:24:26.580540 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:24:26.582637 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:24:26.585085 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:24:26.587146 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:24:26.589213 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:24:26.591211 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:24:26.593278 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:24:26.596189 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:24:26.598393 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:24:26.600526 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/checksums.txt) 2024/04/01 12:24:26.602673 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/columns.txt) 2024/04/01 12:24:26.604772 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/count.txt) 2024/04/01 12:24:26.606829 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.bin) 2024/04/01 12:24:26.609080 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.cmrk3) 2024/04/01 12:24:26.611273 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/primary.cidx) 2024/04/01 12:24:26.613772 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/checksums.txt) 2024/04/01 12:24:26.616265 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/count.txt) 2024/04/01 12:24:26.618488 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.bin) 2024/04/01 12:24:26.620549 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.cmrk3) 2024/04/01 12:24:26.622766 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/primary.cidx) 2024/04/01 12:24:26.643055 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/checksums.txt) 2024/04/01 12:24:26.645367 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.bin) 2024/04/01 12:24:26.647834 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.cmrk3) 2024/04/01 12:24:26.656960 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/primary.cidx) 2024/04/01 12:24:26.659402 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/checksums.txt) 2024/04/01 12:24:26.661901 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/data.bin) 2024/04/01 12:24:26.664083 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/primary.cidx) 2024/04/01 12:24:26.666245 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/checksums.txt) 2024/04/01 12:24:26.668414 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/columns.txt) 2024/04/01 12:24:26.670792 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/count.txt) 2024/04/01 12:24:26.672833 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/data.bin) 2024/04/01 12:24:26.674938 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/data.cmrk3) 2024/04/01 12:24:26.676891 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/metadata_version.txt) 2024/04/01 12:24:26.678981 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/primary.cidx) 2024/04/01 12:24:26.681534 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/arms_frontend_log.sql) 2024/04/01 12:24:26.683935 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/ddd.sql) 2024/04/01 12:24:26.685875 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_alert.sql) 2024/04/01 12:24:26.688339 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_arms_app.sql) 2024/04/01 12:24:26.690769 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_arms_service_group.sql) 2024/04/01 12:24:26.693133 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_arms_service_group_app.sql) 2024/04/01 12:24:26.695236 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_chart.sql) 2024/04/01 12:24:26.697606 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_collect_model.sql) 2024/04/01 12:24:26.699971 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_collect_model_task.sql) 2024/04/01 12:24:26.702230 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_dashboard.sql) 2024/04/01 12:24:26.704493 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_data_index.sql) 2024/04/01 12:24:26.706845 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_host.sql) 2024/04/01 12:24:26.708989 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_inspect_record.sql) 2024/04/01 12:24:26.711131 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_inspect_template.sql) 2024/04/01 12:24:26.713635 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_resource.sql) 2024/04/01 12:24:26.715775 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/dt_resource_optimization_suggest.sql) 2024/04/01 12:24:26.744229 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/easydo_agent_monitor.sql) 2024/04/01 12:24:26.747717 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/easydo_collect_task_monitor.sql) 2024/04/01 12:24:26.750820 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/metrics_summary.sql) 2024/04/01 12:24:26.753187 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/system_attribute_meta.sql) 2024/04/01 12:24:26.755506 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/distributed_1/system_metrics.sql) 2024/04/01 12:24:26.761920 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_alert.sql) 2024/04/01 12:24:26.764783 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_arms_app.sql) 2024/04/01 12:24:26.767319 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_arms_service_group.sql) 2024/04/01 12:24:26.770004 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_arms_service_group_app.sql) 2024/04/01 12:24:26.772438 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_chart.sql) 2024/04/01 12:24:26.774857 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_collect_model.sql) 2024/04/01 12:24:26.777679 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_collect_model_task.sql) 2024/04/01 12:24:26.780135 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_dashboard.sql) 2024/04/01 12:24:26.782563 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_data_index.sql) 2024/04/01 12:24:26.785630 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_host.sql) 2024/04/01 12:24:26.788033 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_inspect_record.sql) 2024/04/01 12:24:26.790684 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_inspect_template.sql) 2024/04/01 12:24:26.793804 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_resource.sql) 2024/04/01 12:24:26.796350 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/dtsmart/dt_resource_optimization_suggest.sql) 2024/04/01 12:24:26.799137 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/arms_frontend_log.sql) 2024/04/01 12:24:26.801697 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/ddd.sql) 2024/04/01 12:24:26.804219 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/easydo_agent_monitor.sql) 2024/04/01 12:24:26.806904 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/easydo_collect_task_monitor.sql) 2024/04/01 12:24:26.809812 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/metrics_summary.sql) 2024/04/01 12:24:26.812724 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/system_attribute_meta.sql) 2024/04/01 12:24:26.815816 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255/metadata/local_1/system_metrics.sql) 2024/04/01 12:24:26.818890 debug remove '/var/lib/clickhouse/backup/shard0-full-20240401122255' logger=RemoveBackupLocal 2024/04/01 12:24:26.818925 debug remove '/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122255' logger=RemoveBackupLocal 2024/04/01 12:24:26.824373 info done backup=shard0-full-20240401122255 duration=385ms location=local logger=RemoveBackupLocal operation=delete 2024/04/01 12:24:26.824502 info clickhouse connection closed logger=clickhouse 2024/04/01 12:24:26.824543 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:24:26.825606 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:24:26.826542 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:26.829199 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:24:26.830444 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:24:26.831268 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:24:26.831300 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:24:26.833237 info SELECT name, engine FROM system.databases WHERE NOT match(name,'^(system|INFORMATION_SCHEMA|information_schema|_temporary_and_external_tables)$') logger=clickhouse 2024/04/01 12:24:26.835147 info SHOW CREATE DATABASE `_internal_ddl_health_check_` logger=clickhouse 2024/04/01 12:24:26.836110 info SHOW CREATE DATABASE `alert` logger=clickhouse 2024/04/01 12:24:26.837008 info SHOW CREATE DATABASE `default` logger=clickhouse 2024/04/01 12:24:26.837825 info SHOW CREATE DATABASE `distributed_1` logger=clickhouse 2024/04/01 12:24:26.838579 info SHOW CREATE DATABASE `dtsmart` logger=clickhouse 2024/04/01 12:24:26.839405 info SHOW CREATE DATABASE `local_1` logger=clickhouse 2024/04/01 12:24:26.840205 info SELECT name, count(*) as is_present FROM system.settings WHERE name IN (?, ?) GROUP BY name with args [display_secrets_in_show_and_select show_table_uuid_in_table_create_query_if_not_nil] logger=clickhouse 2024/04/01 12:24:26.843271 info SELECT name FROM system.databases WHERE engine IN ('MySQL','PostgreSQL','MaterializedPostgreSQL') logger=clickhouse 2024/04/01 12:24:26.844526 info SELECT countIf(name='data_path') is_data_path_present, countIf(name='data_paths') is_data_paths_present, countIf(name='uuid') is_uuid_present, countIf(name='create_table_query') is_create_table_query_present, countIf(name='total_bytes') is_total_bytes_present FROM system.columns WHERE database='system' AND table='tables' logger=clickhouse 2024/04/01 12:24:26.873654 info SELECT database, name, engine , data_paths , uuid , create_table_query , coalesce(total_bytes, 0) AS total_bytes FROM system.tables WHERE is_temporary = 0 ORDER BY total_bytes DESC SETTINGS show_table_uuid_in_table_create_query_if_not_nil=1 logger=clickhouse 2024/04/01 12:24:26.897481 info SELECT metadata_path FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1; logger=clickhouse 2024/04/01 12:24:26.900086 info SELECT count() as cnt FROM system.columns WHERE database='system' AND table='functions' AND name='create_query' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:26.902350 info SELECT name, create_query FROM system.functions WHERE create_query!='' logger=clickhouse 2024/04/01 12:24:26.905200 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:24:26.907714 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:24:26.910862 info BACKUP TABLE `local_1`.`system_metrics`, TABLE `local_1`.`ddd`, TABLE `local_1`.`arms_frontend_log`, TABLE `local_1`.`metrics_summary`, TABLE `local_1`.`system_attribute_meta`, TABLE `local_1`.`easydo_collect_task_monitor`, TABLE `local_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`dt_chart`, TABLE `distributed_1`.`dt_collect_model`, TABLE `distributed_1`.`dt_collect_model_task`, TABLE `distributed_1`.`dt_dashboard`, TABLE `distributed_1`.`dt_data_index`, TABLE `distributed_1`.`dt_host`, TABLE `distributed_1`.`dt_inspect_record`, TABLE `distributed_1`.`dt_inspect_template`, TABLE `distributed_1`.`dt_resource`, TABLE `distributed_1`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`easydo_agent_monitor`, TABLE `distributed_1`.`easydo_collect_task_monitor`, TABLE `distributed_1`.`metrics_summary`, TABLE `distributed_1`.`system_attribute_meta`, TABLE `distributed_1`.`system_metrics`, TABLE `dtsmart`.`dt_alert`, TABLE `dtsmart`.`dt_arms_app`, TABLE `dtsmart`.`dt_arms_service_group`, TABLE `dtsmart`.`dt_arms_service_group_app`, TABLE `dtsmart`.`dt_chart`, TABLE `dtsmart`.`dt_collect_model`, TABLE `dtsmart`.`dt_collect_model_task`, TABLE `dtsmart`.`dt_dashboard`, TABLE `dtsmart`.`dt_data_index`, TABLE `dtsmart`.`dt_host`, TABLE `dtsmart`.`dt_inspect_record`, TABLE `dtsmart`.`dt_inspect_template`, TABLE `dtsmart`.`dt_resource`, TABLE `dtsmart`.`dt_resource_optimization_suggest`, TABLE `distributed_1`.`dt_arms_app`, TABLE `distributed_1`.`dt_arms_service_group`, TABLE `distributed_1`.`dt_arms_service_group_app`, TABLE `distributed_1`.`dt_alert`, TABLE `distributed_1`.`ddd`, TABLE `distributed_1`.`arms_frontend_log` TO Disk(?,?) with args [backups_s3 shard0-full-20240401122426] logger=clickhouse 2024/04/01 12:24:27.468860 info SELECT sum(total_bytes) AS backup_data_size FROM system.tables WHERE concat(database,'.',name) IN ('local_1.system_metrics', 'local_1.ddd', 'local_1.arms_frontend_log', 'local_1.metrics_summary', 'local_1.system_attribute_meta', 'local_1.easydo_collect_task_monitor', 'local_1.easydo_agent_monitor', 'distributed_1.dt_chart', 'distributed_1.dt_collect_model', 'distributed_1.dt_collect_model_task', 'distributed_1.dt_dashboard', 'distributed_1.dt_data_index', 'distributed_1.dt_host', 'distributed_1.dt_inspect_record', 'distributed_1.dt_inspect_template', 'distributed_1.dt_resource', 'distributed_1.dt_resource_optimization_suggest', 'distributed_1.easydo_agent_monitor', 'distributed_1.easydo_collect_task_monitor', 'distributed_1.metrics_summary', 'distributed_1.system_attribute_meta', 'distributed_1.system_metrics', 'dtsmart.dt_alert', 'dtsmart.dt_arms_app', 'dtsmart.dt_arms_service_group', 'dtsmart.dt_arms_service_group_app', 'dtsmart.dt_chart', 'dtsmart.dt_collect_model', 'dtsmart.dt_collect_model_task', 'dtsmart.dt_dashboard', 'dtsmart.dt_data_index', 'dtsmart.dt_host', 'dtsmart.dt_inspect_record', 'dtsmart.dt_inspect_template', 'dtsmart.dt_resource', 'dtsmart.dt_resource_optimization_suggest', 'distributed_1.dt_arms_app', 'distributed_1.dt_arms_service_group', 'distributed_1.dt_arms_service_group_app', 'distributed_1.dt_alert', 'distributed_1.ddd', 'distributed_1.arms_frontend_log') logger=clickhouse 2024/04/01 12:24:27.501083 debug calculate parts list from embedded backup disk backup=shard0-full-20240401122426 logger=backuper operation=create 2024/04/01 12:24:27.505904 info SELECT value FROM `system`.`build_options` where name='VERSION_DESCRIBE' logger=clickhouse 2024/04/01 12:24:27.507656 info done backup=shard0-full-20240401122426 duration=677ms logger=backuper operation=create_embedded 2024/04/01 12:24:27.507751 info clickhouse connection closed logger=clickhouse 2024/04/01 12:24:27.507798 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:24:27.508594 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:24:27.508634 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:24:27.511319 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:24:27.514454 info SELECT max(toInt64(bytes_on_disk * 1.02)) AS max_file_size FROM system.parts logger=clickhouse 2024/04/01 12:24:27.542605 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:27.544727 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:24:27.545605 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:24:27.547500 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:24:27.551572 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:24:27.552775 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C2278690B70406, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:24:27.552906 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:24:27.553373 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/arms_frontend_log: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553465 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/ddd: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553589 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553701 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553788 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553873 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.553959 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554040 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554125 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554205 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554282 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554364 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554444 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554548 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554630 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554730 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554817 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/easydo_agent_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554898 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/easydo_collect_task_monitor: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.554979 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/metrics_summary: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555068 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/system_attribute_meta: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555239 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/distributed_1/system_metrics: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555380 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_alert: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555462 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_arms_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555561 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_arms_service_group: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555642 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_arms_service_group_app: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555733 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_chart: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555835 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_collect_model: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.555921 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_collect_model_task: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.556008 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_dashboard: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.556103 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_data_index: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.556513 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_host: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.556627 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_inspect_record: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.556805 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_inspect_template: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.557143 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_resource: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.557240 warn open /var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/dtsmart/dt_resource_optimization_suggest: no such file or directory logger=getTableListByPatternLocal 2024/04/01 12:24:27.558491 warn /var/lib/clickhouse/backup/shard0-full-20240401122426/upload.state empty, will continue from scratch error: open /var/lib/clickhouse/backup/shard0-full-20240401122426/upload.state: no such file or directory logger=resumable 2024/04/01 12:24:27.558536 debug prepare table concurrent semaphore with concurrency=2 len(tablesForUpload)=42 backup=shard0-full-20240401122426 operation=upload 2024/04/01 12:24:27.558595 debug start local_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=3 logger=uploadTableData 2024/04/01 12:24:27.558634 debug start dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:24:27.558674 debug finish dtsmart.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:24:27.558822 debug start upload 6 files to shard0-full-20240401122426/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:24:27.558856 debug start upload 7 files to shard0-full-20240401122426/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:24:46.987914 debug API call GET /metrics logger=server 2024/04/01 12:25:16.988756 debug API call GET /metrics logger=server 2024/04/01 12:25:46.988082 debug API call GET /metrics logger=server 2024/04/01 12:25:57.581070 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2279B86BB3BB3, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:25:57.581110 error UploadPath return error: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2279B86BB2347, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=uploadTableData 2024/04/01 12:25:57.581118 debug start dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.581183 debug finish dtsmart.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.581132 debug start upload 3 files to shard0-full-20240401122426/shadow/local_1/easydo_collect_task_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.581575 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.581593 debug start dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.581611 debug finish dtsmart.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.581648 debug start dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.581682 debug finish dtsmart.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.581898 debug start local_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:25:57.581981 debug start local_1.ddd with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:25:57.582148 debug start upload 7 files to shard0-full-20240401122426/shadow/local_1/easydo_agent_monitor/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.582168 debug start upload 7 files to shard0-full-20240401122426/shadow/local_1/ddd/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.582464 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.582494 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.582508 debug start local_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=5 logger=uploadTableData 2024/04/01 12:25:57.582530 debug start dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.582557 debug finish dtsmart.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.582878 debug start dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.582904 debug finish dtsmart.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.582982 debug start upload 6 files to shard0-full-20240401122426/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583042 debug start upload 11 files to shard0-full-20240401122426/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583219 debug start local_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=4 logger=uploadTableData 2024/04/01 12:25:57.583294 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.583398 debug start upload 4 files to shard0-full-20240401122426/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583467 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.583506 debug start upload 4 files to shard0-full-20240401122426/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583510 debug start upload 6 files to shard0-full-20240401122426/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583697 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.583738 debug start upload 4 files to shard0-full-20240401122426/shadow/local_1/arms_frontend_log/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583512 debug start upload 7 files to shard0-full-20240401122426/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.583796 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.583988 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584015 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584029 debug start local_1.system_metrics with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:25:57.584055 debug start upload 4 files to shard0-full-20240401122426/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.584109 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584139 debug start upload 3 files to shard0-full-20240401122426/shadow/local_1/system_attribute_meta/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.584160 debug start upload 8 files to shard0-full-20240401122426/shadow/local_1/system_metrics/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.584348 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584422 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584431 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584483 debug start local_1.metrics_summary with concurrency=2 len(table.Parts[...])=1 logger=uploadTableData 2024/04/01 12:25:57.584514 debug start dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.584532 debug finish dtsmart.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.584592 debug start upload 7 files to shard0-full-20240401122426/shadow/local_1/metrics_summary/backups_s3 logger=uploadTableData 2024/04/01 12:25:57.584786 debug start dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.584806 debug finish dtsmart.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.584872 error UploadPath return error: context canceled logger=uploadTableData 2024/04/01 12:25:57.584924 debug start dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.584945 debug finish dtsmart.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585036 debug start dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585056 debug finish dtsmart.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585188 debug start dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585210 debug finish dtsmart.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585293 debug start dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585313 debug finish dtsmart.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585455 debug start dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585476 debug finish dtsmart.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585566 debug start dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585585 debug finish dtsmart.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.585757 debug start distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.585778 debug finish distributed_1.dt_data_index with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.586555 debug start distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.586586 debug finish distributed_1.dt_arms_service_group_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.587151 debug start distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.587186 debug finish distributed_1.dt_alert with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.587831 debug start distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.587868 debug finish distributed_1.dt_arms_app with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.587844 debug start distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.587981 debug finish distributed_1.dt_arms_service_group with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.588239 debug start distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.588276 debug finish distributed_1.dt_chart with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.588332 debug start distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.588358 debug finish distributed_1.dt_resource_optimization_suggest with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.588581 debug start distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.588612 debug finish distributed_1.dt_resource with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.588591 debug start distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.588754 debug finish distributed_1.dt_inspect_template with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.588926 debug start distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.588952 debug finish distributed_1.dt_inspect_record with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.589050 debug start distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.589073 debug finish distributed_1.dt_host with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.589189 debug start distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.589211 debug finish distributed_1.dt_collect_model with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.589707 debug start distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.589738 debug finish distributed_1.dt_dashboard with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.589777 debug start distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.589801 debug finish distributed_1.dt_collect_model_task with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590024 debug start distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590042 debug start distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590051 debug finish distributed_1.arms_frontend_log with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590057 debug finish distributed_1.easydo_agent_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590303 debug start distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590324 debug start distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590355 debug finish distributed_1.easydo_collect_task_monitor with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590325 debug finish distributed_1.system_metrics with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590609 debug start distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590631 debug finish distributed_1.metrics_summary with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.590906 debug start distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.590933 debug finish distributed_1.system_attribute_meta with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.591070 debug start distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 logger=uploadTableData 2024/04/01 12:25:57.591102 debug finish distributed_1.ddd with concurrency=2 len(table.Parts[...])=0 uploadedFiles=map[], uploadedBytes=0 logger=uploadTableData 2024/04/01 12:25:57.591436 info clickhouse connection closed logger=clickhouse 2024/04/01 12:25:57.591473 error metrics.ExecuteWithMetrics(create_remote) return error: one of upload table go-routine return error: can't embeeded upload metadata: operation error S3: PutObject, https response error StatusCode: 404, RequestID: 17C2279B86BB206C, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, api error NoSuchBucket: The specified bucket does not exist logger=metrics 2024/04/01 12:25:57.591546 info clickhouse connection prepared: tcp://localhost:9000 run ping logger=clickhouse 2024/04/01 12:25:57.592310 info clickhouse connection open: tcp://localhost:9000 logger=clickhouse 2024/04/01 12:25:57.592348 info SELECT countIf(name='type') AS is_disk_type_present, countIf(name='free_space') AS is_free_space_present, countIf(name='disks') AS is_storage_policy_present FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') logger=clickhouse 2024/04/01 12:25:57.597051 info SELECT d.path, any(d.name) AS name, any(d.type) AS type, min(d.free_space) AS free_space, groupUniqArray(s.policy_name) AS storage_policies FROM system.disks AS d LEFT JOIN (SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name GROUP BY d.path logger=clickhouse 2024/04/01 12:25:57.601260 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:25:57.603285 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:25:57.604396 info SELECT count() AS is_macros_exists FROM system.tables WHERE database='system' AND name='macros' SETTINGS empty_result_for_aggregation_by_empty_set=0 logger=clickhouse 2024/04/01 12:25:57.606808 info SELECT macro, substitution FROM system.macros logger=clickhouse 2024/04/01 12:25:57.613405 debug /tmp/.clickhouse-backup-metadata.cache.S3 load 0 elements logger=s3 2024/04/01 12:25:57.615445 warn BackupList bd.Walk return error: operation error S3: ListObjectsV2, https response error StatusCode: 404, RequestID: 17C2279B88DC2FB9, HostID: 2fd203aa-24e1-4c1a-b55a-529508ffeeb9, NoSuchBucket: logger=s3 2024/04/01 12:25:57.615651 debug /tmp/.clickhouse-backup-metadata.cache.S3 save 0 elements logger=s3 2024/04/01 12:25:57.615760 debug b.skipIfTheSameRemoteBackupPresent return skip=false logger=RemoveBackupLocal 2024/04/01 12:25:57.615843 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/.backup) 2024/04/01 12:25:57.624074 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/checksums.txt) 2024/04/01 12:25:57.626387 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/columns.txt) 2024/04/01 12:25:57.628959 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/count.txt) 2024/04/01 12:25:57.631427 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.bin) 2024/04/01 12:25:57.633685 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/data.cmrk3) 2024/04/01 12:25:57.635980 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/default_compression_codec.txt) 2024/04/01 12:25:57.640875 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/metadata_version.txt) 2024/04/01 12:25:57.659028 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/minmax__at_date_.idx) 2024/04/01 12:25:57.661360 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/partition.dat) 2024/04/01 12:25:57.664275 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_0_190_47/primary.cidx) 2024/04/01 12:25:57.666901 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/checksums.txt) 2024/04/01 12:25:57.669284 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/count.txt) 2024/04/01 12:25:57.672070 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/data.bin) 2024/04/01 12:25:57.674297 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/data.cmrk3) 2024/04/01 12:25:57.676569 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_191_191_0/primary.cidx) 2024/04/01 12:25:57.679023 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/checksums.txt) 2024/04/01 12:25:57.681315 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/data.bin) 2024/04/01 12:25:57.684007 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/data.cmrk3) 2024/04/01 12:25:57.686387 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_192_192_0/primary.cidx) 2024/04/01 12:25:57.688772 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_193_193_0/checksums.txt) 2024/04/01 12:25:57.691311 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_193_193_0/data.bin) 2024/04/01 12:25:57.693655 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_193_193_0/data.cmrk3) 2024/04/01 12:25:57.696083 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_193_193_0/primary.cidx) 2024/04/01 12:25:57.698473 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_194_194_0/checksums.txt) 2024/04/01 12:25:57.701377 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_194_194_0/data.bin) 2024/04/01 12:25:57.703590 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_194_194_0/data.cmrk3) 2024/04/01 12:25:57.705766 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/arms_frontend_log/11ff205cea3146f68e54df56aee5c0a8_194_194_0/primary.cidx) 2024/04/01 12:25:57.708101 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/checksums.txt) 2024/04/01 12:25:57.711138 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/columns.txt) 2024/04/01 12:25:57.714525 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/count.txt) 2024/04/01 12:25:57.717354 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.bin) 2024/04/01 12:25:57.720246 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/data.cmrk3) 2024/04/01 12:25:57.723205 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/ddd/11ff205cea3146f68e54df56aee5c0a8_0_11_2/primary.cidx) 2024/04/01 12:25:57.726574 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:25:57.729245 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:25:57.732367 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:25:57.735134 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:25:57.737949 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:25:57.741269 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_agent_monitor/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:25:57.744220 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/checksums.txt) 2024/04/01 12:25:57.746953 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/columns.txt) 2024/04/01 12:25:57.749553 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/count.txt) 2024/04/01 12:25:57.751923 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.bin) 2024/04/01 12:25:57.754250 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/data.cmrk3) 2024/04/01 12:25:57.756567 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_0_16_4/primary.cidx) 2024/04/01 12:25:57.758989 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/checksums.txt) 2024/04/01 12:25:57.761774 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/count.txt) 2024/04/01 12:25:57.764127 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.bin) 2024/04/01 12:25:57.766895 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/data.cmrk3) 2024/04/01 12:25:57.769233 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_17_17_0/primary.cidx) 2024/04/01 12:25:57.771835 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/checksums.txt) 2024/04/01 12:25:57.774468 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/data.bin) 2024/04/01 12:25:57.777436 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/easydo_collect_task_monitor/11ff205cea3146f68e54df56aee5c0a8_18_18_0/primary.cidx) 2024/04/01 12:25:57.780328 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/checksums.txt) 2024/04/01 12:25:57.783067 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/columns.txt) 2024/04/01 12:25:57.785659 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/count.txt) 2024/04/01 12:25:57.788254 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.bin) 2024/04/01 12:25:57.794976 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/data.cmrk3) 2024/04/01 12:25:57.797343 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/metrics_summary/11ff205cea3146f68e54df56aee5c0a8_0_20_5/primary.cidx) 2024/04/01 12:25:57.799676 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/checksums.txt) 2024/04/01 12:25:57.802680 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/columns.txt) 2024/04/01 12:25:57.804869 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/count.txt) 2024/04/01 12:25:57.807244 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.bin) 2024/04/01 12:25:57.809884 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/data.cmrk3) 2024/04/01 12:25:57.812248 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_0_40_10/primary.cidx) 2024/04/01 12:25:57.814474 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/checksums.txt) 2024/04/01 12:25:57.816619 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/count.txt) 2024/04/01 12:25:57.818874 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.bin) 2024/04/01 12:25:57.821035 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/data.cmrk3) 2024/04/01 12:25:57.823347 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_41_41_0/primary.cidx) 2024/04/01 12:25:57.825886 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/checksums.txt) 2024/04/01 12:25:57.828107 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.bin) 2024/04/01 12:25:57.830445 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/data.cmrk3) 2024/04/01 12:25:57.844466 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_42_42_0/primary.cidx) 2024/04/01 12:25:57.846853 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/checksums.txt) 2024/04/01 12:25:57.849142 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/data.bin) 2024/04/01 12:25:57.851870 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_attribute_meta/11ff205cea3146f68e54df56aee5c0a8_43_43_0/primary.cidx) 2024/04/01 12:25:57.854271 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/checksums.txt) 2024/04/01 12:25:57.856542 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/columns.txt) 2024/04/01 12:25:57.859159 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/count.txt) 2024/04/01 12:25:57.861563 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/data.bin) 2024/04/01 12:25:57.863958 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/data.cmrk3) 2024/04/01 12:25:57.866035 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/metadata_version.txt) 2024/04/01 12:25:57.868232 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/data/local_1/system_metrics/11ff205cea3146f68e54df56aee5c0a8_0_223_42/primary.cidx) 2024/04/01 12:25:57.871790 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/arms_frontend_log.sql) 2024/04/01 12:25:57.874312 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/ddd.sql) 2024/04/01 12:25:57.876791 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_alert.sql) 2024/04/01 12:25:57.878965 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_arms_app.sql) 2024/04/01 12:25:57.881324 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_arms_service_group.sql) 2024/04/01 12:25:57.883650 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_arms_service_group_app.sql) 2024/04/01 12:25:57.885989 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_chart.sql) 2024/04/01 12:25:57.888134 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_collect_model.sql) 2024/04/01 12:25:57.890459 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_collect_model_task.sql) 2024/04/01 12:25:57.892723 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_dashboard.sql) 2024/04/01 12:25:57.895041 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_data_index.sql) 2024/04/01 12:25:57.899659 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_host.sql) 2024/04/01 12:25:57.901825 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_inspect_record.sql) 2024/04/01 12:25:57.903989 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_inspect_template.sql) 2024/04/01 12:25:57.906499 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_resource.sql) 2024/04/01 12:25:57.908914 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/dt_resource_optimization_suggest.sql) 2024/04/01 12:25:57.911833 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/easydo_agent_monitor.sql) 2024/04/01 12:25:57.914079 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/easydo_collect_task_monitor.sql) 2024/04/01 12:25:57.916591 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/metrics_summary.sql) 2024/04/01 12:25:57.919329 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/system_attribute_meta.sql) 2024/04/01 12:25:57.922012 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/distributed_1/system_metrics.sql) 2024/04/01 12:25:57.925034 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_alert.sql) 2024/04/01 12:25:57.943068 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_arms_app.sql) 2024/04/01 12:25:57.945579 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_arms_service_group.sql) 2024/04/01 12:25:57.948258 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_arms_service_group_app.sql) 2024/04/01 12:25:57.951440 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_chart.sql) 2024/04/01 12:25:57.954070 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_collect_model.sql) 2024/04/01 12:25:57.956607 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_collect_model_task.sql) 2024/04/01 12:25:57.959250 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_dashboard.sql) 2024/04/01 12:25:57.961387 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_data_index.sql) 2024/04/01 12:25:57.963739 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_host.sql) 2024/04/01 12:25:57.965967 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_inspect_record.sql) 2024/04/01 12:25:57.968353 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_inspect_template.sql) 2024/04/01 12:25:57.970756 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_resource.sql) 2024/04/01 12:25:57.972963 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/dtsmart/dt_resource_optimization_suggest.sql) 2024/04/01 12:25:57.975450 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/arms_frontend_log.sql) 2024/04/01 12:25:57.978070 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/ddd.sql) 2024/04/01 12:25:57.980389 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/easydo_agent_monitor.sql) 2024/04/01 12:25:57.982695 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/easydo_collect_task_monitor.sql) 2024/04/01 12:25:57.984885 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/metrics_summary.sql) 2024/04/01 12:25:57.987190 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/system_attribute_meta.sql) 2024/04/01 12:25:57.989527 debug object_disk.ReadMetadataFromFile(/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426/metadata/local_1/system_metrics.sql) 2024/04/01 12:25:57.992253 debug remove '/var/lib/clickhouse/backup/shard0-full-20240401122426' logger=RemoveBackupLocal 2024/04/01 12:25:57.992291 debug remove '/var/lib/clickhouse/disks/backups_s3/shard0-full-20240401122426' logger=RemoveBackupLocal 2024/04/01 12:25:57.997737 info done backup=shard0-full-20240401122426 duration=406ms location=local logger=RemoveBackupLocal operation=delete 2024/04/01 12:25:57.997849 info clickhouse connection closed logger=clickhouse 2024/04/01 12:25:57.997885 error Watch error: too many errors create_remote: 4, delete local: 0, during watch full_interval: 24h, abort watching logger=server 2024/04/01 12:25:57.997937 debug api.status.stop -> status.commands[0] == {ActionRowStatus:{Command:watch --watch-interval="8h" --full-interval="24h" Status:error Start:2024-04-01 12:19:53 Finish:2024-04-01 12:25:57 Error:too many errors create_remote: 4, delete local: 0, during watch full_interval: 24h, abort watching} Ctx: Cancel:} logger=status 2024/04/01 12:26:16.988715 debug API call GET /metrics logger=server 2024/04/01 12:26:46.989096 debug API call GET /metrics logger=server 2024/04/01 12:27:16.987938 debug API call GET /metrics logger=server