|
1 | 1 | import logging
|
2 | 2 | import signal
|
3 | 3 | import time
|
| 4 | +import datetime |
4 | 5 |
|
5 | 6 | import redis
|
6 | 7 | from celery.exceptions import SoftTimeLimitExceeded, TimeLimitExceeded
|
7 | 8 | from celery.result import AsyncResult
|
8 | 9 | from celery.utils.log import get_task_logger
|
9 | 10 | from six import text_type
|
| 11 | +from sqlalchemy.orm import load_only |
10 | 12 |
|
11 |
| -from redash import models, redis_connection, settings, statsd_client |
| 13 | +from redash import models, redis_connection, settings, statsd_client, utils |
| 14 | +from redash.models import TableMetadata, ColumnMetadata, db |
12 | 15 | from redash.query_runner import InterruptException
|
13 | 16 | from redash.tasks.alerts import check_alerts_for_query
|
14 | 17 | from redash.utils import gen_query_hash, json_dumps, json_loads, utcnow, mustache_render
|
@@ -229,13 +232,143 @@ def cleanup_query_results():
|
229 | 232 | logger.info("Deleted %d unused query results.", deleted_count)
|
230 | 233 |
|
231 | 234 |
|
| 235 | +@celery.task(name="redash.tasks.get_table_sample_data") |
| 236 | +def get_table_sample_data(data_source_id, table, table_id): |
| 237 | + ds = models.DataSource.get_by_id(data_source_id) |
| 238 | + sample = ds.query_runner.get_table_sample(table['name']) |
| 239 | + if not sample: |
| 240 | + return |
| 241 | + |
| 242 | + # If a column exists, add a sample to it. |
| 243 | + for i, column in enumerate(table['columns']): |
| 244 | + persisted_column = ColumnMetadata.query.filter( |
| 245 | + ColumnMetadata.name == column, |
| 246 | + ColumnMetadata.table_id == table_id, |
| 247 | + ).options(load_only('id')).first() |
| 248 | + |
| 249 | + if persisted_column: |
| 250 | + column_example = str(sample.get(column, None)) |
| 251 | + if column_example and len(column_example) > 4000: |
| 252 | + column_example = u'{}...'.format(column_example[:4000]) |
| 253 | + |
| 254 | + ColumnMetadata.query.filter( |
| 255 | + ColumnMetadata.id == persisted_column.id, |
| 256 | + ).update({ |
| 257 | + 'example': column_example, |
| 258 | + }) |
| 259 | + models.db.session.commit() |
| 260 | + |
| 261 | +def cleanup_data_in_table(table_model): |
| 262 | + removed_metadata = table_model.query.filter( |
| 263 | + table_model.exists == False, |
| 264 | + ).options(load_only('updated_at')) |
| 265 | + |
| 266 | + for removed_metadata_row in removed_metadata: |
| 267 | + is_old_data = ( |
| 268 | + utils.utcnow() - removed_metadata_row.updated_at |
| 269 | + ) > datetime.timedelta(days=settings.SCHEMA_METADATA_TTL_DAYS) |
| 270 | + |
| 271 | + table_model.query.filter( |
| 272 | + table_model.id == removed_metadata_row.id, |
| 273 | + ).delete() |
| 274 | + |
| 275 | + db.session.commit() |
| 276 | + |
| 277 | +@celery.task(name="redash.tasks.cleanup_schema_metadata") |
| 278 | +def cleanup_schema_metadata(): |
| 279 | + cleanup_data_in_table(TableMetadata) |
| 280 | + cleanup_data_in_table(ColumnMetadata) |
| 281 | + |
232 | 282 | @celery.task(name="redash.tasks.refresh_schema", time_limit=90, soft_time_limit=60)
|
233 | 283 | def refresh_schema(data_source_id):
|
234 | 284 | ds = models.DataSource.get_by_id(data_source_id)
|
235 | 285 | logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
|
236 | 286 | start_time = time.time()
|
| 287 | + |
237 | 288 | try:
|
238 |
| - ds.get_schema(refresh=True) |
| 289 | + existing_tables = set() |
| 290 | + schema = ds.query_runner.get_schema(get_stats=True) |
| 291 | + for table in schema: |
| 292 | + table_name = table['name'] |
| 293 | + existing_tables.add(table_name) |
| 294 | + |
| 295 | + # Assume that there will only exist 1 table with a given name for a given data source so we use first() |
| 296 | + persisted_table = TableMetadata.query.filter( |
| 297 | + TableMetadata.name == table_name, |
| 298 | + TableMetadata.data_source_id == ds.id, |
| 299 | + ).first() |
| 300 | + |
| 301 | + if persisted_table: |
| 302 | + TableMetadata.query.filter( |
| 303 | + TableMetadata.id == persisted_table.id, |
| 304 | + ).update({"exists": True}) |
| 305 | + else: |
| 306 | + metadata = 'metadata' in table |
| 307 | + persisted_table = TableMetadata( |
| 308 | + org_id=ds.org_id, |
| 309 | + name=table_name, |
| 310 | + data_source_id=ds.id, |
| 311 | + column_metadata=metadata |
| 312 | + ) |
| 313 | + models.db.session.add(persisted_table) |
| 314 | + models.db.session.flush() |
| 315 | + |
| 316 | + existing_columns = set() |
| 317 | + for i, column in enumerate(table['columns']): |
| 318 | + existing_columns.add(column) |
| 319 | + column_metadata = { |
| 320 | + 'org_id': ds.org_id, |
| 321 | + 'table_id': persisted_table.id, |
| 322 | + 'name': column, |
| 323 | + 'type': None, |
| 324 | + 'example': None, |
| 325 | + 'exists': True |
| 326 | + } |
| 327 | + if 'metadata' in table: |
| 328 | + column_metadata['type'] = table['metadata'][i]['type'] |
| 329 | + |
| 330 | + # If the column exists, update it, otherwise create a new one. |
| 331 | + persisted_column = ColumnMetadata.query.filter( |
| 332 | + ColumnMetadata.name == column, |
| 333 | + ColumnMetadata.table_id == persisted_table.id, |
| 334 | + ).options(load_only('id')).first() |
| 335 | + if persisted_column: |
| 336 | + ColumnMetadata.query.filter( |
| 337 | + ColumnMetadata.id == persisted_column.id, |
| 338 | + ).update(column_metadata) |
| 339 | + else: |
| 340 | + models.db.session.add(ColumnMetadata(**column_metadata)) |
| 341 | + models.db.session.commit() |
| 342 | + |
| 343 | + get_table_sample_data.apply_async( |
| 344 | + args=(data_source_id, table, persisted_table.id), |
| 345 | + queue=settings.SCHEMAS_REFRESH_QUEUE |
| 346 | + ) |
| 347 | + |
| 348 | + # If a column did not exist, set the 'column_exists' flag to false. |
| 349 | + existing_columns_list = tuple(existing_columns) |
| 350 | + ColumnMetadata.query.filter( |
| 351 | + ColumnMetadata.exists == True, |
| 352 | + ColumnMetadata.table_id == persisted_table.id, |
| 353 | + ~ColumnMetadata.name.in_(existing_columns_list), |
| 354 | + ).update({ |
| 355 | + "exists": False, |
| 356 | + "updated_at": db.func.now() |
| 357 | + }, synchronize_session='fetch') |
| 358 | + |
| 359 | + # If a table did not exist in the get_schema() response above, set the 'exists' flag to false. |
| 360 | + existing_tables_list = tuple(existing_tables) |
| 361 | + tables_to_update = TableMetadata.query.filter( |
| 362 | + TableMetadata.exists == True, |
| 363 | + TableMetadata.data_source_id == ds.id, |
| 364 | + ~TableMetadata.name.in_(existing_tables_list) |
| 365 | + ).update({ |
| 366 | + "exists": False, |
| 367 | + "updated_at": db.func.now() |
| 368 | + }, synchronize_session='fetch') |
| 369 | + |
| 370 | + models.db.session.commit() |
| 371 | + |
239 | 372 | logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
|
240 | 373 | statsd_client.incr('refresh_schema.success')
|
241 | 374 | except SoftTimeLimitExceeded:
|
|
0 commit comments