diff --git a/common/amundsen_common/models/index_map.py b/common/amundsen_common/models/index_map.py index 4deeba243c..0b260840b1 100644 --- a/common/amundsen_common/models/index_map.py +++ b/common/amundsen_common/models/index_map.py @@ -104,7 +104,16 @@ "programmatic_descriptions": { "type": "text", "analyzer": "simple" - } + }, + "update_frequency": { + "type": "text", + "analyzer": "simple", + "fields": { + "raw": { + "type": "keyword" + } + } + }, } } } diff --git a/common/amundsen_common/models/table.py b/common/amundsen_common/models/table.py index f597eee81f..81c8c6d245 100644 --- a/common/amundsen_common/models/table.py +++ b/common/amundsen_common/models/table.py @@ -208,6 +208,7 @@ class Table: sources: Optional[List[Source]] = None is_view: Optional[bool] = attr.ib(default=None, converter=default_if_none) programmatic_descriptions: List[ProgrammaticDescription] = [] + update_frequency: Optional[str] = None common_joins: Optional[List[SqlJoin]] = None common_filters: Optional[List[SqlWhere]] = None diff --git a/common/setup.py b/common/setup.py index 7cbc4111ac..a8273b1179 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ from setuptools import find_packages, setup -__version__ = '0.31.0+foodtruck.2' +__version__ = '0.31.0+foodtruck.3' requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements-dev.txt') diff --git a/databuilder/databuilder/models/table_elasticsearch_document.py b/databuilder/databuilder/models/table_elasticsearch_document.py index 1f35335764..57ca51f005 100644 --- a/databuilder/databuilder/models/table_elasticsearch_document.py +++ b/databuilder/databuilder/models/table_elasticsearch_document.py @@ -27,7 +27,7 @@ def __init__(self, badges: Optional[List[str]] = None, display_name: Optional[str] = None, schema_description: Optional[str] = None, - programmatic_descriptions: List[str] = [], + programmatic_descriptions: List[str] = [] ) -> None: self.database = database self.cluster = cluster diff --git a/databuilder/databuilder/models/table_update_frequency.py b/databuilder/databuilder/models/table_update_frequency.py index d77442c65a..7d5157b8ac 100644 --- a/databuilder/databuilder/models/table_update_frequency.py +++ b/databuilder/databuilder/models/table_update_frequency.py @@ -14,6 +14,8 @@ class UpdateFrequency(Enum): + ANNUALLY = "annually" + QUARTERLY = "quarterly" MONTHLY = "monthly" WEEKLY = "weekly" DAILY = "daily" diff --git a/frontend/amundsen_application/api/utils/metadata_utils.py b/frontend/amundsen_application/api/utils/metadata_utils.py index 02fa41490d..2644ddd02b 100644 --- a/frontend/amundsen_application/api/utils/metadata_utils.py +++ b/frontend/amundsen_application/api/utils/metadata_utils.py @@ -140,6 +140,9 @@ def marshall_table_full(table_dict: Dict) -> Dict: prog_descriptions = results['programmatic_descriptions'] results['programmatic_descriptions'] = _convert_prog_descriptions(prog_descriptions) + update_frequency = results['update_frequency'] + results['update_frequency'] = update_frequency + columns = results['columns'] for col in columns: # Set column key to guarantee it is available on the frontend diff --git a/frontend/amundsen_application/static/js/config/index.spec.ts b/frontend/amundsen_application/static/js/config/index.spec.ts index a0c88c492e..e429551def 100644 --- a/frontend/amundsen_application/static/js/config/index.spec.ts +++ b/frontend/amundsen_application/static/js/config/index.spec.ts @@ -979,6 +979,7 @@ describe('generateExploreUrl', () => { resource_reports: [], watermarks: [], programmatic_descriptions: {}, + update_frequency: null }; it('calls `exploreUrlGenerator` with table metadata', () => { @@ -1136,6 +1137,7 @@ describe('getColumnLineageLink', () => { resource_reports: [], watermarks: [], programmatic_descriptions: {}, + update_frequency: null }; const columnName = 'column_name'; const actual = ConfigUtils.getColumnLineageLink(tableData, columnName); diff --git a/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts b/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts index ba906c2e51..02489a390f 100644 --- a/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts +++ b/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts @@ -72,6 +72,7 @@ export const initialTableDataState: TableMetadata = { resource_reports: [], watermarks: [], programmatic_descriptions: {}, + update_frequency: null }; export const emptyQualityChecks = { diff --git a/frontend/amundsen_application/static/js/fixtures/globalState.ts b/frontend/amundsen_application/static/js/fixtures/globalState.ts index ea7220a167..ec572ca46a 100644 --- a/frontend/amundsen_application/static/js/fixtures/globalState.ts +++ b/frontend/amundsen_application/static/js/fixtures/globalState.ts @@ -261,6 +261,7 @@ const globalState: GlobalState = { resource_reports: [], watermarks: [], programmatic_descriptions: {}, + update_frequency: null }, tableOwners: { isLoading: true, diff --git a/frontend/amundsen_application/static/js/fixtures/metadata/table.ts b/frontend/amundsen_application/static/js/fixtures/metadata/table.ts index ebac591a1e..b501ca3f8a 100644 --- a/frontend/amundsen_application/static/js/fixtures/metadata/table.ts +++ b/frontend/amundsen_application/static/js/fixtures/metadata/table.ts @@ -117,6 +117,7 @@ export const tableMetadata: TableMetadata = { value: '2020-03-05', }, programmatic_descriptions: {}, + update_frequency: null, schema: 'base', sources: [{ source: diff --git a/frontend/amundsen_application/static/js/interfaces/TableMetadata.ts b/frontend/amundsen_application/static/js/interfaces/TableMetadata.ts index 4dd5a64297..2a94ecd621 100644 --- a/frontend/amundsen_application/static/js/interfaces/TableMetadata.ts +++ b/frontend/amundsen_application/static/js/interfaces/TableMetadata.ts @@ -115,6 +115,7 @@ export interface TableMetadata { resource_reports: ResourceReport[]; watermarks: Watermark[]; programmatic_descriptions: TableProgrammaticDescriptions; + update_frequency?: string | null; } export interface UpdateOwnerPayload { diff --git a/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx b/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx index fccb7bd9bb..c01c9ad596 100644 --- a/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx +++ b/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx @@ -602,7 +602,7 @@ export class TableDetail extends React.Component< ) : ( `Snowflake Shares` ); - + tabInfo.push({ content: ( {previewEnabled() && ( - )} + )} @@ -841,7 +841,11 @@ export class TableDetail extends React.Component< {!!data.last_updated_timestamp && ( - {Constants.LAST_UPDATED_TITLE} + { + data.update_frequency != null + ? `${Constants.LAST_UPDATED_TITLE} (${data.update_frequency})` + : Constants.LAST_UPDATED_TITLE + } {formatDateTimeShort({ diff --git a/metadata/metadata_service/proxy/neo4j_proxy.py b/metadata/metadata_service/proxy/neo4j_proxy.py index 6aaeca72e9..5baf6ae9dc 100644 --- a/metadata/metadata_service/proxy/neo4j_proxy.py +++ b/metadata/metadata_service/proxy/neo4j_proxy.py @@ -162,7 +162,7 @@ def get_table(self, *, table_uri: str) -> Table: owners = self._exec_owners_query(table_uri) wmk_results, table_writer, table_apps, timestamp_value, tags, sources, \ - badges, prog_descs, resource_reports = self._exec_table_query(table_uri) + badges, prog_descs, update_frequency, resource_reports = self._exec_table_query(table_uri) joins, filters = self._exec_table_query_query(table_uri) @@ -185,6 +185,7 @@ def get_table(self, *, table_uri: str) -> Table: sources=sources, is_view=self._safe_get(last_neo4j_record, 'table', 'is_view'), programmatic_descriptions=prog_descs, + update_frequency=update_frequency, common_joins=joins, common_filters=filters, resource_reports=resource_reports @@ -385,6 +386,7 @@ def _get_table_query_statement(self) -> str: OPTIONAL MATCH (table)-[:HAS_BADGE]->(badge:Badge) OPTIONAL MATCH (table)-[:SOURCE]->(src:Source) OPTIONAL MATCH (table)-[:DESCRIPTION]->(prog_descriptions:Programmatic_Description) + OPTIONAL MATCH (table)-[:UPDATE_FREQUENCY]->(update_frequency:Update_Frequency) OPTIONAL MATCH (table)-[:HAS_REPORT]->(resource_reports:Report) RETURN collect(distinct wmk) as wmk_records, collect(distinct app_producer) as producing_apps, @@ -395,6 +397,7 @@ def _get_table_query_statement(self) -> str: collect(distinct badge) as badge_records, collect(distinct src) as sources, collect(distinct prog_descriptions) as prog_descriptions, + update_frequency.frequency as update_frequency, collect(distinct resource_reports) as resource_reports """) return table_level_query @@ -496,7 +499,7 @@ def _exec_table_query(self, table_uri: str) -> Tuple: # The owners seem to have been replaced by a separate query. I left the owners in this query, # but we are not extracting it here. - + # owner_record = [] # for owner in table_records.get('owner_records', []): @@ -514,6 +517,8 @@ def _exec_table_query(self, table_uri: str) -> Tuple: table_records.get('prog_descriptions', []) ) + update_frequency = table_records['update_frequency'] + resource_reports = self._extract_resource_reports_from_query(table_records.get('resource_reports', [])) # This owners seem to have been replaced by a separate query. I left the owners in this query, @@ -523,7 +528,7 @@ def _exec_table_query(self, table_uri: str) -> Tuple: # tags, sources, badges, prog_descriptions, resource_reports return wmk_results, table_writer, table_apps, timestamp_value,\ - tags, sources, badges, prog_descriptions, resource_reports + tags, sources, badges, prog_descriptions, update_frequency, resource_reports def _get_table_query_query_statement(self) -> str: table_query_level_query = textwrap.dedent(""" @@ -1827,7 +1832,7 @@ def _get_user_resource_relationship_clause(relation_type: UserResourceRel, id: s else: raise NotImplementedError(f'The relation type {relation_type} is not defined!') return relation - + def _get_dashboard_by_user_relation_query_statement(self, user_email: str, relation_type: UserResourceRel) -> str: rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type, id='', @@ -2736,8 +2741,8 @@ def get_resource_generation_code(self, *, uri: str, resource_type: ResourceType) return GenerationCode(key=query_result['key'], text=query_result['text'], source=query_result['source']) - - + + def _get_snowflake_table_shares_query_statement(self) -> str: snowflake_table_share_query = textwrap.dedent("""\ MATCH (table:Table {key: $table_key}) @@ -2747,7 +2752,7 @@ def _get_snowflake_table_shares_query_statement(self) -> str: """) return snowflake_table_share_query - @timer_with_counter + @timer_with_counter def get_snowflake_table_shares(self, *, table_uri: str) -> Union[List[SnowflakeTableShare], None]: snowflake_table_share_query = self._get_snowflake_table_shares_query_statement() records = self._execute_cypher_query(statement=snowflake_table_share_query, @@ -2758,7 +2763,7 @@ def get_snowflake_table_shares(self, *, table_uri: str) -> Union[List[SnowflakeT snowflake_table_shares = [] for record in records: - + listing_global_name = record.get('listing_global_name', None) if listing_global_name: snowflake_listing = SnowflakeListing(global_name=listing_global_name, @@ -2766,11 +2771,11 @@ def get_snowflake_table_shares(self, *, table_uri: str) -> Union[List[SnowflakeT title=record.get('listing_title', None), subtitle=record.get('listing_subtitle', None), description=record.get('description', None)) - + snowflake_table_share = SnowflakeTableShare(owner_account=record['share_owner_account'], name=record['share_name'], listing=snowflake_listing) - + snowflake_table_shares.append(snowflake_table_share) return snowflake_table_shares diff --git a/requirements-common.txt b/requirements-common.txt index ce0751fdba..90d8f6519b 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -5,7 +5,7 @@ # It is recommended to always pin the exact version (not range) - otherwise common upgrade won't trigger unit tests # on all repositories reyling on this file and any issues that arise from common upgrade might be missed. -amundsen-common==0.31.0+foodtruck.2 +amundsen-common==0.31.0+foodtruck.3 # amundsen-common>=0.27.0 attrs>=19.1.0 boto3==1.17.23 diff --git a/search/search_service/api/swagger_doc/template.yml b/search/search_service/api/swagger_doc/template.yml index c39a309522..b8d4400352 100644 --- a/search/search_service/api/swagger_doc/template.yml +++ b/search/search_service/api/swagger_doc/template.yml @@ -120,6 +120,10 @@ components: type: string description: 'list of programmatic descriptions' example: ['programmatic description1', 'programmatic description2'] + update_frequency: + type: string + description: 'frequency of updates' + example: 'monthly' tags: type: array items: diff --git a/search/search_service/models/table.py b/search/search_service/models/table.py index dc5006bb44..99e8020bd8 100644 --- a/search/search_service/models/table.py +++ b/search/search_service/models/table.py @@ -33,6 +33,7 @@ class Table(Base): column_names: Optional[List[str]] = None column_descriptions: Optional[List[str]] = None programmatic_descriptions: Optional[List[str]] = None + update_frequency: Optional[str] = None # The following are search-only properties: total_usage: int = 0 schema_description: Optional[str] = attr.ib(default=None) @@ -68,6 +69,7 @@ def get_attrs(cls) -> Set: 'last_updated_timestamp', 'display_name', 'programmatic_descriptions', + 'update_frequency', 'total_usage', 'schema_description' } diff --git a/search/search_service/proxy/elasticsearch.py b/search/search_service/proxy/elasticsearch.py index 08ff61aa58..81439a6766 100644 --- a/search/search_service/proxy/elasticsearch.py +++ b/search/search_service/proxy/elasticsearch.py @@ -177,7 +177,8 @@ def get_table_search_query(self, query_term: str) -> dict: "column_descriptions", "tags", "badges", - "programmatic_descriptions"], + "programmatic_descriptions", + "update_frequency"], } }, "field_value_factor": { @@ -297,6 +298,7 @@ def _get_search_result(self, page_index: int, 'display_name': 'display name', 'last_updated_timestamp': 12345678, 'programmatic_descriptions': [], + 'update_frequency', 'schema_description': None, 'tags': ['tag1', 'tag2'], 'badges': [],