update frequency added to ui

common/amundsen_common/models/index_map.py

@@ -104,7 +104,16 @@
                 "programmatic_descriptions": {
                     "type": "text",
                     "analyzer": "simple"
-                }
+                },
+                "update_frequency": {
+                    "type": "text",
+                    "analyzer": "simple",
+                    "fields": {
+                        "raw": {
+                            "type": "keyword"
+                        }
+                    }
+                },
             }
         }
     }

common/amundsen_common/models/table.py

@@ -208,6 +208,7 @@ class Table:
     sources: Optional[List[Source]] = None
     is_view: Optional[bool] = attr.ib(default=None, converter=default_if_none)
     programmatic_descriptions: List[ProgrammaticDescription] = []
+    update_frequency: Optional[str] = None
     common_joins: Optional[List[SqlJoin]] = None
     common_filters: Optional[List[SqlWhere]] = None
 

common/setup.py

@@ -4,7 +4,7 @@
 
 from setuptools import find_packages, setup
 
-__version__ = '0.31.0+foodtruck.2'
+__version__ = '0.31.0+foodtruck.3'
 
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements-dev.txt')

databuilder/databuilder/models/table_elasticsearch_document.py

@@ -27,7 +27,7 @@ def __init__(self,
                  badges: Optional[List[str]] = None,
                  display_name: Optional[str] = None,
                  schema_description: Optional[str] = None,
-                 programmatic_descriptions: List[str] = [],
+                 programmatic_descriptions: List[str] = []
                  ) -> None:
         self.database = database
         self.cluster = cluster

databuilder/databuilder/models/table_update_frequency.py

@@ -14,6 +14,8 @@
 
 
 class UpdateFrequency(Enum):
+    ANNUALLY = "annually"
+    QUARTERLY = "quarterly"
     MONTHLY = "monthly"
     WEEKLY = "weekly"
     DAILY = "daily"

frontend/amundsen_application/api/utils/metadata_utils.py

@@ -140,6 +140,9 @@ def marshall_table_full(table_dict: Dict) -> Dict:
     prog_descriptions = results['programmatic_descriptions']
     results['programmatic_descriptions'] = _convert_prog_descriptions(prog_descriptions)
 
+    update_frequency = results['update_frequency']
+    results['update_frequency'] = update_frequency
+
     columns = results['columns']
     for col in columns:
         # Set column key to guarantee it is available on the frontend

frontend/amundsen_application/static/js/config/index.spec.ts

@@ -979,6 +979,7 @@ describe('generateExploreUrl', () => {
     resource_reports: [],
     watermarks: [],
     programmatic_descriptions: {},
+    update_frequency: null
   };
 
   it('calls `exploreUrlGenerator` with table metadata', () => {
@@ -1136,6 +1137,7 @@ describe('getColumnLineageLink', () => {
       resource_reports: [],
       watermarks: [],
       programmatic_descriptions: {},
+      update_frequency: null
     };
     const columnName = 'column_name';
     const actual = ConfigUtils.getColumnLineageLink(tableData, columnName);

frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts

@@ -72,6 +72,7 @@ export const initialTableDataState: TableMetadata = {
   resource_reports: [],
   watermarks: [],
   programmatic_descriptions: {},
+  update_frequency: null
 };
 
 export const emptyQualityChecks = {

frontend/amundsen_application/static/js/fixtures/globalState.ts

@@ -261,6 +261,7 @@ const globalState: GlobalState = {
       resource_reports: [],
       watermarks: [],
       programmatic_descriptions: {},
+      update_frequency: null
     },
     tableOwners: {
       isLoading: true,

frontend/amundsen_application/static/js/fixtures/metadata/table.ts

@@ -117,6 +117,7 @@ export const tableMetadata: TableMetadata = {
     value: '2020-03-05',
   },
   programmatic_descriptions: {},
+  update_frequency: null,
   schema: 'base',
   sources: [{
     source:

frontend/amundsen_application/static/js/interfaces/TableMetadata.ts

@@ -115,6 +115,7 @@ export interface TableMetadata {
   resource_reports: ResourceReport[];
   watermarks: Watermark[];
   programmatic_descriptions: TableProgrammaticDescriptions;
+  update_frequency?: string | null;
 }
 
 export interface UpdateOwnerPayload {

frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx

@@ -602,7 +602,7 @@ export class TableDetail extends React.Component<
       ) : (
         `Snowflake Shares`
       );
-      
+
       tabInfo.push({
         content: (
           <SnowflakeSharesList
@@ -800,7 +800,7 @@ export class TableDetail extends React.Component<
               <TableReportsDropdown resourceReports={data.resource_reports} />
               {previewEnabled() && (
                 <DataPreviewButton modalTitle={this.getDisplayName()} />
-              )}              
+              )}
               <ExploreButton tableData={data} />
             </div>
           </header>
@@ -841,7 +841,11 @@ export class TableDetail extends React.Component<
                   {!!data.last_updated_timestamp && (
                     <section className="metadata-section">
                       <div className="section-title">
-                        {Constants.LAST_UPDATED_TITLE}
+                        {
+                          data.update_frequency != null
+                          ? `${Constants.LAST_UPDATED_TITLE} (${data.update_frequency})`
+                          : Constants.LAST_UPDATED_TITLE
+                        }
                       </div>
                       <time className="time-body-text">
                         {formatDateTimeShort({

metadata/metadata_service/proxy/neo4j_proxy.py

@@ -162,7 +162,7 @@ def get_table(self, *, table_uri: str) -> Table:
         owners = self._exec_owners_query(table_uri)
 
         wmk_results, table_writer, table_apps, timestamp_value, tags, sources, \
-            badges, prog_descs, resource_reports = self._exec_table_query(table_uri)
+            badges, prog_descs, update_frequency, resource_reports = self._exec_table_query(table_uri)
 
         joins, filters = self._exec_table_query_query(table_uri)
 
@@ -185,6 +185,7 @@ def get_table(self, *, table_uri: str) -> Table:
                       sources=sources,
                       is_view=self._safe_get(last_neo4j_record, 'table', 'is_view'),
                       programmatic_descriptions=prog_descs,
+                      update_frequency=update_frequency,
                       common_joins=joins,
                       common_filters=filters,
                       resource_reports=resource_reports
@@ -385,6 +386,7 @@ def _get_table_query_statement(self) -> str:
             OPTIONAL MATCH (table)-[:HAS_BADGE]->(badge:Badge)
             OPTIONAL MATCH (table)-[:SOURCE]->(src:Source)
             OPTIONAL MATCH (table)-[:DESCRIPTION]->(prog_descriptions:Programmatic_Description)
+            OPTIONAL MATCH (table)-[:UPDATE_FREQUENCY]->(update_frequency:Update_Frequency)
             OPTIONAL MATCH (table)-[:HAS_REPORT]->(resource_reports:Report)
             RETURN collect(distinct wmk) as wmk_records,
             collect(distinct app_producer) as producing_apps,
@@ -395,6 +397,7 @@ def _get_table_query_statement(self) -> str:
             collect(distinct badge) as badge_records,
             collect(distinct src) as sources,
             collect(distinct prog_descriptions) as prog_descriptions,
+            update_frequency.frequency as update_frequency,
             collect(distinct resource_reports) as resource_reports
         """)
         return table_level_query
@@ -496,7 +499,7 @@ def _exec_table_query(self, table_uri: str) -> Tuple:
 
         # The owners seem to have been replaced by a separate query.  I left the owners in this query,
         # but we are not extracting it here.
-        
+
         # owner_record = []
 
         # for owner in table_records.get('owner_records', []):
@@ -514,6 +517,8 @@ def _exec_table_query(self, table_uri: str) -> Tuple:
             table_records.get('prog_descriptions', [])
         )
 
+        update_frequency = table_records['update_frequency']
+
         resource_reports = self._extract_resource_reports_from_query(table_records.get('resource_reports', []))
 
         # This owners seem to have been replaced by a separate query.  I left the owners in this query,
@@ -523,7 +528,7 @@ def _exec_table_query(self, table_uri: str) -> Tuple:
         #     tags, sources, badges, prog_descriptions, resource_reports
 
         return wmk_results, table_writer, table_apps, timestamp_value,\
-            tags, sources, badges, prog_descriptions, resource_reports
+            tags, sources, badges, prog_descriptions, update_frequency, resource_reports
 
     def _get_table_query_query_statement(self) -> str:
         table_query_level_query = textwrap.dedent("""
@@ -1827,7 +1832,7 @@ def _get_user_resource_relationship_clause(relation_type: UserResourceRel, id: s
         else:
             raise NotImplementedError(f'The relation type {relation_type} is not defined!')
         return relation
-        
+
     def _get_dashboard_by_user_relation_query_statement(self, user_email: str, relation_type: UserResourceRel) -> str:
         rel_clause: str = self._get_user_resource_relationship_clause(relation_type=relation_type,
                                                                       id='',
@@ -2736,8 +2741,8 @@ def get_resource_generation_code(self, *, uri: str, resource_type: ResourceType)
         return GenerationCode(key=query_result['key'],
                               text=query_result['text'],
                               source=query_result['source'])
-    
-    
+
+
     def _get_snowflake_table_shares_query_statement(self) -> str:
         snowflake_table_share_query = textwrap.dedent("""\
             MATCH (table:Table {key: $table_key})
@@ -2747,7 +2752,7 @@ def _get_snowflake_table_shares_query_statement(self) -> str:
         """)
         return snowflake_table_share_query
 
-    @timer_with_counter    
+    @timer_with_counter
     def get_snowflake_table_shares(self, *, table_uri: str) -> Union[List[SnowflakeTableShare], None]:
         snowflake_table_share_query = self._get_snowflake_table_shares_query_statement()
         records = self._execute_cypher_query(statement=snowflake_table_share_query,
@@ -2758,19 +2763,19 @@ def get_snowflake_table_shares(self, *, table_uri: str) -> Union[List[SnowflakeT
 
         snowflake_table_shares = []
         for record in records:
-            
+
             listing_global_name = record.get('listing_global_name', None)
             if listing_global_name:
                 snowflake_listing = SnowflakeListing(global_name=listing_global_name,
                                                      name=record.get('listing_name', None),
                                                      title=record.get('listing_title', None),
                                                      subtitle=record.get('listing_subtitle', None),
                                                      description=record.get('description', None))
-            
+
             snowflake_table_share = SnowflakeTableShare(owner_account=record['share_owner_account'],
                                                         name=record['share_name'],
                                                         listing=snowflake_listing)
-            
+
             snowflake_table_shares.append(snowflake_table_share)
 
         return snowflake_table_shares

requirements-common.txt

@@ -5,7 +5,7 @@
 
 # It is recommended to always pin the exact version (not range) - otherwise common upgrade won't trigger unit tests
 # on all repositories reyling on this file and any issues that arise from common upgrade might be missed.
-amundsen-common==0.31.0+foodtruck.2
+amundsen-common==0.31.0+foodtruck.3
 # amundsen-common>=0.27.0
 attrs>=19.1.0
 boto3==1.17.23

search/search_service/api/swagger_doc/template.yml

@@ -120,6 +120,10 @@ components:
             type: string
           description: 'list of programmatic descriptions'
           example: ['programmatic description1', 'programmatic description2']
+        update_frequency:
+          type: string
+          description: 'frequency of updates'
+          example: 'monthly'
         tags:
           type: array
           items:

search/search_service/models/table.py

@@ -33,6 +33,7 @@ class Table(Base):
     column_names: Optional[List[str]] = None
     column_descriptions: Optional[List[str]] = None
     programmatic_descriptions: Optional[List[str]] = None
+    update_frequency: Optional[str] = None
     # The following are search-only properties:
     total_usage: int = 0
     schema_description: Optional[str] = attr.ib(default=None)
@@ -68,6 +69,7 @@ def get_attrs(cls) -> Set:
             'last_updated_timestamp',
             'display_name',
             'programmatic_descriptions',
+            'update_frequency',
             'total_usage',
             'schema_description'
         }

search/search_service/proxy/elasticsearch.py

@@ -177,7 +177,8 @@ def get_table_search_query(self, query_term: str) -> dict:
                                    "column_descriptions",
                                    "tags",
                                    "badges",
-                                   "programmatic_descriptions"],
+                                   "programmatic_descriptions",
+                                   "update_frequency"],
                     }
                 },
                 "field_value_factor": {
@@ -297,6 +298,7 @@ def _get_search_result(self, page_index: int,
                         'display_name': 'display name',
                         'last_updated_timestamp': 12345678,
                         'programmatic_descriptions': [],
+                        'update_frequency',
                         'schema_description': None,
                         'tags': ['tag1', 'tag2'],
                         'badges': [],