Wed 2 Oct 12:42 am #19 - Added source class fror near GCP BigQuery

local-data-platform/local_data_platform/source/gcp/gcp_bigquery/source_gcp_bigquery.py

@@ -1,9 +1,23 @@
-from google.cloud import bigquery
+from source.gcp.gcp_connection import GCPBigQueryConnection
 
-class SourceGCPBigQuery :
+class SourceGCPBigQuery:
+    """
+    A class to interact with Google BigQuery.
+    client : GCPBigQueryConnection
+        An instance of GCPBigQueryConnection to interact with BigQuery.
+    """
+    def __init__(self):
+        self.client = GCPBigQueryConnection('path/to/credentials.json', 'your-project-id')
 
-    def fetch_data(query):
-        client = bigquery.Client()
-        query_job = client.query(query)
-        return query_job.to_dataframe()
-
+    def fetch_data(self, query):
+        """
+        Executes a given SQL query on Google BigQuery and returns the result as a pandas DataFrame.
+
+        Args:
+            query (str): The SQL query to be executed.
+
+        Returns:
+            pandas.DataFrame: The result of the query as a DataFrame.
+        """
+        query_job = self.client.query(query)
+        return query_job.to_dataframe()

local-data-platform/local_data_platform/source/gcp/gcp_connection.py

@@ -2,23 +2,59 @@
 from google.oauth2 import service_account
 
 class GCPBigQueryConnection:
+    """
+    GCPBigQueryConnection is a class that facilitates connection to Google Cloud Platform's BigQuery service using service account credentials.
+
+    Attributes:
+        credentials_path (str): The file path to the service account credentials JSON file.
+        project_id (str): The GCP project ID.
+
+    Methods:
+        __init__(credentials_path, project_id):
+            Initializes the GCPBigQueryConnection with the given credentials path and project ID.
+        
+        _create_client():
+        
+        query(query_string):
+            Executes a SQL query on BigQuery and returns the results.
+
+
+        Example usage:
+        connection = GCPBigQueryConnection('/path/to/credentials.json', 'your-project-id')
+        results = connection.query('SELECT * FROM your_dataset.your_table')
+        for row in results:
+            print(row)    
+    """
     def __init__(self, credentials_path, project_id):
         self.credentials_path = credentials_path
         self.project_id = project_id
         self.client = self._create_client()
 
     def _create_client(self):
+        """
+        Creates a BigQuery client using service account credentials.
+
+        This method reads the service account credentials from a file specified
+        by `self.credentials_path` and uses them to create a BigQuery client
+        for the project specified by `self.project_id`.
+
+        Returns:
+            bigquery.Client: An authenticated BigQuery client.
+        """
         credentials = service_account.Credentials.from_service_account_file(self.credentials_path)
         client = bigquery.Client(credentials=credentials, project=self.project_id)
         return client
 
     def query(self, query_string):
+        """
+        Executes a SQL query on the GCP BigQuery client and returns the results.
+
+        Args:
+            query_string (str): The SQL query to be executed.
+
+        Returns:
+            google.cloud.bigquery.table.RowIterator: An iterator over the rows in the query result.
+        """
         query_job = self.client.query(query_string)
         results = query_job.result()
         return results
-
-# Example usage:
-# connection = GCPBigQueryConnection('/path/to/credentials.json', 'your-project-id')
-# results = connection.query('SELECT * FROM your_dataset.your_table')
-# for row in results:
-#     print(row)

local-data-platform/local_data_platform/source/near/source_near_gcp.py

@@ -1,11 +1,23 @@
-from local_data_platform.source.gcp.gcp_bigquery import SourceGCPBigQuery
+from local_data_platform.source.gcp.gcp_bigquery.source_gcp_bigquery import SourceGCPBigQuery
+"""
+This module defines the Source_Near_GCP class, which inherits from SourceGCPBigQuery
+and provides functionality to fetch transaction data from the NEAR protocol on Google Cloud Platform.
 
+Classes:
+    Source_Near_GCP: A class to fetch NEAR protocol transaction data from GCP BigQuery.
+
+Methods:
+    fetch_near_transaction(self):
+        Fetches NEAR protocol transaction data from the previous day.
+        Returns:
+            DataFrame: A pandas DataFrame containing the transaction data.
+"""
 class Source_Near_GCP(SourceGCPBigQuery):
 
     def fetch_near_transaction(self):
         query = """
             SELECT *
-            FROM `nearprotocol.near.transaction`
-            WHERE DATE(block_timestamp) = DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY)
+            FROM `bigquery-public-data.crypto_near.transactions`
+            LIMIT 1000;
         """
-        return self.fetch(query)
+        return self.fetch_data(query)