From c221d3bd7e9191f1b01636bb218e73354a6e4dec Mon Sep 17 00:00:00 2001
From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com>
Date: Sun, 8 Mar 2020 20:22:35 +0900
Subject: [PATCH 01/28] Adding Youtube API.

---
 README.md                                 |  5 ++
 build.py                                  |  1 +
 requirements.txt                          |  4 +-
 src/main/python/mlsearch/api_requester.py | 82 ++++++++++++++++++++---
 src/main/python/mlsearch/config.py        | 10 ++-
 src/main/python/mlsearch/helper.py        |  4 +-
 src/main/python/mlsearch/protocol.py      | 14 +++-
 src/main/scripts/mlsearch                 | 26 +++++--
 8 files changed, 129 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index 4366464..0e32da6 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,10 @@ Optional Parameters:
                         Paper with code repository password.
   -t GITHUB_ACC_TOKEN, --github_acc_token GITHUB_ACC_TOKEN
                         Github access token.
+  -yk YOUTUBE_DEV_KEY, --youtube_dev_key YOUTUBE_DEV_KEY
+                        Youtube developer key.
+  -np NEXT_PAGE_TOKEN, --next_page_token NEXT_PAGE_TOKEN
+                        Next page token for Youtube API.
 ```
 
 <hr>
@@ -51,4 +55,5 @@ api_request = APIRequest(source, query,
     init_idx, count)
 api_request.pwc_auth_info = ('user_name', 'password')
 api_request.github_acc_token = 'token'
+api_request.youtube_developer_key = 'you_key'
 ```
\ No newline at end of file
diff --git a/build.py b/build.py
index cb91233..dd34c76 100644
--- a/build.py
+++ b/build.py
@@ -18,3 +18,4 @@ def set_properties(project):
     project.build_depends_on("mock")
     project.build_depends_on("requests")
     project.build_depends_on("pygithub")
+    project.build_depends_on("google-api-python-client")
diff --git a/requirements.txt b/requirements.txt
index 656236f..c118007 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,4 @@
 PyGithub==1.43.8
-pybuilder
\ No newline at end of file
+pybuilder
+requests
+google-api-python-client
\ No newline at end of file
diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 33076c2..d543fdf 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -2,6 +2,7 @@
 from mlsearch.protocol import Protocol
 from github import Github
 from requests.auth import HTTPBasicAuth
+import googleapiclient.discovery
 import json
 import requests
 # import scholarly
@@ -10,7 +11,7 @@
 class APIRequest():
     """For handling the different Valid API requests."""
 
-    def __init__(self, source, query, init_idx, count):
+    def __init__(self, source, query, init_idx, count, next_page_token=None):
         """
         Initialization for the class.
 
@@ -18,10 +19,12 @@ def __init__(self, source, query, init_idx, count):
         :param  query:      The query for searching.
         :param  init_idx:   The initial pagination index.
         :param  count:      The number of records to be fetched.
+        :param  next_page_token: The current page token for youtube API.
         """
 
         self.params = {'query':query, 'init_idx':init_idx, 
-                            'count':count, 'source': source}
+                            'count':count, 'source': source,
+                            'next_page_token': next_page_token}
         self.params_model = {'query':str, 'init_idx':int, 
                                   'count':int}
         # Load the configuration file
@@ -32,7 +35,8 @@ def __init__(self, source, query, init_idx, count):
         self.data = {
             'response_code': 201,
             'content': None,
-            'has_next_page': False}
+            'has_next_page': False,
+            'next_page_token': None}
 
     @property
     def github_acc_token(self):
@@ -43,6 +47,14 @@ def github_acc_token(self, access_token):
         if access_token:
             self._config.GITHUB_ACC_TOKEN = access_token
 
+    @property
+    def youtube_developer_key(self):
+        return self._config.YOUTUBE_DEVELOPER_KEY
+
+    @youtube_developer_key.setter
+    def youtube_developer_key(self, developer_key):
+        if developer_key:
+            self._config.YOUTUBE_DEVELOPER_KEY = developer_key
 
     @property
     def pwc_auth_info(self):
@@ -70,7 +82,7 @@ def _validate_params(self):
                 raise TypeError(
                     f'Invalid type for {item}. {typ} is expected but ' 
                     f'{type(self.params[item])} is given.')
-
+        
         if self.params['source'] not in self._config.VALID_API_SOURCE:
             raise ValueError(
                 f"Invalid value for {self.params['source']}. "
@@ -118,8 +130,8 @@ def _fetch_github(self) -> [Protocol]:
             }
             results.append(Protocol(data))
             
-            self.data['response_code'] = 200
-            self.data['content'] = [proto.to_JSON() for proto in results]
+        self.data['response_code'] = 200
+        self.data['content'] = [proto.to_JSON() for proto in results]
 
     def _fetch_paperwithcode(self) -> [Protocol]:
         """Fetch Paper with Code Repository"""
@@ -161,7 +173,58 @@ def _fetch_paperwithcode(self) -> [Protocol]:
             self.data['content'] = [proto.to_JSON() for proto in results]
 
         self.data['response_code'] = query_result.status_code
-        
+    
+    def _fetch_youtube(self, next_page_token=None) -> [Protocol]:
+        """Fetch the Youtube Repository"""
+        results = []
+        youtube = googleapiclient.discovery.build(
+            self._config.YOUTUBE_SERVICE_NAME, 
+            self._config.YOUTUBE_API_VERSION, 
+            developerKey = self._config.YOUTUBE_DEVELOPER_KEY)
+        request = youtube.search().list(
+            part=self._config.YOUTUBE_PART,
+            maxResults=self.params['count'],
+            order=self._config.YOUTUBE_ORDER,
+            q=self.params['query'],
+            safeSearch=self._config.YOUTUBE_SAFESEARCH,
+            pageToken=next_page_token
+        )
+        response = request.execute()
+
+        if 'items' in response and len(response['items']) > 0:
+            for item in response['items']:
+                data = {
+                    'video_id': item.get(
+                        'id', dict({'videoId': None})
+                        ).get('videoId', None),
+                    'title': item.get(
+                        'snippet', dict({'title': None})
+                        ).get('title', None),
+                    'description': item.get(
+                        'snippet',dict({'description': None})
+                        ).get('description', None),
+                    'channel_id': item.get(
+                        'snippet',dict({'channelId': None})
+                        ).get('channelId', None),
+                    'channel_title': item.get(
+                        'snippet',dict({'channelTitle': None})
+                        ).get('channelTitle', None),
+                    'live_broadcast_content': item.get(
+                        'snippet',dict({'liveBroadcastContent': None})
+                        ).get('liveBroadcastContent', None),
+                    'published_datetime': item.get(
+                        'snippet',dict({'publishedAt': None})
+                        ).get('publishedAt', None),
+                    'thumbnails': item.get(
+                        'snippet',dict({'thumbnails': None})
+                        ).get('thumbnails', None),
+                }
+                results.append(Protocol(data))
+            self.data['next_page_token'] = response.get('nextPageToken', None)
+            self.data['content'] = [proto.to_JSON() for proto in results]
+            self.data['has_next_page'] = response.get('pageInfo', dict({'totalResults':0})).get('totalResults', 0) > 0
+        self.data['response_code'] = 200
+
     def fetch_data(self) -> json:
         """Fetch the data from designated API source."""
 
@@ -170,7 +233,10 @@ def fetch_data(self) -> json:
                 self._fetch_paperwithcode()
 
             if self.params.get('source', '') == 'github':
-                responses = self._fetch_github()
+                self._fetch_github()
+
+            if self.params.get('source', '') == 'youtube':
+                self._fetch_youtube(self.params.get('next_page_token', None))
 
             # TODO: Implement the function for Coursera. However, this function
             # may be handled by the backend server.
diff --git a/src/main/python/mlsearch/config.py b/src/main/python/mlsearch/config.py
index fdbc95a..c53574f 100644
--- a/src/main/python/mlsearch/config.py
+++ b/src/main/python/mlsearch/config.py
@@ -13,4 +13,12 @@ class Config(object):
     GITHUB_URL = os.environ.get('GITHUB_URL') or "in:readme+in:description"
 
     # AIP Source
-    VALID_API_SOURCE = ['paperwithcode', 'github', 'coursera']
\ No newline at end of file
+    VALID_API_SOURCE = ['paperwithcode', 'github', 'coursera', 'youtube']
+
+    # Youtube configuration
+    YOUTUBE_SERVICE_NAME = os.environ.get('YOUTUBE_SERVICE_NAME') or "youtube"
+    YOUTUBE_API_VERSION = os.environ.get('YOUTUBE_API_VERSION') or "v3"
+    YOUTUBE_DEVELOPER_KEY = os.environ.get('YOUTUBE_DEVELOPER_KEY') or None
+    YOUTUBE_ORDER = os.environ.get('YOUTUBE_ORDER') or "relevance"
+    YOUTUBE_SAFESEARCH = os.environ.get('YOUTUBE_SAFESEARCH') or "strict"
+    YOUTUBE_PART = os.environ.get('YOUTUBE_PART') or "snippet"
\ No newline at end of file
diff --git a/src/main/python/mlsearch/helper.py b/src/main/python/mlsearch/helper.py
index e53bef8..65ed55a 100644
--- a/src/main/python/mlsearch/helper.py
+++ b/src/main/python/mlsearch/helper.py
@@ -40,7 +40,8 @@ def parse_parameters(event):
     :return:            dict(
                             'query', 'init_idx',
                             'count', 'source',
-                            'cookies', 'timestamp')
+                            'cookies', 'timestamp',
+                            'next_page_token')
     """
     try:
         param = dict()
@@ -50,6 +51,7 @@ def parse_parameters(event):
         param['source'] = event['source']
         param['cookies'] = event['cookies']
         param['timestamp'] = event['timestamp']
+        param['next_page_token'] = event['next_page_token']
         
         if param['init_idx'] >= 0 and param['count'] > 0:
             return param
diff --git a/src/main/python/mlsearch/protocol.py b/src/main/python/mlsearch/protocol.py
index 05c67aa..826cc46 100644
--- a/src/main/python/mlsearch/protocol.py
+++ b/src/main/python/mlsearch/protocol.py
@@ -25,7 +25,13 @@ def __init__(self, kwargs):
             'partners_v1', 'instructors_v1',
             
             # Source Flag
-            'source'
+            'source',
+
+            # Youtube
+            'video_id',
+            'channel_id', 'channel_title',
+            'live_broadcast_content', 'published_datetime',
+            'thumbnails',
         ]
         
         for param in kwargs:
@@ -54,6 +60,12 @@ def __init__(self, kwargs):
         self.instructors_v1 = kwargs.get('instructors_v1', None)
         self.source = kwargs.get('source', None)
         self.pwc_url = kwargs.get('pwc_url', None)
+        self.video_id = kwargs.get('video_id', None)
+        self.channel_id = kwargs.get('channel_id', None)
+        self.channel_title = kwargs.get('channel_title', None)
+        self.live_broadcast_content = kwargs.get('live_broadcast_content', None)
+        self.published_datetime = kwargs.get('published_datetime', None)
+        self.thumbnails = kwargs.get('thumbnails', dict())
     
     def to_JSON(self):
         """Transform the Protocol object to JSON object."""
diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 085c385..f732398 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -22,6 +22,8 @@ ap.add_argument('-tm', '--timestamp', required=True, help="Timestamp of requesti
 ap.add_argument('-u', '--pwc_user', required=False, help="Paper with code repository user name.")
 ap.add_argument('-p', '--pwc_password', required=False, help="Paper with code repository password.")
 ap.add_argument('-t', '--github_acc_token', required=False, help="Github access token.")
+ap.add_argument('-yk', '--youtube_dev_key', required=False, help="Youtube developer key.")
+ap.add_argument('-np', '--next_page_token', required=False, help="Next page token for Youtube API.")
 args = vars(ap.parse_args())
 
 def main(event):
@@ -35,23 +37,33 @@ def main(event):
         param_names = [
             'query', 'init_idx',
             'count', 'source',
-            'cookies', 'timestamp']
+            'cookies', 'timestamp',
+            'next_page_token']
         response_msg = hp.response('success', 200)
-        
         if hp.is_valid_parameters(event, param_names):
             params = hp.parse_parameters(event)
             if params.values():
-                api_request = APIRequest(params['source'], params['query'], params['init_idx'], params['count'])
+                api_request = APIRequest(
+                    params['source'], 
+                    params['query'], 
+                    params['init_idx'], 
+                    params['count'],
+                    params['next_page_token'])
                 if 'pwc_user'in event and 'pwc_password' in event:
                     api_request.pwc_auth_info = (event['pwc_user'], event['pwc_password'])
                 if 'github_acc_token' in event:
                     api_request.github_acc_token = event['github_acc_token']
+                if 'youtube_developer_key' in event:
+                    api_request.youtube_developer_key = event['youtube_developer_key']
                 data = api_request.fetch_data()
                 response_msg = hp.response(
                     message=data.get('content',''),
                     status_code=data.get('response_code'),
                     headers=headers,
-                    optional_attributes={'has_next_page': data.get('has_next_page', False)})
+                    optional_attributes={
+                        'has_next_page': data.get('has_next_page', False),
+                        'next_page_token': data.get('next_page_token', None)})
+
                 return response_msg
 
         response_msg = hp.response('Invalid parameters.', 400)
@@ -59,6 +71,7 @@ def main(event):
 
     except (ValueError, TypeError):
         response_msg = hp.response('Invalid parameters.', 400)
+        return response_msg
 
     except Exception as ex:
         response_msg = hp.response(str(ex), 500)
@@ -71,7 +84,8 @@ if __name__ == "__main__":
         'count': args['count'],
         'source': args['source'],
         'cookies': args['cookies'],
-        'timestamp': args['timestamp']
+        'timestamp': args['timestamp'],
+        'next_page_token': args['next_page_token']
     }
 
     if args['pwc_user']:
@@ -80,6 +94,8 @@ if __name__ == "__main__":
         event['pwc_password'] = args['pwc_password']
     if args['github_acc_token']:
         event['github_acc_token'] = args['github_acc_token']
+    if args['youtube_dev_key']:
+        event['youtube_developer_key'] = args['youtube_dev_key']
 
     result = main(event)
     pp = pprint.PrettyPrinter(indent=2)

From 6752e92f3e1af8e01158ca6f6e94ad20c75b6606 Mon Sep 17 00:00:00 2001
From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com>
Date: Sun, 8 Mar 2020 20:42:32 +0900
Subject: [PATCH 02/28] Modify Error Message

---
 src/main/scripts/mlsearch | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index f732398..26a92c1 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -69,8 +69,8 @@ def main(event):
         response_msg = hp.response('Invalid parameters.', 400)
         return response_msg
 
-    except (ValueError, TypeError):
-        response_msg = hp.response('Invalid parameters.', 400)
+    except (ValueError, TypeError) as ex:
+        response_msg = hp.response(str(ex), 400)
         return response_msg
 
     except Exception as ex:

From a07a9be62b6bc1ce4f7bd94c889f34db9e9495d4 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 00:52:02 +0900
Subject: [PATCH 03/28] Update README.md

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0e32da6..74ff24b 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ Optional Parameters:
                         Github access token.
   -yk YOUTUBE_DEV_KEY, --youtube_dev_key YOUTUBE_DEV_KEY
                         Youtube developer key.
-  -np NEXT_PAGE_TOKEN, --next_page_token NEXT_PAGE_TOKEN
+  -ynpt NEXT_PAGE_TOKEN, --next_page_token NEXT_PAGE_TOKEN
                         Next page token for Youtube API.
 ```
 
@@ -56,4 +56,4 @@ api_request = APIRequest(source, query,
 api_request.pwc_auth_info = ('user_name', 'password')
 api_request.github_acc_token = 'token'
 api_request.youtube_developer_key = 'you_key'
-```
\ No newline at end of file
+```

From 7ce44d4a1bcb25374f0162b245c495dd18a45a22 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:06:06 +0900
Subject: [PATCH 04/28] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 74ff24b..131e886 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Optional Parameters:
                         Paper with code repository user name.
   -p PWC_PASSWORD, --pwc_password PWC_PASSWORD
                         Paper with code repository password.
-  -t GITHUB_ACC_TOKEN, --github_acc_token GITHUB_ACC_TOKEN
+  -gt GITHUB_ACC_TOKEN, --github_acc_token GITHUB_ACC_TOKEN
                         Github access token.
   -yk YOUTUBE_DEV_KEY, --youtube_dev_key YOUTUBE_DEV_KEY
                         Youtube developer key.

From 0b023aa8f0777e63ed3bc206db6451f083c549b2 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:06:13 +0900
Subject: [PATCH 05/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index d543fdf..a7bcc66 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -24,7 +24,7 @@ def __init__(self, source, query, init_idx, count, next_page_token=None):
 
         self.params = {'query':query, 'init_idx':init_idx, 
                             'count':count, 'source': source,
-                            'next_page_token': next_page_token}
+                            'y_next_page_token': next_page_token}
         self.params_model = {'query':str, 'init_idx':int, 
                                   'count':int}
         # Load the configuration file
@@ -247,4 +247,4 @@ def fetch_data(self) -> json:
             self.data['response_code'] = 500
             self.data['content'] = str(ex)
 
-        return self.data
\ No newline at end of file
+        return self.data

From c3af9373ef0d810903d5660ab6c4232b267bf7f9 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:06:39 +0900
Subject: [PATCH 06/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index a7bcc66..f739398 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -82,7 +82,6 @@ def _validate_params(self):
                 raise TypeError(
                     f'Invalid type for {item}. {typ} is expected but ' 
                     f'{type(self.params[item])} is given.')
-        
         if self.params['source'] not in self._config.VALID_API_SOURCE:
             raise ValueError(
                 f"Invalid value for {self.params['source']}. "

From 1dc2ccc9e882993ee49f451e10bee00a9c58c29b Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:06:49 +0900
Subject: [PATCH 07/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 26a92c1..09f672d 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -19,11 +19,11 @@ ap.add_argument('-c', '--count', required=True, help="Total number of results to
 ap.add_argument('-s', '--source', required=True, help="Source API to be looking for.")
 ap.add_argument('-ck', '--cookies', required=True, help="Cookies of current user.")
 ap.add_argument('-tm', '--timestamp', required=True, help="Timestamp of requesting API.")
-ap.add_argument('-u', '--pwc_user', required=False, help="Paper with code repository user name.")
-ap.add_argument('-p', '--pwc_password', required=False, help="Paper with code repository password.")
-ap.add_argument('-t', '--github_acc_token', required=False, help="Github access token.")
+ap.add_argument('-pu', '--pwc_user', required=False, help="Paper with code repository user name.")
+ap.add_argument('-pp', '--pwc_password', required=False, help="Paper with code repository password.")
+ap.add_argument('-gt', '--github_acc_token', required=False, help="Github access token.")
 ap.add_argument('-yk', '--youtube_dev_key', required=False, help="Youtube developer key.")
-ap.add_argument('-np', '--next_page_token', required=False, help="Next page token for Youtube API.")
+ap.add_argument('-yntp', '--next_page_token', required=False, help="Next page token for Youtube API.")
 args = vars(ap.parse_args())
 
 def main(event):
@@ -99,4 +99,4 @@ if __name__ == "__main__":
 
     result = main(event)
     pp = pprint.PrettyPrinter(indent=2)
-    pp.pprint(result)
\ No newline at end of file
+    pp.pprint(result)

From e1769fa1e9415848f1be09ff53dc3cb832567806 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:06:56 +0900
Subject: [PATCH 08/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 09f672d..f39614a 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -38,7 +38,7 @@ def main(event):
             'query', 'init_idx',
             'count', 'source',
             'cookies', 'timestamp',
-            'next_page_token']
+            'y_next_page_token']
         response_msg = hp.response('success', 200)
         if hp.is_valid_parameters(event, param_names):
             params = hp.parse_parameters(event)

From 294ae5466bb63dcde497823418642194b05c0464 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:13:27 +0900
Subject: [PATCH 09/28] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 131e886..aec3533 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ Optional Parameters:
 
   -u PWC_USER, --pwc_user PWC_USER
                         Paper with code repository user name.
-  -p PWC_PASSWORD, --pwc_password PWC_PASSWORD
+  -pp PWC_PASSWORD, --pwc_password PWC_PASSWORD
                         Paper with code repository password.
   -gt GITHUB_ACC_TOKEN, --github_acc_token GITHUB_ACC_TOKEN
                         Github access token.

From 8b73c026a4f0c6018e3d2e5aca48c811a58e8094 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:13:52 +0900
Subject: [PATCH 10/28] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index aec3533..7411676 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ Required Parameters:
                         Timestamp of requesting API.
 Optional Parameters:
 
-  -u PWC_USER, --pwc_user PWC_USER
+  -pu PWC_USER, --pwc_user PWC_USER
                         Paper with code repository user name.
   -pp PWC_PASSWORD, --pwc_password PWC_PASSWORD
                         Paper with code repository password.

From 5b84d363dcef2f03d2d11761ab247531d2572bdd Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:14:36 +0900
Subject: [PATCH 11/28] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7411676..b94bd1c 100644
--- a/README.md
+++ b/README.md
@@ -55,5 +55,5 @@ api_request = APIRequest(source, query,
     init_idx, count)
 api_request.pwc_auth_info = ('user_name', 'password')
 api_request.github_acc_token = 'token'
-api_request.youtube_developer_key = 'you_key'
+api_request.youtube_developer_key = 'your_key'
 ```

From 816688b5723d67c3c5852f7705564684b96aa94e Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:15:37 +0900
Subject: [PATCH 12/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index f739398..18f663d 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -11,7 +11,7 @@
 class APIRequest():
     """For handling the different Valid API requests."""
 
-    def __init__(self, source, query, init_idx, count, next_page_token=None):
+    def __init__(self, source, query, init_idx, count, y_next_page_token=None):
         """
         Initialization for the class.
 

From 11733f0d30cc90251f0da98c728936442db04020 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:16:01 +0900
Subject: [PATCH 13/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 18f663d..0b30c85 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -19,7 +19,7 @@ def __init__(self, source, query, init_idx, count, y_next_page_token=None):
         :param  query:      The query for searching.
         :param  init_idx:   The initial pagination index.
         :param  count:      The number of records to be fetched.
-        :param  next_page_token: The current page token for youtube API.
+        :param  y_next_page_token: The current page token for youtube API.
         """
 
         self.params = {'query':query, 'init_idx':init_idx, 

From c8479bbf32b7f3b3bc53c8f4c3f4ca48934b70c8 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:16:22 +0900
Subject: [PATCH 14/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 0b30c85..10b3ca7 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -36,7 +36,7 @@ def __init__(self, source, query, init_idx, count, y_next_page_token=None):
             'response_code': 201,
             'content': None,
             'has_next_page': False,
-            'next_page_token': None}
+            'y_next_page_token': None}
 
     @property
     def github_acc_token(self):

From d0693c0585ffac0e61c148d74dd25c0ca8604328 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:17:59 +0900
Subject: [PATCH 15/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 10b3ca7..e654a6b 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -186,7 +186,7 @@ def _fetch_youtube(self, next_page_token=None) -> [Protocol]:
             order=self._config.YOUTUBE_ORDER,
             q=self.params['query'],
             safeSearch=self._config.YOUTUBE_SAFESEARCH,
-            pageToken=next_page_token
+            pageToken=y_next_page_token
         )
         response = request.execute()
 

From 94d8da9e98ff4480c62c2a6097d6eaa49875a0d1 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:18:57 +0900
Subject: [PATCH 16/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index e654a6b..7ad449d 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -24,7 +24,7 @@ def __init__(self, source, query, init_idx, count, y_next_page_token=None):
 
         self.params = {'query':query, 'init_idx':init_idx, 
                             'count':count, 'source': source,
-                            'y_next_page_token': next_page_token}
+                            'y_next_page_token': y_next_page_token}
         self.params_model = {'query':str, 'init_idx':int, 
                                   'count':int}
         # Load the configuration file

From fda79778aec0068dd47b7da822ec34568f938311 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:19:32 +0900
Subject: [PATCH 17/28] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b94bd1c..c425d65 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ Optional Parameters:
                         Github access token.
   -yk YOUTUBE_DEV_KEY, --youtube_dev_key YOUTUBE_DEV_KEY
                         Youtube developer key.
-  -ynpt NEXT_PAGE_TOKEN, --next_page_token NEXT_PAGE_TOKEN
+  -ynpt NEXT_PAGE_TOKEN, --y_next_page_token NEXT_PAGE_TOKEN
                         Next page token for Youtube API.
 ```
 

From 27458cd5b6a9271ef5d6e503a55003ff7a95099c Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:20:30 +0900
Subject: [PATCH 18/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 7ad449d..a55d08d 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -173,7 +173,7 @@ def _fetch_paperwithcode(self) -> [Protocol]:
 
         self.data['response_code'] = query_result.status_code
     
-    def _fetch_youtube(self, next_page_token=None) -> [Protocol]:
+    def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]:
         """Fetch the Youtube Repository"""
         results = []
         youtube = googleapiclient.discovery.build(

From d5590ff0893c0dece7bf9f8d03172192912ca74e Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:21:09 +0900
Subject: [PATCH 19/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index a55d08d..7bfc4da 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -219,7 +219,7 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]:
                         ).get('thumbnails', None),
                 }
                 results.append(Protocol(data))
-            self.data['next_page_token'] = response.get('nextPageToken', None)
+            self.data['y_next_page_token'] = response.get('nextPageToken', None)
             self.data['content'] = [proto.to_JSON() for proto in results]
             self.data['has_next_page'] = response.get('pageInfo', dict({'totalResults':0})).get('totalResults', 0) > 0
         self.data['response_code'] = 200

From 9dcd0120420353eece96620b345312c325e43216 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:21:39 +0900
Subject: [PATCH 20/28] Update src/main/python/mlsearch/api_requester.py

---
 src/main/python/mlsearch/api_requester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 7bfc4da..19814fa 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -235,7 +235,7 @@ def fetch_data(self) -> json:
                 self._fetch_github()
 
             if self.params.get('source', '') == 'youtube':
-                self._fetch_youtube(self.params.get('next_page_token', None))
+                self._fetch_youtube(self.params.get('y_next_page_token', None))
 
             # TODO: Implement the function for Coursera. However, this function
             # may be handled by the backend server.

From a71c9cf2c1fbeeeafe876ab2c860d34da655cd26 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:23:45 +0900
Subject: [PATCH 21/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index f39614a..8a87694 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -85,7 +85,7 @@ if __name__ == "__main__":
         'source': args['source'],
         'cookies': args['cookies'],
         'timestamp': args['timestamp'],
-        'next_page_token': args['next_page_token']
+        'y_next_page_token': args['y_next_page_token']
     }
 
     if args['pwc_user']:

From c37544f03ed380910b6f96c45531e9e3d5a1cebd Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:24:04 +0900
Subject: [PATCH 22/28] Update src/main/python/mlsearch/helper.py

---
 src/main/python/mlsearch/helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/python/mlsearch/helper.py b/src/main/python/mlsearch/helper.py
index 65ed55a..834cfa7 100644
--- a/src/main/python/mlsearch/helper.py
+++ b/src/main/python/mlsearch/helper.py
@@ -41,7 +41,7 @@ def parse_parameters(event):
                             'query', 'init_idx',
                             'count', 'source',
                             'cookies', 'timestamp',
-                            'next_page_token')
+                            'y_next_page_token')
     """
     try:
         param = dict()
@@ -59,4 +59,4 @@ def parse_parameters(event):
             return dict()
             
     except:
-        return dict()
\ No newline at end of file
+        return dict()

From 84a4763d1f6ba0d9cc350cffa13951d705c6a5bb Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:24:14 +0900
Subject: [PATCH 23/28] Update src/main/python/mlsearch/helper.py

---
 src/main/python/mlsearch/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/python/mlsearch/helper.py b/src/main/python/mlsearch/helper.py
index 834cfa7..111cf5f 100644
--- a/src/main/python/mlsearch/helper.py
+++ b/src/main/python/mlsearch/helper.py
@@ -51,7 +51,7 @@ def parse_parameters(event):
         param['source'] = event['source']
         param['cookies'] = event['cookies']
         param['timestamp'] = event['timestamp']
-        param['next_page_token'] = event['next_page_token']
+        param['y_next_page_token'] = event['y_next_page_token']
         
         if param['init_idx'] >= 0 and param['count'] > 0:
             return param

From 8c2c87274cd270bfbc046dadf52fc4d35339e666 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:24:21 +0900
Subject: [PATCH 24/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 8a87694..9a7d383 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -23,7 +23,7 @@ ap.add_argument('-pu', '--pwc_user', required=False, help="Paper with code repos
 ap.add_argument('-pp', '--pwc_password', required=False, help="Paper with code repository password.")
 ap.add_argument('-gt', '--github_acc_token', required=False, help="Github access token.")
 ap.add_argument('-yk', '--youtube_dev_key', required=False, help="Youtube developer key.")
-ap.add_argument('-yntp', '--next_page_token', required=False, help="Next page token for Youtube API.")
+ap.add_argument('-yntp', '--y_next_page_token', required=False, help="Next page token for Youtube API.")
 args = vars(ap.parse_args())
 
 def main(event):

From f4375c5dcc64bd4c6239ef3a25c18b28eef17a3d Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:24:29 +0900
Subject: [PATCH 25/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 9a7d383..2def556 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -48,7 +48,7 @@ def main(event):
                     params['query'], 
                     params['init_idx'], 
                     params['count'],
-                    params['next_page_token'])
+                    params['y_next_page_token'])
                 if 'pwc_user'in event and 'pwc_password' in event:
                     api_request.pwc_auth_info = (event['pwc_user'], event['pwc_password'])
                 if 'github_acc_token' in event:

From c391a3b1abe8705e059733b90e86425a6c5be28d Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:24:36 +0900
Subject: [PATCH 26/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 2def556..89e2606 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -62,7 +62,7 @@ def main(event):
                     headers=headers,
                     optional_attributes={
                         'has_next_page': data.get('has_next_page', False),
-                        'next_page_token': data.get('next_page_token', None)})
+                        'next_page_token': data.get('y_next_page_token', None)})
 
                 return response_msg
 

From 0d2fbe42e93685e09675a8d6f7624ab1a6b27fb5 Mon Sep 17 00:00:00 2001
From: Sai Htaung Kham <31495612+saihtaungkham@users.noreply.github.com>
Date: Thu, 12 Mar 2020 01:25:45 +0900
Subject: [PATCH 27/28] Update src/main/scripts/mlsearch

---
 src/main/scripts/mlsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch
index 89e2606..0d1b98e 100644
--- a/src/main/scripts/mlsearch
+++ b/src/main/scripts/mlsearch
@@ -62,7 +62,7 @@ def main(event):
                     headers=headers,
                     optional_attributes={
                         'has_next_page': data.get('has_next_page', False),
-                        'next_page_token': data.get('y_next_page_token', None)})
+                        'y_next_page_token': data.get('y_next_page_token', None)})
 
                 return response_msg
 

From 1c07c84fe697bf7ef1b81294198ae29a9cd8187e Mon Sep 17 00:00:00 2001
From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com>
Date: Tue, 14 Apr 2020 19:22:28 +0900
Subject: [PATCH 28/28] Fix missing source in youtube.

---
 src/main/python/mlsearch/api_requester.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py
index 19814fa..3077ba5 100644
--- a/src/main/python/mlsearch/api_requester.py
+++ b/src/main/python/mlsearch/api_requester.py
@@ -217,6 +217,7 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]:
                     'thumbnails': item.get(
                         'snippet',dict({'thumbnails': None})
                         ).get('thumbnails', None),
+                    'source': self.params.get('source', ''),
                 }
                 results.append(Protocol(data))
             self.data['y_next_page_token'] = response.get('nextPageToken', None)