[api] basic webapp setup, docs init

api/crawlers/__init__.py

@@ -0,0 +1 @@
+from client import Client

api/crawlers/client.py

@@ -0,0 +1,32 @@
+import requests
+import os
+from errors import ClientError
+
+default_api_url = 'http://localhost:5000'
+
+class Client:
+ def __init__(self, **kwargs):
+ self.init(**kwargs)
+
+ def init(self, **kwargs):
+ api_token = kwargs.get('api_token')
+ api_url = kwargs.get('api_url')
+ if api_token is None:
+ raise ClientError('Initilization the client failed, missing API Token')
+ else:
+ self.api_url = api_url or default_api_url
+ self.token = api_token 
+ # self.headers = {'Authorization': f'Bearer {self.token}'} 
+ # headers need to be implemented yet
+ self.update_url = self.api_url + '/add'
+
+ def update(self, article: dict):
+ # r = requests.post(update_url, headers=self.headers, json=article)
+ r = requests.post(update_url, json=article)
+ status = r.status_code 
+ if status == 200:
+ return r.json()
+ else:
+ raise Exception(f'Failed to update with {status_code}')
+
+

api/crawlers/errors.py

@@ -0,0 +1,2 @@
+class ClientError(Exception):
+ pass

api/helpers/__init__.py

@@ -0,0 +1,3 @@
+from .storage import Storage
+from . import config
+from .webapp import app

api/helpers/admin.py

@@ -0,0 +1,3 @@
+class Token:
+ def __init__(self):
+ pass

api/helpers/auth.py

@@ -0,0 +1 @@
+from jose import jwt

api/api/helpers/config.py → api/helpers/config.py

@@ -1,5 +1,5 @@
 import os
 
-
 GOOGLE_CLOUD_PROJECT=os.getenv('GOOGLE_CLOUD_PROJECT')
-KIND='news'
+KIND='news'
+NAMESPACE='sample'

api/helpers/exceptions.py

@@ -0,0 +1,18 @@
+from flask import jsonify
+
+class Error(Exception):
+ def __init__(self, code, message, status_code):
+ self.code = code
+ self.message = message
+ self.status_code = status_code
+
+ def json_response(self):
+ res = jsonify({
+ 'code': self.code,
+ 'message': self.message
+ })
+ res.status_code = self.status_code
+ return res
+
+ def __str__(self):
+ return f'Code: {self.code}, Message: {self.message}'

api/helpers/storage.py

@@ -0,0 +1,51 @@
+from google.cloud import datastore
+from . import config as c
+from typing import Dict
+import time
+from datetime import datetime as dt
+class Storage:
+
+ def __init__(self):
+ self.project_id = c.GOOGLE_CLOUD_PROJECT
+ self.client = datastore.Client(self.project_id)
+ self.kind = c.KIND
+ self.namespace = c.NAMESPACE
+
+ def _doc(self, data):
+ doc = {}
+ doc['updated_on'] = str(time.time())
+ doc['source'] = data.get('source') or 'unknown'
+ doc['source_url'] = data.get('source_url')
+ doc['headline'] = data.get('headline')
+ doc['story_url'] = data.get('story_url')
+ doc['short_desc'] = data.get('short_desc') or 'N/A'
+ doc['category'] = data.get('category') or 'Misc'
+ return doc
+
+ def _put(self, entity, doc):
+ with self.client.transaction():
+ entity.update(doc)
+ self.client.put(entity)
+
+ def _create(self, data):
+ id = str(dt.now()).replace(" ", "|")
+ key = self.client.key(self.kind, id, namespace=self.namespace)
+ entity = self.client.get(key)
+ entity = datastore.Entity(key=key)
+ article = self._doc(data)
+ self._put(entity, article)
+
+ def update(self, data):
+ self._create(data)
+
+ def fetch(self, filters):
+ if not filters:
+ q = self.client.query(kind=self.kind,namespace=self.namespace)
+ else:
+ q = self.client.query(kind=self.kind,namespace=self.namespace)
+ for filter in filters.keys():
+ q.add_filter(filter,'=', filters.get(filter))
+ # TODO: Decide on pagination and add a limit to fetch
+ queried = list(q.fetch())
+ return queried
+

api/helpers/webapp.py

@@ -0,0 +1,47 @@
+from flask import Flask, request, jsonify, url_for
+from . import Storage
+
+
+app = Flask(__name__)
+s = Storage()
+
+@app.before_request
+def check_auth():
+ pass
+
+@app.after_request
+def add_cors_headers(response):
+ response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization, x-datatool-site'
+ response.headers['Access-Control-Allow-Methods'] = 'GET,POST,PUT,DELETE'
+ response.headers['Access-Control-Allow-Origin'] = '*'
+
+ return response
+
+@app.route('/')
+def index():
+ return jsonify({
+ 'name': 'hackernewscloneapi'
+ })
+
+
+@app.route('/add', methods=['POST'])
+def add():
+ data = request.json
+ if not data:
+ return jsonify({}), 400
+ if 'headline' not in data:
+ return jsonify({'error_message': 'missing article headline in payload'}), 412
+ if 'source' not in data:
+ return jsonify({'error_message': 'missing article source in payload'}), 412
+ if 'source_url' not in data:
+ return jsonify({'error_message': 'missing source url in payload'}), 412
+ if 'story_url' not in data:
+ return jsonify({'error_message': 'missing story_url in payload'}), 412
+ s.update(data)
+ return jsonify({'message':'article added to datastore'}),201
+
+@app.route('/news', methods=['GET', 'POST'])
+def get_news():
+ filters = request.json
+ articles = s.fetch(filters=filters)
+ return jsonify(articles)

api/run.py

@@ -1,7 +1,7 @@
 # !usr/bin/env python3
-from api.webapp import app
+from helpers import app
 
 
 
 if __name__ == "__main__":
- app.run()
+ app.run(debug=True)

docs/api.md

@@ -0,0 +1,93 @@
+# hnclone API
+
+This document lists the implemented resources on the API
+
+## List of Contents
+* [Articles](##Articles)
+* [Crawlers](##Crawlers)
+
+## Articles
+Article here refers to the news article as a dictionary.
+
+### Adding Articles to the Datastore
+Using the client
+
+ from client import Client
+ c = Client(api_token='...', api_url='...')
+ article = {'..':'..'} # The structure of the article should be as follows.
+ c.update(article)
+
+or
+
+ POST /add
+ Content-Type: application/json
+
+ {
+ "source": "...",
+ "source_url": "...",
+ "headline":"...",
+ "story_url":"...",
+ "short_desc":"...",
+ "category":"...",
+ "updated_on":"..."
+ } 
+
+ HTTP/1.1 201 Created
+ Content-Type: application/json
+ {
+ "message": "article added"
+ }
+ ---
+ HTTP/1.1 412 Precondition Failed
+ If any of the keys among 'headline', 'source', 'source_url', 'story_url' turns out to be empty/missing
+
+
+### Fetching Articles
+
+ GET /news
+ ---
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+
+ [
+ {
+ "source": "hn",
+ "source_url": "xyx.com",
+ "headline":"alpha beta gamma ",
+ "story_url":"alpha.betagama",
+ "short_desc":"q delta",
+ "category":"science",
+ "updated_on":"..."
+
+ }
+ {
+ "source": "ab",
+ .....
+ }
+ ]
+
+ POST /news
+ Content-Type: application/json
+ {
+ "source": 'abc'
+ ....
+ }
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ [
+ {
+ "source" : 'abc',
+ ...
+ }
+ {
+ "source" : 'abc',
+ ...
+ }
+ ]
+
+
+
+
+## Crawlers
+ #TODO