Apollo GraphQL server skeleton (#335)

* checkpoint

* Tweaks

* Fix comments

* Typo fix

* Re-structured directories. Added support for DB migration

* Update readme to use preformatted code sections.

* Postgres wasn’t returning name/description/price for listings

* DB inserting should be idempotent.

It should be fine to rerun the listener and reindex all listings.

Author: Franck

daemon/listener/.gitignore renamed to daemon/indexing/.gitignore

@@ -0,0 +1,4 @@
+# See https://docs.npmjs.com/misc/developers#keeping-files-out-of-your-package for more about ignoring files.
+
+# dependencies
+/node_modules

daemon/indexing/.npmignore

@@ -0,0 +1,47 @@
+This directory contains code for indexing servers:
+ - listener: Server that listens to events emitted by Origin contracts and indexes them.
+ - apollo: GraphQL server for indexed data
+ - lib: library for indexing data in various backend. Currently Postgres and Elasticsearch are supported.
+
+To start the listener:
+======================
+
+Use origin-box to start an origin-js container.
+
+    docker-compose up origin-js
+    
+If you want to index data in Postgres:
+
+    docker-compose up postgres  # start the postgres container.
+
+    # create the postgres DB schema:
+    docker exec -ti -w /app/daemon/indexing origin-js node node_modules/db-migrate/bin/db-migrate -e origin-box-genesis db:create indexing
+    docker exec -ti -w /app/daemon/indexing origin-js node node_modules/db-migrate/bin/db-migrate up
+
+If you want to index data in Elasticsearch, start the elasticsearch container.
+
+    docker-compose up elasticsearch
+
+Start the listener in the the origin-js container. Use --elasticsearch and/or --db options to pick the indexer(s).
+
+    docker exec -ti origin-js node daemon/indexing/listener/listener.js --elasticsearch --db
+
+You should see messages in the console indicating events are being indexed.
+
+
+To start the Apollo GraphQL server:
+===================================
+
+You will need to update the origin-box:docker-compose.yml. For the image origin-js, proxy port 4000 to 4000 for the Apollo server. [TODO: update origin-box config]
+
+Use origin-box to start an origin-js container.
+
+    docker-compose up origin-js
+
+Start the Apollo server in the origin-js container
+
+    docker exec -ti origin-js node daemon/indexing/apollo/index.js
+
+ The server should start and you can point your browser to http://localhost:4000 to access the GraphQL playground.
+
+

daemon/indexing/README.md

@@ -0,0 +1,48 @@
+const { ApolloServer, gql } = require('apollo-server')
+
+var search = require('../lib/search.js')
+var db = require('../lib/db.js')
+
+/*
+ * Implentation of a GraphQL server using the Apollo framework.
+ * See https://www.apollographql.com/server
+ *
+ */
+
+// Type definitions define the "shape" of the data and specify
+// which ways the data can be fetched from the GraphQL server.
+const typeDefs = gql`
+  type Listing {
+    id: String
+    name: String
+    description: String
+    price: Float
+  }
+
+  # The "Query" type is the root of all GraphQL queries.
+  type Query {
+    Listings(query: String): [Listing],
+  }
+`
+
+// Resolvers define the technique for fetching the types in the schema.
+const resolvers = {
+  Query: {
+    Listings(root, args, context, info) {
+      if (args.query) {
+        return search.Listing.search(args.query)
+      } else {
+        return db.Listing.all()
+      }
+    },
+  },
+}
+
+// Start ApolloServer by passing type definitions (typeDefs) and the resolvers
+// responsible for fetching the data for those types.
+const server = new ApolloServer({ typeDefs, resolvers })
+
+// The `listen` method launches a web-server.
+server.listen().then(({ url }) => {
+  console.log(`Apollo server ready at ${url}`)
+})

daemon/indexing/apollo/index.js

@@ -0,0 +1,17 @@
+{
+  "defaultEnv": "origin-box",
+  "sql-file": true,
+  "origin-box-genesis": {
+    "driver": "pg",
+    "user": "origin",
+    "password": "origin",
+    "host": "postgres"
+  },
+  "origin-box": {
+    "driver": "pg",
+    "user": "origin",
+    "password": "origin",
+    "host": "postgres",
+    "database": "indexing"
+  }
+}

daemon/indexing/database.json

@@ -4,15 +4,13 @@ const { Pool } = require('pg')
   Module to interface with Postgres database.
  */
 
-const dbName = 'graphql'
-
 // TODO(franck): dynamically configure client.
 const pool = new Pool(
   {
-    host: 'localhost',
-    database: dbName,
-    //user: 'franck',
-    //password: 'franck',
+    host: 'postgres',
+    database: 'indexing',
+    user: 'origin',
+    password: 'origin',
   })
 
 
@@ -36,7 +34,17 @@ class Listing {
    */
   static async all() {
     const res = await pool.query(`SELECT * FROM ${Listing.table}`, [])
-    return res.rows
+    // Match the format of the data coming from elasticsearch
+    const results = res.rows.map((row)=>{
+      const json = JSON.parse(row.data)
+      return {
+        id: row.id,
+        name: json.name,
+        description: json.description,
+        price: json.price
+      }
+    })
+    return results
   }
 
   /*
@@ -47,8 +55,10 @@ class Listing {
    * @returns The listingId indexed.
    */
   static async insert(listingId, listing) {
+    // TODO: Check that we are not replacing new data with old
     const res = await pool.query(
-      `INSERT INTO ${Listing.table}(id, data) VALUES($1, $2)`, [listingId, listing])
+      `INSERT INTO ${Listing.table}(id, data) VALUES($1, $2)
+      ON CONFLICT (id) DO UPDATE SET data = excluded.data`, [listingId, listing])
     console.log(`Added row ${listingId} to listing table.`)
     return listingId
   }

daemon/lib/db.js renamed to daemon/indexing/lib/db.js

@@ -8,7 +8,7 @@ var elasticsearch = require('elasticsearch')
 // TODO(franck): dynamically configure client.
 var client = new elasticsearch.Client({
   hosts: [
-    'localhost:9200/'
+    'elasticsearch:9200/'
   ]
 })
 

daemon/lib/search.js renamed to daemon/indexing/lib/search.js

@@ -2,13 +2,18 @@ const http = require('http')
 const urllib = require('url')
 const Web3 = require('web3')
 
-const Origin = require('../../dist/index') // FIXME: replace with origin-js package
+const Origin = require('../../../dist/index') // FIXME: replace with origin-js package
 const search = require ('../lib/search.js')
 const db = require('../lib//db.js')
 
 const web3Provider = new Web3.providers.HttpProvider('http://localhost:8545')
 const web3 = new Web3(web3Provider)
-const o = new Origin({ web3 })
+const o = new Origin({
+  ipfsDomain: 'origin-js',
+  ipfsGatewayProtocol: 'http',
+  ipfsGatewayPort: 8080,
+  web3,
+})
 
 // Origin Listener
 // ---------------
@@ -49,9 +54,15 @@ const generateOfferId = log => {
   ].join('-')
 }
 const getListingDetails = async log => {
+  const listingId = generateListingId(log)
+  console.log("CALLING getListing for ID ", listingId)
+  const listing = await o.marketplace.getListing(listingId)
   return {
-    listing: await o.marketplace.getListing(generateListingId(log))
+    listing: listing,
   }
+  //return {
+  //  listing: await o.marketplace.getListing(generateListingId(log))
+  //}
 }
 const getOfferDetails = async log => {
   return {
@@ -174,6 +185,8 @@ async function runBatch(opts, context) {
 // handleLog - annotates, runs rule, and ouputs a particular log
 async function handleLog(log, rule, contractVersion, context) {
   console.log(`Processing log blockNumber=${log.blockNumber} transactionIndex=${log.transactionIndex}`)
+  console.log("LOG=", log)
+
   log.decoded = web3.eth.abi.decodeLog(
     rule.eventAbi.inputs,
     log.data,

daemon/listener/listener.js renamed to daemon/indexing/listener/listener.js

@@ -0,0 +1,53 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+var fs = require('fs');
+var path = require('path');
+var Promise;
+
+/**
+  * We receive the dbmigrate dependency from dbmigrate initially.
+  * This enables us to not have to rely on NODE_PATH.
+  */
+exports.setup = function(options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+  Promise = options.Promise;
+};
+
+exports.up = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20180815061230-add-listing-table-up.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports.down = function(db) {
+  var filePath = path.join(__dirname, 'sqls', '20180815061230-add-listing-table-down.sql');
+  return new Promise( function( resolve, reject ) {
+    fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){
+      if (err) return reject(err);
+      console.log('received data: ' + data);
+
+      resolve(data);
+    });
+  })
+  .then(function(data) {
+    return db.runSql(data);
+  });
+};
+
+exports._meta = {
+  "version": 1
+};

daemon/indexing/migrations/20180815061230-add-listing-table.js

@@ -0,0 +1 @@
+DROP TABLE listing;

daemon/indexing/migrations/sqls/20180815061230-add-listing-table-down.sql

@@ -0,0 +1,4 @@
+CREATE TABLE listing (
+  id        VARCHAR(32) PRIMARY KEY,
+  data      VARCHAR(4096)
+);