Write to influxdb.

.gitignore

@@ -4,3 +4,4 @@ dist/
 .idea
 *.iml
 output/
+influxdb/

docker-compose.yml

@@ -1,12 +1,42 @@
 version: '3'
 services:
-  shimmersea-price-archiver:
+  archiver:
     build: .
     image: shimmersea-price-archiver
     container_name: shimmersea-price-archiver
     restart: unless-stopped
+    depends_on:
+      influxdb:
+        condition: service_healthy
+        restart: true
     volumes:
-      - /opt/shimmersea-price-archiver/output:/output
+      - ${OUTPUT_DIR:-/opt/shimmersea-price-archiver/output}:/output
       - /etc/localtime:/etc/localtime:ro
     environment:
-      CRON: '*/5 * * * *'
+      - CRON=${CRON:-*/5 * * * *}
+      - INFLUXDB_URL=http://influxdb:8086
+      - INFLUXDB_ADMIN_TOKEN=${INFLUXDB_ADMIN_TOKEN}
+      - INFLUXDB_ORG=${INFLUXDB_ORG:-someorg}
+      - INFLUXDB_BUCKET=${INFLUXDB_BUCKET:-shimersea-prices}
+
+  influxdb:
+    image: influxdb:2.0
+    container_name: shimmersea-price-archiver.influxdb
+    restart: unless-stopped
+    ports:
+      - '9999:8086'
+    volumes:
+      - ${INFLUXDB_DATA_DIR:-/opt/shimmersea-price-archiver/influxdb/data}:/var/lib/influxdb2
+      - ${INFLUXDB_CONFIG_DIR:-/opt/shimmersea-price-archiver/influxdb/config}:/etc/influxdb2
+    environment:
+      - DOCKER_INFLUXDB_INIT_MODE=setup
+      - DOCKER_INFLUXDB_INIT_USERNAME=${INFLUXDB_USERNAME}
+      - DOCKER_INFLUXDB_INIT_PASSWORD=${INFLUXDB_PASSWORD}
+      - DOCKER_INFLUXDB_INIT_ORG=${INFLUXDB_ORG:-someorg}
+      - DOCKER_INFLUXDB_INIT_BUCKET=${INFLUXDB_BUCKET:-shimersea-prices}
+      - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${INFLUXDB_ADMIN_TOKEN}
+    healthcheck:
+      test: "curl -f http://localhost:8086/ping"
+      interval: 5s
+      timeout: 10s
+      retries: 5

package.json

@@ -9,6 +9,7 @@
   "author": "bogenlos",
   "license": "ISC",
   "dependencies": {
+    "@influxdata/influxdb-client": "^1.33.2",
     "axios": "^1.5.1",
     "big.js": "^6.2.1",
     "dotenv": "^16.3.1",

src/index.ts

@@ -22,6 +22,12 @@ export interface TokenPairPrices {
 const RPC_ENDPOINT = 'https://json-rpc.evm.shimmer.network';
 const SHIMMER_SEA_ROUTER_ADDRESS = '0x3EdAFd0258F75E0F49d570B1b28a1F7A042bcEC3';
 
+const getTimestampWithoutSeconds = () => {
+  const timestamp = new Date();
+  timestamp.setSeconds(0, 0);
+  return timestamp;
+};
+
 const printTokenPairPrices = (tokenPairPrices: TokenPairPrices) => {
   console.log(`${tokenPairPrices.timestamp.toUTCString()}`);
   console.log(
@@ -37,13 +43,13 @@ const printTokenPairPrices = (tokenPairPrices: TokenPairPrices) => {
   console.log('----------');
 };
 
-const readSmrUsdPrice = async () => {
+const readSmrUsdPrice = async (timestamp: Date) => {
   try {
     const response = await axios.get<undefined, { data: { mid: number; timestamp: number } }>(
       'https://api.bitfinex.com/v1/pubticker/smrusd',
     );
     const tokenPairPrices = {
-      timestamp: new Date(),
+      timestamp,
       symbol1: 'SMR',
       symbol2: 'USD',
       amount1: 1n,
@@ -62,14 +68,14 @@ const readSmrUsdPrice = async () => {
           .toFixed(0),
       ),
     };
-    persist(tokenPairPrices);
+    await persist(tokenPairPrices);
     printTokenPairPrices(tokenPairPrices);
   } catch (e) {
     console.error('Error fetching smr/usd from ', e);
   }
 };
 
-const readTokenPairPrices = async () => {
+const readTokenPairPrices = async (timestamp: Date) => {
   const jsonRpcProvider = new JsonRpcProvider(RPC_ENDPOINT);
   const shimmerSeaRouterContract = new Contract(SHIMMER_SEA_ROUTER_ADDRESS, shimmerSeaRouterAbi, jsonRpcProvider);
   const getAmountsOut = shimmerSeaRouterContract.getFunction('getAmountsOut');
@@ -85,7 +91,7 @@ const readTokenPairPrices = async () => {
     ]);
 
     const tokenPairPrices = {
-      timestamp: new Date(),
+      timestamp,
       symbol1: token1.symbol,
       symbol2: token2.symbol,
       amount1: BigInt(token1.amount),
@@ -95,14 +101,15 @@ const readTokenPairPrices = async () => {
       price1To2: token1ToToken2[1],
       price2To1: token2ToToken1[1],
     };
-    persist(tokenPairPrices);
+    await persist(tokenPairPrices);
     printTokenPairPrices(tokenPairPrices);
   }
 };
 
 const cronExpression = process.env.CRON ?? '*/10 * * * *';
 cron.schedule(cronExpression, async () => {
-  await Promise.all([readSmrUsdPrice(), readTokenPairPrices()]);
+  const timestamp = getTimestampWithoutSeconds();
+  await Promise.all([readSmrUsdPrice(timestamp), readTokenPairPrices(timestamp)]);
 });
 
 console.log('Running with config:');
@@ -111,5 +118,6 @@ console.log(`  OUTPUT_DIR ${process.env.OUTPUT_DIR ?? './output'}`);
 console.log('----------');
 
 (async () => {
-  await Promise.all([readSmrUsdPrice(), readTokenPairPrices()]);
+  const timestamp = getTimestampWithoutSeconds();
+  await Promise.all([readSmrUsdPrice(timestamp), readTokenPairPrices(timestamp)]);
 })();

src/persistence.ts

@@ -1,19 +1,63 @@
+import { InfluxDB, Point } from '@influxdata/influxdb-client';
+import 'dotenv/config';
 import fs from 'fs';
 import { TokenPairPrices } from './index';
 
-const persist = (tokenPairPrices: TokenPairPrices) => {
+const url = process.env.INFLUXDB_URL;
+const token = process.env.INFLUXDB_ADMIN_TOKEN;
+const org = process.env.INFLUXDB_ORG;
+const bucket = process.env.INFLUXDB_BUCKET;
+
+const maxUInt64 = 2n ** 64n - 1n;
+
+const toMaxUInt64 = (value: bigint, decimals: number) => {
+  let valueUInt64 = value;
+  let reducedDecimals = decimals;
+  while (valueUInt64 > maxUInt64) {
+    valueUInt64 = valueUInt64 / 10n;
+    reducedDecimals--;
+  }
+  console.log(`Converted ${value} with ${decimals} decimals to ${valueUInt64} with ${reducedDecimals} decimals`);
+  return { valueUInt64, reducedDecimals };
+};
+
+const persistToInflux = async (tokenPairPrices: TokenPairPrices) => {
+  const writeApi = new InfluxDB({ url, token }).getWriteApi(org, bucket, 's');
+
+  const uint64_1to2 = toMaxUInt64(tokenPairPrices.price1To2, tokenPairPrices.decimals2);
+  const uint64_2to1 = toMaxUInt64(tokenPairPrices.price2To1, tokenPairPrices.decimals1);
+
+  const pricePoint = new Point('price')
+    .tag('symbol1', tokenPairPrices.symbol1)
+    .tag('symbol2', tokenPairPrices.symbol2)
+    .uintField('amount1', tokenPairPrices.amount1)
+    .uintField('amount2', tokenPairPrices.amount2)
+    .uintField('1to2', uint64_1to2.valueUInt64)
+    .uintField('2to1', uint64_2to1.valueUInt64)
+    .intField('1to2decimals', uint64_1to2.reducedDecimals)
+    .intField('2to1decimals', uint64_2to1.reducedDecimals)
+    .intField('decimals1_str', tokenPairPrices.decimals1)
+    .intField('decimals2_str', tokenPairPrices.decimals2)
+    .stringField('1to2_str', tokenPairPrices.price1To2.toString())
+    .stringField('2to1_str', tokenPairPrices.price2To1.toString())
+    .timestamp(tokenPairPrices.timestamp);
+
+  writeApi.writePoint(pricePoint);
+  await writeApi.flush();
+  console.log(`Persisted ${tokenPairPrices.symbol1}-${tokenPairPrices.symbol2} to InfluxDB`);
+};
+
+const persistToFile = async (tokenPairPrices: TokenPairPrices) => {
   const timestamp = tokenPairPrices.timestamp;
   const year = timestamp.getUTCFullYear();
-  const month = `0${timestamp.getUTCMonth()}`.substring(-2);
-  const day = `0${timestamp.getUTCDate()}`.substring(-2);
-
-  const outputDir = (process.env.OUTPUT_DIR ?? './output') + `/${year}/${month}`;
-  fs.mkdirSync(`${outputDir}`, { recursive: true });
+  let month = `0${timestamp.getUTCMonth() + 1}`;
+  month = month.length > 2 ? month.substring(1) : month;
+  let day = `0${timestamp.getUTCDate()}`;
+  day = day.length > 2 ? day.substring(1) : day;
 
-  const outputFile = `${outputDir}/${tokenPairPrices.symbol1}-${tokenPairPrices.symbol2}_${year}-${month}-${day}.json`;
   const data = {
-    unix: Math.floor(timestamp.getTime() / 1000),
-    iso: timestamp.toISOString(),
+    unix: Math.floor(tokenPairPrices.timestamp.getTime() / 1000),
+    iso: tokenPairPrices.timestamp.toISOString(),
     amount1: tokenPairPrices.amount1.toString(),
     amount2: tokenPairPrices.amount2.toString(),
     decimals1: tokenPairPrices.decimals1,
@@ -22,6 +66,11 @@ const persist = (tokenPairPrices: TokenPairPrices) => {
     '2to1': tokenPairPrices.price2To1.toString(),
   };
 
+  const outputDir = (process.env.OUTPUT_DIR ?? './output') + `/${year}/${month}`;
+  fs.mkdirSync(`${outputDir}`, { recursive: true });
+
+  const outputFile = `${outputDir}/${tokenPairPrices.symbol1}-${tokenPairPrices.symbol2}_${year}-${month}-${day}.json`;
+
   if (!fs.existsSync(outputFile)) {
     fs.writeFileSync(outputFile, JSON.stringify([data]));
   } else {
@@ -33,4 +82,11 @@ const persist = (tokenPairPrices: TokenPairPrices) => {
   console.log(`Persisted ${tokenPairPrices.symbol1}-${tokenPairPrices.symbol2} to ${outputFile}`);
 };
 
+const persist = async (tokenPairPrices: TokenPairPrices) => {
+  await persistToFile(tokenPairPrices);
+  if (process.env.INFLUXDB_URL) {
+    await persistToInflux(tokenPairPrices);
+  }
+};
+
 export default persist;