Merge pull request #513 from near/main

Prod Release 17/01/24

.github/workflows/block-streamer-ci.yml

@@ -19,6 +19,8 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Protoc
       uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Check
       working-directory: ./block-streamer
       run: cargo check
@@ -29,6 +31,8 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Protoc
       uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Test
       working-directory: ./block-streamer
       run: cargo test
@@ -41,6 +45,8 @@ jobs:
       uses: actions/checkout@v3
     - name: Install Protoc
       uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
@@ -60,6 +66,8 @@ jobs:
       uses: actions/checkout@v3
     - name: Install Protoc
       uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:

.github/workflows/coordinator-ci.yml

@@ -0,0 +1,81 @@
+name: Coordinator
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - "coordinator/**"
+  pull_request:
+    paths:
+      - "coordinator/**"
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Check
+      working-directory: ./coordinator
+      run: cargo check
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Test
+      working-directory: ./coordinator
+      run: cargo test
+
+
+  format:
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.70.0
+        override: true
+        profile: minimal
+        components: rustfmt
+    - name: Check formatting
+      working-directory: ./coordinator
+      run: |
+        cargo fmt -- --check
+
+  clippy:
+    runs-on: ubuntu-20.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.70.0
+        override: true
+        profile: minimal
+        components: clippy
+    - name: Clippy check
+      working-directory: ./coordinator
+      run: |
+        cargo clippy

block-server/handler.js

@@ -35,20 +35,16 @@ const normalizeBlockHeight = function(block_height) {
 }
 
 const fetchStreamerMessage = async function(block_height, options) {
-    const blockPromise = fetchBlockPromise(block_height, options);
-    // hardcoding 4 shards to test performance
-    const shardsPromises = await fetchShardsPromises(block_height, 4, options); // block.chunks.length)
+    const block = await fetchBlockPromise(block_height, options);
+    const shards = await Promise.all(fetchShardsPromises(block_height, block.chunks.length, options))
 
-    const results = await Promise.all([blockPromise, ...shardsPromises]);
-    const block = results.shift();
-    const shards = results;
     return {
-        block: block,
-        shards: shards,
+        block,
+        shards,
     };
 }
 
-const fetchShardsPromises = async function(block_height, number_of_shards, options) {
+const fetchShardsPromises = function(block_height, number_of_shards, options) {
     return ([...Array(number_of_shards).keys()].map((shard_id) =>
         fetchShardPromise(block_height, shard_id, options)));
 }
@@ -101,4 +97,4 @@ const renameUnderscoreFieldsToCamelCase = function(value) {
         return newValue;
     }
     return value;
-}
+}

block-streamer/Cargo.toml

@@ -24,9 +24,9 @@ tokio-stream = "0.1.14"
 tonic = "0.10.2"
 wildmatch = "2.1.1"
 
-registry-types = { path = "../registry/types" }
+registry-types = { path = "../registry/types", features = ["near-primitives"] }
 
-near-lake-framework = "0.7.4"
+near-lake-framework = "=0.7.4"
 
 [build-dependencies]
 tonic-build = "0.10"

block-streamer/examples/start_stream.rs

@@ -12,6 +12,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             start_block_height: 106700000,
             account_id: "morgs.near".to_string(),
             function_name: "test".to_string(),
+            version: 0,
+            redis_stream: "morgs.near/test:block_stream".to_string(),
             rule: Some(Rule::ActionAnyRule(ActionAnyRule {
                 affected_account_id: "social.near".to_string(),
                 status: Status::Success.into(),

block-streamer/proto/block_streamer.proto

@@ -2,48 +2,52 @@ syntax = "proto3";
 
 package blockstreamer;
 
-// The BlockStreamer service provides RPCs to manage BlockStream instances.
+// The BlockStreamer service provides RPCs to manage BlockStream instances
 service BlockStreamer {
-    // Starts a new BlockStream process.
+    // Starts a new BlockStream process
     rpc StartStream (StartStreamRequest) returns (StartStreamResponse);
 
-    // Stops an existing BlockStream process.
+    // Stops an existing BlockStream process
     rpc StopStream (StopStreamRequest) returns (StopStreamResponse);
 
-    // Lists all current BlockStream processes.
+    // Lists all current BlockStream processes
     rpc ListStreams (ListStreamsRequest) returns (ListStreamsResponse);
 }
 
-// Request message for starting a BlockStream.
+// Request message for starting a BlockStream
 message StartStreamRequest {
-    // Which block height to start from.
+    // Which block height to start from
     uint64 start_block_height = 1;
-    // The account ID which the indexer is defined under
+    // Account ID which the indexer is defined under
     string account_id = 2;
-    // The name of the indexer
+    // Name of the indexer
     string function_name = 3;
-    // The filter rule to apply to incoming blocks
+    // Block height corresponding to the created/updated height of the indexer
+    uint64 version = 4;
+    // Key of Redis Stream to publish blocks to
+    string redis_stream = 5;
+    // Filter rule to apply to incoming blocks
     oneof rule {
-        ActionAnyRule action_any_rule = 4;
-        ActionFunctionCallRule action_function_call_rule = 5;
+        ActionAnyRule action_any_rule = 6;
+        ActionFunctionCallRule action_function_call_rule = 7;
     }
 }
 
 // Match any action against the specified account
 message ActionAnyRule {
-    // The account ID pattern to match against
+    // Account ID pattern to match against
     string affected_account_id = 1;
-    // The status of the action to match against
+    // Status of the action to match against
     Status status = 2;
 }
 
 // Match a specific function call against the specified account
 message ActionFunctionCallRule {
-    // The account ID pattern to match against
+    // Account ID pattern to match against
     string affected_account_id = 1;
-    // The function name to match against
+    // Function name to match against
     string function_name = 2;
-    // The status of the action to match against
+    // Status of the action to match against
     Status status = 3;
 }
 
@@ -54,40 +58,43 @@ enum Status {
     STATUS_ANY = 3;
 }
 
-// Response message for starting a BlockStream.
+// Response message for starting a BlockStream
 message StartStreamResponse {
-    // ID or handle of the started BlockStream.
+    // ID or handle of the started BlockStream
     string stream_id = 1;
 }
 
-// Request message for stopping a BlockStream.
+// Request message for stopping a BlockStream
 message StopStreamRequest {
-    // ID or handle of the BlockStream to stop.
+    // ID or handle of the BlockStream to stop
     string stream_id = 1;
 }
 
-// Response message for stopping a BlockStream.
+// Response message for stopping a BlockStream
 message StopStreamResponse {
-    // Confirmation message or status.
+    // Confirmation message or status
     string status = 1;
 }
 
-// Request message for listing BlockStreams.
+// Request message for listing BlockStreams
 message ListStreamsRequest {
-    // Optional filters or parameters for listing streams.
+    // Optional filters or parameters for listing streams
 }
 
-// Response message for listing BlockStreams.
+// Response message for listing BlockStreams
 message ListStreamsResponse {
-    // List of active BlockStreams.
+    // List of active BlockStreams
     repeated StreamInfo streams = 1;
 }
 
-// Information about a single BlockStream instance.
+// Information about a single BlockStream instance
 message StreamInfo {
+    // ID or handle of the BlockStream
     string stream_id = 1;
-    int64 start_block_height = 2;
-    string indexer_name = 3;
-    string chain_id = 4;
-    string status = 5;
+    // Account ID of the indexer
+    string account_id = 3;
+    // Function name of the indexer
+    string function_name = 4;
+    // Block height corresponding to the created/updated height of the indexer
+    uint64 version = 5;
 }