🚚 release

.github/CODEOWNERS

@@ -1 +1 @@
-*                     @leoparente @mfiedorowicz @natm
+*                     @leoparente @mfiedorowicz

.github/golangci.yaml

@@ -15,6 +15,7 @@ linters:
     - govet
     - gosimple
     - bodyclose
+    - gci
 
 issues:
   exclude-use-default: false
@@ -26,3 +27,11 @@ issues:
 
 severity:
   default-severity: error
+
+linters-settings:
+  gci:
+    sections:
+      - standard
+      - default
+      - prefix(github.com/netboxlabs/diode)
+    custom-order: true

Makefile

@@ -12,3 +12,12 @@ gen-diode-sdk-python:
 .PHONY: gen-diode-server-go
 gen-diode-server-go:
 	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.server.go.yaml
+
+.PHONY: gen-diode-netbox-plugin-reconciler-sdk-python
+gen-diode-netbox-plugin-reconciler-sdk-python:
+	@cd diode-proto/ && buf format -w && buf generate --template buf.gen.netbox-plugin.reconciler.sdk.py.yaml --include-imports
+	@find ../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/diode/v1 \( -name '*.py' -o -name '*.pyi' \) -execdir mv {} ../../v1/ \;
+	@rm -rf ../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/diode
+	@find ../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/ \( -name '*.py' -o -name '*.pyi' \) \
+	-exec sed -i.bak -e 's/^from diode.v1/from netbox_diode_plugin.reconciler.sdk.v1/' \
+	-e 's/^from validate/from netbox_diode_plugin.reconciler.sdk.validate/' {} \; -exec rm -f {}.bak \;

diode-proto/buf.gen.netbox-plugin.reconciler.sdk.py.yaml

@@ -0,0 +1,10 @@
+version: v2
+inputs:
+  - proto_file: diode/v1/reconciler.proto
+plugins:
+  - remote: buf.build/protocolbuffers/python:v26.1
+    out: ../../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/
+  - remote: buf.build/protocolbuffers/pyi:v26.1
+    out: ../../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/
+  - remote: buf.build/grpc/python:v1.62.1
+    out: ../../diode-netbox-plugin/netbox_diode_plugin/reconciler/sdk/

diode-proto/diode/v1/reconciler.proto

@@ -37,8 +37,8 @@ message RetrieveIngestionDataSourcesResponse {
   repeated IngestionDataSource ingestion_data_sources = 1;
 }
 
-// ChangeSetError represents an error when applying a change set
-message ChangeSetError {
+// IngestionError represents an error occurring while processing an ingestion entity
+message IngestionError {
   message Details {
     message Error {
       string error = 1; //key value pair of the error
@@ -54,44 +54,61 @@ message ChangeSetError {
 }
 
 enum State {
-  NEW = 0;
-  RECONCILED = 1;
-  FAILED = 2;
-  NO_CHANGES = 3;
+  UNSPECIFIED = 0;
+  QUEUED = 1;
+  RECONCILED = 2;
+  FAILED = 3;
+  NO_CHANGES = 4;
+}
+
+// Ingestion metrics
+message IngestionMetrics {
+  int32 total = 1;
+  int32 queued = 2;
+  int32 reconciled = 3;
+  int32 failed = 4;
+  int32 no_changes = 5;
+}
+
+// A change set
+message ChangeSet {
+  string id = 1; // A change set ID
+  bytes data = 2; // Binary data representing the change set
 }
 
 // An ingestion log
 message IngestionLog {
-  string data_type = 1;
-  State state = 2;
-  string request_id = 3;
-  int64 ingestion_ts = 4;
-  string producer_app_name = 5;
-  string producer_app_version = 6;
-  string sdk_name = 7;
-  string sdk_version = 8;
-  diode.v1.Entity entity = 9;
-  ChangeSetError error = 10;
+  string id = 1;
+  string data_type = 2;
+  State state = 3;
+  string request_id = 4;
+  int64 ingestion_ts = 5;
+  string producer_app_name = 6;
+  string producer_app_version = 7;
+  string sdk_name = 8;
+  string sdk_version = 9;
+  diode.v1.Entity entity = 10;
+  IngestionError error = 11;
+  ChangeSet change_set = 12;
 }
 
 // The request to retrieve ingestion logs
 message RetrieveIngestionLogsRequest {
-  int32 page_size = 1 [(validate.rules).int32 = {
-    gte: 1
-    lte: 1000
-  }]; // Number of logs per page, default is 100
+  optional int32 page_size = 1; // Number of logs per page, default is 100
   optional State state = 2; // Optional filter by state field
   string data_type = 3; // Optional filter by data type field
   string request_id = 4; // Optional filter by request ID
   int64 ingestion_ts_start = 5; // Optional start of ingestion timestamp range
   int64 ingestion_ts_end = 6; // Optional end of ingestion timestamp range
   string page_token = 7; // Token to fetch the next page of results
+  bool only_metrics = 8; // Flag to return only the ingestion metrics
 }
 
 // The response from the retrieve ingestion logs request
 message RetrieveIngestionLogsResponse {
   repeated IngestionLog logs = 1; // List of ingestion logs
-  string next_page_token = 2; // Token for the next page of results, if any
+  IngestionMetrics metrics = 2; // ingestion metrics
+  string next_page_token = 3; // Token for the next page of results, if any
 }
 
 // Reconciler service API

diode-server/README.md

@@ -67,6 +67,7 @@ Edit the `.env` to match your environment:
 * `NETBOX_TO_DIODE_API_KEY`: API key generated with the Diode NetBox plugin installation
 * `INGESTER_TO_RECONCILER_API_KEY`: API key to authorize RPC calls between the Ingester and Reconciler services (at
   least 40 characters, example generation with shell command: `openssl rand -base64 40 | head -c 40`)
+* `MIGRATION_ENABLED`: Set to `false` to disable the migration, default is `true`
 
 ### Running the Diode server
 

diode-server/docker/docker-compose.yaml

@@ -3,18 +3,24 @@ services:
   ingress-nginx:
     image: nginx:latest
     command: >
-      /bin/sh -c "echo 'upstream diode {
+      /bin/sh -c "echo 'upstream diode-ingester {
         server diode-ingester:8081;
       }
-
+      upstream diode-reconciler {
+        server diode-reconciler:8081;
+      }
       server {
         listen 80;
         http2 on;
         server_name localhost;
         client_max_body_size 25m;
-        location /diode {
+        location /diode/diode.v1.IngesterService {
           rewrite /diode/(.*) /$$1 break;
-          grpc_pass grpc://diode;
+          grpc_pass grpc://diode-ingester;
+        }
+        location /diode/diode.v1.ReconcilerService {
+          rewrite /diode/(.*) /$$1 break;
+          grpc_pass grpc://diode-reconciler;
         }
       }'
       > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
@@ -39,8 +45,7 @@ services:
       - INGESTER_TO_RECONCILER_API_KEY=${INGESTER_TO_RECONCILER_API_KEY}
       - SENTRY_DSN=${SENTRY_DSN}
     restart: always
-    ports:
-      - "8081:8081"
+    ports: [ ]
     depends_on:
       - diode-redis
       - diode-reconciler
@@ -52,12 +57,14 @@ services:
       - REDIS_HOST=${REDIS_HOST}
       - REDIS_PORT=${REDIS_PORT}
       - NETBOX_DIODE_PLUGIN_API_BASE_URL=${NETBOX_DIODE_PLUGIN_API_BASE_URL}
+      - NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY=${NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY}
       - DIODE_TO_NETBOX_API_KEY=${DIODE_TO_NETBOX_API_KEY}
       - NETBOX_TO_DIODE_API_KEY=${NETBOX_TO_DIODE_API_KEY}
       - INGESTER_TO_RECONCILER_API_KEY=${INGESTER_TO_RECONCILER_API_KEY}
       - DIODE_API_KEY=${DIODE_API_KEY}
       - LOGGING_LEVEL=${LOGGING_LEVEL}
       - SENTRY_DSN=${SENTRY_DSN}
+      - MIGRATION_ENABLED=${MIGRATION_ENABLED}
     restart: always
     ports: [ ]
     depends_on:
@@ -74,21 +81,6 @@ services:
     ports: [ ]
     volumes:
       - diode-redis-data:/data
-  diode-redis-cli:
-    image: redis/redis-stack-server:latest
-    links:
-      - diode-redis
-    entrypoint:
-      - sh
-      - -c
-      - |
-        redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" <<EOF
-        FT.CREATE ingest-entity ON JSON PREFIX 1 "ingest-entity:" SCHEMA \
-        $.data_type AS data_type TEXT \
-        $.state AS state NUMERIC \
-        $.request_id AS request_id TEXT \
-        $.ingestion_ts AS ingestion_ts NUMERIC SORTABLE
-        EOF
 volumes:
   diode-redis-data:
     driver: local

diode-server/docker/sample.env

@@ -6,9 +6,11 @@ REDIS_PORT=6378
 RECONCILER_GRPC_HOST=diode-reconciler
 RECONCILER_GRPC_PORT=8081
 NETBOX_DIODE_PLUGIN_API_BASE_URL=http://NETBOX_HOST/api/plugins/diode
+NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY=false
 DIODE_TO_NETBOX_API_KEY=1368dbad13e418d5a443d93cf255edde03a2a754
 NETBOX_TO_DIODE_API_KEY=1e99338b8cab5fc637bc55f390bda1446f619c42
 DIODE_API_KEY=5a52c45ee8231156cb620d193b0291912dd15433
 INGESTER_TO_RECONCILER_API_KEY=sXjJZe6BBzVuovrVyyH4Q3vbceqvDwh2kC3DRpML
 LOGGING_LEVEL=DEBUG
 SENTRY_DSN=
+MIGRATION_ENABLED=true