Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

panic: runtime error: invalid memory address or nil pointer dereference typing.go:36 #1054

Open
kholisrag opened this issue Nov 19, 2024 · 12 comments
Open

panic: runtime error: invalid memory address or nil pointer dereference typing.go:36 #1054

kholisrag opened this issue Nov 19, 2024 · 12 comments

Comments

@kholisrag
Copy link

hi, wanna report,
I got below error

Nov 19 05:51:10.500 INF [DDL] Executing query query="CREATE TABLE IF NOT EXISTS `gcp-project-id`.`kholistest_stage_mysql_db_Example1Store`.`instantsell_booking_model_counter___artie_hdddt_1732017070` (`id` int,`date_period` date,`model_id` int,`total_booking` int,`total_booking_summary` int,`is_latest_record` int,`created` datetime,`storage` string,`__artie_delete` bool,`__artie_only_set_delete` bool,PRIMARY KEY (`id`) NOT ENFORCED) OPTIONS (expiration_timestamp = TIMESTAMP(\"2024-11-19 11:51:10 UTC\"))"
Nov 19 05:51:10.587 INF [DDL] Executing query query="CREATE TABLE IF NOT EXISTS `gcp-project-id`.`kholistest_stage_mysql_db_Example1Store`.`inventory_cycle_count_snapshot___artie_1it4q_1732017070` (`id` int,`pending_no_location_refund_count` int,`pending_no_location_non_refund_count` int,`pending_storage_count` int,`created` datetime,`modified` datetime,`time_index` int,`__artie_delete` bool,`__artie_only_set_delete` bool,PRIMARY KEY (`id`) NOT ENFORCED) OPTIONS (expiration_timestamp = TIMESTAMP(\"2024-11-19 11:51:10 UTC\"))"
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x1bd627e]

goroutine 3434 [running]:
github.com/artie-labs/transfer/clients/bigquery/dialect.BigQueryDialect.DataTypeForKind({}, {{0x27e189a, 0x7}, 0x0, 0x0, 0xc0000f9918}, 0xc0?)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/dialect/typing.go:36 +0x1be
github.com/artie-labs/transfer/lib/destination/ddl.BuildCreateTableSQL({0x2bbe768, 0x4211120}, {0x2b993f8, 0xc000988d20}, 0x1, {0xc000ada990, 0xb}, {0xc000bcf808, 0x38, 0x3f})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/lib/destination/ddl/ddl.go:37 +0x41d
github.com/artie-labs/transfer/clients/shared.CreateTable({0x2b98c50, 0x4211120}, {0x2bb9140, 0xc00012ba00}, 0xc00060c780, 0xc000988b40, {0x2b993f8, 0xc000988d20}, 0x1)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/shared/table.go:19 +0xe5
github.com/artie-labs/transfer/clients/bigquery.(*Store).PrepareTemporaryTable(0xc00012ba00, {0x2b98c50, 0x4211120}, 0xc00060c780, 0xc0008937d0?, {0x2b993f8, 0xc000988d20}, {0x0?, 0x0?}, {{0x0, ...}, ...}, ...)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/bigquery.go:82 +0x8d
github.com/artie-labs/transfer/clients/shared.Merge({0x2b98c50, 0x4211120}, {0x2bb9140, 0xc00012ba00}, 0xc00060c780, {0x1, {0x0, 0x0, 0x0}, 0x1})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/shared/merge.go:67 +0x806
github.com/artie-labs/transfer/clients/bigquery.(*Store).Merge(0xc00060c780?, {0x2b98c50?, 0x4211120?}, 0x746163696c706572?)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/merge.go:34 +0x245
github.com/artie-labs/transfer/processes/consumer.flush({0x2b98c50, 0x4211120}, {0x7df58a657998, 0xc00012ba00}, 0xc00094d3b0)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:120 +0xcb
github.com/artie-labs/transfer/processes/consumer.Flush.func1.1(0x2b90b20?, {0xc00006e7c0?, 0x46af1d?})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:92 +0x27
github.com/artie-labs/transfer/lib/retry.WithRetriesAndResult[...]({0x2b90b20?, 0xc00006e7c0}, 0xc0011cdef0)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/lib/retry/retry.go:98 +0xed
github.com/artie-labs/transfer/processes/consumer.Flush.func1({0xc000adb4a0, 0x9}, 0xc00094d3b0)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:91 +0x6c6
created by github.com/artie-labs/transfer/processes/consumer.Flush in goroutine 11
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:57 +0x45b

any idea to debug further?
@Tang8330
Copy link
Contributor

What does the column type look like in BigQuery?

@Tang8330
Copy link
Contributor

Did you create the table in BigQuery? Or did Transfer create the table?

@kholisrag
Copy link
Author

2024-11-19_13-06

left is the artie temp table, right is the destination tables

Did you create the table in BigQuery?

I created manually for the table in bigquery

@kholisrag
Copy link
Author

Did you create the table in BigQuery?

just tested to let the artie create the tables, seems to still have same error

Nov 19 06:15:06.144 INF [DDL] Executing query query="CREATE TABLE IF NOT EXISTS `gcp-project-id`.`kholistest_stage_mysql_db_Example1Store`.`inventory_cycle_count_snapshot___artie_zzo1w_1732018506` (`id` int,`pending_no_location_refund_count` int,`pending_no_location_non_refund_count` int,`pending_storage_count` int,`created` datetime,`modified` datetime,`time_index` int,`__artie_delete` bool,`__artie_only_set_delete` bool,PRIMARY KEY (`id`) NOT ENFORCED) OPTIONS (expiration_timestamp = TIMESTAMP(\"2024-11-19 12:15:06 UTC\"))"
Nov 19 06:15:06.249 INF [DDL] Executing query query="CREATE TABLE IF NOT EXISTS `gcp-project-id`.`kholistest_stage_mysql_db_Example1Store`.`instantsell_booking_model_counter___artie_xsfmw_1732018506` (`id` int,`date_period` date,`model_id` int,`total_booking` int,`total_booking_summary` int,`is_latest_record` int,`created` datetime,`storage` string,`__artie_delete` bool,`__artie_only_set_delete` bool,PRIMARY KEY (`id`) NOT ENFORCED) OPTIONS (expiration_timestamp = TIMESTAMP(\"2024-11-19 12:15:06 UTC\"))"
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x1bd627e]

goroutine 4367 [running]:
github.com/artie-labs/transfer/clients/bigquery/dialect.BigQueryDialect.DataTypeForKind({}, {{0x27e189a, 0x7}, 0x0, 0x0, 0xc0001dde18}, 0x40?)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/dialect/typing.go:36 +0x1be
github.com/artie-labs/transfer/lib/destination/ddl.BuildCreateTableSQL({0x2bbe768, 0x4211120}, {0x2b993f8, 0xc0018123c0}, 0x1, {0xc000b632d0, 0xb}, {0xc000a46808, 0x38, 0x3f})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/lib/destination/ddl/ddl.go:37 +0x41d
github.com/artie-labs/transfer/clients/shared.CreateTable({0x2b98c50, 0x4211120}, {0x2bb9140, 0xc0007e91e0}, 0xc0000ff080, 0xc001812120, {0x2b993f8, 0xc0018123c0}, 0x1)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/shared/table.go:19 +0xe5
github.com/artie-labs/transfer/clients/bigquery.(*Store).PrepareTemporaryTable(0xc0007e91e0, {0x2b98c50, 0x4211120}, 0xc0000ff080, 0xc0007c00c0?, {0x2b993f8, 0xc0018123c0}, {0x0?, 0x0?}, {{0x0, ...}, ...}, ...)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/bigquery.go:82 +0x8d
github.com/artie-labs/transfer/clients/shared.Merge({0x2b98c50, 0x4211120}, {0x2bb9140, 0xc0007e91e0}, 0xc0000ff080, {0x1, {0x0, 0x0, 0x0}, 0x1})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/shared/merge.go:67 +0x806
github.com/artie-labs/transfer/clients/bigquery.(*Store).Merge(0xc0000ff080?, {0x2b98c50?, 0x4211120?}, 0x0?)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/clients/bigquery/merge.go:34 +0x245
github.com/artie-labs/transfer/processes/consumer.flush({0x2b98c50, 0x4211120}, {0x7e01ca0c33b8, 0xc0007e91e0}, 0xc000276780)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:120 +0xcb
github.com/artie-labs/transfer/processes/consumer.Flush.func1.1(0x2b90b20?, {0xc000a618a0?, 0x46af1d?})
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:92 +0x27
github.com/artie-labs/transfer/lib/retry.WithRetriesAndResult[...]({0x2b90b20?, 0xc000a618a0}, 0xc0012b1ef0)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/lib/retry/retry.go:98 +0xed
github.com/artie-labs/transfer/processes/consumer.Flush.func1({0xc0007b20e0, 0x9}, 0xc000276780)
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:91 +0x6c6
created by github.com/artie-labs/transfer/processes/consumer.Flush in goroutine 35
	/Users/robintang/code/go/src/github.com/artie-labs/transfer/processes/consumer/flush.go:57 +0x45b

@Tang8330
Copy link
Contributor

Can you reduce the buffer size and post the Kafka messages in here?

@Tang8330
Copy link
Contributor

You somehow got a decimal column that does not have any precision or scale set.

@kholisrag
Copy link
Author

Can you reduce the buffer size and post the Kafka messages in here?

you mean artie bufferRows ?

@kholisrag
Copy link
Author

this is my config for artie-transfer

mode: replication
flushIntervalSeconds: 120
bufferRows: 10000
flushSizeKb: 256000
outputSource: bigquery
bigquery:
  pathToCredentials: /app/secrets/sa.json
  location: "asia-southeast1"
  projectID: "gcp-project-id"
  defaultDataset: "kholistest_stage_mysql_db_Laku6Store"
queue: kafka
kafka:
  bootstrapServer: db-replication-kafka-kafka-bootstrap:9092
  groupID: "group_dbz_artie_mysql_dev-server-01_Example1Store_3"
...

@Tang8330
Copy link
Contributor

Yes, buffer rows. Also a way to repro this

@kholisrag
Copy link
Author

this is for the kafka message on key and value

key:

{
  "schema": {
    "type": "struct",
    "fields": [
      {
        "type": "int32",
        "optional": false,
        "field": "id"
      }
    ],
    "optional": false,
    "name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory_cycle_count_snapshot.Key"
  },
  "payload": {
    "id": 403240
  }
}

value:

{
  "schema": {
    "type": "struct",
    "fields": [
      {
        "type": "struct",
        "fields": [
          {
            "type": "int32",
            "optional": false,
            "field": "id"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_no_location_refund_count"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_no_location_non_refund_count"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_storage_count"
          },
          {
            "type": "int64",
            "optional": false,
            "name": "io.debezium.time.Timestamp",
            "version": 1,
            "field": "created"
          },
          {
            "type": "int64",
            "optional": false,
            "name": "io.debezium.time.Timestamp",
            "version": 1,
            "field": "modified"
          },
          {
            "type": "int64",
            "optional": true,
            "field": "time_index"
          }
        ],
        "optional": true,
        "name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory_cycle_count_snapshot.Value",
        "field": "before"
      },
      {
        "type": "struct",
        "fields": [
          {
            "type": "int32",
            "optional": false,
            "field": "id"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_no_location_refund_count"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_no_location_non_refund_count"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "pending_storage_count"
          },
          {
            "type": "int64",
            "optional": false,
            "name": "io.debezium.time.Timestamp",
            "version": 1,
            "field": "created"
          },
          {
            "type": "int64",
            "optional": false,
            "name": "io.debezium.time.Timestamp",
            "version": 1,
            "field": "modified"
          },
          {
            "type": "int64",
            "optional": true,
            "field": "time_index"
          }
        ],
        "optional": true,
        "name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory_cycle_count_snapshot.Value",
        "field": "after"
      },
      {
        "type": "struct",
        "fields": [
          {
            "type": "string",
            "optional": false,
            "field": "version"
          },
          {
            "type": "string",
            "optional": false,
            "field": "connector"
          },
          {
            "type": "string",
            "optional": false,
            "field": "name"
          },
          {
            "type": "int64",
            "optional": false,
            "field": "ts_ms"
          },
          {
            "type": "string",
            "optional": true,
            "name": "io.debezium.data.Enum",
            "version": 1,
            "parameters": {
              "allowed": "true,last,false,incremental"
            },
            "default": "false",
            "field": "snapshot"
          },
          {
            "type": "string",
            "optional": false,
            "field": "db"
          },
          {
            "type": "string",
            "optional": true,
            "field": "sequence"
          },
          {
            "type": "int64",
            "optional": true,
            "field": "ts_us"
          },
          {
            "type": "int64",
            "optional": true,
            "field": "ts_ns"
          },
          {
            "type": "string",
            "optional": true,
            "field": "table"
          },
          {
            "type": "int64",
            "optional": false,
            "field": "server_id"
          },
          {
            "type": "string",
            "optional": true,
            "field": "gtid"
          },
          {
            "type": "string",
            "optional": false,
            "field": "file"
          },
          {
            "type": "int64",
            "optional": false,
            "field": "pos"
          },
          {
            "type": "int32",
            "optional": false,
            "field": "row"
          },
          {
            "type": "int64",
            "optional": true,
            "field": "thread"
          },
          {
            "type": "string",
            "optional": true,
            "field": "query"
          }
        ],
        "optional": false,
        "name": "io.debezium.connector.mysql.Source",
        "field": "source"
      },
      {
        "type": "struct",
        "fields": [
          {
            "type": "string",
            "optional": false,
            "field": "id"
          },
          {
            "type": "int64",
            "optional": false,
            "field": "total_order"
          },
          {
            "type": "int64",
            "optional": false,
            "field": "data_collection_order"
          }
        ],
        "optional": true,
        "name": "event.block",
        "version": 1,
        "field": "transaction"
      },
      {
        "type": "string",
        "optional": false,
        "field": "op"
      },
      {
        "type": "int64",
        "optional": true,
        "field": "ts_ms"
      },
      {
        "type": "int64",
        "optional": true,
        "field": "ts_us"
      },
      {
        "type": "int64",
        "optional": true,
        "field": "ts_ns"
      }
    ],
    "optional": false,
    "name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory_cycle_count_snapshot.Envelope",
    "version": 2
  },
  "payload": {
    "before": null,
    "after": {
      "id": 403240,
      "pending_no_location_refund_count": 2,
      "pending_no_location_non_refund_count": 130,
      "pending_storage_count": 948,
      "created": 1731976200000,
      "modified": 1731976200000,
      "time_index": 202411190030
    },
    "source": {
      "version": "3.0.1.Final",
      "connector": "mysql",
      "name": "dbz_artie_mysql_dev-server-01_Example1Store",
      "ts_ms": 1731947400000,
      "snapshot": "false",
      "db": "Example1Store",
      "sequence": null,
      "ts_us": 1731947400000000,
      "ts_ns": 1731947400000000000,
      "table": "inventory_cycle_count_snapshot",
      "server_id": 1,
      "gtid": null,
      "file": "mysql-bin.000465",
      "pos": 290994,
      "row": 0,
      "thread": 7092030,
      "query": null
    },
    "transaction": null,
    "op": "c",
    "ts_ms": 1731994774995,
    "ts_us": 1731994774995351,
    "ts_ns": 1731994774995351677
  }
}

@kholisrag
Copy link
Author

kholisrag commented Nov 19, 2024 •
edited
Loading

this is what I got when I trying to run using vscode debugger

2024-11-19_18-28
2024-11-19_17-13

seem related to the decimal type
in this line of code https://github.com/artie-labs/transfer/blob/master/clients/bigquery/dialect/typing.go#L35-L37
there is some edge case that we need to address.

@kholisrag
Copy link
Author

Update:

seem this is the problematic kafka message after debugging with vscode debugger

{
	"schema": {
		"type": "struct",
		"fields": [
			{
				"type": "struct",
				"fields": [
					{
						"type": "int64",
						"optional": false,
						"field": "id"
					},
					{
						"type": "string",
						"optional": false,
						"field": "display_name"
					},
					{
						"type": "string",
						"optional": false,
						"field": "sku"
					},
					{
						"type": "string",
						"optional": true,
						"field": "serial"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "product_id"
					},
					{
						"type": "string",
						"optional": false,
						"field": "imei"
					},
					{
						"type": "string",
						"optional": false,
						"field": "product_type"
					},
					{
						"type": "string",
						"optional": true,
						"field": "grade_type"
					},
					{
						"type": "string",
						"optional": true,
						"field": "color"
					},
					{
						"type": "string",
						"optional": true,
						"field": "storage"
					},
					{
						"type": "string",
						"optional": true,
						"field": "ram"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "arrival_date"
					},
					{
						"type": "string",
						"optional": false,
						"field": "invoice_reference"
					},
					{
						"type": "string",
						"optional": false,
						"field": "supplier"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "price"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "repair_cost"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "good_price_at_inpro"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "good_price_at_order"
					},
					{
						"type": "int64",
						"optional": false,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "created"
					},
					{
						"type": "int64",
						"optional": false,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "modified"
					},
					{
						"type": "string",
						"optional": false,
						"field": "status"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "order_id"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "out_datetime"
					},
					{
						"type": "string",
						"optional": true,
						"field": "internal_comment"
					},
					{
						"type": "string",
						"optional": true,
						"field": "comment"
					},
					{
						"type": "string",
						"optional": true,
						"field": "test_result_description"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "test_result_date"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "pre_in_pro_date"
					},
					{
						"type": "int64",
						"optional": false,
						"default": 2,
						"field": "is_indonesia"
					},
					{
						"type": "string",
						"optional": true,
						"field": "model_number"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "master_model_id"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "bin_id"
					},
					{
						"type": "string",
						"optional": true,
						"field": "promoter_code"
					},
					{
						"type": "string",
						"optional": true,
						"field": "in_condition"
					},
					{
						"type": "string",
						"optional": true,
						"field": "traded_phone"
					},
					{
						"type": "string",
						"optional": true,
						"field": "code_letter"
					},
					{
						"type": "string",
						"optional": true,
						"field": "code_number"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "dealer_id"
					},
					{
						"type": "string",
						"optional": true,
						"field": "trade_in_unique_code"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "with_box"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "box_price"
					},
					{
						"type": "string",
						"optional": true,
						"field": "location"
					},
					{
						"type": "string",
						"optional": true,
						"field": "source_name"
					},
					{
						"type": "string",
						"optional": true,
						"field": "source_channel"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "imei_return_count"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "payment_datetime"
					},
					{
						"type": "string",
						"optional": true,
						"field": "payment_status"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "aycs_bin_id"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "is_rebola_refunded_sku"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "exlcude_from_regular_bidding"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "exclude_from_regular_bidding"
					},
					{
						"type": "string",
						"optional": false,
						"default": "id",
						"field": "region_id"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "bidding_promo_date_reference"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "received_by_customer_datetime"
					}
				],
				"optional": true,
				"name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory.Value",
				"field": "before"
			},
			{
				"type": "struct",
				"fields": [
					{
						"type": "int64",
						"optional": false,
						"field": "id"
					},
					{
						"type": "string",
						"optional": false,
						"field": "display_name"
					},
					{
						"type": "string",
						"optional": false,
						"field": "sku"
					},
					{
						"type": "string",
						"optional": true,
						"field": "serial"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "product_id"
					},
					{
						"type": "string",
						"optional": false,
						"field": "imei"
					},
					{
						"type": "string",
						"optional": false,
						"field": "product_type"
					},
					{
						"type": "string",
						"optional": true,
						"field": "grade_type"
					},
					{
						"type": "string",
						"optional": true,
						"field": "color"
					},
					{
						"type": "string",
						"optional": true,
						"field": "storage"
					},
					{
						"type": "string",
						"optional": true,
						"field": "ram"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "arrival_date"
					},
					{
						"type": "string",
						"optional": false,
						"field": "invoice_reference"
					},
					{
						"type": "string",
						"optional": false,
						"field": "supplier"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "price"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "repair_cost"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "good_price_at_inpro"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "good_price_at_order"
					},
					{
						"type": "int64",
						"optional": false,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "created"
					},
					{
						"type": "int64",
						"optional": false,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "modified"
					},
					{
						"type": "string",
						"optional": false,
						"field": "status"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "order_id"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "out_datetime"
					},
					{
						"type": "string",
						"optional": true,
						"field": "internal_comment"
					},
					{
						"type": "string",
						"optional": true,
						"field": "comment"
					},
					{
						"type": "string",
						"optional": true,
						"field": "test_result_description"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "test_result_date"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "pre_in_pro_date"
					},
					{
						"type": "int64",
						"optional": false,
						"default": 2,
						"field": "is_indonesia"
					},
					{
						"type": "string",
						"optional": true,
						"field": "model_number"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "master_model_id"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "bin_id"
					},
					{
						"type": "string",
						"optional": true,
						"field": "promoter_code"
					},
					{
						"type": "string",
						"optional": true,
						"field": "in_condition"
					},
					{
						"type": "string",
						"optional": true,
						"field": "traded_phone"
					},
					{
						"type": "string",
						"optional": true,
						"field": "code_letter"
					},
					{
						"type": "string",
						"optional": true,
						"field": "code_number"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "dealer_id"
					},
					{
						"type": "string",
						"optional": true,
						"field": "trade_in_unique_code"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "with_box"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "box_price"
					},
					{
						"type": "string",
						"optional": true,
						"field": "location"
					},
					{
						"type": "string",
						"optional": true,
						"field": "source_name"
					},
					{
						"type": "string",
						"optional": true,
						"field": "source_channel"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "imei_return_count"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "payment_datetime"
					},
					{
						"type": "string",
						"optional": true,
						"field": "payment_status"
					},
					{
						"type": "int32",
						"optional": true,
						"field": "aycs_bin_id"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "is_rebola_refunded_sku"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "exlcude_from_regular_bidding"
					},
					{
						"type": "int16",
						"optional": false,
						"default": 0,
						"field": "exclude_from_regular_bidding"
					},
					{
						"type": "string",
						"optional": false,
						"default": "id",
						"field": "region_id"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "bidding_promo_date_reference"
					},
					{
						"type": "int64",
						"optional": true,
						"name": "io.debezium.time.Timestamp",
						"version": 1,
						"field": "received_by_customer_datetime"
					}
				],
				"optional": true,
				"name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory.Value",
				"field": "after"
			},
			{
				"type": "struct",
				"fields": [
					{
						"type": "string",
						"optional": false,
						"field": "version"
					},
					{
						"type": "string",
						"optional": false,
						"field": "connector"
					},
					{
						"type": "string",
						"optional": false,
						"field": "name"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "ts_ms"
					},
					{
						"type": "string",
						"optional": true,
						"name": "io.debezium.data.Enum",
						"version": 1,
						"parameters": {
							"allowed": "true,last,false,incremental"
						},
						"default": "false",
						"field": "snapshot"
					},
					{
						"type": "string",
						"optional": false,
						"field": "db"
					},
					{
						"type": "string",
						"optional": true,
						"field": "sequence"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "ts_us"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "ts_ns"
					},
					{
						"type": "string",
						"optional": true,
						"field": "table"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "server_id"
					},
					{
						"type": "string",
						"optional": true,
						"field": "gtid"
					},
					{
						"type": "string",
						"optional": false,
						"field": "file"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "pos"
					},
					{
						"type": "int32",
						"optional": false,
						"field": "row"
					},
					{
						"type": "int64",
						"optional": true,
						"field": "thread"
					},
					{
						"type": "string",
						"optional": true,
						"field": "query"
					}
				],
				"optional": false,
				"name": "io.debezium.connector.mysql.Source",
				"field": "source"
			},
			{
				"type": "struct",
				"fields": [
					{
						"type": "string",
						"optional": false,
						"field": "id"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "total_order"
					},
					{
						"type": "int64",
						"optional": false,
						"field": "data_collection_order"
					}
				],
				"optional": true,
				"name": "event.block",
				"version": 1,
				"field": "transaction"
			},
			{
				"type": "string",
				"optional": false,
				"field": "op"
			},
			{
				"type": "int64",
				"optional": true,
				"field": "ts_ms"
			},
			{
				"type": "int64",
				"optional": true,
				"field": "ts_us"
			},
			{
				"type": "int64",
				"optional": true,
				"field": "ts_ns"
			}
		],
		"optional": false,
		"name": "dbz_artie_mysql_dev-server-01_Example1Store.Example1Store.inventory.Envelope",
		"version": 2
	},
	"payload": {
		"before": null,
		"after": {
			"id": 588564,
			"display_name": "{\"brand\":\"Oppo\",\"model\":null,\"color\":null,\"storage\":\"128gb\"}",
			"sku": "O446751289OWC",
			"serial": null,
			"product_id": null,
			"imei": "866485044675128",
			"product_type": "smartphone",
			"grade_type": "good",
			"color": null,
			"storage": "128gb",
			"ram": "8gb",
			"arrival_date": 1732043815000,
			"invoice_reference": "test 19 Nov 2024",
			"supplier": "",
			"price": 50000000000,
			"repair_cost": null,
			"good_price_at_inpro": null,
			"good_price_at_order": null,
			"created": 1732043815000,
			"modified": 1732043815000,
			"status": "pending",
			"order_id": null,
			"out_datetime": null,
			"internal_comment": null,
			"comment": "[TradingInv-20241119]ref:5d6f4c7e-f7a1-4af1-be49-dfb69d61fe89|src:CYBORG_TRADING_DASHBOARD;",
			"test_result_description": null,
			"test_result_date": null,
			"pre_in_pro_date": 1732043815000,
			"is_indonesia": 1,
			"model_number": null,
			"master_model_id": 841,
			"bin_id": null,
			"promoter_code": null,
			"in_condition": null,
			"traded_phone": null,
			"code_letter": null,
			"code_number": null,
			"dealer_id": null,
			"trade_in_unique_code": null,
			"with_box": null,
			"box_price": null,
			"location": null,
			"source_name": "Rebonics PTE Ltd",
			"source_channel": "sg-rebonics-trading",
			"imei_return_count": 0,
			"payment_datetime": null,
			"payment_status": null,
			"aycs_bin_id": null,
			"is_rebola_refunded_sku": 0,
			"exlcude_from_regular_bidding": 0,
			"exclude_from_regular_bidding": 0,
			"region_id": "sg",
			"bidding_promo_date_reference": null,
			"received_by_customer_datetime": null
		},
		"source": {
			"version": "3.0.1.Final",
			"connector": "mysql",
			"name": "dbz_artie_mysql_dev-server-01_Example1Store",
			"ts_ms": 1732015015000,
			"snapshot": "false",
			"db": "Example1Store",
			"sequence": null,
			"ts_us": 1732015015000000,
			"ts_ns": 1732015015000000000,
			"table": "inventory",
			"server_id": 1,
			"gtid": null,
			"file": "mysql-bin.000466",
			"pos": 359010,
			"row": 0,
			"thread": 7126722,
			"query": null
		},
		"transaction": null,
		"op": "c",
		"ts_ms": 1732015015742,
		"ts_us": 1732015015742325,
		"ts_ns": 1732015015742325347
	}
}

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@Tang8330 @kholisrag