Skip to content

Commit

Permalink
chore: add description in FieldDefinition for LLM semantics
Browse files Browse the repository at this point in the history
  • Loading branch information
@duonganhthu43
duonganhthu43 committed Apr 3, 2024
1 parent cb4c2fe commit 77cc8f2
Show file tree
Hide file tree
Showing 27 changed files with 74 additions and 0 deletions.
1 change: 1 addition & 0 deletions dozer-ingestion/aerospike/src/connector.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,7 @@ impl Connector for AerospikeConnector {
},
nullable: name != "PK",
source: Default::default(),
description: None,
})
.collect(),
primary_index,
Expand Down
13 changes: 13 additions & 0 deletions dozer-ingestion/ethereum/src/log/helper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ pub fn get_contract_event_schemas(
},
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
});
}

Expand Down Expand Up @@ -231,72 +232,84 @@ pub fn get_eth_schema() -> Schema {
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "address".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "topics".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "data".to_string(),
typ: FieldType::Binary,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "block_hash".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "block_number".to_string(),
typ: FieldType::UInt,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "transaction_hash".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "transaction_index".to_string(),
typ: FieldType::Int,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "log_index".to_string(),
typ: FieldType::Int,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "transaction_log_index".to_string(),
typ: FieldType::Int,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "log_type".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "removed".to_string(),
typ: FieldType::Boolean,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],

Expand Down
8 changes: 8 additions & 0 deletions dozer-ingestion/ethereum/src/trace/helper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,48 +93,56 @@ pub fn get_trace_schema() -> Schema {
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "from".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "to".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "value".to_string(),
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "gas".to_string(),
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "gas_used".to_string(),
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "input".to_string(),
typ: FieldType::Text,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "output".to_string(),
typ: FieldType::Text,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![],
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/javascript/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ impl Connector for JavaScriptConnector {
typ: FieldType::Json,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
}],
primary_index: vec![],
},
Expand Down
6 changes: 6 additions & 0 deletions dozer-ingestion/kafka/src/debezium/mapper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,24 +317,28 @@ mod tests {
typ: FieldType::Int,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "name".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "description".to_string(),
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "weight".to_string(),
typ: FieldType::Float,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![],
Expand Down Expand Up @@ -406,12 +410,14 @@ mod tests {
typ: FieldType::Int,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "name".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![],
Expand Down
3 changes: 3 additions & 0 deletions dozer-ingestion/kafka/src/debezium/schema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ pub fn map_schema(
typ,
nullable: f.optional.map_or(false, |o| o),
source: SourceDefinition::Dynamic,
description: None,
})
})
.collect(),
Expand Down Expand Up @@ -196,12 +197,14 @@ mod tests {
typ: FieldType::Int,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "name".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![0],
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/kafka/src/debezium/schema_registry.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ impl SchemaRegistry {
typ,
nullable,
source: SourceDefinition::Dynamic,
description: None,
})
})
.collect();
Expand Down
2 changes: 2 additions & 0 deletions dozer-ingestion/kafka/src/no_schema_registry_basic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,14 @@ impl NoSchemaRegistryBasic {
typ: FieldType::String,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "message".to_string(),
typ: FieldType::String,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![0],
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/kafka/src/schema_registry_basic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ impl SchemaRegistryBasic {
typ,
nullable,
source: SourceDefinition::Dynamic,
description: None,
})
})
.collect();
Expand Down
2 changes: 2 additions & 0 deletions dozer-ingestion/mongodb/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,12 +501,14 @@ impl Connector for MongodbConnector {
typ: FieldType::Json,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "data".to_owned(),
typ: FieldType::Json,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![0],
Expand Down
6 changes: 6 additions & 0 deletions dozer-ingestion/mysql/src/connector.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ impl Connector for MySQLConnector {
typ,
nullable,
source: SourceDefinition::Dynamic,
description: None,
}
},
)
Expand Down Expand Up @@ -840,18 +841,21 @@ mod tests {
typ: FieldType::Int,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "c2".into(),
typ: FieldType::Text,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "c3".into(),
typ: FieldType::Float,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![0],
Expand All @@ -866,12 +870,14 @@ mod tests {
typ: FieldType::Int,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
FieldDefinition {
name: "value".into(),
typ: FieldType::Json,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
},
],
primary_index: vec![0],
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/object-store/src/schema_helper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ pub fn map_schema_to_dozer<'a, I: Iterator<Item = &'a Arc<Field>>>(
typ: mapped_field_type,
nullable: field.is_nullable(),
source: SourceDefinition::Dynamic,
description: None,
})
})
.collect()
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/oracle/src/connector/mapping.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@ pub fn decide_schema(
connection: connection.to_string(),
name: table_name.clone(),
},
description: None,
}),
Err(err) => return Err(Error::DataType(err.clone())),
}
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/postgres/src/helper.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ pub fn convert_column_to_field(column: &Column) -> Result<FieldDefinition, Postg
typ,
nullable: true,
source: SourceDefinition::Dynamic,
description: None,
})
}

Expand Down
3 changes: 3 additions & 0 deletions dozer-ingestion/postgres/src/schema/sorter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ mod tests {
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
false,
);
Expand All @@ -113,6 +114,7 @@ mod tests {
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
true,
);
Expand All @@ -122,6 +124,7 @@ mod tests {
typ: FieldType::UInt,
nullable: false,
source: SourceDefinition::Dynamic,
description: None,
},
false,
);
Expand Down
1 change: 1 addition & 0 deletions dozer-ingestion/snowflake/src/connection/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ impl<'env> Client<'env> {
typ,
nullable: *nullable,
source: SourceDefinition::Dynamic,
description: None,
});
}
}
Expand Down
2 changes: 2 additions & 0 deletions dozer-ingestion/tests/test_suite/data.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ pub fn records_without_primary_key() -> (FieldsAndPk, Vec<Vec<Field>>) {
typ: FieldType::Int,
nullable: false,
source: Default::default(),
description: None,
},
FieldDefinition {
name: "uint".to_string(),
typ: FieldType::UInt,
nullable: false,
source: Default::default(),
description: None,
},
];

Expand Down
Loading

0 comments on commit 77cc8f2

Please sign in to comment.