Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support path parameterization for file data #113

Merged
merged 6 commits into from
Feb 6, 2023
Merged

support path parameterization for file data #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a2bdd36
add arguments for PARQUET、ORC、JSON、CSV ，In order to realize parameter…
Codelone Nov 21, 2022
5fe8cd9
-p is empty, throw exception
Codelone Nov 21, 2022
7546f5b
Merge branch 'master' of https://github.com/vesoft-inc/nebula-exchange
Codelone Dec 2, 2022
2a2ecb1
Merge branch 'master' of https://github.com/vesoft-inc/nebula-exchange
Codelone Dec 27, 2022
801e392
change variable name, `file` to `variable`, `path` to `param`, and f…
Codelone Dec 27, 2022
afbb1f4
add test case for the variable path
Codelone Dec 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion exchange-common/src/main/scala/com/vesoft/exchange/common/Package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,6 @@ final case class Argument(config: String = "application.conf",
hive: Boolean = false,
directly: Boolean = false,
dry: Boolean = false,
reload: String = "")
reload: String = "",
variable: Boolean = false,
param: String = "")
51 changes: 40 additions & 11 deletions exchange-common/src/main/scala/com/vesoft/exchange/common/config/Configs.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,16 @@ object Configs {
* @param configPath
* @return
*/
def parse(configPath: String): Configs = {
def parse(configPath: String, variable: Boolean = false, param: String = ""): Configs = {
var config: Config = null
var paths: Map[String,String] = null
if (variable) {
if (param.isEmpty) throw new IllegalArgumentException(s"-p must to set ")
paths = param.split(",").map(path => {
val kv = path.split("=")
(kv(0), kv(1))
}).toMap
}
if (configPath.startsWith("hdfs://")) {
val hadoopConfig: Configuration = new Configuration()
val fs: FileSystem = org.apache.hadoop.fs.FileSystem.get(hadoopConfig)
Expand Down Expand Up @@ -391,7 +399,7 @@ object Configs {
}

val sourceCategory = toSourceCategory(tagConfig.getString("type.source"))
val sourceConfig = dataSourceConfig(sourceCategory, tagConfig, nebulaConfig)
val sourceConfig = dataSourceConfig(sourceCategory, tagConfig, nebulaConfig, variable, paths)
LOG.info(s"Source Config ${sourceConfig}")
hasKafka = sourceCategory == SourceCategory.KAFKA

Expand Down Expand Up @@ -455,7 +463,7 @@ object Configs {
edgeConfig.hasPath("longitude")

val sourceCategory = toSourceCategory(edgeConfig.getString("type.source"))
val sourceConfig = dataSourceConfig(sourceCategory, edgeConfig, nebulaConfig)
val sourceConfig = dataSourceConfig(sourceCategory, edgeConfig, nebulaConfig, variable, paths)
LOG.info(s"Source Config ${sourceConfig}")
hasKafka = sourceCategory == SourceCategory.KAFKA

Expand Down Expand Up @@ -614,14 +622,19 @@ object Configs {
*/
private[this] def dataSourceConfig(category: SourceCategory.Value,
config: Config,
nebulaConfig: Config): DataSourceConfigEntry = {
nebulaConfig: Config,
variable: Boolean,
paths: Map[String,String]): DataSourceConfigEntry = {
category match {
case SourceCategory.PARQUET =>
FileBaseSourceConfigEntry(SourceCategory.PARQUET, config.getString("path"))
if (variable) FileBaseSourceConfigEntry(SourceCategory.PARQUET, paths(config.getString("path")))
else FileBaseSourceConfigEntry(SourceCategory.PARQUET, config.getString("path"))
case SourceCategory.ORC =>
FileBaseSourceConfigEntry(SourceCategory.ORC, config.getString("path"))
if (variable) FileBaseSourceConfigEntry(SourceCategory.ORC, paths(config.getString("path")))
else FileBaseSourceConfigEntry(SourceCategory.ORC, config.getString("path"))
case SourceCategory.JSON =>
FileBaseSourceConfigEntry(SourceCategory.JSON, config.getString("path"))
if (variable) FileBaseSourceConfigEntry(SourceCategory.JSON, paths(config.getString("path")))
else FileBaseSourceConfigEntry(SourceCategory.JSON, config.getString("path"))
case SourceCategory.CSV =>
val separator =
if (config.hasPath("separator"))
Expand All @@ -632,10 +645,16 @@ object Configs {
config.getBoolean("header")
else
false
FileBaseSourceConfigEntry(SourceCategory.CSV,
config.getString("path"),
Some(separator),
Some(header))
if (variable)
FileBaseSourceConfigEntry(SourceCategory.CSV,
paths(config.getString("path")),
Some(separator),
Some(header))
else
FileBaseSourceConfigEntry(SourceCategory.CSV,
config.getString("path"),
Some(separator),
Some(header))
case SourceCategory.HIVE =>
HiveSourceConfigEntry(SourceCategory.HIVE, config.getString("exec"))
case SourceCategory.NEO4J =>
Expand Down Expand Up @@ -966,6 +985,16 @@ object Configs {
.valueName("<path>")
.action((x, c) => c.copy(reload = x))
.text("reload path")

opt[Unit]('v', "variable")
.action((_, c) => c.copy(variable = true))
.text("enable file param")

opt[String]('p', "param")
.valueName("<param>")
.action((x, c) => c.copy(param = x))
.text("file param path")

}
parser.parse(args, Argument())
}
Expand Down
60 changes: 60 additions & 0 deletions exchange-common/src/test/scala/com/vesoft/exchange/common/config/ConfigsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,66 @@ class ConfigsSuite {
}
}


@Test
def configsWithVariableSuite(): Unit = {
val args = List("-c", "src/test/resources/application.conf", "-v", "-p", "path0=/app/test1.parquet,path1=/app/test2.csv,path2=/app/test2.json,path3=/app/test3.json")
val options = Configs.parser(args.toArray, "test")
val c: Argument = options match {
case Some(config) => config
case _ =>
assert(false)
sys.exit(-1)
}
assert(c.variable)

val configs = Configs.parse(c.config, c.variable, c.param)
val tagsConfig = configs.tagsConfig
val edgesConfig = configs.edgesConfig
for (tagConfig <- tagsConfig) {
val source = tagConfig.dataSourceConfigEntry

source.category match {
case SourceCategory.CSV => {
val csv = tagConfig.dataSourceConfigEntry.asInstanceOf[FileBaseSourceConfigEntry]
assert(csv.path.equals("/app/test2.csv"))
}
case SourceCategory.JSON => {
val json = tagConfig.dataSourceConfigEntry.asInstanceOf[FileDataSourceConfigEntry]
assert(json.path.equals("/app/test3.json"))
}
case SourceCategory.PARQUET => {
val parquet = tagConfig.dataSourceConfigEntry.asInstanceOf[FileDataSourceConfigEntry]
assert(parquet.path.equals("/app/test1.parquet"))
}

case _ => {}
}
}

for (edgeConfig <- edgesConfig) {
val source = edgeConfig.dataSourceConfigEntry
val sink = edgeConfig.dataSinkConfigEntry
assert(sink.category == SinkCategory.CLIENT || sink.category == SinkCategory.SST)

source.category match {
case SourceCategory.CSV => {
val csv = edgeConfig.dataSourceConfigEntry.asInstanceOf[FileBaseSourceConfigEntry]
assert(csv.path.equals("/app/test2.csv"))
}
case SourceCategory.JSON => {
val json = edgeConfig.dataSourceConfigEntry.asInstanceOf[FileDataSourceConfigEntry]
assert(json.path.equals("/app/test2.json"))
}
case SourceCategory.PARQUET => {
val parquet = edgeConfig.dataSourceConfigEntry.asInstanceOf[FileDataSourceConfigEntry]
assert(parquet.path.equals("/app/test1.parquet"))
}
case _ => {}
}
}

}
/**
* correct com.vesoft.exchange.common.config
*/
Expand Down
2 changes: 1 addition & 1 deletion nebula-exchange_spark_2.2/src/main/scala/com/vesoft/nebula/exchange/Exchange.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ object Exchange {
sys.exit(-1)
}

val configs = Configs.parse(c.config)
val configs = Configs.parse(c.config, c.variable, c.param)
LOG.info(s"Config ${configs}")

val session = SparkSession
Expand Down
2 changes: 1 addition & 1 deletion nebula-exchange_spark_2.4/src/main/scala/com/vesoft/nebula/exchange/Exchange.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ object Exchange {
sys.exit(-1)
}

val configs = Configs.parse(c.config)
val configs = Configs.parse(c.config, c.variable, c.param)
LOG.info(s"Config ${configs}")

val session = SparkSession
Expand Down
2 changes: 1 addition & 1 deletion nebula-exchange_spark_3.0/src/main/scala/com/vesoft/nebula/exchange/Exchange.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ object Exchange {
sys.exit(-1)
}

val configs = Configs.parse(c.config)
val configs = Configs.parse(c.config, c.variable, c.param)
LOG.info(s"Config ${configs}")

val session = SparkSession
Expand Down