Skip to content

[SPARK-15711][SQL] Ban CREATE TEMPORARY TABLE USING AS SELECT #13451

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
clockfly wants to merge 3 commits into from
Closed

[SPARK-15711][SQL] Ban CREATE TEMPORARY TABLE USING AS SELECT #13451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2f0355f
Ban CREATE TEMP TABLE USING AS SELECT for now
clockfly Jun 1, 2016
a94dd89
fix the exception type.
clockfly Jun 2, 2016
aaca7f0
fix style.
clockfly Jun 2, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,6 @@ final class DataFrameWriter private[sql](df: DataFrame) {
CreateTableUsingAsSelect(
tableIdent,
source,
temporary = false,
partitioningColumns.map(_.toArray).getOrElse(Array.empty[String]),
getBucketSpec,
mode,
Expand Down
9 changes: 5 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,17 +317,19 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
// Get the backing query.
val query = plan(ctx.query)

if (temp) {
Copy link
Contributor

@tejasapatil tejasapatil Jun 2, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can you please rename this to temporary

Copy link
Contributor Author

@clockfly clockfly Jun 2, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably not in the scope of this PR, temp is a existing variable defined at https://github.com/clockfly/spark/blob/aaca7f01882de19c9dda47500f7d4f03712d3383/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala#L304.

throw operationNotAllowed("CREATE TEMPORARY TABLE ... USING ... AS query", ctx)
}

// Determine the storage mode.
val mode = if (ifNotExists) {
SaveMode.Ignore
} else if (temp) {
SaveMode.Overwrite
} else {
SaveMode.ErrorIfExists
}

CreateTableUsingAsSelect(
table, provider, temp, partitionColumnNames, bucketSpec, mode, options, query)
table, provider, partitionColumnNames, bucketSpec, mode, options, query)
} else {
val struct = Option(ctx.colTypeList()).map(createStructType)
CreateTableUsing(
Expand Down Expand Up @@ -960,7 +962,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
CreateTableUsingAsSelect(
tableIdent = tableDesc.identifier,
provider = conf.defaultDataSourceName,
temporary = false,
partitionColumns = tableDesc.partitionColumnNames.toArray,
bucketSpec = None,
mode = mode,
Expand Down
10 changes: 1 addition & 9 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,15 +397,7 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
throw new AnalysisException(
"allowExisting should be set to false when creating a temporary table.")

case c: CreateTableUsingAsSelect if c.temporary && c.partitionColumns.nonEmpty =>
sys.error("Cannot create temporary partitioned table.")

case c: CreateTableUsingAsSelect if c.temporary =>
val cmd = CreateTempTableUsingAsSelectCommand(
c.tableIdent, c.provider, Array.empty[String], c.mode, c.options, c.child)
ExecutedCommandExec(cmd) :: Nil

case c: CreateTableUsingAsSelect if !c.temporary =>
case c: CreateTableUsingAsSelect =>
val cmd =
CreateDataSourceTableAsSelectCommand(
c.tableIdent,
Expand Down
32 changes: 0 additions & 32 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@ case class CreateTableUsing(
case class CreateTableUsingAsSelect(
tableIdent: TableIdentifier,
provider: String,
temporary: Boolean,
partitionColumns: Array[String],
bucketSpec: Option[BucketSpec],
mode: SaveMode,
Expand Down Expand Up @@ -91,37 +90,6 @@ case class CreateTempTableUsing(
}
}

case class CreateTempTableUsingAsSelectCommand(
tableIdent: TableIdentifier,
provider: String,
partitionColumns: Array[String],
mode: SaveMode,
options: Map[String, String],
query: LogicalPlan) extends RunnableCommand {

if (tableIdent.database.isDefined) {
throw new AnalysisException(
s"Temporary table '$tableIdent' should not have specified a database")
}

override def run(sparkSession: SparkSession): Seq[Row] = {
val df = Dataset.ofRows(sparkSession, query)
val dataSource = DataSource(
sparkSession,
className = provider,
partitionColumns = partitionColumns,
bucketSpec = None,
options = options)
val result = dataSource.write(mode, df)
sparkSession.sessionState.catalog.createTempView(
tableIdent.table,
Dataset.ofRows(sparkSession, LogicalRelation(result)).logicalPlan,
overrideIfExists = true)

Seq.empty[Row]
}
}

case class RefreshTable(tableIdent: TableIdentifier)
extends RunnableCommand {

Expand Down
Loading