Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2461,6 +2461,60 @@ class Dataset[T] private[sql] (val session: SparkSession, private[sql] val plan:
new DataFrameWriterV2[T](table, this)
}

def unpersist(blocking: Boolean): this.type = {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you skip persist for a reason?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

persist is implemented in the Python side I believe

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I now remember:

def persist(): this.type
def persist(newLevel: StorageLevel): this.type

There is a StorageLevel class in the signature which is from the core module. Does Scala client now can reuse the core 's classes (IIUC scala client is aiming to get rid of core dependency eventually)?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, we can use the storage enum. We will move the enum to common/util later on.

throw new UnsupportedOperationException("unpersist() is not implemented.")
}

def unpersist(): this.type = unpersist(blocking = false)

def cache(): this.type = {
throw new UnsupportedOperationException("cache() is not implemented.")
}

def withWatermark(eventTime: String, delayThreshold: String): Dataset[T] = {
throw new UnsupportedOperationException("withWatermark is not implemented.")
}

def observe(name: String, expr: Column, exprs: Column*): Dataset[T] = {
throw new UnsupportedOperationException("observe is not implemented.")
}

def foreach(f: T => Unit): Unit = {
throw new UnsupportedOperationException("foreach is not implemented.")
}

def foreachPartition(f: Iterator[T] => Unit): Unit = {
throw new UnsupportedOperationException("foreach is not implemented.")
}

def checkpoint(): Dataset[T] = {
throw new UnsupportedOperationException("checkpoint is not implemented.")
}

def checkpoint(eager: Boolean): Dataset[T] = {
throw new UnsupportedOperationException("checkpoint is not implemented.")
}

def localCheckpoint(): Dataset[T] = {
throw new UnsupportedOperationException("localCheckpoint is not implemented.")
}

def localCheckpoint(eager: Boolean): Dataset[T] = {
throw new UnsupportedOperationException("localCheckpoint is not implemented.")
}

def sameSemantics(other: Dataset[T]): Boolean = {
throw new UnsupportedOperationException("sameSemantics is not implemented.")
}

def semanticHash(): Int = {
throw new UnsupportedOperationException("semanticHash is not implemented.")
}

def toJSON: Dataset[String] = {
throw new UnsupportedOperationException("toJSON is not implemented.")
}

private[sql] def analyze: proto.AnalyzePlanResponse = {
session.analyze(plan, proto.Explain.ExplainMode.SIMPLE)
}
Expand Down