Skip to content

[SPARK-44702][SQL] Fix ExpressionEncoder derivation for case class fi… #58863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Yuvraj2172 wants to merge 1 commit into
base: master
Choose a base branch
from
+28 −2
Open

[SPARK-44702][SQL] Fix ExpressionEncoder derivation for case class fi… #58863

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions sql/api/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,10 @@ trait ScalaReflection extends Logging {
def getConstructorParameters(tpe: Type): Seq[(String, Type)] = {
val dealiasedTpe = tpe.dealias
val formalTypeArgs = dealiasedTpe.typeSymbol.asClass.typeParams
val TypeRef(_, _, actualTypeArgs) = dealiasedTpe
val actualTypeArgs = dealiasedTpe match {
case TypeRef(_, _, args) => args
case _ => Nil
}
val params = constructParams(dealiasedTpe)
params.map { p =>
val paramTpe = p.typeSignature
Expand Down Expand Up @@ -514,7 +517,14 @@ trait ScalaReflection extends Logging {

protected def constructParams(tpe: Type): Seq[Symbol] = {
val constructorSymbol = tpe.member(termNames.CONSTRUCTOR) match {
case NoSymbol => getCompanionConstructor(tpe)
case NoSymbol =>
// A compound type formed with `with` (e.g. `Foo with Tag`) has no constructor of its
// own. Its erasure collapses it to its single concrete class parent (traits erase to
// interfaces), whose constructor is the one we actually want.
tpe.erasure.member(termNames.CONSTRUCTOR) match {
case NoSymbol => getCompanionConstructor(tpe)
case sym => sym
}
case sym => sym
}
val params = if (constructorSymbol.isMethod) {
Expand Down
16 changes: 16 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/ScalaReflectionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,12 @@ object FooEnum extends Enumeration {

case class FooClassWithEnum(i: Int, e: FooEnum)

trait TraitTag

case class InnerDataWithTraitTag(x: Int)

case class OuterDataWithTraitTag(y: InnerDataWithTraitTag with TraitTag)

object TestingUDT {
@SQLUserDefinedType(udt = classOf[NestedStructUDT])
class NestedStruct(val a: Integer, val b: Long, val c: Double)
Expand Down Expand Up @@ -220,6 +226,16 @@ class ScalaReflectionSuite extends SparkFunSuite {
nullable = true))
}

test("SPARK-44702: case class field type tagged with a trait") {
val schema = schemaFor[OuterDataWithTraitTag]
assert(schema === Schema(
StructType(Seq(
StructField("y", StructType(Seq(
StructField("x", IntegerType, nullable = false))),
nullable = true))),
nullable = true))
}

test("nullable data") {
val schema = schemaFor[NullableData]
assert(schema === Schema(
Expand Down