Skip to content

Commit c5983c1

Browse files
c21cloud-fan
authored andcommitted
[SPARK-38018][SQL][3.2] Fix ColumnVectorUtils.populate to handle CalendarIntervalType correctly
### What changes were proposed in this pull request? This is a backport of #35314 to branch 3.2. See that original PR for context. ### Why are the changes needed? To fix potential correctness issue. ### Does this PR introduce _any_ user-facing change? No but fix the exiting correctness issue when reading partition column with CalendarInterval type. ### How was this patch tested? Added unit test in `ColumnVectorSuite.scala`. Closes #37114 from c21/branch-3.2. Authored-by: Cheng Su <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent 32aff86 commit c5983c1

File tree

2 files changed

+12
-2
lines changed

2 files changed

+12
-2
lines changed

sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnVectorUtils.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,8 @@ public static void populate(WritableColumnVector col, InternalRow row, int field
9191
} else if (t instanceof CalendarIntervalType) {
9292
CalendarInterval c = (CalendarInterval)row.get(fieldIdx, t);
9393
col.getChild(0).putInts(0, capacity, c.months);
94-
col.getChild(1).putLongs(0, capacity, c.microseconds);
94+
col.getChild(1).putInts(0, capacity, c.days);
95+
col.getChild(2).putLongs(0, capacity, c.microseconds);
9596
} else if (t instanceof DateType) {
9697
col.putInts(0, capacity, row.getInt(fieldIdx));
9798
} else if (t instanceof TimestampType || t instanceof TimestampNTZType) {

sql/core/src/test/scala/org/apache/spark/sql/execution/vectorized/ColumnVectorSuite.scala

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.columnar.ColumnAccessor
2525
import org.apache.spark.sql.execution.columnar.compression.ColumnBuilderHelper
2626
import org.apache.spark.sql.types._
2727
import org.apache.spark.sql.vectorized.ColumnarArray
28-
import org.apache.spark.unsafe.types.UTF8String
28+
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
2929

3030
class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
3131
private def withVector(
@@ -536,5 +536,14 @@ class ColumnVectorSuite extends SparkFunSuite with BeforeAndAfterEach {
536536
}
537537
}
538538
}
539+
540+
test("SPARK-38018: ColumnVectorUtils.populate to handle CalendarIntervalType correctly") {
541+
val vector = new OnHeapColumnVector(5, CalendarIntervalType)
542+
val row = new SpecificInternalRow(Array(CalendarIntervalType))
543+
val interval = new CalendarInterval(3, 5, 1000000)
544+
row.setInterval(0, interval)
545+
ColumnVectorUtils.populate(vector, row, 0)
546+
assert(vector.getInterval(0) === interval)
547+
}
539548
}
540549

0 commit comments

Comments
 (0)