From 9c6dca8bc9f7d2256edd7ed60848881da677fb76 Mon Sep 17 00:00:00 2001 From: Mariam-Almesfer Date: Tue, 20 Oct 2026 19:05:54 +0300 Subject: [PATCH] [GLUTEN-10134][VL] Add ANSI support for GetTimestamp via function overlay --- .../velox/VeloxSparkPlanExecApi.scala | 17 ++ .../GetTimestampAnsiValidateSuite.scala | 248 ++++++++++++++++++ .../functions/overlay/CheckedGetTimestamp.h | 122 +++++++++ .../operators/functions/overlay/README.md | 6 + .../overlay/RegisterFunctionOverlay.cc | 8 + cpp/velox/tests/SparkFunctionTest.cc | 81 ++++++ .../gluten/backendsapi/SparkPlanExecApi.scala | 8 + .../expression/ExpressionConverter.scala | 7 + .../gluten/expression/ExpressionNames.scala | 2 + 9 files changed, 499 insertions(+) create mode 100644 backends-velox/src/test/scala/org/apache/gluten/functions/GetTimestampAnsiValidateSuite.scala create mode 100644 cpp/velox/operators/functions/overlay/CheckedGetTimestamp.h diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 387215ca668..65e488a5d4d 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -305,6 +305,23 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging { GenericExpressionTransformer(substraitExprName, children, expr) } + /** + * Picks checked_get_timestamp from the Spark expression's own frozen failOnError, not the + * session's current ANSI setting -- so to_timestamp and try_to_timestamp keep their own behavior + * even in the same query. Not conditioned on dataType: checked_get_timestamp has no TIMESTAMP_NTZ + * registration either, so an NTZ expression falls back the same way regardless. + */ + override def genGetTimestampTransformer( + substraitExprName: String, + children: Seq[ExpressionTransformer], + expr: Expression): ExpressionTransformer = { + val name = expr match { + case g: GetTimestamp if g.failOnError => ExpressionNames.CHECKED_GET_TIMESTAMP + case _ => substraitExprName + } + GenericExpressionTransformer(name, children, expr) + } + override def genDateDiffTransformer( substraitExprName: String, endDate: ExpressionTransformer, diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/GetTimestampAnsiValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/GetTimestampAnsiValidateSuite.scala new file mode 100644 index 00000000000..0581ec5b9cf --- /dev/null +++ b/backends-velox/src/test/scala/org/apache/gluten/functions/GetTimestampAnsiValidateSuite.scala @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.functions + +import org.apache.gluten.config.GlutenConfig +import org.apache.gluten.execution.ProjectExecTransformer + +import org.apache.spark.SPARK_VERSION +import org.apache.spark.SparkConf +import org.apache.spark.SparkException +import org.apache.spark.sql.Row +import org.apache.spark.sql.internal.SQLConf + +import scala.util.{Failure, Success, Try} + +// Covers GetTimestamp (to_date/to_timestamp with a format, and their try_ variants) under the +// checked_get_timestamp / get_timestamp split -- see +// VeloxSparkPlanExecApi.genGetTimestampTransformer. +class GetTimestampAnsiValidateSuite extends FunctionsValidateSuite { + + disableFallbackCheck + + override protected def sparkConf: SparkConf = { + super.sparkConf + // Without this, Gluten falls back to Spark whenever ANSI is on, and every assertion below + // would silently exercise the fallback path instead of the native one. + .set(GlutenConfig.GLUTEN_ANSI_FALLBACK_ENABLED.key, "false") + .set(SQLConf.ANSI_ENABLED.key, "true") + } + + // try_to_date only exists from Spark 4.1 (TryToDateExpressionBuilder); try_to_timestamp and + // to_timestamp_ntz need no such guard, confirmed back to 3.4. + private def tryToDateSupported: Boolean = { + val parts = SPARK_VERSION.split("\\.") + val major = parts(0).toInt + val minor = if (parts.length > 1) parts(1).toInt else 0 + major > 4 || (major == 4 && minor >= 1) + } + + // Writes `rows` to a real parquet file and reads it back, so `str`/`fmt` are genuine + // row-dependent column values rather than literals Spark could constant-fold away. + private def withTimestampStrTable(rows: (String, String)*)(f: => Unit): Unit = { + withTempPath { + path => + val values = rows + .map { + case (str, fmt) => + val strSql = if (str == null) "CAST(NULL AS STRING)" else s"'$str'" + val fmtSql = if (fmt == null) "CAST(NULL AS STRING)" else s"'$fmt'" + s"($strSql, $fmtSql)" + } + .mkString(", ") + sql(s"SELECT * FROM VALUES $values AS t(str, fmt)").write.parquet(path.getCanonicalPath) + withTempView("ts_str_tab") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("ts_str_tab") + f + } + } + } + + // Checks the plan is native *before* the throw: a plan that fell back to Spark would also + // throw under ANSI on, for an unrelated reason, so checking only after collect() wouldn't + // prove anything. Also checks the message and that vanilla Spark rejects the same input. + private def checkNativeThrows(sqlStr: String, messageContains: String): Unit = { + val df = sql(sqlStr) + checkGlutenPlan[ProjectExecTransformer](df) + val nativeError = intercept[SparkException](df.collect()) + assert(nativeError.getMessage.contains(messageContains), nativeError.getMessage) + withSQLConf(vanillaSparkConfs(): _*) { + intercept[Exception](sql(sqlStr).collect()) + } + } + + // Checks the plan fell back to Spark (no signature is registered for this case) and that the + // result or error still matches vanilla Spark. + private def checkFallsBackMatchingVanilla(sqlStr: String): Unit = { + val df = sql(sqlStr) + assert(!df.queryExecution.executedPlan.exists(_.isInstanceOf[ProjectExecTransformer])) + Try(df.collect()) match { + case Success(rows) => + // withSQLConf returns Unit before Spark 4.0, so the result is captured in a var. + var vanilla: Array[Row] = null + withSQLConf(vanillaSparkConfs(): _*) { + vanilla = sql(sqlStr).collect() + } + assert(rows.sameElements(vanilla)) + case Failure(_) => + withSQLConf(vanillaSparkConfs(): _*) { + intercept[Exception](sql(sqlStr).collect()) + } + } + } + + test("to_timestamp ansi on: valid input, constant and column format") { + withTimestampStrTable(("2024-01-01 10:30:00", "yyyy-MM-dd HH:mm:ss")) { + runQueryAndCompare("SELECT to_timestamp(str, 'yyyy-MM-dd HH:mm:ss') FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + runQueryAndCompare("SELECT to_timestamp(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + + test("to_timestamp ansi on: invalid data throws, constant and column format") { + withTimestampStrTable(("not a date", "yyyy-MM-dd HH:mm:ss")) { + checkNativeThrows( + "SELECT to_timestamp(str, 'yyyy-MM-dd HH:mm:ss') FROM ts_str_tab", + "Invalid date format") + checkNativeThrows("SELECT to_timestamp(str, fmt) FROM ts_str_tab", "Invalid date format") + } + } + + test("to_timestamp ansi off: invalid data returns null natively") { + withTimestampStrTable(("not a date", "yyyy-MM-dd HH:mm:ss")) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + runQueryAndCompare("SELECT to_timestamp(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + assert(sql("SELECT to_timestamp(str, fmt) FROM ts_str_tab").collect()(0).isNullAt(0)) + } + } + } + + test("to_timestamp: invalid format pattern throws under both ANSI settings") { + // 'b' is not a supported Joda specifier: a format-pattern failure, never gated by ANSI. + withTimestampStrTable(("2024-01-01", "yyyy-MM-dd bb")) { + checkNativeThrows( + "SELECT to_timestamp(str, fmt) FROM ts_str_tab", + "Specifier b is not supported") + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + checkNativeThrows( + "SELECT to_timestamp(str, fmt) FROM ts_str_tab", + "Specifier b is not supported") + } + } + } + + test("to_timestamp ansi on: NULL string and NULL format each stay NULL, no error") { + withTimestampStrTable((null, "yyyy-MM-dd"), ("2024-01-01", null)) { + runQueryAndCompare("SELECT to_timestamp(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + val rows = sql("SELECT to_timestamp(str, fmt) FROM ts_str_tab ORDER BY str").collect() + assert(rows(0).isNullAt(0) && rows(1).isNullAt(0)) + } + } + + test("try_to_timestamp ansi on: invalid data returns null natively, not an error") { + // try_to_timestamp always analyzes with failOnError = false, regardless of ANSI. + withTimestampStrTable(("not a date", "yyyy-MM-dd")) { + runQueryAndCompare("SELECT try_to_timestamp(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + assert(sql("SELECT try_to_timestamp(str, fmt) FROM ts_str_tab").collect()(0).isNullAt(0)) + } + } + + test("to_timestamp throws while try_to_timestamp succeeds, same query, same native projection") { + // One row, two columns: a valid string for to_timestamp, an invalid one for try_to_timestamp. + withTempPath { + path => + sql("SELECT '2024-01-01' AS valid_str, 'not a date' AS invalid_str, 'yyyy-MM-dd' AS fmt") + .write + .parquet(path.getCanonicalPath) + withTempView("mixed_tab") { + spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("mixed_tab") + val query = + "SELECT to_timestamp(valid_str, fmt), try_to_timestamp(invalid_str, fmt) FROM mixed_tab" + val df = sql(query) + checkGlutenPlan[ProjectExecTransformer](df) + val row = df.collect()(0) + assert(!row.isNullAt(0) && row.getTimestamp(0) != null) + assert(row.isNullAt(1)) + + // Confirms the invalid column alone still throws -- the success above isn't masking it. + checkNativeThrows( + "SELECT to_timestamp(invalid_str, fmt) FROM mixed_tab", + "Invalid date format") + } + } + } + + test("to_date ansi on: valid input, constant and column format") { + withTimestampStrTable(("2024-01-01", "yyyy-MM-dd")) { + runQueryAndCompare("SELECT to_date(str, 'yyyy-MM-dd') FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + runQueryAndCompare("SELECT to_date(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + } + } + + test("to_date ansi on: invalid input throws, constant and column format") { + withTimestampStrTable(("not a date", "yyyy-MM-dd")) { + checkNativeThrows("SELECT to_date(str, 'yyyy-MM-dd') FROM ts_str_tab", "Invalid date format") + checkNativeThrows("SELECT to_date(str, fmt) FROM ts_str_tab", "Invalid date format") + } + } + + test("try_to_date ansi on: invalid data returns null natively, where supported") { + assume(tryToDateSupported, s"try_to_date requires Spark 4.1+, running $SPARK_VERSION") + withTimestampStrTable(("not a date", "yyyy-MM-dd")) { + runQueryAndCompare("SELECT try_to_date(str, fmt) FROM ts_str_tab") { + checkGlutenPlan[ProjectExecTransformer] + } + assert(sql("SELECT try_to_date(str, fmt) FROM ts_str_tab").collect()(0).isNullAt(0)) + } + } + + // dataType = TIMESTAMP_NTZ is out of scope: neither get_timestamp nor checked_get_timestamp + // has an NTZ registration, so these fall back. + test("to_timestamp_ntz: falls back, matches vanilla Spark") { + withTimestampStrTable(("2024-01-01", "yyyy-MM-dd")) { + checkFallsBackMatchingVanilla("SELECT to_timestamp_ntz(str, fmt) FROM ts_str_tab") + } + } + + test( + "to_timestamp with spark.sql.timestampType=TIMESTAMP_NTZ: falls back, matches vanilla Spark") { + withSQLConf(SQLConf.TIMESTAMP_TYPE.key -> "TIMESTAMP_NTZ") { + withTimestampStrTable(("2024-01-01", "yyyy-MM-dd")) { + checkFallsBackMatchingVanilla("SELECT to_timestamp(str, fmt) FROM ts_str_tab") + } + } + } + + test("to_timestamp_ntz ansi on, invalid input: falls back, matches vanilla Spark's error") { + withTimestampStrTable(("not a date", "yyyy-MM-dd")) { + checkFallsBackMatchingVanilla("SELECT to_timestamp_ntz(str, fmt) FROM ts_str_tab") + } + } +} diff --git a/cpp/velox/operators/functions/overlay/CheckedGetTimestamp.h b/cpp/velox/operators/functions/overlay/CheckedGetTimestamp.h new file mode 100644 index 00000000000..1f9aa65534b --- /dev/null +++ b/cpp/velox/operators/functions/overlay/CheckedGetTimestamp.h @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "velox/functions/lib/DateTimeFormatter.h" +#include "velox/functions/sparksql/SparkQueryConfig.h" +#include "velox/functions/sparksql/TimestampUtils.h" +#include "velox/type/tz/TimeZoneMap.h" + +namespace gluten { + +namespace { +// Mirrors facebook::velox::functions::sparksql::detail::initializeFormatter, +// calling buildSimpleDateTimeFormatter/buildJodaDateTimeFormatter directly +// instead of through detail::getDateTimeFormatter: that function is defined +// (not just declared) in DateTimeFunctions.h without `inline`, and is already +// compiled into libvelox.a via RegisterDatetime.cpp, so including that header +// here causes a multiple-definition link error. +inline std::shared_ptr initializeCheckedFormatter( + std::string_view format, + bool legacyFormatter) { + auto formatter = legacyFormatter ? facebook::velox::functions::buildSimpleDateTimeFormatter(format, /*lenient=*/false) + : facebook::velox::functions::buildJodaDateTimeFormatter(format); + if (formatter.hasError()) { + if (legacyFormatter) { + return nullptr; + } + VELOX_USER_FAIL(formatter.error().message()); + } + return formatter.value(); +} +} // namespace + +/// checked_get_timestamp(string, format) -> timestamp +/// +/// get_timestamp's failOnError = true sibling, picked by +/// VeloxSparkPlanExecApi.genGetTimestampTransformer from the Spark +/// expression's own frozen failOnError field. Identical to +/// GetTimestampFunction except it throws on a parse failure instead of +/// returning NULL; an invalid format pattern is unchanged (never ANSI-gated). +template +struct CheckedGetTimestampFunction { + VELOX_DEFINE_FUNCTION_TYPES(T); + + FOLLY_ALWAYS_INLINE void initialize( + const std::vector& /*inputTypes*/, + const facebook::velox::core::QueryConfig& config, + const arg_type* /*input*/, + const arg_type* format) { + legacyFormatter_ = facebook::velox::functions::sparksql::SparkQueryConfig{config}.legacyDateFormatter(); + auto sessionTimezoneName = config.sessionTimezone(); + if (!sessionTimezoneName.empty()) { + sessionTimeZone_ = facebook::velox::tz::locateZone(sessionTimezoneName); + } + if (format != nullptr) { + auto formatter = initializeCheckedFormatter(std::string_view(*format), legacyFormatter_); + if (formatter) { + formatter_ = formatter; + } else { + invalidFormat_ = true; + } + isConstantTimeFormat_ = true; + } + } + + FOLLY_ALWAYS_INLINE bool call( + out_type& result, + const arg_type& input, + const arg_type& format) { + if (invalidFormat_) { + // Invalid pattern under the legacy formatter: never ANSI-gated. + return false; + } + if (!isConstantTimeFormat_) { + auto formatter = initializeCheckedFormatter(std::string_view(format), legacyFormatter_); + if (formatter) { + formatter_ = formatter; + } else { + return false; + } + } + auto dateTimeResult = formatter_->parse(std::string_view(input)); + if (dateTimeResult.hasError()) { + // The one site ANSI mode gates; get_timestamp returns NULL here instead. + VELOX_USER_FAIL("{}", dateTimeResult.error().message()); + } + facebook::velox::functions::sparksql::toGMTWithGapCorrection( + (*dateTimeResult).timestamp, *getTimeZone(*dateTimeResult)); + result = (*dateTimeResult).timestamp; + return true; + } + + private: + const facebook::velox::tz::TimeZone* getTimeZone(const facebook::velox::functions::DateTimeResult& result) const { + // If timezone was not parsed, fallback to the session timezone. If + // there's no session timezone, fallback to 0 (GMT). + return result.timezone ? result.timezone : sessionTimeZone_; + } + + std::shared_ptr formatter_{nullptr}; + bool isConstantTimeFormat_{false}; + const facebook::velox::tz::TimeZone* sessionTimeZone_{facebook::velox::tz::locateZone(0)}; // default to GMT. + bool legacyFormatter_{false}; + bool invalidFormat_{false}; +}; + +} // namespace gluten diff --git a/cpp/velox/operators/functions/overlay/README.md b/cpp/velox/operators/functions/overlay/README.md index c265c22158d..ecc79187120 100644 --- a/cpp/velox/operators/functions/overlay/README.md +++ b/cpp/velox/operators/functions/overlay/README.md @@ -26,6 +26,12 @@ the overlay takes precedence over the Velox implementation. Velox's function authoring APIs (simple function, vector function, aggregate, or window function). See `Round.h` for a simple-function example and [Velox scalar functions guide](https://github.com/facebookincubator/velox/blob/main/velox/docs/develop/scalar-functions.rst). + When a Spark expression freezes ANSI/failOnError-like behavior as a field + captured at analysis time, prefer registering a second, differently-named + function and having the backend-specific transformer pick between the two + from that frozen field -- see `CheckedGetTimestamp.h` and + `genGetTimestampTransformer`, which mirrors `MakeTimestamp`'s existing + `make_timestamp`/`try_make_timestamp` split. 2. Register it in `RegisterFunctionOverlay.cc` inside `registerFunctionOverlay()`. Use the same name Gluten's Substrait plan conversion emits (Spark's function name in most cases). diff --git a/cpp/velox/operators/functions/overlay/RegisterFunctionOverlay.cc b/cpp/velox/operators/functions/overlay/RegisterFunctionOverlay.cc index d5d40975311..f5a5b5dee1c 100644 --- a/cpp/velox/operators/functions/overlay/RegisterFunctionOverlay.cc +++ b/cpp/velox/operators/functions/overlay/RegisterFunctionOverlay.cc @@ -16,6 +16,7 @@ */ #include "operators/functions/overlay/RegisterFunctionOverlay.h" +#include "operators/functions/overlay/CheckedGetTimestamp.h" #include "operators/functions/overlay/GetMapValue.h" #include "operators/functions/overlay/Round.h" #include "velox/functions/lib/RegistrationHelpers.h" @@ -37,6 +38,12 @@ void registerRoundFunction() { velox::registerFunction({"round"}); } +// get_timestamp's failOnError = true sibling; see VeloxSparkPlanExecApi.genGetTimestampTransformer. +void registerCheckedGetTimestampFunction() { + velox::registerFunction( + {"checked_get_timestamp"}); +} + } // namespace void registerFunctionOverlay() { @@ -46,6 +53,7 @@ void registerFunctionOverlay() { // canPushdown(), so remaining filters like m['k'].x = v extract m["k"] and // do not defeat map-key pruning. registerSparkGetMapValueFunction("get_map_value"); + registerCheckedGetTimestampFunction(); } } // namespace gluten diff --git a/cpp/velox/tests/SparkFunctionTest.cc b/cpp/velox/tests/SparkFunctionTest.cc index 0384a69beee..19e911d71fb 100644 --- a/cpp/velox/tests/SparkFunctionTest.cc +++ b/cpp/velox/tests/SparkFunctionTest.cc @@ -177,3 +177,84 @@ TEST_F(SparkFunctionTest, getMapValueSemantics) { result = evaluate("get_map_value(c0, 2)", makeRowVector({intKeyed})); facebook::velox::test::assertEqualVectors(makeNullableFlatVector({"b", std::nullopt}), result); } + +// checked_get_timestamp is get_timestamp's failOnError = true sibling (see +// CheckedGetTimestamp.h). Exercise both names directly. +TEST_F(SparkFunctionTest, checkedGetTimestampMatchesGetTimestampOnValidInput) { + auto input = makeRowVector({ + makeFlatVector({"2024-01-01", "2024-06-15 10:30:00"}), + makeFlatVector({"yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss"}), + }); + auto getTimestampResult = evaluate("get_timestamp(c0, c1)", input); + auto checkedResult = evaluate("checked_get_timestamp(c0, c1)", input); + facebook::velox::test::assertEqualVectors(getTimestampResult, checkedResult); +} + +TEST_F(SparkFunctionTest, checkedGetTimestampThrowsOnInvalidData) { + // get_timestamp stays NULL on the exact same input. + auto input = makeRowVector({ + makeFlatVector({"not a date"}), + makeFlatVector({"yyyy-MM-dd"}), + }); + VELOX_ASSERT_THROW(evaluate("checked_get_timestamp(c0, c1)", input), "Invalid date format"); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("get_timestamp(c0, c1)", input)); +} + +TEST_F(SparkFunctionTest, checkedGetTimestampNullInputStaysNull) { + // Not an error, even though checked_get_timestamp throws for other invalid input. + auto input = makeRowVector({ + makeNullableFlatVector({std::nullopt}), + makeFlatVector({"yyyy-MM-dd"}), + }); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("checked_get_timestamp(c0, c1)", input)); +} + +TEST_F(SparkFunctionTest, checkedGetTimestampInvalidPatternPreservesLegacyVsJodaDistinction) { + // An invalid format pattern is never ANSI-gated: legacy stays NULL, Joda still throws, + // unchanged from get_timestamp for both names. + auto input = makeRowVector({ + makeFlatVector({"2024-01-01"}), + makeFlatVector({"yyyy-MM-dd bb"}), // 'b' is not a supported Joda specifier. + }); + + queryCtx_->testingOverrideConfigUnsafe( + {{functions::sparksql::SparkQueryConfig::qualify(functions::sparksql::SparkQueryConfig::kLegacyDateFormatter), + "true"}}); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("checked_get_timestamp(c0, c1)", input)); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("get_timestamp(c0, c1)", input)); + + queryCtx_->testingOverrideConfigUnsafe( + {{functions::sparksql::SparkQueryConfig::qualify(functions::sparksql::SparkQueryConfig::kLegacyDateFormatter), + "false"}}); + VELOX_ASSERT_THROW(evaluate("checked_get_timestamp(c0, c1)", input), "Specifier b is not supported"); + VELOX_ASSERT_THROW(evaluate("get_timestamp(c0, c1)", input), "Specifier b is not supported"); +} + +TEST_F(SparkFunctionTest, checkedGetTimestampConstantFormat) { + // A literal format takes the isConstantTimeFormat_ path (formatter built once in initialize()). + auto columnFormatInput = + makeRowVector({makeFlatVector({"2024-01-01"}), makeFlatVector({"yyyy-MM-dd"})}); + facebook::velox::test::assertEqualVectors( + evaluate("checked_get_timestamp(c0, 'yyyy-MM-dd')", columnFormatInput), + evaluate("checked_get_timestamp(c0, c1)", columnFormatInput)); + + auto invalidDataConstantFormat = makeRowVector({makeFlatVector({"not a date"})}); + VELOX_ASSERT_THROW( + evaluate("checked_get_timestamp(c0, 'yyyy-MM-dd')", invalidDataConstantFormat), "Invalid date format"); +} + +TEST_F(SparkFunctionTest, checkedGetTimestampNullFormatStaysNull) { + // A NULL format, not just a NULL input string, also propagates to NULL. + auto input = makeRowVector({ + makeFlatVector({"2024-01-01"}), + makeNullableFlatVector({std::nullopt}), + }); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("checked_get_timestamp(c0, c1)", input)); + facebook::velox::test::assertEqualVectors( + makeNullableFlatVector({std::nullopt}), evaluate("get_timestamp(c0, c1)", input)); +} diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 1bbd92f3c55..c080e9986ba 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -365,6 +365,14 @@ trait SparkPlanExecApi { throw new GlutenNotSupportException("make_timestamp is not supported") } + /** Transform get_timestamp to Substrait. Default is a passthrough; Velox overrides it. */ + def genGetTimestampTransformer( + substraitExprName: String, + children: Seq[ExpressionTransformer], + expr: Expression): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, children, expr) + } + def genRegexpReplaceTransformer( substraitExprName: String, children: Seq[ExpressionTransformer], diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index 0464734c323..a3bae1a5ad3 100644 --- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -709,6 +709,13 @@ object ExpressionConverter extends SQLConfHelper with Logging { makeTimestampExprName, m.children.map(replaceWithExpressionTransformer0(_, attributeSeq, expressionsMap)), m) + case g: GetTimestamp => + // checked_get_timestamp vs. get_timestamp is Velox-specific, so it's picked in + // VeloxSparkPlanExecApi, not here. + BackendsApiManager.getSparkPlanExecApiInstance.genGetTimestampTransformer( + substraitExprName, + g.children.map(replaceWithExpressionTransformer0(_, attributeSeq, expressionsMap)), + g) case tsAdd: BinaryExpression if tsAdd.getClass.getSimpleName.equals("TimestampAdd") => BackendsApiManager.getSparkPlanExecApiInstance.genTimestampAddTransformer( substraitExprName, diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index 50e7c0097f4..4d65cbdb231 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -226,6 +226,8 @@ object ExpressionNames { final val TRUNC = "trunc" final val DATE_TRUNC = "date_trunc" final val GET_TIMESTAMP = "get_timestamp" // for function: to_date/to_timestamp + // Velox-only: get_timestamp's failOnError = true sibling. See VeloxSparkPlanExecApi. + final val CHECKED_GET_TIMESTAMP = "checked_get_timestamp" final val TIMESTAMP_ADD = "timestampadd" final val TIMESTAMP_DIFF = "timestampdiff" final val NEXT_DAY = "next_day"