Skip to content

Changed isObjectNotFoundException and isSyntaxErrorBestEffort to be O… #53456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
SavicStefan wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Changed isObjectNotFoundException and isSyntaxErrorBestEffort to be O… #53456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d02c02b
Changed isObjectNotFoundException and isSyntaxErrorBestEffort to be O…
Dec 12, 2025
18dc1c1
Fixed stuff
Dec 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ object JDBCRDD extends Logging {
try {
getQueryOutputSchema(fullQuery, options, dialect, conn)
} catch {
case e: SQLException if dialect.isSyntaxErrorBestEffort(e) =>
case e: SQLException if dialect.isSyntaxErrorBestEffort(e).contains(true) =>
throw new SparkException(
errorClass = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_OUTPUT_SCHEMA_RESOLUTION",
messageParameters = Map(
Expand Down Expand Up @@ -334,7 +334,7 @@ class JDBCRDD(
try {
stmt.executeQuery()
} catch {
case e: SQLException if dialect.isSyntaxErrorBestEffort(e) =>
case e: SQLException if dialect.isSyntaxErrorBestEffort(e).contains(true) =>
throw new SparkException(
errorClass = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_QUERY_EXECUTION",
messageParameters = Map(
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ object JdbcUtils extends Logging with SQLConfHelper {

executionResult match {
case Success(_) => true
case Failure(e: SQLException) if dialect.isObjectNotFoundException(e) => false
case Failure(e: SQLException) if dialect.isObjectNotFoundException(e).contains(true) => false
case Failure(e) => throw e // Re-throw unexpected exceptions
}
}
Expand Down
2 changes: 1 addition & 1 deletion sql/core/src/main/scala/org/apache/spark/sql/jdbc/AggregatedDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ private class AggregatedDialect(dialects: List[JdbcDialect])
dialects.head.getSchemaQuery(table)
}

override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
dialects.head.isSyntaxErrorBestEffort(exception)
}

Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/DB2Dialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ private case class DB2Dialect() extends JdbcDialect with SQLConfHelper with NoLe
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getErrorCode == -204
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getErrorCode == -204)
}

class DB2SQLBuilder extends JDBCSQLBuilder {
Expand Down Expand Up @@ -123,8 +123,8 @@ private case class DB2Dialect() extends JdbcDialect with SQLConfHelper with NoLe
override def isCascadingTruncateTable(): Option[Boolean] = Some(false)

// See https://www.ibm.com/docs/en/db2-for-zos/12.0.0?topic=codes-sqlstate-values-common-error
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

// scalastyle:off line.size.limit
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/DatabricksDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ private case class DatabricksDialect() extends JdbcDialect with NoLegacyJDBCErro
url.startsWith("jdbc:databricks")
}

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getSQLState == "42P01" || e.getSQLState == "42704"
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getSQLState == "42P01" || e.getSQLState == "42704")
}

override def getCatalystType(
Expand All @@ -54,8 +54,8 @@ private case class DatabricksDialect() extends JdbcDialect with NoLegacyJDBCErro
}

// See https://docs.databricks.com/aws/en/error-messages/sqlstates
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

override def quoteIdentifier(colName: String): String = {
Expand Down
10 changes: 5 additions & 5 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/DerbyDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ private case class DerbyDialect() extends JdbcDialect with NoLegacyJDBCError {
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getSQLState.equalsIgnoreCase("42Y07") ||
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getSQLState.equalsIgnoreCase("42Y07") ||
e.getSQLState.equalsIgnoreCase("42X05") ||
e.getSQLState.equalsIgnoreCase("X0X05")
e.getSQLState.equalsIgnoreCase("X0X05"))
}

override def getCatalystType(
Expand Down Expand Up @@ -69,8 +69,8 @@ private case class DerbyDialect() extends JdbcDialect with NoLegacyJDBCError {
override def isCascadingTruncateTable(): Option[Boolean] = Some(false)

// See https://db.apache.org/derby/docs/10.15/ref/rrefexcept71493.html
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

// See https://db.apache.org/derby/docs/10.15/ref/rrefsqljrenametablestatement.html
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/H2Dialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,13 @@ private[sql] case class H2Dialect() extends JdbcDialect with NoLegacyJDBCError {
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
Set(42102, 42103, 42104, 90079).contains(e.getErrorCode)
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(Set(42102, 42103, 42104, 90079).contains(e.getErrorCode))
}

// See https://www.h2database.com/javadoc/org/h2/api/ErrorCode.html
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

override def getCatalystType(
Expand Down
6 changes: 4 additions & 2 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,7 @@ abstract class JdbcDialect extends Serializable with Logging {
* @return true if the exception is confidently identified as a syntax error; false otherwise.
*/
@Since("4.1.0")
def isSyntaxErrorBestEffort(exception: java.sql.SQLException): Boolean = false
def isSyntaxErrorBestEffort(exception: java.sql.SQLException): Option[Boolean] = None

/**
* Rename an existing table.
Expand Down Expand Up @@ -783,7 +783,9 @@ abstract class JdbcDialect extends Serializable with Logging {
}

@Since("4.1.0")
def isObjectNotFoundException(e: SQLException): Boolean = true
def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Option(e.getSQLState).map(_.startsWith("42"))
Copy link
Contributor

@urosstan-db urosstan-db Dec 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note for reviewers: we want to have some generic implementation here that covers almost all SQL engines (42000 group usually means no access, no object found or syntax error on most engines, but since we don't expect syntax errors when table parameter is provided, we should be fine with this approach).

The most used callsite for tableExists now is in createRelation on write path. It checks whether tableExists, and it table does not exist, it goes on create table path. We should be fine even if we catch some other error than object not found in tableExists, since createTable would probably fail as well.

}

/**
* Gets a dialect exception, classifies it and wraps it by `AnalysisException`.
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/MsSqlServerDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ private case class MsSqlServerDialect() extends JdbcDialect with NoLegacyJDBCErr
override def canHandle(url: String): Boolean =
url.toLowerCase(Locale.ROOT).startsWith("jdbc:sqlserver")

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getErrorCode == 208
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getErrorCode == 208)
}

// Microsoft SQL Server does not have the boolean type.
Expand Down Expand Up @@ -273,7 +273,7 @@ private case class MsSqlServerDialect() extends JdbcDialect with NoLegacyJDBCErr
override def getJdbcSQLQueryBuilder(options: JDBCOptions): JdbcSQLQueryBuilder =
new MsSqlServerSQLQueryBuilder(this, options)

override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
val exceptionMessage = Option(exception.getMessage)
.map(_.toLowerCase(Locale.ROOT))
.getOrElse("")
Expand All @@ -282,7 +282,7 @@ private case class MsSqlServerDialect() extends JdbcDialect with NoLegacyJDBCErr
// most syntax errors so we have to base off the exception message.
// https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
// scalastyle:on line.size.limit
exceptionMessage.contains("incorrect syntax") || exceptionMessage.contains("syntax error")
Some(exceptionMessage.contains("incorrect syntax") || exceptionMessage.contains("syntax error"))
}

override def supportsLimit: Boolean = true
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,8 @@ private case class MySQLDialect() extends JdbcDialect with SQLConfHelper with No
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getErrorCode == 1146
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getErrorCode == 1146)
}

class MySQLSQLBuilder extends JDBCSQLBuilder {
Expand Down Expand Up @@ -227,8 +227,8 @@ private case class MySQLDialect() extends JdbcDialect with SQLConfHelper with No
override def isCascadingTruncateTable(): Option[Boolean] = Some(false)

// See https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
"42000".equals(exception.getSQLState)
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Some("42000".equals(exception.getSQLState))
}

// See https://dev.mysql.com/doc/refman/8.0/en/alter-table.html
Expand Down
10 changes: 5 additions & 5 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ private case class OracleDialect() extends JdbcDialect with SQLConfHelper with N
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getMessage.contains("ORA-00942") ||
e.getMessage.contains("ORA-39165")
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getMessage.contains("ORA-00942") ||
e.getMessage.contains("ORA-39165"))
}

class OracleSQLBuilder extends JDBCSQLBuilder {
Expand Down Expand Up @@ -198,8 +198,8 @@ private case class OracleDialect() extends JdbcDialect with SQLConfHelper with N
override def isCascadingTruncateTable(): Option[Boolean] = Some(false)

// See https://docs.oracle.com/cd/E11882_01/appdev.112/e10827/appd.htm#g642406
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
"42000".equals(exception.getSQLState)
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Some("42000".equals(exception.getSQLState))
}

/**
Expand Down
10 changes: 5 additions & 5 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ private case class PostgresDialect()
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getSQLState == "42P01" ||
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getSQLState == "42P01" ||
e.getSQLState == "3F000" ||
e.getSQLState == "42704"
e.getSQLState == "42704")
}

override def getCatalystType(
Expand Down Expand Up @@ -243,8 +243,8 @@ private case class PostgresDialect()
}

// See https://www.postgresql.org/docs/current/errcodes-appendix.html
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

// SHOW INDEX syntax
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/SnowflakeDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ private case class SnowflakeDialect() extends JdbcDialect with NoLegacyJDBCError
override def canHandle(url: String): Boolean =
url.toLowerCase(Locale.ROOT).startsWith("jdbc:snowflake")

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getSQLState == "002003"
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getSQLState == "002003")
}

override def getJDBCType(dt: DataType): Option[JdbcType] = dt match {
Expand All @@ -39,10 +39,10 @@ private case class SnowflakeDialect() extends JdbcDialect with NoLegacyJDBCError
case _ => JdbcUtils.getCommonJDBCType(dt)
}

override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
// There is no official documentation for SQL state in Snowflake, but they follow ANSI standard
// where 42000 SQLState is used for syntax errors.
// Manual tests also show that this is the error state for syntax error
"42000".equals(exception.getSQLState)
Some("42000".equals(exception.getSQLState))
}
}
8 changes: 4 additions & 4 deletions sql/core/src/main/scala/org/apache/spark/sql/jdbc/TeradataDialect.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ private case class TeradataDialect() extends JdbcDialect with NoLegacyJDBCError
override def isSupportedFunction(funcName: String): Boolean =
supportedFunctions.contains(funcName)

override def isObjectNotFoundException(e: SQLException): Boolean = {
e.getErrorCode == 3807
override def isObjectNotFoundException(e: SQLException): Option[Boolean] = {
Some(e.getErrorCode == 3807)
}

override def getJDBCType(dt: DataType): Option[JdbcType] = dt match {
Expand All @@ -56,8 +56,8 @@ private case class TeradataDialect() extends JdbcDialect with NoLegacyJDBCError
// scalastyle:off line.size.limit
// See https://docs.teradata.com/r/Enterprise_IntelliFlex_VMware/SQL-Stored-Procedures-and-Embedded-SQL/SQLSTATE-Mappings/SQLSTATE-Codes
// scalastyle:on line.size.limit
override def isSyntaxErrorBestEffort(exception: SQLException): Boolean = {
Option(exception.getSQLState).exists(_.startsWith("42"))
override def isSyntaxErrorBestEffort(exception: SQLException): Option[Boolean] = {
Option(exception.getSQLState).map(_.startsWith("42"))
}

/**
Expand Down