diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 1dad20fdb5130..8e6ffb0a30c50 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -6912,6 +6912,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val ORACLE_NUMBER_DEFAULT_SCALE = + buildConf("spark.sql.oracle.numberDefaultScale") + .doc("Default scale for Oracle bare NUMBER columns with no precision/scale " + + "or FLOAT columns (JDBC scale -127). Oracle reports these as precision 0 " + + "or scale -127 via JDBC metadata. Spark maps them to DecimalType(38, scale).") + .version("4.1.0") + .intConf + .checkValue(s => s >= 0 && s <= 38, "oracle numberDefaultScale must be between 0 and 38") + .createWithDefault(10) + val LEGACY_DB2_TIMESTAMP_MAPPING_ENABLED = buildConf("spark.sql.legacy.db2.numericMapping.enabled") .internal() @@ -9104,6 +9114,8 @@ class SQLConf extends Serializable with Logging with SqlApiConf { def legacyOracleTimestampMappingEnabled: Boolean = getConf(LEGACY_ORACLE_TIMESTAMP_MAPPING_ENABLED) + def oracleNumberDefaultScale: Int = getConf(ORACLE_NUMBER_DEFAULT_SCALE) + def legacyDB2numericMappingEnabled: Boolean = getConf(LEGACY_DB2_TIMESTAMP_MAPPING_ENABLED) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala index 46080dd57b1dd..251b9bf0e1a41 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala @@ -160,12 +160,20 @@ private case class OracleDialect() extends JdbcDialect with SQLConfHelper with N // https://github.com/apache/spark/pull/8780#issuecomment-145598968 // and // https://github.com/apache/spark/pull/8780#issuecomment-144541760 - case 0 => Option(DecimalType(DecimalType.MAX_PRECISION, 10)) + case 0 => + logWarning(s"Bare Oracle NUMBER with no precision/scale mapped to " + + s"DecimalType(38, ${conf.oracleNumberDefaultScale}). Set " + + s"spark.sql.oracle.numberDefaultScale to adjust.") + Option(DecimalType(DecimalType.MAX_PRECISION, conf.oracleNumberDefaultScale)) // Handle FLOAT fields in a special way because JDBC ResultSetMetaData converts // this to NUMERIC with -127 scale // Not sure if there is a more robust way to identify the field as a float (or other // numeric types that do not specify a scale. - case _ if scale == -127L => Option(DecimalType(DecimalType.MAX_PRECISION, 10)) + case _ if scale == -127L => + logWarning(s"Oracle NUMERIC/FLOAT with scale -127 mapped to " + + s"DecimalType(38, ${conf.oracleNumberDefaultScale}). Set " + + s"spark.sql.oracle.numberDefaultScale to adjust.") + Option(DecimalType(DecimalType.MAX_PRECISION, conf.oracleNumberDefaultScale)) case _ => None } case TIMESTAMP_TZ | TIMESTAMP_LTZ =>