Execute Stored Procedures Directly (#2154)

* Initial implementation for exec cstmt directly

* Regression fix, continue to do extra metadata lookup call to fetch SP param name info

* Fixes p1

* Fixes p2

* Fixes p3

* Fixes p4

* Updated test to test erroring out on retrieving out param value

* Revert "Fixes p4"

This reverts commit b19dfb0.

* UDF fix

* Accounted for premature closing of statements

* Datetime out of range fix

* Cstmts executed internally by the driver for XA transactions send out params as non-PLP over the wire for nvarchar and binary types

* Accounted for zero param sproc

* Fixed setting/registering of out of order params

* Datetime fractional seconds correction

* DTC tests

* Zero param sproc test; Error test for casting of sproc return value

* Test shuffled ordering of registering and setting cstmt params

* UDF test

* Skip return values from RPC on statement close if there are any

* Test mix of index a param names for cstmt

* Removed static callRpcDirectly variable

* Test cleanup

* Fixed invalid param test

* Fixed test for JDK 8

* Formatting; Removed comma counting

* Removed getter/setter enum

* Removed comments regarding Yukon in IOBuffer

* Changed returnValueIsAccessed to isReturnValueAccessed

* Fixed intellij wildcard imports

* Renamed DTC test class and test

* Revert "Removed getter/setter enum"

This reverts commit 2b35e53.

* New connection string property to toggle sp_sproc_columns calls

* Deleted DTC test

* Removed non-driver error from TestResource

* Fixed switch case for date/time types in dtv class

* SQLServerConnTest and RequestBoundaryMethodTest update

* Removed DTC test group

* Fixed docs/comments for new CS prop

* Fixed setting/registering of when using only named parameters

* RPC call check considers whether the call is an internal encryption query call

* Switched from useFastCallableStatements to useFlexibleCallableStatements

* Fixed typo in CS prop setter/getter

* Include enclave package in RPC call

* Code Review Changes; Fixed threading issue

* Merge conflict fix p2.

* Update src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerDataSource.java

Co-authored-by: David Engel <v-davidengel@microsoft.com>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerCallableStatement.java

Co-authored-by: David Engel <v-davidengel@microsoft.com>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java

Co-authored-by: David Engel <v-davidengel@microsoft.com>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java

Co-authored-by: David Engel <v-davidengel@microsoft.com>

* PR review; initial named parameter and index parameter restrictions

---------

Co-authored-by: David Engel <v-davidengel@microsoft.com>

pom.xml

@@ -43,7 +43,7 @@
  xAzureSQLDW - - - - For tests not compatible with Azure Data Warehouse - 
  xAzureSQLMI - - - - For tests not compatible with Azure SQL Managed Instance 
  NTLM - - - - - - - For tests using NTLM Authentication mode (excluded by default)
- kerberos -  - - - - For tests using Kerberos authentication (excluded by default)
+ kerberos  - - - - - For tests using Kerberos authentication (excluded by default)
  reqExternalSetup - For tests requiring external setup (excluded by default)
  clientCertAuth - - For tests requiring client certificate authentication 
  setup (excluded by default) - - - - - - - - - - - - - - - - - - - - - - - 

src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java

@@ -466,6 +466,7 @@ static final String getEncryptionLevel(int level) {
  final static int COLINFO_STATUS_DIFFERENT_NAME = 0x20;
 
  final static int MAX_FRACTIONAL_SECONDS_SCALE = 7;
+ final static int DEFAULT_FRACTIONAL_SECONDS_SCALE = 3;
 
  final static Timestamp MAX_TIMESTAMP = Timestamp.valueOf("2079-06-06 23:59:59");
  final static Timestamp MIN_TIMESTAMP = Timestamp.valueOf("1900-01-01 00:00:00");
@@ -4788,7 +4789,7 @@ void writeVMaxHeader(long headerLength, boolean isNull, SQLCollation collation)
  * Utility for internal writeRPCString calls
  */
  void writeRPCStringUnicode(String sValue) throws SQLServerException {
- writeRPCStringUnicode(null, sValue, false, null);
+ writeRPCStringUnicode(null, sValue, false, null, false);
  }
 
  /**
@@ -4803,8 +4804,8 @@ void writeRPCStringUnicode(String sValue) throws SQLServerException {
  * @param collation
  * the collation of the data value
  */
- void writeRPCStringUnicode(String sName, String sValue, boolean bOut,
- SQLCollation collation) throws SQLServerException {
+ void writeRPCStringUnicode(String sName, String sValue, boolean bOut, SQLCollation collation,
+ boolean isNonPLP) throws SQLServerException {
  boolean bValueNull = (sValue == null);
  int nValueLen = bValueNull ? 0 : (2 * sValue.length());
  // Textual RPC requires a collation. If none is provided, as is the case when
@@ -4816,10 +4817,9 @@ void writeRPCStringUnicode(String sName, String sValue, boolean bOut,
  * Use PLP encoding if either OUT params were specified or if the user query exceeds
  * DataTypes.SHORT_VARTYPE_MAX_BYTES
  */
- if (nValueLen > DataTypes.SHORT_VARTYPE_MAX_BYTES || bOut) {
+ if ((nValueLen > DataTypes.SHORT_VARTYPE_MAX_BYTES || bOut) && !isNonPLP) {
  writeRPCNameValType(sName, bOut, TDSType.NVARCHAR);
 
- // Handle Yukon v*max type header here.
  writeVMaxHeader(nValueLen, // Length
  bValueNull, // Is null?
  collation);
@@ -5418,7 +5418,6 @@ void writeTVPColumnMetaData(TVP value) throws SQLServerException {
  // Use PLP encoding on Yukon and later with long values
  if (!isShortValue) // PLP
  {
- // Handle Yukon v*max type header here.
  writeShort((short) 0xFFFF);
  con.getDatabaseCollation().writeCollation(this);
  } else // non PLP
@@ -5436,7 +5435,6 @@ void writeTVPColumnMetaData(TVP value) throws SQLServerException {
  isShortValue = pair.getValue().precision <= DataTypes.SHORT_VARTYPE_MAX_BYTES;
  // Use PLP encoding on Yukon and later with long values
  if (!isShortValue) // PLP
- // Handle Yukon v*max type header here.
  writeShort((short) 0xFFFF);
  else // non PLP
  writeShort((short) DataTypes.SHORT_VARTYPE_MAX_BYTES);
@@ -5571,8 +5569,8 @@ void writeCryptoMetaData() throws SQLServerException {
  writeByte(cryptoMeta.normalizationRuleVersion);
  }
 
- void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcType,
- SQLCollation collation) throws SQLServerException {
+ void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcType, SQLCollation collation,
+ boolean isNonPLP) throws SQLServerException {
  boolean bValueNull = (bValue == null);
  int nValueLen = bValueNull ? 0 : bValue.length;
  boolean isShortValue = (nValueLen <= DataTypes.SHORT_VARTYPE_MAX_BYTES);
@@ -5618,8 +5616,7 @@ void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcT
 
  writeRPCNameValType(sName, bOut, tdsType);
 
- if (usePLP) {
- // Handle Yukon v*max type header here.
+ if (usePLP && !isNonPLP) {
  writeVMaxHeader(nValueLen, bValueNull, collation);
 
  // Send the data.
@@ -6400,7 +6397,6 @@ void writeRPCInputStream(String sName, InputStream stream, long streamLength, bo
 
  writeRPCNameValType(sName, bOut, jdbcType.isTextual() ? TDSType.BIGVARCHAR : TDSType.BIGVARBINARY);
 
- // Handle Yukon v*max type header here.
  writeVMaxHeader(streamLength, false, jdbcType.isTextual() ? collation : null);
  }
 
@@ -6540,7 +6536,6 @@ void writeRPCReaderUnicode(String sName, Reader re, long reLength, boolean bOut,
 
  writeRPCNameValType(sName, bOut, TDSType.NVARCHAR);
 
- // Handle Yukon v*max type header here.
  writeVMaxHeader(
  (DataTypes.UNKNOWN_STREAM_LENGTH == reLength) ? DataTypes.UNKNOWN_STREAM_LENGTH : 2 * reLength, // Length
  // (in
@@ -6995,6 +6990,35 @@ final short peekStatusFlag() {
  return 0;
  }
 
+ final int peekReturnValueStatus() throws SQLServerException {
+ // Ensure that we have a packet to read from.
+ if (!ensurePayload()) {
+ throwInvalidTDS();
+ }
+
+ // In order to parse the 'status' value, we need to skip over the following properties in the TDS packet
+ // payload: TDS token type (1 byte value), ordinal/length (2 byte value), parameter name length value (1 byte value) and
+ // the number of bytes that make the parameter name (need to be calculated).
+ //
+ // 'offset' starts at 4 because tdsTokenType + ordinal/length + parameter name length value is 4 bytes. So, we
+ // skip 4 bytes immediateley.
+ int offset = 4;
+ int paramNameLength = currentPacket.payload[payloadOffset + 3];
+
+ // Check if parameter name is set. If it's set, it should be > 0. In which case, we add the
+ // additional bytes to skip.
+ if (paramNameLength > 0) {
+ // Each character in unicode is 2 bytes
+ offset += 2 * paramNameLength;
+ }
+
+ if (payloadOffset + offset <= currentPacket.payloadLength) {
+ return currentPacket.payload[payloadOffset + offset] & 0xFF;
+ }
+
+ return -1;
+ }
+
  final int readUnsignedByte() throws SQLServerException {
  // Ensure that we have a packet to read from.
  if (!ensurePayload())

src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerConnection.java

@@ -496,5 +496,5 @@ CallableStatement prepareCall(String sql, int nType, int nConcur, int nHold,
  * @param calcBigDecimalScale
  * A boolean that indicates if the driver should calculate scale from inputted big decimal values.
  */
- void setCalcBigDecimalScale(boolean computeBigDecimal);
+ void setCalcBigDecimalScale(boolean calcBigDecimalScale);
 }

src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerDataSource.java

@@ -609,6 +609,25 @@ public interface ISQLServerDataSource extends javax.sql.CommonDataSource {
  */
  boolean getUseDefaultGSSCredential();
 
+ /**
+ * Sets whether or not sp_sproc_columns will be used for parameter name lookup.
+ *
+ * @param useFlexibleCallableStatements
+ * When set to false, sp_sproc_columns is not used for parameter name lookup
+ * in callable statements. This eliminates a round trip to the server but imposes limitations
+ * on how parameters are set. When set to false, applications must either reference
+ * parameters by name or by index, not both. Parameters must also be set in the same
+ * order as the stored procedure definition.
+ */
+ void setUseFlexibleCallableStatements(boolean useFlexibleCallableStatements);
+
+ /**
+ * Returns whether or not sp_sproc_columns is being used for parameter name lookup.
+ *
+ * @return useFlexibleCallableStatements
+ */
+ boolean getUseFlexibleCallableStatements();
+
  /**
  * Sets the GSSCredential.
  *

src/main/java/com/microsoft/sqlserver/jdbc/Parameter.java

@@ -37,6 +37,8 @@ final class Parameter {
  // For unencrypted parameters cryptometa will be null. For encrypted parameters it will hold encryption metadata.
  CryptoMetadata cryptoMeta = null;
 
+ boolean isNonPLP = false;
+
  TypeInfo getTypeInfo() {
  return typeInfo;
  }
@@ -48,6 +50,7 @@ final CryptoMetadata getCryptoMetadata() {
  private boolean shouldHonorAEForParameter = false;
  private boolean userProvidesPrecision = false;
  private boolean userProvidesScale = false;
+ private boolean isReturnValue = false;
 
  // The parameter type definition
  private String typeDefinition = null;
@@ -70,6 +73,60 @@ boolean isOutput() {
  return null != registeredOutDTV;
  }
 
+ /**
+ * Returns true/false if the parameter is of return type
+ *
+ * @return isReturnValue
+ */
+ boolean isReturnValue() {
+ return isReturnValue;
+ }
+
+ /**
+ * Sets the parameter to be of return type
+ *
+ * @param isReturnValue
+ */
+ void setReturnValue(boolean isReturnValue) {
+ this.isReturnValue = isReturnValue;
+ }
+
+ /**
+ * Sets the name of the parameter
+ *
+ * @param name
+ */
+ void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Retrieve the name of the parameter
+ *
+ * @return
+ */
+ String getName() {
+ return this.name;
+ }
+
+ /**
+ * Returns the `registeredOutDTV` instance of the parameter
+ *
+ * @return registeredOutDTV
+ */
+ DTV getRegisteredOutDTV() {
+ return this.registeredOutDTV;
+ }
+
+ /**
+ * Returns the `inputDTV` instance of the parameter
+ *
+ * @return inputDTV
+ */
+ DTV getInputDTV() {
+ return this.inputDTV;
+ }
+
  // Since a parameter can have only one type definition for both sending its value to the server (IN)
  // and getting its value from the server (OUT), we use the JDBC type of the IN parameter value if there
  // is one; otherwise we use the registered OUT param JDBC type.
@@ -246,7 +303,7 @@ void setFromReturnStatus(int returnStatus, SQLServerConnection con) throws SQLSe
  if (null == getterDTV)
  getterDTV = new DTV();
 
- getterDTV.setValue(null, JDBCType.INTEGER, returnStatus, JavaType.INTEGER, null, null, null, con,
+ getterDTV.setValue(null, this.getJdbcType(), returnStatus, JavaType.INTEGER, null, null, null, con,
  getForceEncryption());
  }
 
@@ -387,10 +444,14 @@ boolean isValueGotten() {
 
  Object getValue(JDBCType jdbcType, InputStreamGetterArgs getterArgs, Calendar cal, TDSReader tdsReader,
  SQLServerStatement statement) throws SQLServerException {
- if (null == getterDTV)
+ if (null == getterDTV) {
  getterDTV = new DTV();
+ }
+
+ if (null != tdsReader) {
+ deriveTypeInfo(tdsReader);
+ }
 
- deriveTypeInfo(tdsReader);
  // If the parameter is not encrypted or column encryption is turned off (either at connection or
  // statement level), cryptoMeta would be null.
  return getterDTV.getValue(jdbcType, outScale, getterArgs, cal, typeInfo, cryptoMeta, tdsReader, statement);
@@ -1206,15 +1267,17 @@ String getTypeDefinition(SQLServerConnection con, TDSReader tdsReader) throws SQ
  return typeDefinition;
  }
 
- void sendByRPC(TDSWriter tdsWriter, SQLServerStatement statement) throws SQLServerException {
+ void sendByRPC(TDSWriter tdsWriter, boolean callRPCDirectly,
+ SQLServerStatement statement) throws SQLServerException {
  assert null != inputDTV : "Parameter was neither set nor registered";
  SQLServerConnection conn = statement.connection;
 
  try {
+ inputDTV.isNonPLP = isNonPLP;
  inputDTV.sendCryptoMetaData(this.cryptoMeta, tdsWriter);
  inputDTV.setJdbcTypeSetByUser(getJdbcTypeSetByUser(), getValueLength());
- inputDTV.sendByRPC(name, null, conn.getDatabaseCollation(), valueLength, isOutput() ? outScale : scale,
- isOutput(), tdsWriter, statement);
+ inputDTV.sendByRPC(callRPCDirectly ? name : null, null, conn.getDatabaseCollation(), valueLength,
+ isOutput() ? outScale : scale, isOutput(), tdsWriter, statement);
  } finally {
  // reset the cryptoMeta in IOBuffer
  inputDTV.sendCryptoMetaData(null, tdsWriter);