Skip to content

Commit 21f7bd8

Browse files
samsharma2700samsharma2700
authored
Feature | Add SqlBulkCopyOptions.CacheMetadata flag (#3939)
1 parent ee71014 commit 21f7bd8

11 files changed

Lines changed: 1119 additions & 6 deletions

File tree

‎doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,35 @@ This code is provided to demonstrate the syntax for using **SqlBulkCopy** only.
235235
]]></format>
236236
</example>
237237
</Close>
238+
<ClearCachedMetadata>
239+
<summary>
240+
Clears the cached destination table metadata when using the
241+
<see cref="F:Microsoft.Data.SqlClient.SqlBulkCopyOptions.CacheMetadata" />
242+
option.
243+
</summary>
244+
<remarks>
245+
<para>
246+
Call this method when you know the destination table schema
247+
has changed and you want to force the next
248+
<c>WriteToServer</c> operation to refresh the metadata
249+
from the server.
250+
</para>
251+
<para>
252+
The cache is automatically invalidated when the
253+
<see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" />
254+
property is changed to a different table name.
255+
</para>
256+
<para>
257+
The cache is not automatically invalidated when the
258+
connection context changes. Call this method if the
259+
underlying
260+
<see cref="T:Microsoft.Data.SqlClient.SqlConnection" />
261+
changes database (for example, via
262+
<see cref="M:System.Data.Common.DbConnection.ChangeDatabase(System.String)" />)
263+
or reconnects to a different server due to failover.
264+
</para>
265+
</remarks>
266+
</ClearCachedMetadata>
238267
<EnableStreaming>
239268
<summary>
240269
Enables or disables a <see cref="T:Microsoft.Data.SqlClient.SqlBulkCopy" /> object to stream data from an <see cref="T:System.Data.IDataReader" /> object

‎doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,5 +68,46 @@ To see how the option changes the way the bulk load works, run the sample with t
6868
When specified, each batch of the bulk-copy operation will occur within a transaction. If you indicate this option and also provide a <see cref="T:Microsoft.Data.SqlClient.SqlTransaction" /> object to the constructor, an <see cref="T:System.ArgumentException" /> occurs.
6969
</summary>
7070
</UseInternalTransaction>
71+
<CacheMetadata>
72+
<summary>
73+
<para>
74+
When specified, <b>CacheMetadata</b> caches destination table
75+
metadata after the first bulk copy operation, allowing
76+
subsequent operations to the same table to skip the metadata
77+
discovery query. This can improve performance when performing
78+
multiple bulk copy operations to the same destination table.
79+
</para>
80+
<para>
81+
<b>Warning:</b> Use this option only when you are certain the
82+
destination table schema will not change between bulk copy
83+
operations. If the table schema changes (columns added,
84+
removed, or modified), using cached metadata may result in
85+
data corruption, failed operations, or unexpected behavior.
86+
Call
87+
<see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.ClearCachedMetadata" />
88+
to clear the cache if the schema changes.
89+
</para>
90+
<para>
91+
The cache is automatically invalidated when
92+
<see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.DestinationTableName" />
93+
is changed to a different table. Changing
94+
<see cref="P:Microsoft.Data.SqlClient.SqlBulkCopy.ColumnMappings" />
95+
between operations does not require cache invalidation
96+
because the cached metadata describes only the destination
97+
table schema, not the source-to-destination column mapping.
98+
</para>
99+
<para>
100+
The cache is not automatically invalidated when the
101+
connection context changes. If the underlying
102+
<see cref="T:Microsoft.Data.SqlClient.SqlConnection" />
103+
changes database (for example, via
104+
<see cref="M:System.Data.Common.DbConnection.ChangeDatabase(System.String)" />)
105+
or reconnects to a different server due to failover, callers
106+
should call
107+
<see cref="M:Microsoft.Data.SqlClient.SqlBulkCopy.ClearCachedMetadata" />
108+
to ensure the metadata is refreshed.
109+
</para>
110+
</summary>
111+
</CacheMetadata>
71112
</members>
72113
</docs>

‎src/Microsoft.Data.SqlClient/ref/Microsoft.Data.SqlClient.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,8 @@ public SqlBulkCopy(string connectionString, Microsoft.Data.SqlClient.SqlBulkCopy
212212
public event Microsoft.Data.SqlClient.SqlRowsCopiedEventHandler SqlRowsCopied { add { } remove { } }
213213
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
214214
public void Close() { }
215+
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/ClearCachedMetadata/*'/>
216+
public void ClearCachedMetadata() { }
215217
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/System.IDisposable.Dispose/*'/>
216218
void System.IDisposable.Dispose() { }
217219
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/WriteToServer[@name="DbDataReaderParameter"]/*'/>
@@ -343,6 +345,8 @@ public enum SqlBulkCopyOptions
343345
{
344346
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
345347
AllowEncryptedValueModifications = 64,
348+
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
349+
CacheMetadata = 128,
346350
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CheckConstraints/*'/>
347351
CheckConstraints = 2,
348352
/// <include file='../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/Default/*'/>

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopy.cs‎

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,13 @@ private int RowNumber
235235

236236
private SourceColumnMetadata[] _currentRowMetadata;
237237

238+
// Metadata caching fields for CacheMetadata option
239+
internal BulkCopySimpleResultSet CachedMetadata { get; private set; }
240+
// Per-operation clone of the destination table metadata, used when CacheMetadata is
241+
// enabled so that column-pruning in AnalyzeTargetAndCreateUpdateBulkCommand does not
242+
// mutate the cached BulkCopySimpleResultSet.
243+
private _SqlMetaDataSet _operationMetaData;
244+
238245
#if DEBUG
239246
internal static bool s_setAlwaysTaskOnWrite; //when set and in DEBUG mode, TdsParser::WriteBulkCopyValue will always return a task
240247
internal static bool SetAlwaysTaskOnWrite
@@ -353,6 +360,12 @@ public string DestinationTableName
353360
{
354361
throw ADP.ArgumentOutOfRange(nameof(DestinationTableName));
355362
}
363+
else if (string.Equals(_destinationTableName, value, StringComparison.Ordinal))
364+
{
365+
return;
366+
}
367+
368+
CachedMetadata = null;
356369
_destinationTableName = value;
357370
}
358371
}
@@ -497,6 +510,14 @@ IF EXISTS (SELECT TOP 1 * FROM sys.all_columns WHERE [object_id] = OBJECT_ID('sy
497510
// We need to have a _parser.RunAsync to make it real async.
498511
private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out BulkCopySimpleResultSet result)
499512
{
513+
// Check if we have valid cached metadata for the current destination table
514+
if (CachedMetadata != null)
515+
{
516+
SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Using cached metadata for table '{0}'", _destinationTableName);
517+
result = CachedMetadata;
518+
return null;
519+
}
520+
500521
string TDSCommand = CreateInitialQuery();
501522
SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Initial Query: '{0}'", TDSCommand);
502523
SqlClientEventSource.Log.TryCorrelationTraceEvent("SqlBulkCopy.CreateAndExecuteInitialQueryAsync | Info | Correlation | Object Id {0}, Activity Id {1}", ObjectID, ActivityCorrelator.Current);
@@ -506,6 +527,7 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
506527
{
507528
result = new BulkCopySimpleResultSet();
508529
RunParser(result);
530+
CacheMetadataIfEnabled(result);
509531
return null;
510532
}
511533
else
@@ -523,17 +545,31 @@ private Task<BulkCopySimpleResultSet> CreateAndExecuteInitialQueryAsync(out Bulk
523545
{
524546
var internalResult = new BulkCopySimpleResultSet();
525547
RunParserReliably(internalResult);
548+
CacheMetadataIfEnabled(internalResult);
526549
return internalResult;
527550
}
528551
}, TaskScheduler.Default);
529552
}
530553
}
531554

555+
private void CacheMetadataIfEnabled(BulkCopySimpleResultSet result)
556+
{
557+
if (IsCopyOption(SqlBulkCopyOptions.CacheMetadata))
558+
{
559+
CachedMetadata = result;
560+
SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.CacheMetadataIfEnabled | Info | Cached metadata for table '{0}'", _destinationTableName);
561+
}
562+
}
563+
532564
// Matches associated columns with metadata from initial query.
533565
// Builds and executes the update bulk command.
534-
private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults)
566+
// metaDataSet is passed in by the caller so that when CacheMetadata is enabled, the
567+
// caller can supply a clone, allowing this method to null-prune unmatched/rejected
568+
// columns freely without mutating the shared cache.
569+
private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet internalResults, _SqlMetaDataSet metaDataSet)
535570
{
536571
Debug.Assert(internalResults != null, "Where are the results from the initial query?");
572+
Debug.Assert(metaDataSet != null, "metaDataSet must not be null");
537573

538574
StringBuilder updateBulkCommandText = new StringBuilder();
539575

@@ -577,8 +613,9 @@ private string AnalyzeTargetAndCreateUpdateBulkCommand(BulkCopySimpleResultSet i
577613
// the next column in the command text.
578614
bool appendComma = false;
579615

580-
// Loop over the metadata for each result column.
581-
_SqlMetaDataSet metaDataSet = internalResults[MetaDataResultId].MetaData;
616+
// Loop over the metadata for each result column, null-pruning unmatched/rejected
617+
// columns. metaDataSet is safe to mutate here — see the call site for clone logic.
618+
_operationMetaData = metaDataSet;
582619
_sortedColumnMappings = new List<_ColumnMapping>(metaDataSet.Length);
583620
for (int i = 0; i < metaDataSet.Length; i++)
584621
{
@@ -875,11 +912,18 @@ private void WriteMetaData(BulkCopySimpleResultSet internalResults)
875912
{
876913
_stateObj.SetTimeoutSeconds(BulkCopyTimeout);
877914

878-
_SqlMetaDataSet metadataCollection = internalResults[MetaDataResultId].MetaData;
915+
_SqlMetaDataSet metadataCollection = _operationMetaData ?? internalResults[MetaDataResultId].MetaData;
879916
_stateObj._outputMessageType = TdsEnums.MT_BULK;
880917
_parser.WriteBulkCopyMetaData(metadataCollection, _sortedColumnMappings.Count, _stateObj);
881918
}
882919

920+
/// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/ClearCachedMetadata/*'/>
921+
public void ClearCachedMetadata()
922+
{
923+
CachedMetadata = null;
924+
SqlClientEventSource.Log.TryTraceEvent("SqlBulkCopy.ClearCachedMetadata | Info | Metadata cache cleared");
925+
}
926+
883927
// Terminates the bulk copy operation.
884928
// Must be called at the end of the bulk copy session.
885929
/// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopy.xml' path='docs/members[@name="SqlBulkCopy"]/Close/*'/>
@@ -900,6 +944,8 @@ private void Dispose(bool disposing)
900944
// Dispose dependent objects
901945
_columnMappings = null;
902946
_parser = null;
947+
CachedMetadata = null;
948+
_operationMetaData = null;
903949
try
904950
{
905951
// Just in case there is a lingering transaction (which there shouldn't be)
@@ -2667,7 +2713,7 @@ private Task CopyBatchesAsyncContinued(BulkCopySimpleResultSet internalResults,
26672713

26682714
// Load encryption keys now (if needed)
26692715
_parser.LoadColumnEncryptionKeys(
2670-
internalResults[MetaDataResultId].MetaData,
2716+
_operationMetaData ?? internalResults[MetaDataResultId].MetaData,
26712717
_connection);
26722718

26732719
Task task = CopyRowsAsync(0, _savedBatchSize, cts); // This is copying 1 batch of rows and setting _hasMoreRowToCopy = true/false.
@@ -2840,7 +2886,14 @@ private void WriteToServerInternalRestContinuedAsync(BulkCopySimpleResultSet int
28402886

28412887
try
28422888
{
2843-
updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults);
2889+
// When CacheMetadata is enabled, internalResults IS the cached result set (see
2890+
// CreateAndExecuteInitialQueryAsync). Clone the metadata set so that
2891+
// AnalyzeTargetAndCreateUpdateBulkCommand can null-prune unmatched/rejected
2892+
// columns without mutating the cache across WriteToServer calls.
2893+
_SqlMetaDataSet metaDataSet = CachedMetadata != null
2894+
? internalResults[MetaDataResultId].MetaData.Clone()
2895+
: internalResults[MetaDataResultId].MetaData;
2896+
updateBulkCommandText = AnalyzeTargetAndCreateUpdateBulkCommand(internalResults, metaDataSet);
28442897

28452898
if (_sortedColumnMappings.Count != 0)
28462899
{
@@ -3194,6 +3247,7 @@ private void ResetWriteToServerGlobalVariables()
31943247
_dataTableSource = null;
31953248
_dbDataReaderRowSource = null;
31963249
_isAsyncBulkCopy = false;
3250+
_operationMetaData = null;
31973251
_rowEnumerator = null;
31983252
_rowSource = null;
31993253
_rowSourceType = ValueSourceType.Unspecified;

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlBulkCopyOptions.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ public enum SqlBulkCopyOptions
3333

3434
/// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/AllowEncryptedValueModifications/*'/>
3535
AllowEncryptedValueModifications = 1 << 6,
36+
37+
/// <include file='../../../../../../doc/snippets/Microsoft.Data.SqlClient/SqlBulkCopyOptions.xml' path='docs/members[@name="SqlBulkCopyOptions"]/CacheMetadata/*'/>
38+
CacheMetadata = 1 << 7,
3639
}
3740
}
3841

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserHelperClasses.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,7 @@ private _SqlMetaDataSet(_SqlMetaDataSet original)
359359
_visibleColumnMap = original._visibleColumnMap;
360360
dbColumnSchema = original.dbColumnSchema;
361361
schemaTable = original.schemaTable;
362+
cekTable = original.cekTable;
362363

363364
if (original._metaDataArray == null)
364365
{
@@ -584,6 +585,10 @@ internal virtual void CopyFrom(SqlMetaDataPriv original)
584585
xmlSchemaCollection = new SqlMetaDataXmlSchemaCollection();
585586
xmlSchemaCollection.CopyFrom(original.xmlSchemaCollection);
586587
}
588+
589+
this.isEncrypted = original.isEncrypted;
590+
this.baseTI = original.baseTI;
591+
this.cipherMD = original.cipherMD;
587592
}
588593
}
589594

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@
111111
<Compile Include="SQL\SqlBulkCopyTest\Bug85007.cs" />
112112
<Compile Include="SQL\SqlBulkCopyTest\Bug903514.cs" />
113113
<Compile Include="SQL\SqlBulkCopyTest\Bug98182.cs" />
114+
<Compile Include="SQL\SqlBulkCopyTest\CacheMetadata.cs" />
114115
<Compile Include="SQL\SqlBulkCopyTest\CheckConstraints.cs" />
115116
<Compile Include="SQL\SqlBulkCopyTest\CopyWidenNullInexactNumerics.cs" />
116117
<Compile Include="SQL\SqlBulkCopyTest\DataConversionErrorMessageTest.cs" />

0 commit comments

Comments
 (0)