CSHARP-4149: Add FLE 2 behavior for CreateCollection() and Collection.Drop(). (#790)

CSHARP-4149: Add FLE 2 behavior for CreateCollection() and Collection.Drop().

Author: DmitryLukyanov

src/MongoDB.Driver.Core/Core/Encryption/EncryptedCollectionHelper.cs

@@ -0,0 +1,86 @@
+/* Copyright 2010-present MongoDB Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MongoDB.Bson;
+
+namespace MongoDB.Driver.Encryption
+{
+    internal static class EncryptedCollectionHelper
+    {
+        public static BsonDocument AdditionalCreateIndexDocument { get; } = new BsonDocument("__safeContent__", 1);
+
+        public static void EnsureCollectionsValid(IReadOnlyDictionary<string, BsonDocument> schemaMap, IReadOnlyDictionary<string, BsonDocument> encryptedFieldsMap)
+        {
+            if (schemaMap == null || encryptedFieldsMap == null || schemaMap.Count == 0 || encryptedFieldsMap.Count == 0)
+            {
+                return;
+            }
+
+            var mutualKeys = schemaMap.Keys.Where(k => encryptedFieldsMap.ContainsKey(k));
+            if (mutualKeys.Any())
+            {
+                throw new ArgumentException($"SchemaMap and EncryptedFieldsMap cannot both contain the same collections: {string.Join(", ", mutualKeys)}.");
+            }
+        }
+
+        public static string GetAdditionalCollectionName(BsonDocument encryptedFields, CollectionNamespace mainCollectionNamespace, HelperCollectionForEncryption helperCollection) =>
+            helperCollection switch
+            {
+                HelperCollectionForEncryption.Esc => encryptedFields.GetValue("escCollection", defaultValue: $"enxcol_.{mainCollectionNamespace.CollectionName}.esc").ToString(),
+                HelperCollectionForEncryption.Ecc => encryptedFields.GetValue("eccCollection", defaultValue: $"enxcol_.{mainCollectionNamespace.CollectionName}.ecc").ToString(),
+                HelperCollectionForEncryption.Ecos => encryptedFields.GetValue("ecocCollection", defaultValue: $"enxcol_.{mainCollectionNamespace.CollectionName}.ecoc").ToString(),
+                _ => throw new InvalidOperationException($"Not supported encryption helper collection {helperCollection}."),
+            };
+
+        public static bool TryGetEffectiveEncryptedFields(CollectionNamespace collectionNamespace, BsonDocument encryptedFields, IReadOnlyDictionary<string, BsonDocument> encryptedFieldsMap, out BsonDocument effectiveEncryptedFields)
+        {
+            if (encryptedFields != null)
+            {
+                effectiveEncryptedFields = encryptedFields;
+                return true;
+            }
+
+            if (encryptedFieldsMap != null)
+            {
+                return encryptedFieldsMap.TryGetValue(collectionNamespace.ToString(), out effectiveEncryptedFields);
+            }
+
+            effectiveEncryptedFields = null;
+            return false;
+        }
+
+        public static BsonDocument GetEffectiveEncryptedFields(CollectionNamespace collectionNamespace, BsonDocument encryptedFields, IReadOnlyDictionary<string, BsonDocument> encryptedFieldsMap)
+        {
+            if (TryGetEffectiveEncryptedFields(collectionNamespace, encryptedFields, encryptedFieldsMap, out var effectiveEncryptedFields))
+            {
+                return effectiveEncryptedFields;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        public enum HelperCollectionForEncryption
+        {
+            Esc,
+            Ecc,
+            Ecos
+        }
+    }
+}

src/MongoDB.Driver.Core/Core/Operations/CompositeWriteOperation.cs

@@ -0,0 +1,65 @@
+/* Copyright 2010-present MongoDB Inc.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MongoDB.Driver.Core.Bindings;
+using MongoDB.Driver.Core.Misc;
+
+namespace MongoDB.Driver.Core.Operations
+{
+    internal sealed class CompositeWriteOperation<TResult> : IWriteOperation<TResult>
+    {
+        private readonly (IWriteOperation<TResult> Operation, bool IsMainOperation)[] _operations;
+
+        public CompositeWriteOperation(params (IWriteOperation<TResult>, bool IsMainOperation)[] operations)
+        {
+            _operations = Ensure.IsNotNull(operations, nameof(operations));
+            Ensure.IsGreaterThanZero(operations.Length, nameof(operations.Length));
+            Ensure.That(operations.Count(o => o.IsMainOperation) == 1, message: $"{nameof(CompositeWriteOperation<TResult>)} must have a single main operation.");
+        }
+
+        public TResult Execute(IWriteBinding binding, CancellationToken cancellationToken)
+        {
+            TResult result = default;
+            foreach (var operationInfo in _operations)
+            {
+                var itemResult = operationInfo.Operation.Execute(binding, cancellationToken);
+                if (operationInfo.IsMainOperation)
+                {
+                    result = itemResult;
+                }
+            }
+
+            return result;
+        }
+
+        public async Task<TResult> ExecuteAsync(IWriteBinding binding, CancellationToken cancellationToken)
+        {
+            TResult result = default;
+            foreach (var operationInfo in _operations)
+            {
+                var itemResult = await operationInfo.Operation.ExecuteAsync(binding, cancellationToken).ConfigureAwait(false);
+                if (operationInfo.IsMainOperation)
+                {
+                    result = itemResult;
+                }
+            }
+
+            return result;
+        }
+    }
+}

src/MongoDB.Driver.Core/Core/Operations/CreateCollectionOperation.cs

@@ -19,9 +19,10 @@
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Serializers;
 using MongoDB.Driver.Core.Bindings;
-using MongoDB.Driver.Core.Connections;
 using MongoDB.Driver.Core.Misc;
 using MongoDB.Driver.Core.WireProtocol.Messages.Encoders;
+using MongoDB.Driver.Encryption;
+using static MongoDB.Driver.Encryption.EncryptedCollectionHelper;
 
 namespace MongoDB.Driver.Core.Operations
 {
@@ -30,12 +31,48 @@ namespace MongoDB.Driver.Core.Operations
     /// </summary>
     public class CreateCollectionOperation : IWriteOperation<BsonDocument>
     {
+        #region static
+        internal static IWriteOperation<BsonDocument> CreateEncryptedCreateCollectionOperationIfConfigured(
+            CollectionNamespace collectionNamespace,
+            BsonDocument encryptedFields,
+            MessageEncoderSettings messageEncoderSettings,
+            Action<CreateCollectionOperation> createCollectionOperationConfigurator)
+        {
+            var mainOperation = new CreateCollectionOperation(
+                collectionNamespace,
+                messageEncoderSettings)
+            {
+                EncryptedFields = encryptedFields
+            };
+
+            createCollectionOperationConfigurator?.Invoke(mainOperation);
+
+            if (encryptedFields != null)
+            {
+                return new CompositeWriteOperation<BsonDocument>(
+                    (CreateInnerCollectionOperation(EncryptedCollectionHelper.GetAdditionalCollectionName(encryptedFields, collectionNamespace, HelperCollectionForEncryption.Esc)), IsMainOperation: false),
+                    (CreateInnerCollectionOperation(EncryptedCollectionHelper.GetAdditionalCollectionName(encryptedFields, collectionNamespace, HelperCollectionForEncryption.Ecc)), IsMainOperation: false),
+                    (CreateInnerCollectionOperation(EncryptedCollectionHelper.GetAdditionalCollectionName(encryptedFields, collectionNamespace, HelperCollectionForEncryption.Ecos)), IsMainOperation: false),
+                    (mainOperation, IsMainOperation: true),
+                    (new CreateIndexesOperation(collectionNamespace, new[] { new CreateIndexRequest(EncryptedCollectionHelper.AdditionalCreateIndexDocument) }, messageEncoderSettings), IsMainOperation: false));
+            }
+            else
+            {
+                return mainOperation;
+            }
+
+            CreateCollectionOperation CreateInnerCollectionOperation(string collectionName)
+                => new CreateCollectionOperation(new CollectionNamespace(collectionNamespace.DatabaseNamespace.DatabaseName, collectionName), messageEncoderSettings);
+        }
+        #endregion
+
         // fields
         private bool? _autoIndexId;
         private bool? _capped;
         private Collation _collation;
         private readonly CollectionNamespace _collectionNamespace;
         private BsonValue _comment;
+        private BsonDocument _encryptedFields;
         private TimeSpan? _expireAfter;
         private BsonDocument _indexOptionDefaults;
         private long? _maxDocuments;
@@ -125,6 +162,12 @@ public CollectionNamespace CollectionNamespace
             get { return _collectionNamespace; }
         }
 
+        internal BsonDocument EncryptedFields
+        {
+            get { return _encryptedFields; }
+            private set { _encryptedFields = value; }
+        }
+
         /// <summary>
         /// Gets or sets the expiration timespan for time series collections. Used to automatically delete documents in time series collections.
         /// See https://www.mongodb.com/docs/manual/reference/command/create/ for supported options and https://www.mongodb.com/docs/manual/core/timeseries-collections/
@@ -282,7 +325,7 @@ public WriteConcern WriteConcern
         }
 
         // methods
-        internal BsonDocument CreateCommand(ICoreSessionHandle session, ConnectionDescription connectionDescription)
+        internal BsonDocument CreateCommand(ICoreSessionHandle session)
         {
             var flags = GetFlags();
             var writeConcern = WriteConcernHelper.GetEffectiveWriteConcern(session, _writeConcern);
@@ -303,7 +346,8 @@ internal BsonDocument CreateCommand(ICoreSessionHandle session, ConnectionDescri
                 { "comment",  _comment, _comment != null },
                 { "writeConcern", writeConcern, writeConcern != null },
                 { "expireAfterSeconds", () => _expireAfter.Value.TotalSeconds, _expireAfter.HasValue },
-                { "timeseries", () => _timeSeriesOptions.ToBsonDocument(), _timeSeriesOptions != null }
+                { "timeseries", () => _timeSeriesOptions.ToBsonDocument(), _timeSeriesOptions != null },
+                { "encryptedFields", _encryptedFields, _encryptedFields != null }
             };
         }
 
@@ -337,7 +381,7 @@ public BsonDocument Execute(IWriteBinding binding, CancellationToken cancellatio
             using (var channel = channelSource.GetChannel(cancellationToken))
             using (var channelBinding = new ChannelReadWriteBinding(channelSource.Server, channel, binding.Session.Fork()))
             {
-                var operation = CreateOperation(channelBinding.Session, channel.ConnectionDescription);
+                var operation = CreateOperation(channelBinding.Session);
                 return operation.Execute(channelBinding, cancellationToken);
             }
         }
@@ -351,14 +395,14 @@ public async Task<BsonDocument> ExecuteAsync(IWriteBinding binding, Cancellation
             using (var channel = await channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false))
             using (var channelBinding = new ChannelReadWriteBinding(channelSource.Server, channel, binding.Session.Fork()))
             {
-                var operation = CreateOperation(channelBinding.Session, channel.ConnectionDescription);
+                var operation = CreateOperation(channelBinding.Session);
                 return await operation.ExecuteAsync(channelBinding, cancellationToken).ConfigureAwait(false);
             }
         }
 
-        private WriteCommandOperation<BsonDocument> CreateOperation(ICoreSessionHandle session, ConnectionDescription connectionDescription)
+        private WriteCommandOperation<BsonDocument> CreateOperation(ICoreSessionHandle session)
         {
-            var command = CreateCommand(session, connectionDescription);
+            var command = CreateCommand(session);
             return new WriteCommandOperation<BsonDocument>(_collectionNamespace.DatabaseNamespace, command, BsonDocumentSerializer.Instance, _messageEncoderSettings);
         }
 

src/MongoDB.Driver.Core/Core/Operations/CreateIndexesOperation.cs

@@ -19,7 +19,9 @@
 using System.Threading;
 using System.Threading.Tasks;
 using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Serializers;
 using MongoDB.Driver.Core.Bindings;
+using MongoDB.Driver.Core.Connections;
 using MongoDB.Driver.Core.Events;
 using MongoDB.Driver.Core.Misc;
 using MongoDB.Driver.Core.WireProtocol.Messages.Encoders;
@@ -33,8 +35,8 @@ public class CreateIndexesOperation : IWriteOperation<BsonDocument>
     {
         // fields
         private readonly CollectionNamespace _collectionNamespace;
-        private CreateIndexCommitQuorum _commitQuorum;
         private BsonValue _comment;
+        private CreateIndexCommitQuorum _commitQuorum;
         private TimeSpan? _maxTime;
         private readonly MessageEncoderSettings _messageEncoderSettings;
         private readonly IEnumerable<CreateIndexRequest> _requests;
@@ -72,6 +74,9 @@ public CollectionNamespace CollectionNamespace
         /// <summary>
         /// Gets or sets the comment.
         /// </summary>
+        /// <value>
+        /// The comment.
+        /// </value>
         public BsonValue Comment
         {
             get { return _comment; }
@@ -122,10 +127,10 @@ public WriteConcern WriteConcern
         }
 
         /// <summary>
-        /// Gets or sets the max time.
+        /// Gets or sets the MaxTime.
         /// </summary>
-        /// <value>
-        /// The max time
+        /// <value> 
+        /// The maxtime.
         /// </value>
         public TimeSpan? MaxTime
         {
@@ -142,7 +147,7 @@ public BsonDocument Execute(IWriteBinding binding, CancellationToken cancellatio
             using (var channel = channelSource.GetChannel(cancellationToken))
             using (var channelBinding = new ChannelReadWriteBinding(channelSource.Server, channel, binding.Session.Fork()))
             {
-                var operation = CreateOperation();
+                var operation = CreateOperation(channelBinding.Session, channel.ConnectionDescription);
                 return operation.Execute(channelBinding, cancellationToken);
             }
         }
@@ -155,21 +160,38 @@ public async Task<BsonDocument> ExecuteAsync(IWriteBinding binding, Cancellation
             using (var channel = await channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false))
             using (var channelBinding = new ChannelReadWriteBinding(channelSource.Server, channel, binding.Session.Fork()))
             {
-                var operation = CreateOperation();
+                var operation = CreateOperation(channelBinding.Session, channel.ConnectionDescription);
                 return await operation.ExecuteAsync(channelBinding, cancellationToken).ConfigureAwait(false);
             }
         }
 
-        // internal methods
-        internal IWriteOperation<BsonDocument> CreateOperation()
+        // private methods
+        internal BsonDocument CreateCommand(ICoreSessionHandle session, ConnectionDescription connectionDescription)
         {
-            return new CreateIndexesUsingCommandOperation(_collectionNamespace, _requests, _messageEncoderSettings)
+            var maxWireVersion = connectionDescription.MaxWireVersion;
+            var writeConcern = WriteConcernHelper.GetEffectiveWriteConcern(session, _writeConcern);
+            if (_commitQuorum != null)
             {
-                Comment = _comment,
-                CommitQuorum = _commitQuorum,
-                MaxTime = _maxTime,
-                WriteConcern = _writeConcern
+                Feature.CreateIndexCommitQuorum.ThrowIfNotSupported(maxWireVersion);
+            }
+
+            return new BsonDocument
+            {
+                { "createIndexes", _collectionNamespace.CollectionName },
+                { "indexes", new BsonArray(_requests.Select(request => request.CreateIndexDocument())) },
+                { "maxTimeMS", () => MaxTimeHelper.ToMaxTimeMS(_maxTime.Value), _maxTime.HasValue },
+                { "writeConcern", writeConcern, writeConcern != null },
+                { "comment", _comment, _comment != null },
+                { "commitQuorum", () => _commitQuorum.ToBsonValue(), _commitQuorum != null }
             };
         }
+
+        private WriteCommandOperation<BsonDocument> CreateOperation(ICoreSessionHandle session, ConnectionDescription connectionDescription)
+        {
+            var databaseNamespace = _collectionNamespace.DatabaseNamespace;
+            var command = CreateCommand(session, connectionDescription);
+            var resultSerializer = BsonDocumentSerializer.Instance;
+            return new WriteCommandOperation<BsonDocument>(databaseNamespace, command, resultSerializer, _messageEncoderSettings);
+        }
     }
 }