CSHARP-4153: Add FLE 2 API to AutoEncryptionOpts. (#793)

CSHARP-4160: Add FLE 2 test for compactStructuredEncryptionData.
CSHARP-4157: Add FLE 2 API to ClientEncryptionOpts.
CSHARp-4175: Ban comparisons to encrypted fields in collection validator and partialFilterExpression.
Co-authored-by: Boris <[email protected]>

Author: DmitryLukyanov

build.cake

@@ -416,10 +416,37 @@ Task("TestCsfleWithMockedKms")
         );
     });
 
-Task("TestCsfleWithMockedKmsnet472").IsDependentOn("TestCsfleWithMockedKms");
+Task("TestCsfleWithMockedKmsNet472").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard20").IsDependentOn("TestCsfleWithMockedKms");
 Task("TestCsfleWithMockedKmsNetStandard21").IsDependentOn("TestCsfleWithMockedKms");
 
+Task("TestMongocryptd")
+    .IsDependentOn("Build")
+    .DoesForEach(
+        items: GetFiles("./**/*.Tests.csproj"),
+        action: (BuildConfig buildConfig, Path testProject) =>
+    {
+        var settings = new DotNetCoreTestSettings
+        {
+            NoBuild = true,
+            NoRestore = true,
+            Configuration = configuration,
+            Loggers = CreateLoggers(),
+            ArgumentCustomization = args => args.Append("-- RunConfiguration.TargetPlatform=x64"),
+            Filter = "Category=\"CSFLE\"",
+            Framework = buildConfig.Framework
+        };
+
+        DotNetCoreTest(
+            testProject.FullPath,
+            settings
+        );
+    });
+
+Task("TestMongocryptdNet472").IsDependentOn("TestMongocryptd");
+Task("TestMongocryptdNetStandard20").IsDependentOn("TestMongocryptd");
+Task("TestMongocryptdNetStandard21").IsDependentOn("TestMongocryptd");
+
 Task("Docs")
     .IsDependentOn("ApiDocs")
     .IsDependentOn("RefDocs");

evergreen/evergreen.yml

@@ -13,8 +13,8 @@ stepback: true
 command_type: system
 
 # Protect ourself against rogue test case, or curl gone wild, that runs forever
-# 45 minutes: 20 minutes is a normal test run + up to 10 minutes for test setup + 15 minutes for longer macOS tests
-exec_timeout_secs: 2700
+# 60 minutes: 20 minutes is a normal test run + up to 10 minutes for test setup + 15 minutes for longer macOS tests + 15 minutes for longer macOS 1015 tests
+exec_timeout_secs: 3600
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -292,6 +292,8 @@ functions:
           . ./evergreen/set-virtualenv.sh
           . ./evergreen/set-temp-fle-aws-creds.sh
           ${PREPARE_SHELL}
+          OS=${OS} \
+            . ./evergreen/fetch-crypt_shared-library.sh
           OS=${OS} \
             evergreen/add-ca-certs.sh
           AUTH=${AUTH} \
@@ -339,6 +341,41 @@ functions:
           OS=${OS} \
             evergreen/cleanup-test-resources.sh
 
+  run-mongocryptd-tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: mongo-csharp-driver
+        script: |
+          set +x
+          export FLE_AWS_ACCESS_KEY_ID=${FLE_AWS_ACCESS_KEY_ID}
+          export FLE_AWS_SECRET_ACCESS_KEY=${FLE_AWS_SECRET_ACCESS_KEY}
+          export FLE_AZURE_TENANT_ID=${FLE_AZURE_TENANT_ID}
+          export FLE_AZURE_CLIENT_ID=${FLE_AZURE_CLIENT_ID}
+          export FLE_AZURE_CLIENT_SECRET=${FLE_AZURE_CLIENT_SECRET}
+          export FLE_GCP_EMAIL=${FLE_GCP_EMAIL}
+          export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
+          . ./evergreen/set-virtualenv.sh
+          . ./evergreen/set-temp-fle-aws-creds.sh
+          ${PREPARE_SHELL}
+          OS=${OS} \
+            evergreen/add-ca-certs.sh
+          AUTH=${AUTH} \
+          SSL=${SSL} \
+          MONGODB_URI="${MONGODB_URI}" \
+          TOPOLOGY=${TOPOLOGY} \
+          OS=${OS} \
+          COMPRESSOR=${COMPRESSOR} \
+          CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \
+          REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
+          TEST_MONGOCRYPTD="true" \
+          TARGET="TestMongocryptd" \
+          FRAMEWORK=${FRAMEWORK} \
+            evergreen/run-tests.sh
+          echo "Skipping certificate removal..."
+          OS=${OS} \
+            evergreen/cleanup-test-resources.sh
+
   run-atlas-connectivity-tests:
     - command: shell.exec
       type: test
@@ -835,6 +872,27 @@ tasks:
           vars:
             FRAMEWORK: netstandard21
 
+    - name: test-mongocryptd-net472
+      commands:
+        - func: bootstrap-mongo-orchestration
+        - func: run-mongocryptd-tests
+          vars:
+            FRAMEWORK: net472
+
+    - name: test-mongocryptd-netstandard20
+      commands:
+        - func: bootstrap-mongo-orchestration
+        - func: run-mongocryptd-tests
+          vars:
+            FRAMEWORK: netstandard20
+
+    - name: test-mongocryptd-netstandard21
+      commands:
+        - func: bootstrap-mongo-orchestration
+        - func: run-mongocryptd-tests
+          vars:
+            FRAMEWORK: netstandard21
+
     - name: test-kms-tls-mocked-net472
       commands:
         - func: start-kms-mock-servers
@@ -1371,11 +1429,11 @@ axes:
         variables:
           OS: "ubuntu-1804"
         run_on: ubuntu1804-test
-      - id: "macos-1014"
-        display_name: "macOS 10.14"
+      - id: "macos-1015"
+        display_name: "macOS 10.15"
         variables:
-          OS: "macos-1014"
-        run_on: macos-1014
+          OS: "macos-1015"
+        run_on: macos-1015
 
   - id: topology
     display_name: Topology
@@ -1447,10 +1505,10 @@ buildvariants:
     - ubuntu1804-test
   tasks:
     - name: compile
-- name: macos1014-compile
-  display_name: "macOS 10.14 compile"
+- name: macos1015-compile
+  display_name: "macOS 10.15 compile"
   run_on:
-    - macos-1014
+    - macos-1015
   tasks:
     - name: compile
 
@@ -1499,7 +1557,7 @@ buildvariants:
      - name: test-netstandard21
 
 - matrix_name: "tests-snappy-compression-macOS"
-  matrix_spec: { compressor : "snappy", auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "rapid", "latest"], topology: "standalone", os: "macos-1014" }
+  matrix_spec: { compressor : "snappy", auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "rapid", "latest"], topology: "standalone", os: "macos-1015" }
   display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} "
   tags: ["tests-variant"]
   tasks:
@@ -1523,7 +1581,7 @@ buildvariants:
      - name: test-netstandard21
 
 - matrix_name: "tests-zstandard-compression-macOS"
-  matrix_spec: { compressor : "zstandard", auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "rapid", "latest"], topology: "standalone", os: "macos-1014" }
+  matrix_spec: { compressor : "zstandard", auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "rapid", "latest"], topology: "standalone", os: "macos-1015" }
   display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} "
   tags: ["tests-variant"]
   tasks:
@@ -1547,14 +1605,14 @@ buildvariants:
      - name: test-netstandard21
 
 - matrix_name: "secure-tests-macOS"
-  matrix_spec: { version: ["5.0", "6.0", "rapid", "latest"], topology: "replicaset", auth: "auth", ssl: "ssl", os: "macos-1014" }
+  matrix_spec: { version: ["5.0", "6.0", "rapid", "latest"], topology: "replicaset", auth: "auth", ssl: "ssl", os: "macos-1015" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${os}"
   tags: ["tests-variant"]
   tasks:
      - name: test-netstandard21
 
 - matrix_name: "unsecure-tests-macOS"
-  matrix_spec: { version: ["5.0", "6.0", "rapid", "latest"], topology: "replicaset", auth: "noauth", ssl: "nossl", os: "macos-1014" }
+  matrix_spec: { version: ["5.0", "6.0", "rapid", "latest"], topology: "replicaset", auth: "noauth", ssl: "nossl", os: "macos-1015" }
   display_name: "${version} ${topology} ${auth} ${ssl} ${os}"
   tags: ["tests-variant"]
   tasks:
@@ -1661,21 +1719,42 @@ buildvariants:
 
 - matrix_name: "csfle-with-mocked-kms-tests-windows"
   matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
-  display_name: "CSFLE Mocked KMS ${os}"
+  display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
     - name: test-kms-tls-mocked-net472
     - name: test-kms-tls-mocked-netstandard20
     - name: test-kms-tls-mocked-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-linux"
   matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
-  display_name: "CSFLE Mocked KMS ${os}"
+  display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
     - name: test-kms-tls-mocked-netstandard20
     - name: test-kms-tls-mocked-netstandard21
 
 - matrix_name: "csfle-with-mocked-kms-tests-macOS"
-  matrix_spec: { os: "macos-1014", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
-  display_name: "CSFLE Mocked KMS ${os}"
+  matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "5.0", "6.0", "rapid", "latest" ], topology: ["standalone"] }
+  display_name: "CSFLE Mocked KMS ${version} ${os}"
   tasks:
     - name: test-kms-tls-mocked-netstandard21
+
+- matrix_name: "csfle1-windows"
+  matrix_spec: { os: "windows-64", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
+  display_name: "mongocryptd ${version} ${os}"
+  tasks:
+    - name: test-mongocryptd-net472
+    - name: test-mongocryptd-netstandard20
+    - name: test-mongocryptd-netstandard21
+
+- matrix_name: "csfle1-linux"
+  matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
+  display_name: "mongocryptd ${version} ${os}"
+  tasks:
+    - name: test-mongocryptd-netstandard20
+    - name: test-mongocryptd-netstandard21
+
+- matrix_name: "csfle1-macOS"
+  matrix_spec: { os: "macos-1015", ssl: "nossl", version: [ "4.2", "4.4", "5.0", "6.0", "latest" ], topology: ["replicaset"] }
+  display_name: "mongocryptd ${version} ${os}"
+  tasks:
+    - name: test-mongocryptd-netstandard21

evergreen/fetch-crypt_shared-library.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Fetch csfle shared library.
+#
+# Environment variables used as input:
+#   OS                                               The current operating system
+#   DRIVERS_TOOLS
+#
+# Environment variables produced as output:
+#   MONGODB_CSFLE_PATH                               The MONGODB_CSFLE_PATH path
+
+set -o xtrace   # Write all commands first to stderr
+set -o errexit  # Exit the script with an error if any of the commands fail
+
+
+PYTHON=$(OS=${OS} ${PROJECT_DIRECTORY}/evergreen/get-python-path.sh)
+$PYTHON -u ${DRIVERS_TOOLS}/.evergreen/mongodl.py --component crypt_shared --out ${DRIVERS_TOOLS}/evergreen/csfle --version 6.0.0-rc8
+
+if [[ "$OS" =~ Windows|windows ]]; then
+    export CRYPT_SHARED_LIB_PATH="${DRIVERS_TOOLS}/evergreen/csfle/bin/mongo_crypt_v1.dll"
+elif [[ "$OS" =~ Mac|mac ]]; then
+    export CRYPT_SHARED_LIB_PATH="${DRIVERS_TOOLS}/evergreen/csfle/lib/mongo_crypt_v1.dylib"
+else
+    export CRYPT_SHARED_LIB_PATH="${DRIVERS_TOOLS}/evergreen/csfle/lib/mongo_crypt_v1.so"
+fi
+
+echo "crypt shared library path $CRYPT_SHARED_LIB_PATH"

evergreen/run-tests.sh

@@ -68,6 +68,8 @@ provision_compressor () {
 ############################################
 #            Main Program                  #
 ############################################
+echo "CRYPT_SHARED_LIB_PATH:" $CRYPT_SHARED_LIB_PATH
+echo "TEST_MONGOCRYPTD:" $TEST_MONGOCRYPTD
 echo "Initial MongoDB URI:" $MONGODB_URI
 echo "Framework: " $FRAMEWORK
 

src/MongoDB.Driver.Core/Core/Clusters/Cluster.cs

@@ -187,6 +187,7 @@ protected virtual void Dispose(bool disposing)
                 UpdateClusterDescription(newClusterDescription);
 
                 _rapidHeartbeatTimer.Dispose();
+                _cryptClient?.Dispose();
             }
         }
 
@@ -222,9 +223,9 @@ public virtual void Initialize()
             ThrowIfDisposed();
             if (_state.TryChange(State.Initial, State.Open))
             {
-                if (_settings.KmsProviders != null || _settings.SchemaMap != null)
+                if (_settings.CryptClientSettings != null)
                 {
-                    _cryptClient = CryptClientCreator.CreateCryptClient(_settings.KmsProviders, _settings.SchemaMap);
+                    _cryptClient = CryptClientCreator.CreateCryptClient(_settings.CryptClientSettings);
                 }
             }
         }