Merge branch 'master' into test-hosting-and-project-number

Author: joehan

CHANGELOG.md

@@ -1,4 +1,6 @@
+- Fixed a bug when deploying firestore indexes failed due to broken index comparison logic (#8859)
 - Added prefix support for multi-instance Cloud Functions extension parameters. (#8911)
 - Fixed a bug when `firebase deploy --only dataconnect` doesn't include GQL in nested folders (#8981)
 - Make it possible to init a dataconnect project in non interactive mode (#8993)
 - Added 2 new MCP tools for crashlytics `get_sample_crash_for_issue` and `get_issue_details` (#8995)
+- Use Gemini to generate schema and seed_data.gql in `firebase init dataconnect` (#8988)

src/dataconnect/client.ts

@@ -41,24 +41,31 @@ export async function createService(
   projectId: string,
   locationId: string,
   serviceId: string,
-): Promise<types.Service> {
-  const op = await dataconnectClient().post<types.Service, types.Service>(
-    `/projects/${projectId}/locations/${locationId}/services`,
-    {
-      name: `projects/${projectId}/locations/${locationId}/services/${serviceId}`,
-    },
-    {
-      queryParams: {
-        service_id: serviceId,
+): Promise<types.Service | undefined> {
+  try {
+    const op = await dataconnectClient().post<types.Service, types.Service>(
+      `/projects/${projectId}/locations/${locationId}/services`,
+      {
+        name: `projects/${projectId}/locations/${locationId}/services/${serviceId}`,
       },
-    },
-  );
-  const pollRes = await operationPoller.pollOperation<types.Service>({
-    apiOrigin: dataconnectOrigin(),
-    apiVersion: DATACONNECT_API_VERSION,
-    operationResourceName: op.body.name,
-  });
-  return pollRes;
+      {
+        queryParams: {
+          service_id: serviceId,
+        },
+      },
+    );
+    const pollRes = await operationPoller.pollOperation<types.Service>({
+      apiOrigin: dataconnectOrigin(),
+      apiVersion: DATACONNECT_API_VERSION,
+      operationResourceName: op.body.name,
+    });
+    return pollRes;
+  } catch (err: any) {
+    if (err.status !== 409) {
+      throw err;
+    }
+    return undefined; // Service already exists
+  }
 }
 
 export async function deleteService(serviceName: string): Promise<types.Service> {

src/dataconnect/ensureApis.ts

@@ -1,21 +1,13 @@
 import * as api from "../api";
-import { check, ensure } from "../ensureApiEnabled";
+import { ensure } from "../ensureApiEnabled";
 
 const prefix = "dataconnect";
 
-export async function isApiEnabled(projectId: string): Promise<boolean> {
-  return await check(projectId, api.dataconnectOrigin(), prefix);
-}
-
 export async function ensureApis(projectId: string): Promise<void> {
-  await ensure(projectId, api.dataconnectOrigin(), prefix);
-  await ensure(projectId, api.cloudSQLAdminOrigin(), prefix);
-}
-
-export async function ensureSparkApis(projectId: string): Promise<void> {
-  // These are the APIs that can be enabled without a billing account.
-  await ensure(projectId, api.cloudSQLAdminOrigin(), prefix);
-  await ensure(projectId, api.dataconnectOrigin(), prefix);
+  await Promise.all([
+    ensure(projectId, api.dataconnectOrigin(), prefix),
+    ensure(projectId, api.cloudSQLAdminOrigin(), prefix),
+  ]);
 }
 
 export async function ensureGIFApis(projectId: string): Promise<void> {

src/dataconnect/freeTrial.ts

@@ -12,32 +12,32 @@ const FREE_TRIAL_METRIC = "sqladmin.googleapis.com/fdc_lifetime_free_trial_per_p
 
 // Checks whether there is already a free trial instance on a project.
 export async function checkFreeTrialInstanceUsed(projectId: string): Promise<boolean> {
-  utils.logLabeledBullet("dataconnect", "Checking Cloud SQL no cost trial eligibility...");
   const past7d = new Date();
   past7d.setDate(past7d.getDate() - 7);
   const query: CmQuery = {
     filter: `metric.type="serviceruntime.googleapis.com/quota/allocation/usage" AND metric.label.quota_metric = "${FREE_TRIAL_METRIC}"`,
     "interval.endTime": new Date().toJSON(),
     "interval.startTime": past7d.toJSON(),
   };
+  let used = true;
   try {
     const ts = await queryTimeSeries(query, projectId);
-    let used = true;
     if (ts.length) {
       used = ts[0].points.some((p) => p.value.int64Value);
     }
-    if (used) {
-      utils.logLabeledWarning(
-        "dataconnect",
-        "CloudSQL no cost trial has already been used on this project.",
-      );
-    }
-    return used;
   } catch (err: any) {
     // If the metric doesn't exist, free trial is not used.
+    used = false;
+  }
+  if (used) {
+    utils.logLabeledWarning(
+      "dataconnect",
+      "CloudSQL no cost trial has already been used on this project.",
+    );
+  } else {
     utils.logLabeledSuccess("dataconnect", "CloudSQL no cost trial available!");
-    return false;
   }
+  return used;
 }
 
 export async function getFreeTrialInstanceId(projectId: string): Promise<string | undefined> {
@@ -84,9 +84,11 @@ export function printFreeTrialUnavailable(
 }
 
 export function upgradeInstructions(projectId: string): string {
-  return `If you'd like to provision a CloudSQL Postgres instance on the Firebase Data Connect no-cost trial:
-1. Please upgrade to the pay-as-you-go (Blaze) billing plan. Visit the following page:
-     https://console.firebase.google.com/project/${projectId}/usage/details
-2. Run ${clc.bold("firebase init dataconnect")} again to configure the Cloud SQL instance.
-3. Run ${clc.bold("firebase deploy --only dataconnect")} to deploy your Data Connect service.`;
+  return `To provision a CloudSQL Postgres instance on the Firebase Data Connect no-cost trial:
+
+  1. Please upgrade to the pay-as-you-go (Blaze) billing plan. Visit the following page:
+
+      https://console.firebase.google.com/project/${projectId}/usage/details
+
+  2. Run ${clc.bold("firebase deploy --only dataconnect")} to deploy your Data Connect service.`;
 }

src/dataconnect/provisionCloudSql.ts

@@ -32,7 +32,8 @@ export async function provisionCloudSql(args: {
   } = args;
   try {
     const existingInstance = await cloudSqlAdminClient.getInstance(projectId, instanceId);
-    silent || utils.logLabeledBullet("dataconnect", `Found existing instance ${instanceId}.`);
+    silent ||
+      utils.logLabeledBullet("dataconnect", `Found existing Cloud SQL instance ${instanceId}.`);
     connectionName = existingInstance?.connectionName || "";
     const why = getUpdateReason(existingInstance, enableGoogleMlIntegration);
     if (why) {
@@ -61,7 +62,6 @@ export async function provisionCloudSql(args: {
     if (err.status !== 404) {
       throw err;
     }
-    cmekWarning();
     const cta = dryRun ? "It will be created on your next deploy" : "Creating it now.";
     const freeTrialUsed = await checkFreeTrialInstanceUsed(projectId);
     silent ||
@@ -95,7 +95,7 @@ export async function provisionCloudSql(args: {
         silent ||
           utils.logLabeledBullet(
             "dataconnect",
-            "Cloud SQL instance creation started - it should be ready shortly. Database and users will be created on your next deploy.",
+            "Cloud SQL instance creation started. While it is being set up, your data will be saved in a temporary database. When it is ready, your data will be migrated.",
           );
         return connectionName;
       }
@@ -111,17 +111,11 @@ export async function provisionCloudSql(args: {
         silent ||
           utils.logLabeledBullet(
             "dataconnect",
-            `Database ${databaseId} not found. It will be created on your next deploy.`,
+            `Postgres database ${databaseId} not found. It will be created on your next deploy.`,
           );
       } else {
-        // Create the database if not found.
-        silent ||
-          utils.logLabeledBullet(
-            "dataconnect",
-            `Database ${databaseId} not found, creating it now...`,
-          );
         await cloudSqlAdminClient.createDatabase(projectId, instanceId, databaseId);
-        silent || utils.logLabeledBullet("dataconnect", `Database ${databaseId} created.`);
+        silent || utils.logLabeledBullet("dataconnect", `Postgres database ${databaseId} created.`);
       }
     } else {
       // Skip it if the database is not accessible.
@@ -171,11 +165,3 @@ export function getUpdateReason(instance: Instance, requireGoogleMlIntegration:
 
   return reason;
 }
-
-function cmekWarning() {
-  const message =
-    "Cloud SQL instances created via the Firebase CLI do not support customer managed encryption keys.\n" +
-    "If you'd like to use a CMEK to encrypt your data, first create a CMEK encrypted instance (https://cloud.google.com/sql/docs/postgres/configure-cmek#createcmekinstance).\n" +
-    "Then, edit your `dataconnect.yaml` file to use the encrypted instance and redeploy.";
-  utils.logLabeledWarning("dataconnect", message);
-}

src/dataconnect/types.ts

@@ -35,6 +35,7 @@ export interface PostgreSql {
   database?: string;
   cloudSql?: CloudSqlInstance;
   schemaValidation?: SchemaValidation | "NONE" | "SQL_SCHEMA_VALIDATION_UNSPECIFIED";
+  schemaMigration?: "MIGRATE_COMPATIBLE";
 }
 
 export interface CloudSqlInstance {

src/firestore/api.ts

@@ -5,6 +5,7 @@ import * as utils from "../utils";
 import * as validator from "./validator";
 
 import * as types from "./api-types";
+import { DatabaseEdition, Density } from "./api-types";
 import * as Spec from "./api-spec";
 import * as sort from "./api-sort";
 import * as util from "./util";
@@ -13,6 +14,7 @@ import { firestoreOrigin } from "../api";
 import { FirebaseError } from "../error";
 import { Client } from "../apiv2";
 import { PrettyPrint } from "./pretty-print";
+import { optionalValueMatches } from "../functional";
 
 export class FirestoreApi {
   apiClient = new Client({ urlPrefix: firestoreOrigin(), apiVersion: "v1" });
@@ -74,8 +76,10 @@ export class FirestoreApi {
       databaseId,
     );
 
+    const database = await this.getDatabase(options.project, databaseId);
+    const edition = database.databaseEdition ?? DatabaseEdition.STANDARD;
     const indexesToDelete = existingIndexes.filter((index) => {
-      return !indexesToDeploy.some((spec) => this.indexMatchesSpec(index, spec));
+      return !indexesToDeploy.some((spec) => this.indexMatchesSpec(index, spec, edition));
     });
 
     // We only want to delete fields where there is nothing in the local file with the same
@@ -127,7 +131,7 @@ export class FirestoreApi {
     }
 
     for (const index of indexesToDeploy) {
-      const exists = existingIndexes.some((x) => this.indexMatchesSpec(x, index));
+      const exists = existingIndexes.some((x) => this.indexMatchesSpec(x, index, edition));
       if (exists) {
         logger.debug(`Skipping existing index: ${JSON.stringify(index)}`);
       } else {
@@ -325,8 +329,11 @@ export class FirestoreApi {
       validator.assertType("multikey", index.multikey, "boolean");
     }
 
-    if (index.unique) {
+    if (index.unique !== undefined) {
       validator.assertType("unique", index.unique, "boolean");
+      // TODO(b/439901837): Remove this check and update indexMatchesSpec once
+      //  unique index configuration is supported.
+      throw new FirebaseError("The `unique` index configuration is not supported yet.");
     }
 
     validator.assertHas(index, "fields");
@@ -479,10 +486,51 @@ export class FirestoreApi {
     return this.apiClient.delete(`/${url}`);
   }
 
+  /**
+   * Returns true if the given ApiScope values match.
+   * If either one is undefined, the default value is used for comparison.
+   * @param lhs the first ApiScope value.
+   * @param rhs the second ApiScope value.
+   */
+  optionalApiScopeMatches(
+    lhs: types.ApiScope | undefined,
+    rhs: types.ApiScope | undefined,
+  ): boolean {
+    return optionalValueMatches<types.ApiScope>(lhs, rhs, types.ApiScope.ANY_API);
+  }
+
+  /**
+   * Returns true if the given Density values match.
+   * If either one is undefined, the default value is used for comparison based on Database Edition.
+   * @param lhs the first Density value.
+   * @param rhs the second Density value.
+   * @param edition the database edition used to determine the default value.
+   */
+  optionalDensityMatches(
+    lhs: Density | undefined,
+    rhs: Density | undefined,
+    edition: types.DatabaseEdition,
+  ): boolean {
+    const defaultValue =
+      edition === DatabaseEdition.STANDARD ? types.Density.SPARSE_ALL : types.Density.DENSE;
+    return optionalValueMatches<types.Density>(lhs, rhs, defaultValue);
+  }
+
+  /**
+   * Returns true if the given Multikey values match.
+   * If either one is undefined, the default value is used for comparison.
+   * @param lhs the first Multikey value.
+   * @param rhs the second Multikey value.
+   */
+  optionalMultikeyMatches(lhs: boolean | undefined, rhs: boolean | undefined): boolean {
+    const defaultValue = false;
+    return optionalValueMatches<boolean>(lhs, rhs, defaultValue);
+  }
+
   /**
    * Determine if an API Index and a Spec Index are functionally equivalent.
    */
-  indexMatchesSpec(index: types.Index, spec: Spec.Index): boolean {
+  indexMatchesSpec(index: types.Index, spec: Spec.Index, edition: types.DatabaseEdition): boolean {
     const collection = util.parseIndexName(index.name).collectionGroupId;
     if (collection !== spec.collectionGroup) {
       return false;
@@ -492,21 +540,24 @@ export class FirestoreApi {
       return false;
     }
 
-    if (index.apiScope !== spec.apiScope) {
+    // apiScope is an optional value and may be missing in firestore.indexes.json,
+    // and may also be missing from the server value (when default is picked).
+    if (!this.optionalApiScopeMatches(index.apiScope, spec.apiScope)) {
       return false;
     }
 
-    if (index.density !== spec.density) {
+    // density is an optional value and may be missing in firestore.indexes.json,
+    // and may also be missing from the server value (when default is picked).
+    if (!this.optionalDensityMatches(index.density, spec.density, edition)) {
       return false;
     }
-
-    if (index.multikey !== spec.multikey) {
+    // multikey is an optional value and may be missing in firestore.indexes.json,
+    // and may also be missing from the server value (when default is picked).
+    if (!this.optionalMultikeyMatches(index.multikey, spec.multikey)) {
       return false;
     }
 
-    if (index.unique !== spec.unique) {
-      return false;
-    }
+    // TODO(b/439901837): Compare `unique` index configuration when it's supported.
 
     if (index.fields.length !== spec.fields.length) {
       return false;
@@ -529,7 +580,8 @@ export class FirestoreApi {
         return false;
       }
 
-      if (iField.vectorConfig !== sField.vectorConfig) {
+      // Note: vectorConfig is an object, and using '!==' should not be used.
+      if (!utils.deepEqual(iField.vectorConfig, sField.vectorConfig)) {
         return false;
       }