Skip to content

Commit 08d00a1

Browse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: Add Kafka-based sources to IngestionDataSourceSettings proto and IngestionFailureEvent proto (#2007)
* chore: remove redundant gRPC service config file feat: specify retry configuration PiperOrigin-RevId: 712493958 Source-Link: googleapis/googleapis@29aea41 Source-Link: googleapis/googleapis-gen@04bcbab Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMDRiY2JhYjQ1MDU5ODllOTg0YjE0MDNkNDM4ZmZmYzAzMTIxNDRhZiJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: Add Kafka-based sources to IngestionDataSourceSettings proto and IngestionFailureEvent proto PiperOrigin-RevId: 712896662 Source-Link: googleapis/googleapis@b062cd1 Source-Link: googleapis/googleapis-gen@49fbaeb Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNDlmYmFlYjBjNmIxMjc1NWE2MmY0NGYyZGNiOTdhNWQxZWU3ZDgwZSJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent c8e5811 commit 08d00a1

12 files changed

+4652
-752
lines changed

‎.jsdoc.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2024 Google LLC
1+
// Copyright 2025 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ module.exports = {
4040
includePattern: '\\.js$'
4141
},
4242
templates: {
43-
copyright: 'Copyright 2024 Google LLC',
43+
copyright: 'Copyright 2025 Google LLC',
4444
includeDate: false,
4545
sourceFiles: false,
4646
systemName: '@google-cloud/pubsub',

‎protos/google/pubsub/v1/pubsub.proto

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,13 +323,181 @@ message IngestionDataSourceSettings {
323323
string match_glob = 9 [(google.api.field_behavior) = OPTIONAL];
324324
}
325325

326+
// Ingestion settings for Azure Event Hubs.
327+
message AzureEventHubs {
328+
// Possible states for managed ingestion from Event Hubs.
329+
enum State {
330+
// Default value. This value is unused.
331+
STATE_UNSPECIFIED = 0;
332+
333+
// Ingestion is active.
334+
ACTIVE = 1;
335+
336+
// Permission denied encountered while consuming data from Event Hubs.
337+
// This can happen when `client_id`, or `tenant_id` are invalid. Or the
338+
// right permissions haven't been granted.
339+
EVENT_HUBS_PERMISSION_DENIED = 2;
340+
341+
// Permission denied encountered while publishing to the topic.
342+
PUBLISH_PERMISSION_DENIED = 3;
343+
344+
// The provided Event Hubs namespace couldn't be found.
345+
NAMESPACE_NOT_FOUND = 4;
346+
347+
// The provided Event Hub couldn't be found.
348+
EVENT_HUB_NOT_FOUND = 5;
349+
350+
// The provided Event Hubs subscription couldn't be found.
351+
SUBSCRIPTION_NOT_FOUND = 6;
352+
353+
// The provided Event Hubs resource group couldn't be found.
354+
RESOURCE_GROUP_NOT_FOUND = 7;
355+
}
356+
357+
// Output only. An output-only field that indicates the state of the Event
358+
// Hubs ingestion source.
359+
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
360+
361+
// Optional. Name of the resource group within the azure subscription.
362+
string resource_group = 2 [(google.api.field_behavior) = OPTIONAL];
363+
364+
// Optional. The name of the Event Hubs namespace.
365+
string namespace = 3 [(google.api.field_behavior) = OPTIONAL];
366+
367+
// Optional. The name of the Event Hub.
368+
string event_hub = 4 [(google.api.field_behavior) = OPTIONAL];
369+
370+
// Optional. The client id of the Azure application that is being used to
371+
// authenticate Pub/Sub.
372+
string client_id = 5 [(google.api.field_behavior) = OPTIONAL];
373+
374+
// Optional. The tenant id of the Azure application that is being used to
375+
// authenticate Pub/Sub.
376+
string tenant_id = 6 [(google.api.field_behavior) = OPTIONAL];
377+
378+
// Optional. The Azure subscription id.
379+
string subscription_id = 7 [(google.api.field_behavior) = OPTIONAL];
380+
381+
// Optional. The GCP service account to be used for Federated Identity
382+
// authentication.
383+
string gcp_service_account = 8 [(google.api.field_behavior) = OPTIONAL];
384+
}
385+
386+
// Ingestion settings for Amazon MSK.
387+
message AwsMsk {
388+
// Possible states for managed ingestion from Amazon MSK.
389+
enum State {
390+
// Default value. This value is unused.
391+
STATE_UNSPECIFIED = 0;
392+
393+
// Ingestion is active.
394+
ACTIVE = 1;
395+
396+
// Permission denied encountered while consuming data from Amazon MSK.
397+
MSK_PERMISSION_DENIED = 2;
398+
399+
// Permission denied encountered while publishing to the topic.
400+
PUBLISH_PERMISSION_DENIED = 3;
401+
402+
// The provided MSK cluster wasn't found.
403+
CLUSTER_NOT_FOUND = 4;
404+
405+
// The provided topic wasn't found.
406+
TOPIC_NOT_FOUND = 5;
407+
}
408+
409+
// Output only. An output-only field that indicates the state of the Amazon
410+
// MSK ingestion source.
411+
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
412+
413+
// Required. The Amazon Resource Name (ARN) that uniquely identifies the
414+
// cluster.
415+
string cluster_arn = 2 [(google.api.field_behavior) = REQUIRED];
416+
417+
// Required. The name of the topic in the Amazon MSK cluster that Pub/Sub
418+
// will import from.
419+
string topic = 3 [(google.api.field_behavior) = REQUIRED];
420+
421+
// Required. AWS role ARN to be used for Federated Identity authentication
422+
// with Amazon MSK. Check the Pub/Sub docs for how to set up this role and
423+
// the required permissions that need to be attached to it.
424+
string aws_role_arn = 4 [(google.api.field_behavior) = REQUIRED];
425+
426+
// Required. The GCP service account to be used for Federated Identity
427+
// authentication with Amazon MSK (via a `AssumeRoleWithWebIdentity` call
428+
// for the provided role). The `aws_role_arn` must be set up with
429+
// `accounts.google.com:sub` equals to this service account number.
430+
string gcp_service_account = 5 [(google.api.field_behavior) = REQUIRED];
431+
}
432+
433+
// Ingestion settings for Confluent Cloud.
434+
message ConfluentCloud {
435+
// Possible states for managed ingestion from Confluent Cloud.
436+
enum State {
437+
// Default value. This value is unused.
438+
STATE_UNSPECIFIED = 0;
439+
440+
// Ingestion is active.
441+
ACTIVE = 1;
442+
443+
// Permission denied encountered while consuming data from Confluent
444+
// Cloud.
445+
CONFLUENT_CLOUD_PERMISSION_DENIED = 2;
446+
447+
// Permission denied encountered while publishing to the topic.
448+
PUBLISH_PERMISSION_DENIED = 3;
449+
450+
// The provided bootstrap server address is unreachable.
451+
UNREACHABLE_BOOTSTRAP_SERVER = 4;
452+
453+
// The provided cluster wasn't found.
454+
CLUSTER_NOT_FOUND = 5;
455+
456+
// The provided topic wasn't found.
457+
TOPIC_NOT_FOUND = 6;
458+
}
459+
460+
// Output only. An output-only field that indicates the state of the
461+
// Confluent Cloud ingestion source.
462+
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
463+
464+
// Required. The address of the bootstrap server. The format is url:port.
465+
string bootstrap_server = 2 [(google.api.field_behavior) = REQUIRED];
466+
467+
// Required. The id of the cluster.
468+
string cluster_id = 3 [(google.api.field_behavior) = REQUIRED];
469+
470+
// Required. The name of the topic in the Confluent Cloud cluster that
471+
// Pub/Sub will import from.
472+
string topic = 4 [(google.api.field_behavior) = REQUIRED];
473+
474+
// Required. The id of the identity pool to be used for Federated Identity
475+
// authentication with Confluent Cloud. See
476+
// https://docs.confluent.io/cloud/current/security/authenticate/workload-identities/identity-providers/oauth/identity-pools.html#add-oauth-identity-pools.
477+
string identity_pool_id = 5 [(google.api.field_behavior) = REQUIRED];
478+
479+
// Required. The GCP service account to be used for Federated Identity
480+
// authentication with `identity_pool_id`.
481+
string gcp_service_account = 6 [(google.api.field_behavior) = REQUIRED];
482+
}
483+
326484
// Only one source type can have settings set.
327485
oneof source {
328486
// Optional. Amazon Kinesis Data Streams.
329487
AwsKinesis aws_kinesis = 1 [(google.api.field_behavior) = OPTIONAL];
330488

331489
// Optional. Cloud Storage.
332490
CloudStorage cloud_storage = 2 [(google.api.field_behavior) = OPTIONAL];
491+
492+
// Optional. Azure Event Hubs.
493+
AzureEventHubs azure_event_hubs = 3
494+
[(google.api.field_behavior) = OPTIONAL];
495+
496+
// Optional. Amazon MSK.
497+
AwsMsk aws_msk = 5 [(google.api.field_behavior) = OPTIONAL];
498+
499+
// Optional. Confluent Cloud.
500+
ConfluentCloud confluent_cloud = 6 [(google.api.field_behavior) = OPTIONAL];
333501
}
334502

335503
// Optional. Platform Logs settings. If unset, no Platform Logs will be
@@ -408,6 +576,78 @@ message IngestionFailureEvent {
408576
}
409577
}
410578

579+
// Failure when ingesting from an Amazon MSK source.
580+
message AwsMskFailureReason {
581+
// Optional. The ARN of the cluster of the topic being ingested from.
582+
string cluster_arn = 1 [(google.api.field_behavior) = OPTIONAL];
583+
584+
// Optional. The name of the Kafka topic being ingested from.
585+
string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
586+
587+
// Optional. The partition ID of the message that failed to be ingested.
588+
int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
589+
590+
// Optional. The offset within the partition of the message that failed to
591+
// be ingested.
592+
int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
593+
594+
// Reason why ingestion failed for the specified message.
595+
oneof reason {
596+
// Optional. The Pub/Sub API limits prevented the desired message from
597+
// being published.
598+
ApiViolationReason api_violation_reason = 5
599+
[(google.api.field_behavior) = OPTIONAL];
600+
}
601+
}
602+
603+
// Failure when ingesting from an Azure Event Hubs source.
604+
message AzureEventHubsFailureReason {
605+
// Optional. The namespace containing the event hub being ingested from.
606+
string namespace = 1 [(google.api.field_behavior) = OPTIONAL];
607+
608+
// Optional. The name of the event hub being ingested from.
609+
string event_hub = 2 [(google.api.field_behavior) = OPTIONAL];
610+
611+
// Optional. The partition ID of the message that failed to be ingested.
612+
int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
613+
614+
// Optional. The offset within the partition of the message that failed to
615+
// be ingested.
616+
int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
617+
618+
// Reason why ingestion failed for the specified message.
619+
oneof reason {
620+
// Optional. The Pub/Sub API limits prevented the desired message from
621+
// being published.
622+
ApiViolationReason api_violation_reason = 5
623+
[(google.api.field_behavior) = OPTIONAL];
624+
}
625+
}
626+
627+
// Failure when ingesting from a Confluent Cloud source.
628+
message ConfluentCloudFailureReason {
629+
// Optional. The cluster ID containing the topic being ingested from.
630+
string cluster_id = 1 [(google.api.field_behavior) = OPTIONAL];
631+
632+
// Optional. The name of the Kafka topic being ingested from.
633+
string kafka_topic = 2 [(google.api.field_behavior) = OPTIONAL];
634+
635+
// Optional. The partition ID of the message that failed to be ingested.
636+
int64 partition_id = 3 [(google.api.field_behavior) = OPTIONAL];
637+
638+
// Optional. The offset within the partition of the message that failed to
639+
// be ingested.
640+
int64 offset = 4 [(google.api.field_behavior) = OPTIONAL];
641+
642+
// Reason why ingestion failed for the specified message.
643+
oneof reason {
644+
// Optional. The Pub/Sub API limits prevented the desired message from
645+
// being published.
646+
ApiViolationReason api_violation_reason = 5
647+
[(google.api.field_behavior) = OPTIONAL];
648+
}
649+
}
650+
411651
// Required. Name of the import topic. Format is:
412652
// projects/{project_name}/topics/{topic_name}.
413653
string topic = 1 [(google.api.field_behavior) = REQUIRED];
@@ -419,6 +659,18 @@ message IngestionFailureEvent {
419659
// Optional. Failure when ingesting from Cloud Storage.
420660
CloudStorageFailure cloud_storage_failure = 3
421661
[(google.api.field_behavior) = OPTIONAL];
662+
663+
// Optional. Failure when ingesting from Amazon MSK.
664+
AwsMskFailureReason aws_msk_failure = 4
665+
[(google.api.field_behavior) = OPTIONAL];
666+
667+
// Optional. Failure when ingesting from Azure Event Hubs.
668+
AzureEventHubsFailureReason azure_event_hubs_failure = 5
669+
[(google.api.field_behavior) = OPTIONAL];
670+
671+
// Optional. Failure when ingesting from Confluent Cloud.
672+
ConfluentCloudFailureReason confluent_cloud_failure = 6
673+
[(google.api.field_behavior) = OPTIONAL];
422674
}
423675
}
424676

0 commit comments

Comments
 (0)