Skip to content

Commit cf8040d

Browse files
artursouzaartursouza
authored
Implement retry and timeout policy for gRPC client. (#889)
* Implement retry and timeout policy for gRPC client. Signed-off-by: Artur Souza <[email protected]> * Fix invoke actor after aborted flow. Signed-off-by: Artur Souza <[email protected]> --------- Signed-off-by: Artur Souza <[email protected]>
1 parent 6d65991 commit cf8040d

File tree

25 files changed

+1023
-116
lines changed

25 files changed

+1023
-116
lines changed

‎.github/workflows/build.yml‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ jobs:
4848
DAPR_INSTALL_URL: https://raw.githubusercontent.com/dapr/cli/v1.11.0-rc.1/install/install.sh
4949
DAPR_CLI_REF:
5050
DAPR_REF:
51+
TOXIPROXY_URL: https://github.com/Shopify/toxiproxy/releases/download/v2.5.0/toxiproxy-server-linux-amd64
5152
steps:
5253
- uses: actions/checkout@v3
5354
- name: Set up OpenJDK ${{ env.JDK_VER }}
@@ -101,14 +102,20 @@ jobs:
101102
docker stop dapr_placement
102103
cd dapr
103104
./dist/linux_amd64/release/placement &
104-
- name: Install Local kafka using docker-compose
105+
- name: Install local Kafka using docker-compose
105106
run: |
106107
docker-compose -f ./sdk-tests/deploy/local-test-kafka.yml up -d
107108
docker ps
108-
- name: Install Local mongo database using docker-compose
109+
- name: Install local Mongo database using docker-compose
109110
run: |
110111
docker-compose -f ./sdk-tests/deploy/local-test-mongo.yml up -d
111112
docker ps
113+
- name: Install local ToxiProxy to simulate connectivity issues to Dapr sidecar
114+
run: |
115+
mkdir -p /home/runner/.local/bin
116+
wget -q ${{ env.TOXIPROXY_URL }} -O /home/runner/.local/bin/toxiproxy-server
117+
chmod +x /home/runner/.local/bin/toxiproxy-server
118+
/home/runner/.local/bin/toxiproxy-server --version
112119
- name: Clean up files
113120
run: mvn clean -B
114121
- name: Build sdk

‎sdk-actors/src/main/java/io/dapr/actors/client/ActorClient.java‎

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import io.dapr.client.DaprApiProtocol;
1717
import io.dapr.client.DaprHttpBuilder;
18+
import io.dapr.client.resiliency.ResiliencyOptions;
1819
import io.dapr.config.Properties;
1920
import io.dapr.utils.Version;
2021
import io.dapr.v1.DaprGrpc;
@@ -46,26 +47,41 @@ public class ActorClient implements AutoCloseable {
4647
* Instantiates a new channel for Dapr sidecar communication.
4748
*/
4849
public ActorClient() {
49-
this(Properties.API_PROTOCOL.get());
50+
this(null);
51+
}
52+
53+
/**
54+
* Instantiates a new channel for Dapr sidecar communication.
55+
*
56+
* @param resiliencyOptions Client resiliency options.
57+
*/
58+
public ActorClient(ResiliencyOptions resiliencyOptions) {
59+
this(Properties.API_PROTOCOL.get(), resiliencyOptions);
5060
}
5161

5262
/**
5363
* Instantiates a new channel for Dapr sidecar communication.
5464
*
5565
* @param apiProtocol Dapr's API protocol.
66+
* @param resiliencyOptions Client resiliency options.
5667
*/
57-
private ActorClient(DaprApiProtocol apiProtocol) {
58-
this(apiProtocol, buildManagedChannel(apiProtocol));
68+
private ActorClient(DaprApiProtocol apiProtocol, ResiliencyOptions resiliencyOptions) {
69+
this(apiProtocol, buildManagedChannel(apiProtocol), resiliencyOptions);
5970
}
6071

6172
/**
6273
* Instantiates a new channel for Dapr sidecar communication.
6374
*
6475
* @param apiProtocol Dapr's API protocol.
76+
* @param grpcManagedChannel gRPC channel.
77+
* @param resiliencyOptions Client resiliency options.
6578
*/
66-
private ActorClient(DaprApiProtocol apiProtocol, ManagedChannel grpcManagedChannel) {
79+
private ActorClient(
80+
DaprApiProtocol apiProtocol,
81+
ManagedChannel grpcManagedChannel,
82+
ResiliencyOptions resiliencyOptions) {
6783
this.grpcManagedChannel = grpcManagedChannel;
68-
this.daprClient = buildDaprClient(apiProtocol, grpcManagedChannel);
84+
this.daprClient = buildDaprClient(apiProtocol, grpcManagedChannel, resiliencyOptions);
6985
}
7086

7187
/**
@@ -119,9 +135,12 @@ private static ManagedChannel buildManagedChannel(DaprApiProtocol apiProtocol) {
119135
* @return an instance of the setup Client
120136
* @throws java.lang.IllegalStateException if any required field is missing
121137
*/
122-
private static DaprClient buildDaprClient(DaprApiProtocol apiProtocol, Channel grpcManagedChannel) {
138+
private static DaprClient buildDaprClient(
139+
DaprApiProtocol apiProtocol,
140+
Channel grpcManagedChannel,
141+
ResiliencyOptions resiliencyOptions) {
123142
switch (apiProtocol) {
124-
case GRPC: return new DaprGrpcClient(DaprGrpc.newStub(grpcManagedChannel));
143+
case GRPC: return new DaprGrpcClient(DaprGrpc.newStub(grpcManagedChannel), resiliencyOptions);
125144
case HTTP: {
126145
LOGGER.warn("HTTP client protocol is deprecated and will be removed in Dapr's Java SDK version 1.10.");
127146
return new DaprHttpClient(new DaprHttpBuilder().build());

‎sdk-actors/src/main/java/io/dapr/actors/client/DaprGrpcClient.java‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,12 @@
1414
package io.dapr.actors.client;
1515

1616
import com.google.protobuf.ByteString;
17+
import io.dapr.client.resiliency.ResiliencyOptions;
1718
import io.dapr.config.Properties;
1819
import io.dapr.exceptions.DaprException;
1920
import io.dapr.internal.opencensus.GrpcWrapper;
21+
import io.dapr.internal.resiliency.RetryPolicy;
22+
import io.dapr.internal.resiliency.TimeoutPolicy;
2023
import io.dapr.v1.DaprGrpc;
2124
import io.dapr.v1.DaprProtos;
2225
import io.grpc.CallOptions;
@@ -39,18 +42,33 @@
3942
*/
4043
class DaprGrpcClient implements DaprClient {
4144

45+
/**
46+
* Timeout policy for SDK calls to Dapr API.
47+
*/
48+
private final TimeoutPolicy timeoutPolicy;
49+
50+
/**
51+
* Retry policy for SDK calls to Dapr API.
52+
*/
53+
private final RetryPolicy retryPolicy;
54+
4255
/**
4356
* The async gRPC stub.
4457
*/
45-
private DaprGrpc.DaprStub client;
58+
private final DaprGrpc.DaprStub client;
4659

4760
/**
4861
* Internal constructor.
4962
*
5063
* @param grpcClient Dapr's GRPC client.
64+
* @param resiliencyOptions Client resiliency options (optional)
5165
*/
52-
DaprGrpcClient(DaprGrpc.DaprStub grpcClient) {
66+
DaprGrpcClient(DaprGrpc.DaprStub grpcClient, ResiliencyOptions resiliencyOptions) {
5367
this.client = intercept(grpcClient);
68+
this.timeoutPolicy = new TimeoutPolicy(
69+
resiliencyOptions == null ? null : resiliencyOptions.getTimeout());
70+
this.retryPolicy = new RetryPolicy(
71+
resiliencyOptions == null ? null : resiliencyOptions.getMaxRetries());
5472
}
5573

5674
/**
@@ -78,14 +96,14 @@ public Mono<byte[]> invoke(String actorType, String actorId, String methodName,
7896
* @param client GRPC client for Dapr.
7997
* @return Client after adding interceptors.
8098
*/
81-
private static DaprGrpc.DaprStub intercept(DaprGrpc.DaprStub client) {
99+
private DaprGrpc.DaprStub intercept(DaprGrpc.DaprStub client) {
82100
ClientInterceptor interceptor = new ClientInterceptor() {
83101
@Override
84102
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
85103
MethodDescriptor<ReqT, RespT> methodDescriptor,
86-
CallOptions callOptions,
104+
CallOptions options,
87105
Channel channel) {
88-
ClientCall<ReqT, RespT> clientCall = channel.newCall(methodDescriptor, callOptions);
106+
ClientCall<ReqT, RespT> clientCall = channel.newCall(methodDescriptor, timeoutPolicy.apply(options));
89107
return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(clientCall) {
90108
@Override
91109
public void start(final Listener<RespT> responseListener, final Metadata metadata) {
@@ -114,7 +132,8 @@ private static DaprGrpc.DaprStub intercept(ContextView context, DaprGrpc.DaprStu
114132
}
115133

116134
private <T> Mono<T> createMono(Consumer<StreamObserver<T>> consumer) {
117-
return Mono.create(sink -> DaprException.wrap(() -> consumer.accept(createStreamObserver(sink))).run());
135+
return retryPolicy.apply(
136+
Mono.create(sink -> DaprException.wrap(() -> consumer.accept(createStreamObserver(sink))).run()));
118137
}
119138

120139
private <T> StreamObserver<T> createStreamObserver(MonoSink<T> sink) {

‎sdk-actors/src/main/java/io/dapr/actors/runtime/AbstractActor.java‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.io.IOException;
2121
import java.time.Duration;
2222
import java.util.UUID;
23+
import java.util.concurrent.atomic.AtomicBoolean;
2324

2425
/**
2526
* Represents the base class for actors.
@@ -28,8 +29,6 @@
2829
*/
2930
public abstract class AbstractActor {
3031

31-
private static final ActorObjectSerializer INTERNAL_SERIALIZER = new ActorObjectSerializer();
32-
3332
/**
3433
* Type of tracing messages.
3534
*/
@@ -58,7 +57,7 @@ public abstract class AbstractActor {
5857
/**
5958
* Internal control to assert method invocation on start and finish in this SDK.
6059
*/
61-
private boolean started;
60+
private final AtomicBoolean started;
6261

6362
/**
6463
* Instantiates a new Actor.
@@ -74,7 +73,7 @@ protected AbstractActor(ActorRuntimeContext runtimeContext, ActorId id) {
7473
runtimeContext.getActorTypeInformation().getName(),
7574
id);
7675
this.actorTrace = runtimeContext.getActorTrace();
77-
this.started = false;
76+
this.started = new AtomicBoolean(false);
7877
}
7978

8079
/**
@@ -250,14 +249,16 @@ protected Mono<Void> saveState() {
250249

251250
/**
252251
* Resets the cached state of this Actor.
252+
*
253+
* @param force Forces the rollback, even if not in a call.
253254
*/
254-
void rollback() {
255-
if (!this.started) {
255+
void rollback(boolean force) {
256+
if (!force && !this.started.get()) {
256257
throw new IllegalStateException("Cannot reset state before starting call.");
257258
}
258259

259260
this.resetState();
260-
this.started = false;
261+
this.started.set(false);
261262
}
262263

263264
/**
@@ -302,11 +303,12 @@ Mono<Void> onDeactivateInternal() {
302303
*/
303304
Mono<Void> onPreActorMethodInternal(ActorMethodContext actorMethodContext) {
304305
return Mono.fromRunnable(() -> {
305-
if (this.started) {
306-
throw new IllegalStateException("Cannot invoke a method before completing previous call.");
306+
if (this.started.get()) {
307+
throw new IllegalStateException(
308+
"Cannot invoke a method before completing previous call. " + getId().toString());
307309
}
308310

309-
this.started = true;
311+
this.started.set(true);
310312
}).then(this.onPreActorMethod(actorMethodContext));
311313
}
312314

@@ -318,14 +320,13 @@ Mono<Void> onPreActorMethodInternal(ActorMethodContext actorMethodContext) {
318320
*/
319321
Mono<Void> onPostActorMethodInternal(ActorMethodContext actorMethodContext) {
320322
return Mono.fromRunnable(() -> {
321-
if (!this.started) {
323+
if (!this.started.get()) {
322324
throw new IllegalStateException("Cannot complete a method before starting a call.");
323325
}
324-
}).then(this.onPostActorMethod(actorMethodContext))
325-
.then(this.saveState())
326-
.then(Mono.fromRunnable(() -> {
327-
this.started = false;
328-
}));
326+
})
327+
.then(this.onPostActorMethod(actorMethodContext))
328+
.then(this.saveState())
329+
.then(Mono.fromRunnable(() -> this.started.set(false)));
329330
}
330331

331332
/**

‎sdk-actors/src/main/java/io/dapr/actors/runtime/ActorManager.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -306,15 +306,16 @@ private <T> Mono<T> invoke(ActorId actorId, ActorMethodContext context, Function
306306
this.runtimeContext.getActorTypeInformation().getName()));
307307
}
308308

309-
return actor.onPreActorMethodInternal(context)
309+
return Mono.fromRunnable(() -> actor.rollback(true))
310+
.onErrorMap(throwable -> {
311+
actor.rollback(false);
312+
return throwable;
313+
})
314+
.then(actor.onPreActorMethodInternal(context))
310315
.then((Mono<Object>) func.apply(actor))
311316
.switchIfEmpty(
312317
actor.onPostActorMethodInternal(context))
313318
.flatMap(r -> actor.onPostActorMethodInternal(context).thenReturn(r))
314-
.onErrorMap(throwable -> {
315-
actor.rollback();
316-
return throwable;
317-
})
318319
.map(o -> (T) o);
319320
} catch (Exception e) {
320321
return Mono.error(e);

‎sdk-actors/src/test/java/io/dapr/actors/client/DaprGrpcClientTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public void setup() throws IOException {
105105
InProcessChannelBuilder.forName(serverName).directExecutor().build());
106106

107107
// Create a HelloWorldClient using the in-process channel;
108-
client = new DaprGrpcClient(DaprGrpc.newStub(channel));
108+
client = new DaprGrpcClient(DaprGrpc.newStub(channel), null);
109109
}
110110

111111
@Test

‎sdk-tests/components/mongo-statestore.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,7 @@ spec:
1111
- name: databaseName
1212
value: local
1313
- name: collectionName
14-
value: testCollection
14+
value: testCollection
15+
scopes:
16+
- grpcstateclientit
17+
- httpstateclientit

‎sdk-tests/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,11 @@
146146
<version>1.3.5</version>
147147
<scope>compile</scope>
148148
</dependency>
149+
<dependency>
150+
<groupId>eu.rekawek.toxiproxy</groupId>
151+
<artifactId>toxiproxy-java</artifactId>
152+
<version>2.1.7</version>
153+
</dependency>
149154
</dependencies>
150155

151156
<build>

‎sdk-tests/src/test/java/io/dapr/it/BaseIT.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
import io.dapr.actors.client.ActorClient;
1717
import io.dapr.client.DaprApiProtocol;
18+
import io.dapr.client.resiliency.ResiliencyOptions;
1819
import org.apache.commons.lang3.tuple.ImmutablePair;
1920
import org.junit.AfterClass;
2021

@@ -194,8 +195,12 @@ public static void cleanUp() throws Exception {
194195
}
195196
}
196197

197-
protected ActorClient newActorClient() {
198-
ActorClient client = new ActorClient();
198+
protected static ActorClient newActorClient() {
199+
return newActorClient(null);
200+
}
201+
202+
protected static ActorClient newActorClient(ResiliencyOptions resiliencyOptions) {
203+
ActorClient client = new ActorClient(resiliencyOptions);
199204
TO_BE_CLOSED.add(client);
200205
return client;
201206
}

‎sdk-tests/src/test/java/io/dapr/it/Command.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,25 @@ public void run() throws InterruptedException, IOException {
8383
}
8484
});
8585

86+
final Thread stderrReader = new Thread(() -> {
87+
try {
88+
try (InputStream stderr = this.process.getErrorStream()) {
89+
try (InputStreamReader isr = new InputStreamReader(stderr)) {
90+
try (BufferedReader br = new BufferedReader(isr)) {
91+
String line;
92+
while ((line = br.readLine()) != null) {
93+
System.err.println(line);
94+
}
95+
}
96+
}
97+
}
98+
} catch (IOException ex) {
99+
throw new RuntimeException(ex);
100+
}
101+
});
102+
86103
stdoutReader.start();
104+
stderrReader.start();
87105
// Waits for success to happen within 1 minute.
88106
finished.tryAcquire(SUCCESS_WAIT_TIMEOUT_MINUTES, TimeUnit.MINUTES);
89107
if (!success.get()) {

0 commit comments

Comments
 (0)