Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit bb01d89

Browse files
zoewanggzoewangg
committedMar 28, 2019
Add benchmarks for async operations
1 parent acc8b54 commit bb01d89

26 files changed

+890
-126
lines changed
 

‎test/sdk-benchmarks/README.md

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,38 @@
1-
# Basic usage
1+
# Sdk Benchmark Harness
22

3+
4+
This module contains sdk benchmark harness using [JMH].
5+
6+
Each benchmark class has a set of default
7+
JMH configurations tailored to SDK's build job and you might need to
8+
adjust them based on your test environment such as increasing warmup iterations
9+
or measurement time in order to get more reliable data.
10+
11+
There are three ways to run benchmarks.
12+
13+
- Using the executable JAR (Preferred usage per JMH site)
314
```bash
15+
mvn clean install -P quick -pl :sdk-benchmarks --am
416

17+
# Run specific benchmark
18+
java -jar target/benchmarks.jar ApacheHttpClientBenchmark
519

6-
# Run 3 warm up iterations, 3 benchmark iterations, 1 fork
20+
# Run all benchmarks: 3 warm up iterations, 3 benchmark iterations, 1 fork
721
java -jar target/benchmarks.jar -wi 3 -i 3 -f 1
822
```
923

10-
See http://openjdk.java.net/projects/code-tools/jmh/ for more information on JMH.
24+
- Using`mvn exec:exec` commands to invoke `BenchmarkRunner` main method
25+
```bash
26+
mvn clean install -P quick -pl :sdk-benchmarks --am
27+
mvn clean install -pl :bom-internal
28+
cd test/sdk-benchmarks
29+
mvn exec:exec
30+
```
31+
32+
- From IDE
33+
34+
You can run the main method within each Benchmark class from your IDE. If you are using Eclipse, you might need to
35+
set up build configurations for JMH annotation, please check [JMH]. Note that the benchmark result from IDE
36+
might not be as reliable as the above approaches and is generally not recommended.
37+
38+
[JMH]: http://openjdk.java.net/projects/code-tools/jmh/

‎test/sdk-benchmarks/pom.xml

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,16 @@
129129
<artifactId>url-connection-client</artifactId>
130130
<version>${awsjavasdk.version}</version>
131131
</dependency>
132+
<dependency>
133+
<groupId>software.amazon.awssdk</groupId>
134+
<artifactId>netty-nio-client</artifactId>
135+
<version>${awsjavasdk.version}</version>
136+
</dependency>
137+
<dependency>
138+
<groupId>io.netty</groupId>
139+
<artifactId>netty-tcnative-boringssl-static</artifactId>
140+
<scope>compile</scope>
141+
</dependency>
132142
<dependency>
133143
<groupId>software.amazon.awssdk</groupId>
134144
<artifactId>regions</artifactId>
@@ -153,14 +163,33 @@
153163
<dependency>
154164
<groupId>org.slf4j</groupId>
155165
<artifactId>slf4j-log4j12</artifactId>
156-
<version>1.7.25</version>
166+
<scope>test</scope>
167+
</dependency>
168+
<dependency>
169+
<groupId>log4j</groupId>
170+
<artifactId>log4j</artifactId>
171+
<scope>test</scope>
157172
</dependency>
158173
<dependency>
159174
<groupId>org.eclipse.jetty</groupId>
160175
<artifactId>jetty-servlet</artifactId>
161-
<version>9.2.22.v20170606</version>
162176
</dependency>
163-
177+
<dependency>
178+
<groupId>org.eclipse.jetty</groupId>
179+
<artifactId>jetty-server</artifactId>
180+
</dependency>
181+
<dependency>
182+
<groupId>org.eclipse.jetty.http2</groupId>
183+
<artifactId>http2-server</artifactId>
184+
</dependency>
185+
<dependency>
186+
<groupId>org.eclipse.jetty</groupId>
187+
<artifactId>jetty-alpn-java-server</artifactId>
188+
</dependency>
189+
<dependency>
190+
<groupId>org.eclipse.jetty.http2</groupId>
191+
<artifactId>http2-hpack</artifactId>
192+
</dependency>
164193
</dependencies>
165194

166195
<dependencyManagement>
@@ -187,6 +216,13 @@
187216
<type>pom</type>
188217
<scope>import</scope>
189218
</dependency>
219+
<dependency>
220+
<groupId>org.eclipse.jetty</groupId>
221+
<artifactId>jetty-bom</artifactId>
222+
<version>9.4.15.v20190215</version>
223+
<type>pom</type>
224+
<scope>import</scope>
225+
</dependency>
190226
</dependencies>
191227
</dependencyManagement>
192228

@@ -271,11 +307,22 @@
271307
</goals>
272308
<configuration>
273309
<finalName>${uberjar.name}</finalName>
310+
<createDependencyReducedPom>false</createDependencyReducedPom>
311+
<artifactSet>
312+
<includes>
313+
<include>*:*</include>
314+
</includes>
315+
</artifactSet>
274316
<transformers>
275317
<transformer
276318
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
277-
<mainClass>org.openjdk.jmh.Main</mainClass>
319+
<manifestEntries>
320+
<Main-Class>org.openjdk.jmh.Main</Main-Class>
321+
</manifestEntries>
278322
</transformer>
323+
<!-- This is important to run h2 server -->
324+
<!-- https://github.com/jetty-project/embedded-jetty-live-war/blob/master/theserver/pom.xml -->
325+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
279326
</transformers>
280327
<filters>
281328
<filter>

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
package software.amazon.awssdk.benchmark;
1717

18-
import java.io.IOException;
18+
import java.util.ArrayList;
1919
import java.util.Arrays;
2020
import java.util.Collection;
2121
import java.util.List;
@@ -24,23 +24,53 @@
2424
import org.openjdk.jmh.runner.RunnerException;
2525
import org.openjdk.jmh.runner.options.ChainedOptionsBuilder;
2626
import org.openjdk.jmh.runner.options.OptionsBuilder;
27+
import software.amazon.awssdk.benchmark.apicall.httpclient.async.NettyClientH1NonTlsBenchmark;
28+
import software.amazon.awssdk.benchmark.apicall.httpclient.async.NettyHttpClientH1Benchmark;
29+
import software.amazon.awssdk.benchmark.apicall.httpclient.async.NettyHttpClientH2Benchmark;
30+
import software.amazon.awssdk.benchmark.apicall.httpclient.sync.ApacheHttpClientBenchmark;
31+
import software.amazon.awssdk.benchmark.apicall.httpclient.sync.UrlConnectionHttpClientBenchmark;
32+
import software.amazon.awssdk.benchmark.apicall.protocol.Ec2ProtocolBenchmark;
33+
import software.amazon.awssdk.benchmark.apicall.protocol.JsonProtocolBenchmark;
34+
import software.amazon.awssdk.benchmark.apicall.protocol.QueryProtocolBenchmark;
35+
import software.amazon.awssdk.benchmark.apicall.protocol.XmlProtocolBenchmark;
36+
import software.amazon.awssdk.benchmark.coldstart.V2DefaultClientCreationBenchmark;
37+
import software.amazon.awssdk.benchmark.coldstart.V2OptimizedClientCreationBenchmark;
2738
import software.amazon.awssdk.utils.Logger;
2839

2940
public class BenchmarkRunner {
3041

31-
private static final List<String> BENCHMARKS_TO_RUN =
32-
Arrays.asList("Ec2ProtocolBenchmark", "JsonProtocolBenchmark", "QueryProtocolBenchmark", "XmlProtocolBenchmark",
33-
"V2OptimizedClientCreationBenchmark", "V1ClientCreationBenchmark", "V2DefaultClientCreationBenchmark",
34-
"ApacheHttpClientBenchmark", "UrlConnectionHttpClientClientBenchmark");
42+
private static final List<String> PROTOCOL_BENCHMARKS = Arrays.asList(
43+
Ec2ProtocolBenchmark.class.getSimpleName(), JsonProtocolBenchmark.class.getSimpleName(),
44+
QueryProtocolBenchmark.class.getSimpleName(), XmlProtocolBenchmark.class.getSimpleName());
45+
46+
private static final List<String> ASYNC_BENCHMARKS = Arrays.asList(
47+
NettyHttpClientH2Benchmark.class.getSimpleName(),
48+
NettyHttpClientH1Benchmark.class.getSimpleName(),
49+
NettyClientH1NonTlsBenchmark.class.getSimpleName());
50+
51+
private static final List<String> SYNC_BENCHMARKS = Arrays.asList(
52+
ApacheHttpClientBenchmark.class.getSimpleName(),
53+
UrlConnectionHttpClientBenchmark.class.getSimpleName());
54+
55+
private static final List<String> COLD_START_BENCHMARKS = Arrays.asList(
56+
V2OptimizedClientCreationBenchmark.class.getSimpleName(),
57+
V2DefaultClientCreationBenchmark.class.getSimpleName());
58+
3559
private static final Logger log = Logger.loggerFor(BenchmarkRunner.class);
60+
3661
private final List<String> benchmarksToRun;
3762

3863
private BenchmarkRunner(List<String> benchmarksToRun) {
3964
this.benchmarksToRun = benchmarksToRun;
4065
}
4166

42-
public static void main(String... args) throws RunnerException, IOException {
43-
BenchmarkRunner runner = new BenchmarkRunner(BENCHMARKS_TO_RUN);
67+
public static void main(String... args) throws RunnerException {
68+
List<String> benchmarksToRun = new ArrayList<>(SYNC_BENCHMARKS);
69+
benchmarksToRun.addAll(PROTOCOL_BENCHMARKS);
70+
benchmarksToRun.addAll(COLD_START_BENCHMARKS);
71+
benchmarksToRun.addAll(ASYNC_BENCHMARKS);
72+
73+
BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun);
4474
runner.runBenchmark();
4575
}
4676

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/httpclient/SdkHttpClientBenchmark.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark.apicall.httpclient;
17+
18+
import org.openjdk.jmh.infra.Blackhole;
19+
20+
/**
21+
* Interface to be used for sdk http client benchmark
22+
*/
23+
public interface SdkHttpClientBenchmark {
24+
25+
/**
26+
* Benchmark for sequential api calls
27+
*
28+
* @param blackhole the blackhole
29+
*/
30+
void sequentialApiCall(Blackhole blackhole);
31+
32+
/**
33+
* Benchmark for concurrent api calls.
34+
*
35+
* <p>Not applies to all sdk http clients such as UrlConnectionHttpClient.
36+
* Running with UrlConnectionHttpClient has high error rate because it doesn't
37+
* support connection pooling.
38+
*
39+
* @param blackhole the blackhole
40+
*/
41+
default void concurrentApiCall(Blackhole blackhole) {}
42+
}

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/httpclient/async/BaseNettyBenchmark.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark.apicall.httpclient.async;
17+
18+
import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.CONCURRENT_CALLS;
19+
import static software.amazon.awssdk.benchmark.utils.BenchmarkUtils.awaitCountdownLatchUninterruptibly;
20+
import static software.amazon.awssdk.benchmark.utils.BenchmarkUtils.countDownUponCompletion;
21+
22+
import java.util.concurrent.CountDownLatch;
23+
import java.util.concurrent.TimeUnit;
24+
import org.openjdk.jmh.annotations.Benchmark;
25+
import org.openjdk.jmh.annotations.OperationsPerInvocation;
26+
import org.openjdk.jmh.infra.Blackhole;
27+
import software.amazon.awssdk.benchmark.apicall.httpclient.SdkHttpClientBenchmark;
28+
import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonAsyncClient;
29+
30+
/**
31+
* Base class for netty benchmark
32+
*/
33+
public abstract class BaseNettyBenchmark implements SdkHttpClientBenchmark {
34+
35+
protected ProtocolRestJsonAsyncClient client;
36+
37+
@Override
38+
@Benchmark
39+
@OperationsPerInvocation(CONCURRENT_CALLS)
40+
public void concurrentApiCall(Blackhole blackhole) {
41+
CountDownLatch countDownLatch = new CountDownLatch(CONCURRENT_CALLS);
42+
for (int i = 0; i < CONCURRENT_CALLS; i++) {
43+
countDownUponCompletion(blackhole, client.allTypes(), countDownLatch);
44+
}
45+
46+
awaitCountdownLatchUninterruptibly(countDownLatch, 10, TimeUnit.SECONDS);
47+
}
48+
49+
@Override
50+
@Benchmark
51+
public void sequentialApiCall(Blackhole blackhole) {
52+
CountDownLatch countDownLatch = new CountDownLatch(1);
53+
countDownUponCompletion(blackhole, client.allTypes(), countDownLatch);
54+
awaitCountdownLatchUninterruptibly(countDownLatch, 1, TimeUnit.SECONDS);
55+
}
56+
}

‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/sync/ApacheHttpClientBenchmark.java renamed to ‎test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apicall/httpclient/async/NettyClientH1NonTlsBenchmark.java

Lines changed: 12 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,10 @@
1313
* permissions and limitations under the License.
1414
*/
1515

16-
package software.amazon.awssdk.benchmark.apicall.sync;
17-
18-
import static software.amazon.awssdk.benchmark.utils.BenchmarkUtil.LOCAL_URI;
19-
import static software.amazon.awssdk.benchmark.utils.BenchmarkUtil.PORT_NUMBER;
16+
package software.amazon.awssdk.benchmark.apicall.httpclient.async;
2017

2118
import java.util.Collection;
2219
import java.util.concurrent.TimeUnit;
23-
import org.openjdk.jmh.annotations.Benchmark;
2420
import org.openjdk.jmh.annotations.BenchmarkMode;
2521
import org.openjdk.jmh.annotations.Fork;
2622
import org.openjdk.jmh.annotations.Level;
@@ -31,58 +27,46 @@
3127
import org.openjdk.jmh.annotations.State;
3228
import org.openjdk.jmh.annotations.TearDown;
3329
import org.openjdk.jmh.annotations.Warmup;
34-
import org.openjdk.jmh.infra.Blackhole;
3530
import org.openjdk.jmh.profile.StackProfiler;
3631
import org.openjdk.jmh.results.RunResult;
3732
import org.openjdk.jmh.runner.Runner;
3833
import org.openjdk.jmh.runner.options.Options;
3934
import org.openjdk.jmh.runner.options.OptionsBuilder;
4035
import software.amazon.awssdk.benchmark.utils.MockServer;
41-
import software.amazon.awssdk.http.SdkHttpClient;
42-
import software.amazon.awssdk.http.apache.ApacheHttpClient;
43-
import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonClient;
36+
import software.amazon.awssdk.services.protocolrestjson.ProtocolRestJsonAsyncClient;
4437

4538
/**
46-
* Benchmarking for running with different http clients.
39+
* Using netty client to test against local mock http server.
4740
*/
48-
@State(Scope.Thread)
41+
@State(Scope.Benchmark)
4942
@Warmup(iterations = 3, time = 15, timeUnit = TimeUnit.SECONDS)
5043
@Measurement(iterations = 5, time = 10, timeUnit = TimeUnit.SECONDS)
5144
@Fork(2) // To reduce difference between each run
5245
@BenchmarkMode(Mode.Throughput)
53-
public class ApacheHttpClientBenchmark implements SdkApiCallBenchmark {
46+
public class NettyClientH1NonTlsBenchmark extends BaseNettyBenchmark {
5447

5548
private MockServer mockServer;
56-
private SdkHttpClient sdkHttpClient;
57-
private ProtocolRestJsonClient client;
5849

5950
@Setup(Level.Trial)
6051
public void setup() throws Exception {
61-
mockServer = new MockServer(PORT_NUMBER);
52+
mockServer = new MockServer();
6253
mockServer.start();
63-
sdkHttpClient = ApacheHttpClient.builder().build();
64-
client = ProtocolRestJsonClient.builder()
65-
.endpointOverride(LOCAL_URI)
66-
.httpClient(sdkHttpClient)
67-
.build();
54+
client = ProtocolRestJsonAsyncClient.builder()
55+
.endpointOverride(mockServer.getHttpUri())
56+
.build();
57+
// Making sure the request actually succeeds
58+
client.allTypes().join();
6859
}
6960

7061
@TearDown(Level.Trial)
7162
public void tearDown() throws Exception {
7263
mockServer.stop();
73-
sdkHttpClient.close();
7464
client.close();
7565
}
7666

77-
@Benchmark
78-
public void apiCall(Blackhole blackhole) {
79-
blackhole.consume(client.allTypes());
80-
}
81-
8267
public static void main(String... args) throws Exception {
83-
8468
Options opt = new OptionsBuilder()
85-
.include(ApacheHttpClientBenchmark.class.getSimpleName())
69+
.include(NettyClientH1NonTlsBenchmark.class.getSimpleName())
8670
.addProfiler(StackProfiler.class)
8771
.build();
8872
Collection<RunResult> run = new Runner(opt).run();

0 commit comments

Comments
 (0)
Please sign in to comment.