Skip to content

Commit bc983e7

Browse files
josecorellatexastony
josecorella
and
texastony
committed
test: add concurrency testing for storage operations (#1132)
Co-authored-by: Tony Knapp <[email protected]>
1 parent 17f37df commit bc983e7

File tree

7 files changed

+892
-0
lines changed

7 files changed

+892
-0
lines changed

‎.github/workflows/library_concurrency_tests.yml

+116
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# This workflow performs Concurrency tests of the MPL in Java.
2+
name: Library Concurrency Tests
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
dafny:
8+
description: "The Dafny version to run"
9+
required: true
10+
type: string
11+
regenerate-code:
12+
description: "Regenerate code using smithy-dafny"
13+
required: false
14+
default: false
15+
type: boolean
16+
17+
jobs:
18+
generateEncryptVectors:
19+
strategy:
20+
matrix:
21+
library: [AwsCryptographicMaterialProviders]
22+
os: [
23+
# https://taskei.amazon.dev/tasks/CrypTool-5283
24+
# windows-latest,
25+
ubuntu-latest,
26+
macos-13,
27+
]
28+
language: [
29+
java,
30+
# net,
31+
# python,
32+
# rust
33+
]
34+
# https://taskei.amazon.dev/tasks/CrypTool-5284
35+
java-versions: [8, 17]
36+
runs-on: ${{ matrix.os }}
37+
permissions:
38+
id-token: write
39+
contents: read
40+
steps:
41+
- name: Support longpaths on Git checkout
42+
run: |
43+
git config --global core.longpaths true
44+
45+
# Test Vectors need to call KMS
46+
- name: Configure AWS Credentials for Tests
47+
uses: aws-actions/configure-aws-credentials@v2
48+
with:
49+
aws-region: us-west-2
50+
role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-MPL-Dafny-Role-us-west-2
51+
role-session-name: ConcurrencyTests
52+
53+
- uses: actions/checkout@v3
54+
# Not all submodules are needed.
55+
# We manually pull the submodule we DO need.
56+
- run: git submodule update --init libraries
57+
- run: git submodule update --init --recursive smithy-dafny
58+
59+
# Setup Java in Rust is needed for running polymorph
60+
- name: Setup Java 17
61+
if: matrix.language == 'java' || matrix.language == 'rust'
62+
uses: actions/setup-java@v3
63+
with:
64+
distribution: "corretto"
65+
java-version: 17
66+
67+
- name: Setup .NET Core SDK '6.0.x'
68+
uses: actions/setup-dotnet@v3
69+
with:
70+
dotnet-version: "6.0.x"
71+
72+
- name: Setup Dafny
73+
uses: dafny-lang/[email protected]
74+
with:
75+
dafny-version: ${{ inputs.dafny }}
76+
77+
- name: Regenerate code using smithy-dafny if necessary
78+
if: ${{ inputs.regenerate-code }}
79+
uses: ./.github/actions/polymorph_codegen
80+
with:
81+
dafny: ${{ inputs.dafny }}
82+
library: ${{ matrix.library }}
83+
diff-generated-code: false
84+
85+
# Build implementation for each runtime
86+
- name: Build ${{ matrix.library }} implementation in Java
87+
shell: bash
88+
working-directory: ./${{ matrix.library }}
89+
run: |
90+
# This works because `node` is installed by default on GHA runners
91+
CORES=$(node -e 'console.log(os.cpus().length)')
92+
make build_java CORES=$CORES
93+
94+
- name: Setup gradle
95+
if: matrix.language == 'java'
96+
uses: gradle/gradle-build-action@v2
97+
with:
98+
gradle-version: 7.2
99+
100+
- name: Setup Java ${{matrix.java-versions}}
101+
uses: actions/setup-java@v3
102+
with:
103+
distribution: "corretto"
104+
java-version: ${{matrix.java-versions}}
105+
106+
- name: Compile Java
107+
uses: gradle/gradle-build-action@v3
108+
with:
109+
arguments: build
110+
build-root-directory: ./${{ matrix.library }}/runtimes/java
111+
112+
- name: Test Java
113+
uses: gradle/gradle-build-action@v3
114+
with:
115+
arguments: testConcurrentExamples
116+
build-root-directory: ./${{ matrix.library }}/runtimes/java

‎AwsCryptographicMaterialProviders/runtimes/java/build.gradle.kts

+24
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,30 @@ val testExamples = task<Test>("testExamples") {
315315
testLogging {
316316
events("passed")
317317
}
318+
filter {
319+
excludeTestsMatching("software.amazon.cryptography.example.hierarchy.concurrent.*")
320+
}
321+
}
322+
323+
val testConcurrentExamples = task<Test>("testConcurrentExamples") {
324+
description = "Runs concurrency tests."
325+
group = "verification"
326+
327+
testClassesDirs = sourceSets["testExamples"].output.classesDirs
328+
classpath = sourceSets["testExamples"].runtimeClasspath + sourceSets["examples"].output + sourceSets.main.get().output
329+
// This will show System.out.println statements
330+
testLogging.showStandardStreams = true
331+
useTestNG() {
332+
suites("src/testExamples/java/software/amazon/cryptography/example/hierarchy/concurrent/testng-parallel.xml")
333+
maxParallelForks = 2
334+
}
335+
336+
testLogging {
337+
events("passed")
338+
}
339+
filter {
340+
includeTestsMatching("software.amazon.cryptography.example.hierarchy.concurrent.*")
341+
}
318342
}
319343

320344
fun buildPom(mavenPublication: MavenPublication) {

‎AwsCryptographicMaterialProviders/runtimes/java/src/testExamples/java/software/amazon/cryptography/example/hierarchy/concurrent/ConcurrentConditionCheckWriteTest.java

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
package software.amazon.cryptography.example.hierarchy.concurrent;
2+
3+
import java.text.DateFormat;
4+
import java.text.SimpleDateFormat;
5+
import java.util.Arrays;
6+
import java.util.Collections;
7+
import java.util.Date;
8+
import java.util.HashMap;
9+
import java.util.List;
10+
import java.util.Map;
11+
import java.util.TimeZone;
12+
import java.util.concurrent.ConcurrentHashMap;
13+
import java.util.concurrent.ConcurrentLinkedDeque;
14+
import org.testng.Assert;
15+
import org.testng.annotations.AfterClass;
16+
import org.testng.annotations.BeforeClass;
17+
import org.testng.annotations.Test;
18+
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
19+
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
20+
import software.amazon.awssdk.services.dynamodb.model.GetItemResponse;
21+
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItem;
22+
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsRequest;
23+
import software.amazon.awssdk.services.dynamodb.model.TransactWriteItemsResponse;
24+
import software.amazon.awssdk.services.dynamodb.model.TransactionCanceledException;
25+
import software.amazon.awssdk.utils.ImmutableMap;
26+
import software.amazon.cryptography.example.DdbHelper;
27+
import software.amazon.cryptography.example.Fixtures;
28+
29+
// These concurrent tests check that DynamoDB behaves the way we expect when
30+
// there are multiple request to write an item to DynamoDB. Our libraries use the
31+
// TransactWriteItems API with a condition check that the primary key we are writing
32+
// does not exist. This will result in either 1. ConditionCheckFailure if an item has been
33+
// written, and we are trying to write over it and 2. TransactionConflict if we are trying to write
34+
// while there is a transaction being committed.
35+
public class ConcurrentConditionCheckWriteTest {
36+
37+
private static final Integer threadCount = 5;
38+
private static final String mLockedId = "concurrency-test-write-key";
39+
private static final Map<String, String> INDEX_EXPR_ATT_NAMES =
40+
ImmutableMap.of("#pk", "branch-key-id");
41+
42+
private static final List<String> identifiers = Collections.unmodifiableList(
43+
Arrays.asList("1", "2", "3", "4", "5")
44+
);
45+
private Map<String, DynamoDbClient> threadIdToDdbClient;
46+
private static Map<String, String> indexToThreadId;
47+
private ConcurrentLinkedDeque<String> unpickedIndices;
48+
49+
@BeforeClass
50+
public void setup() {
51+
threadIdToDdbClient = new ConcurrentHashMap<>(6, 1, threadCount);
52+
identifiers.forEach(id ->
53+
threadIdToDdbClient.put(id, DynamoDbClient.create())
54+
);
55+
indexToThreadId = new ConcurrentHashMap<>(6, 1, threadCount);
56+
unpickedIndices = new ConcurrentLinkedDeque<>(identifiers);
57+
}
58+
59+
@AfterClass
60+
public void teardown() {
61+
DynamoDbClient _ddbClient = DynamoDbClient.create();
62+
identifiers.forEach(id ->
63+
DdbHelper.deleteKeyStoreDdbItem(
64+
mLockedId,
65+
"branch:ACTIVE",
66+
Fixtures.TEST_KEYSTORE_NAME,
67+
_ddbClient,
68+
true
69+
)
70+
);
71+
}
72+
73+
public static Map<String, AttributeValue> indexItem(
74+
final AttributeValue value,
75+
final String timestamp
76+
) {
77+
Map<String, AttributeValue> item = new HashMap<>();
78+
79+
item.put("branch-key-id", AttributeValue.builder().s(mLockedId).build());
80+
item.put("type", AttributeValue.builder().s(indexType()).build());
81+
item.put("value", value);
82+
item.put("timestamp", AttributeValue.builder().s(timestamp).build());
83+
return item;
84+
}
85+
86+
private static String indexType() {
87+
return "branch:ACTIVE";
88+
}
89+
90+
public static TransactWriteItem conditionalWrite(
91+
final AttributeValue value,
92+
final String timestamp
93+
) {
94+
return TransactWriteItem
95+
.builder()
96+
.put(putBuilder ->
97+
putBuilder
98+
.tableName(Fixtures.TEST_KEYSTORE_NAME)
99+
.item(indexItem(value, timestamp))
100+
.conditionExpression("attribute_not_exists(#pk)")
101+
.expressionAttributeNames(INDEX_EXPR_ATT_NAMES)
102+
)
103+
.build();
104+
}
105+
106+
private DynamoDbClient clientForThread(final String threadIdToIndex) {
107+
return threadIdToDdbClient.computeIfAbsent(
108+
threadIdToIndex,
109+
ddbClient -> DynamoDbClient.create()
110+
);
111+
}
112+
113+
@Test(threadPoolSize = 5, invocationCount = 30, timeOut = 1000)
114+
public void TestConcurrentWriteCheck() {
115+
String threadId = String.valueOf(Thread.currentThread().getId());
116+
String threadIdToIndex = indexToThreadId.computeIfAbsent(
117+
threadId,
118+
str -> unpickedIndices.pop()
119+
);
120+
AttributeValue value = AttributeValue.builder().s(threadIdToIndex).build();
121+
TimeZone tz = TimeZone.getTimeZone("UTC");
122+
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSS'Z'"); // Quoted "Z" to indicate UTC, no timezone offset
123+
df.setTimeZone(tz);
124+
String timestamp = df.format(new Date());
125+
126+
System.out.println(
127+
"Thread ID: " +
128+
Thread.currentThread().getId() +
129+
" ThreadIndex: " +
130+
threadIdToIndex +
131+
" Timestamp: " +
132+
timestamp
133+
);
134+
135+
try {
136+
DynamoDbClient client = clientForThread(threadIdToIndex);
137+
TransactWriteItemsResponse transactWriteItemsResponse =
138+
client.transactWriteItems(
139+
TransactWriteItemsRequest
140+
.builder()
141+
.transactItems(conditionalWrite(value, timestamp))
142+
.build()
143+
);
144+
Assert.assertTrue(
145+
transactWriteItemsResponse.sdkHttpResponse().isSuccessful()
146+
);
147+
} catch (TransactionCanceledException exception) {
148+
// We can fail for two reasons, either there's already a transact write in flight
149+
// 0r we have failed the condition check.
150+
exception
151+
.cancellationReasons()
152+
.forEach(cancellationReason -> {
153+
Assert.assertTrue(
154+
(cancellationReason.code().equals("TransactionConflict") ||
155+
cancellationReason.code().equals("ConditionalCheckFailed"))
156+
);
157+
});
158+
}
159+
}
160+
}

‎AwsCryptographicMaterialProviders/runtimes/java/src/testExamples/java/software/amazon/cryptography/example/hierarchy/concurrent/README.md

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved."
2+
[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0"
3+
4+
# AWS Cryptographic Material Providers Library Concurrency Testing Suite
5+
6+
Welcome to the AWS Cryptographic Material Providers Library Concurrency and Parallelization
7+
Testing Suite 🎉!
8+
9+
This testing suite helps set up scenarios that we would like to run in a parallel or multithreaded environment.
10+
11+
Some things to keep in mind when you add tests. Think about how you will be creating resources per
12+
thread and what kind of state you need to keep between tests.
13+
14+
Examples:
15+
16+
- [Test regular DynamoDB Client TransactWrites](./ConcurrentConditionCheckWriteTest.java)
17+
- [Test ACTIVE branch key reads while branch key creation is inflight](./StorageWriteReadConcurrencyTests.java)
18+
- [Test branch key reads while branch key versioning is inflight](./StorageVersionReadConcurrencyTests.java)
19+
20+
[Security issue notifications](./CONTRIBUTING.md#security-issue-notifications)
21+
22+
## Security
23+
24+
If you discover a potential security issue in this project
25+
we ask that you notify AWS/Amazon Security via our
26+
[vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/).
27+
Please **do not** create a public GitHub issue.

0 commit comments

Comments
 (0)