Skip to content

Commit 0b2d6c7

Browse files
MalinAhlbergMalinAhlberg
authored
Merge pull request #1161 from neicnordic/feature/remove-user-from-filepath
Remove user id from filepath
2 parents 556bca7 + e263eb3 commit 0b2d6c7

File tree

17 files changed

+116
-63
lines changed

17 files changed

+116
-63
lines changed

‎.github/integration/tests/sda/01_install_dependencies.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,5 @@ for t in curl expect jq openssh-client postgresql-client xxd; do
1313
apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null
1414
fi
1515
done
16+
17+
pip -q install s3cmd

‎.github/integration/tests/sda/10_upload_test.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ done
2020
## truncate database
2121
psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files, sda.encryption_keys CASCADE;"
2222

23-
pip -q install s3cmd
24-
2523
for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
2624
curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file"
2725
if [ ! -f "$file.c4gh" ]; then

‎.github/integration/tests/sda/20_ingest-verify_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do
3636
jq -r -c -n \
3737
--arg type ingest \
3838
--arg user [email protected] \
39-
--arg filepath test_dummy.org/"$file.c4gh" \
39+
--arg filepath "$file.c4gh" \
4040
--argjson encrypted_checksums "$encrypted_checksums" \
4141
'$ARGS.named|@base64'
4242
)

‎.github/integration/tests/sda/21_cancel_test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ cancel_payload=$(
3434
jq -r -c -n \
3535
--arg type cancel \
3636
--arg user [email protected] \
37-
--arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \
37+
--arg filepath NA12878_20k_b37.bam.c4gh \
3838
--argjson encrypted_checksums "$encrypted_checksums" \
3939
'$ARGS.named|@base64'
4040
)
@@ -65,7 +65,7 @@ ingest_payload=$(
6565
jq -r -c -n \
6666
--arg type ingest \
6767
--arg user [email protected] \
68-
--arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \
68+
--arg filepath NA12878_20k_b37.bam.c4gh \
6969
--argjson encrypted_checksums "$encrypted_checksums" \
7070
'$ARGS.named|@base64'
7171
)

‎.github/integration/tests/sda/22_error_test.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ if [ -n "$PGSSLCERT" ]; then
2929
fi
3030

3131
## get correlation id from message
32-
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NB12878.bam.c4gh';")
32+
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NB12878.bam.c4gh';")
3333

3434
properties=$(
3535
jq -c -n \
@@ -50,7 +50,7 @@ ingest_payload=$(
5050
jq -r -c -n \
5151
--arg type ingest \
5252
--arg user [email protected] \
53-
--arg filepath test_dummy.org/NB12878.bam.c4gh \
53+
--arg filepath NB12878.bam.c4gh \
5454
--argjson encrypted_checksums "$encrypted_checksums" \
5555
'$ARGS.named|@base64'
5656
)
@@ -96,7 +96,7 @@ verify_payload=$(
9696
--arg user [email protected] \
9797
--arg archive_path "$CORRID" \
9898
--arg file_id "$CORRID" \
99-
--arg filepath test_dummy.org/NB12878.bam.c4gh \
99+
--arg filepath NB12878.bam.c4gh \
100100
--argjson encrypted_checksums "$encrypted_checksums" \
101101
--argjson re_verify false \
102102
'$ARGS.named|@base64'

‎.github/integration/tests/sda/30_backup-finalize_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ EOD
9494

9595
# check DB for archive file names
9696
for file in NA12878.bam.c4gh NA12878.bai.c4gh NA12878_20k_b37.bam.c4gh NA12878_20k_b37.bai.c4gh; do
97-
archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = 'test_dummy.org/$file';")
97+
archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = '$file';")
9898
size=$(s3cmd -c direct ls s3://backup/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3)
9999
if [ "$size" -eq 0 ]; then
100100
echo "Failed to get size of $file from backup site"

‎.github/integration/tests/sda/31_cancel_test2.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ ENC_SHA=$(sha256sum NA12878.bam.c4gh | cut -d' ' -f 1)
77
ENC_MD5=$(md5sum NA12878.bam.c4gh | cut -d' ' -f 1)
88

99
## get correlation id from message
10-
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NA12878.bam.c4gh';")
10+
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';")
1111

1212

1313
properties=$(
@@ -30,7 +30,7 @@ cancel_payload=$(
3030
jq -r -c -n \
3131
--arg type cancel \
3232
--arg user [email protected] \
33-
--arg filepath test_dummy.org/NA12878.bam.c4gh \
33+
--arg filepath NA12878.bam.c4gh \
3434
--argjson encrypted_checksums "$encrypted_checksums" \
3535
'$ARGS.named|@base64'
3636
)
@@ -67,7 +67,7 @@ ingest_payload=$(
6767
jq -r -c -n \
6868
--arg type ingest \
6969
--arg user [email protected] \
70-
--arg filepath test_dummy.org/NA12878.bam.c4gh \
70+
--arg filepath NA12878.bam.c4gh \
7171
--argjson encrypted_checksums "$encrypted_checksums" \
7272
'$ARGS.named|@base64'
7373
)
@@ -110,7 +110,7 @@ accession_payload=$(
110110
jq -r -c -n \
111111
--arg type accession \
112112
--arg user [email protected] \
113-
--arg filepath test_dummy.org/NA12878.bam.c4gh \
113+
--arg filepath NA12878.bam.c4gh \
114114
--arg accession_id EGAF74900000001 \
115115
--argjson decrypted_checksums "$decrypted_checksums" \
116116
'$ARGS.named|@base64'

‎.github/integration/tests/sda/32_test_race_condition.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ ENC_SHA=$(sha256sum race_file.c4gh | cut -d' ' -f 1)
1414
ENC_MD5=$(md5sum race_file.c4gh | cut -d' ' -f 1)
1515

1616
## get correlation id from message
17-
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/race_file.c4gh';")
17+
CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'race_file.c4gh';")
1818

1919
properties=$(
2020
jq -c -n \
@@ -36,7 +36,7 @@ accession_payload=$(
3636
jq -r -c -n \
3737
--arg type accession \
3838
--arg user [email protected] \
39-
--arg filepath test_dummy.org/race_file.c4gh \
39+
--arg filepath race_file.c4gh \
4040
--arg accession_id EGAF74900000099 \
4141
--argjson decrypted_checksums "$decrypted_checksums" \
4242
'$ARGS.named|@base64'
@@ -74,7 +74,7 @@ ingest_payload=$(
7474
jq -r -c -n \
7575
--arg type ingest \
7676
--arg user [email protected] \
77-
--arg filepath test_dummy.org/race_file.c4gh \
77+
--arg filepath race_file.c4gh \
7878
--argjson encrypted_checksums "$encrypted_checksums" \
7979
'$ARGS.named|@base64'
8080
)

‎.github/integration/tests/sda/60_api_admin_test.sh‎

Lines changed: 29 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
#!/bin/sh
1+
#!/bin/bash
22
set -e
33
cd shared || true
44

55
token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
6-
# Upload a file and make sure it's listed
6+
## make sure previously uploaded files are listed
77
result="$(curl -sk -L "http://api:8080/users/[email protected]/files" -H "Authorization: Bearer $token" | jq '. | length')"
88
if [ "$result" -ne 2 ]; then
99
echo "wrong number of files returned for user [email protected]"
@@ -41,25 +41,27 @@ fi
4141

4242
# Reupload a file under a different name, test to delete it
4343
s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh
44+
stream_size=$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')
4445

45-
echo "waiting for upload to complete"
4646
URI=http://rabbitmq:15672
4747
if [ -n "$PGSSLCERT" ]; then
4848
URI=https://rabbitmq:15671
4949
fi
50+
stream_size=$((stream_size + 1))
5051
RETRY_TIMES=0
51-
until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 4 ]; do
52-
echo "waiting for upload to complete"
53-
RETRY_TIMES=$((RETRY_TIMES + 1))
54-
if [ "$RETRY_TIMES" -eq 30 ]; then
55-
echo "::error::Time out while waiting for upload to complete"
56-
exit 1
57-
fi
58-
sleep 2
52+
53+
until [ $((stream_size)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; do
54+
echo "waiting for upload to complete"
55+
RETRY_TIMES=$((RETRY_TIMES + 1))
56+
if [ "$RETRY_TIMES" -eq 30 ]; then
57+
echo "Upload did not complete successfully"
58+
exit 1
59+
fi
60+
sleep 2
5961
done
6062

6163
# get the fileId of the new file
62-
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')"
64+
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')"
6365

6466
output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null)
6567
if [ -z "$output" ] ; then
@@ -104,27 +106,23 @@ fi
104106

105107
# Re-upload the file and use the api to ingest it, then try to delete it
106108
s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh
107-
108-
URI=http://rabbitmq:15672
109-
if [ -n "$PGSSLCERT" ]; then
110-
URI=https://rabbitmq:15671
111-
fi
109+
stream_size=$((stream_size + 1))
112110
RETRY_TIMES=0
113-
until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do
114-
echo "waiting for upload to complete"
115-
RETRY_TIMES=$((RETRY_TIMES + 1))
116-
if [ "$RETRY_TIMES" -eq 3 ]; then
117-
echo "::error::Time out while waiting for upload to complete"
118-
#exit 1
119-
break
120-
fi
121-
sleep 2
111+
112+
until [ $((stream_size)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; do
113+
echo "waiting for upload to complete"
114+
RETRY_TIMES=$((RETRY_TIMES + 1))
115+
if [ "$RETRY_TIMES" -eq 30 ]; then
116+
echo "Upload did not complete successfully"
117+
exit 1
118+
fi
119+
sleep 2
122120
done
123121

124122
# Ingest it
125123
new_payload=$(
126124
jq -c -n \
127-
--arg filepath "test_dummy.org/NE12878.bam.c4gh" \
125+
--arg filepath "NE12878.bam.c4gh" \
128126
--arg user "[email protected]" \
129127
'$ARGS.named'
130128
)
@@ -135,7 +133,7 @@ if [ "$resp" != "200" ]; then
135133
exit 1
136134
fi
137135

138-
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')"
136+
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')"
139137
# wait for the fail to get the correct status
140138
RETRY_TIMES=0
141139

@@ -150,9 +148,10 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e
150148
done
151149

152150
# Try to delete file not in inbox
153-
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')"
151+
fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/[email protected]/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')"
154152
resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/[email protected]/$fileid")"
155153
if [ "$resp" != "404" ]; then
156154
echo "Error when deleting the file, expected 404 got: $resp"
157155
exit 1
158-
fi
156+
fi
157+
echo "API admin tests completed successfully"

‎.github/integration/tests/sda/92_handle_file_errors.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ bad_file_payload=$(
4545
jq -r -c -n \
4646
--arg type ingest \
4747
--arg user [email protected] \
48-
--arg filepath test_dummy.org/bad.file.c4gh \
48+
--arg filepath bad.file.c4gh \
4949
--argjson encrypted_checksums "$encrypted_checksums" \
5050
'$ARGS.named|@base64'
5151
)
@@ -77,7 +77,7 @@ missing_file_payload=$(
7777
jq -r -c -n \
7878
--arg type ingest \
7979
--arg user [email protected] \
80-
--arg filepath test_dummy.org/missing.file.c4gh \
80+
--arg filepath missing.file.c4gh \
8181
--argjson encrypted_checksums "$encrypted_checksums" \
8282
'$ARGS.named|@base64'
8383
)
@@ -125,7 +125,7 @@ truncated_file_payload=$(
125125
jq -r -c -n \
126126
--arg type ingest \
127127
--arg user [email protected] \
128-
--arg filepath test_dummy.org/truncated.c4gh \
128+
--arg filepath truncated.c4gh \
129129
--argjson encrypted_checksums "$encrypted_checksums" \
130130
'$ARGS.named|@base64'
131131
)

0 commit comments

Comments
 (0)