From 538b5965d0eb8ea6224f925d26dcf669725ead0d Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Tue, 12 Oct 2021 09:17:30 +0000 Subject: [PATCH 01/20] added bwakit argument and pulling from pcap 5.7.0 --- CHANGES.md | 4 ++++ Dockerfile | 2 +- cwls/cgpmap-bamBaiOut.cwl | 15 +++++++++++---- cwls/cgpmap-bamCsiOut.cwl | 15 +++++++++++---- cwls/cgpmap-cramOut.cwl | 15 +++++++++++---- cwls/mixins/requirements.yml | 2 +- scripts/ds-cgpmap.pl | 4 ++++ scripts/mapping.sh | 6 ++++++ 8 files changed, 49 insertions(+), 14 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c3b3a9b..cc64f90 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,10 @@ ## 3.2.0 +* Update PCAP-core to 5.7.0, which includes bwakit post processing + +## 3.2.0 + * Update PCAP-core to 5.4.0, new ubuntu htslib/samtools 1.11. * `-scramble` deprecated, see `seqslice` * New options to control duplicate marking diff --git a/Dockerfile b/Dockerfile index c2cbf14..dd44070 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/wtsicgp/pcap-core:5.4.0 +FROM quay.io/wtsicgp/pcap-core:5.7.0 LABEL vendor="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" LABEL maintainer="cgphelp@sanger.ac.uk" diff --git a/cwls/cgpmap-bamBaiOut.cwl b/cwls/cgpmap-bamBaiOut.cwl index 131c044..0342e0c 100644 --- a/cwls/cgpmap-bamBaiOut.cwl +++ b/cwls/cgpmap-bamBaiOut.cwl @@ -91,12 +91,19 @@ inputs: prefix: -bwamem2 position: 9 + bwakit: + type: boolean + doc: "Use bwakit post-processing" + inputBinding: + prefix: -bwakit + position: 10 + nomarkdup: type: boolean doc: "Do not mark duplicates" inputBinding: prefix: -nomarkdup - position: 10 + position: 11 dupmode: type: string? @@ -104,7 +111,7 @@ inputs: default: 't' inputBinding: prefix: -dupmode - position: 11 + position: 12 separate: true legacy: @@ -112,7 +119,7 @@ inputs: doc: "Use legacy merge/dupmark from biobambam2 tools, slower, more memory" inputBinding: prefix: -legacy - position: 12 + position: 13 seq_in: type: @@ -121,7 +128,7 @@ inputs: items: File doc: "Can be BAM, CRAM, fastq (paired or interleaved), BAM/CRAM can be mixed together but not FASTQ." inputBinding: - position: 13 + position: 14 outputs: out_bam: diff --git a/cwls/cgpmap-bamCsiOut.cwl b/cwls/cgpmap-bamCsiOut.cwl index e4dc8cd..b27d0bb 100644 --- a/cwls/cgpmap-bamCsiOut.cwl +++ b/cwls/cgpmap-bamCsiOut.cwl @@ -91,12 +91,19 @@ inputs: prefix: -bwamem2 position: 9 + bwakit: + type: boolean + doc: "Use bwakit post-processing" + inputBinding: + prefix: -bwakit + position: 10 + nomarkdup: type: boolean doc: "Do not mark duplicates" inputBinding: prefix: -nomarkdup - position: 10 + position: 11 dupmode: type: string? @@ -104,7 +111,7 @@ inputs: default: 't' inputBinding: prefix: -dupmode - position: 11 + position: 12 separate: true legacy: @@ -112,7 +119,7 @@ inputs: doc: "Use legacy merge/dupmark from biobambam2 tools, slower, more memory" inputBinding: prefix: -legacy - position: 12 + position: 13 seq_in: type: @@ -121,7 +128,7 @@ inputs: items: File doc: "Can be BAM, CRAM, fastq (paired or interleaved), BAM/CRAM can be mixed together but not FASTQ." inputBinding: - position: 13 + position: 14 outputs: out_bam: diff --git a/cwls/cgpmap-cramOut.cwl b/cwls/cgpmap-cramOut.cwl index 8e2b672..bbfa02c 100644 --- a/cwls/cgpmap-cramOut.cwl +++ b/cwls/cgpmap-cramOut.cwl @@ -100,12 +100,19 @@ inputs: prefix: -bwamem2 position: 10 + bwakit: + type: boolean + doc: "Use bwakit post-processing" + inputBinding: + prefix: -bwakit + position: 11 + nomarkdup: type: boolean doc: "Do not mark duplicates" inputBinding: prefix: -nomarkdup - position: 11 + position: 12 dupmode: type: string? @@ -113,7 +120,7 @@ inputs: default: 't' inputBinding: prefix: -dupmode - position: 12 + position: 13 separate: true legacy: @@ -121,7 +128,7 @@ inputs: doc: "Use legacy merge/dupmark from biobambam2 tools, slower, more memory" inputBinding: prefix: -legacy - position: 13 + position: 14 seq_in: type: @@ -130,7 +137,7 @@ inputs: items: File doc: "Can be BAM, CRAM, fastq (paired or interleaved), BAM/CRAM can be mixed together but not FASTQ." inputBinding: - position: 14 + position: 15 outputs: out_cram: diff --git a/cwls/mixins/requirements.yml b/cwls/mixins/requirements.yml index 7a4a3d6..4f03bb6 100644 --- a/cwls/mixins/requirements.yml +++ b/cwls/mixins/requirements.yml @@ -1,2 +1,2 @@ class: DockerRequirement -dockerPull: "quay.io/wtsicgp/dockstore-cgpmap:3.2.0" +dockerPull: "quay.io/wtsicgp/dockstore-cgpmap:3.3.0" diff --git a/scripts/ds-cgpmap.pl b/scripts/ds-cgpmap.pl index 3e2e870..ff6b9ca 100755 --- a/scripts/ds-cgpmap.pl +++ b/scripts/ds-cgpmap.pl @@ -21,6 +21,7 @@ 'f' => 0.05, 'dupmode' => 't', 'bwamem2' => 0, + 'bwakit' => 0, 'nomarkdup' => 0, 'legacy' => 0, 'seqslice' => undef, @@ -40,6 +41,7 @@ 'q|qc' => \$opts{'q'}, 'f|qcf:f' => \$opts{'f'}, 'bm2|bwamem2' => \$opts{'bwamem2'}, + 'kit|bwakit' => \$opts{'seqslice'}, 'n|nomarkdup' => \$opts{'nomarkdup'}, 'd|dupmode:s' => \$opts{'dupmode'}, 'legacy' => \$opts{'legacy'}, @@ -85,6 +87,7 @@ printf $FH "MMQCFRAC=%s\n", $opts{'f'} if(defined $opts{'f'}); printf $FH "DUPMODE=%s\n", $opts{'dupmode'}; printf $FH "BWAMEM2=%d\n", $opts{'bwamem2'} if(defined $opts{'bwamem2'}); +printf $FH "BWAKIT=%d\n", $opts{'bwakit'} if(defined $opts{'bwakit'}); printf $FH "NOMARKDUP=%d\n", $opts{'nomarkdup'} if(defined $opts{'nomarkdup'}); printf $FH "LEGACY=%d\n", $opts{'legacy'} if(defined $opts{'legacy'}); printf $FH "SEQSLICE=%d\n", $opts{'seqslice'} if(defined $opts{'seqslice'}); @@ -125,6 +128,7 @@ =head1 SYNOPSIS Optional parameters: -threads -t Set the number of cpu/cores available [default all]. -bwamem2 -bm2 Use bwa-mem2 instead of bwa (experimental). + -bwakit -kit Run bwakit post alignment processing. -nomarkdup -n Don't mark duplicates [flag] -seqslice -ss seqs_per_slice for CRAM compression [samtools default: 10000] -cram -c Output cram, see '-seqslice' diff --git a/scripts/mapping.sh b/scripts/mapping.sh index f953c83..a0d84ac 100755 --- a/scripts/mapping.sh +++ b/scripts/mapping.sh @@ -58,6 +58,7 @@ else echo -e "\tSEQSLICE : $SEQSLICE" fi echo -e "\tBWAMEM2 : $BWAMEM2" +echo -e "\tBWAKIT : $BWAKIT" echo -e "\tNOMARKDUP : $NOMARKDUP" echo -e "\tLEGACY : $LEGACY" set +u @@ -122,6 +123,11 @@ if [ $BWAMEM2 -gt 0 ]; then ADD_ARGS="$ADD_ARGS --bwamem2" fi +# if BWAKIT set +if [ $BWAKIT -gt 0 ]; then + ADD_ARGS="$ADD_ARGS --bwakit" +fi + # if NOMARKDUP set if [ $NOMARKDUP -gt 0 ]; then ADD_ARGS="$ADD_ARGS --nomarkdup" From f7f15e2552eb8bc7e86ba1178342c3de2e15dab9 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Tue, 12 Oct 2021 09:21:22 +0000 Subject: [PATCH 02/20] typo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index cc64f90..ebcb7e9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # CHANGES -## 3.2.0 +## 3.3.0 * Update PCAP-core to 5.7.0, which includes bwakit post processing From e3e77db5f97452f26ad8984f5366951f266ebd96 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Tue, 19 Oct 2021 10:37:28 +0000 Subject: [PATCH 03/20] Fix incorrect hash key value --- scripts/ds-cgpmap.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ds-cgpmap.pl b/scripts/ds-cgpmap.pl index ff6b9ca..234eb99 100755 --- a/scripts/ds-cgpmap.pl +++ b/scripts/ds-cgpmap.pl @@ -41,7 +41,7 @@ 'q|qc' => \$opts{'q'}, 'f|qcf:f' => \$opts{'f'}, 'bm2|bwamem2' => \$opts{'bwamem2'}, - 'kit|bwakit' => \$opts{'seqslice'}, + 'kit|bwakit' => \$opts{'bwakit'}, 'n|nomarkdup' => \$opts{'nomarkdup'}, 'd|dupmode:s' => \$opts{'dupmode'}, 'legacy' => \$opts{'legacy'}, From 4c782c93c33c38aea689b780ed8807aaa9bb0b92 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 09:19:04 +0000 Subject: [PATCH 04/20] Updated readme and example parameter jsons --- Dockerfile | 2 +- README.md | 83 ++++--------------- examples/cgpmap/bamBaiOut/bam_in.json | 1 + examples/cgpmap/bamBaiOut/cram_in.json | 1 + examples/cgpmap/bamBaiOut/fq_gz_in.json | 1 + .../cgpmap/bamBaiOut/fq_gz_in_qcreads.json | 1 + examples/cgpmap/bamCsiOut/fq_gz_in.json | 1 + examples/cgpmap/cramOut/fq_gz_in.json | 1 + 8 files changed, 21 insertions(+), 70 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd44070..800824d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM quay.io/wtsicgp/pcap-core:5.7.0 LABEL vendor="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute" LABEL maintainer="cgphelp@sanger.ac.uk" LABEL uk.ac.sanger.cgp.description="PCAP-core for dockstore.org" -LABEL uk.ac.sanger.cgp.version="3.2.0" +LABEL uk.ac.sanger.cgp.version="3.3.0" USER root diff --git a/README.md b/README.md index decab56..49dbc04 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,11 @@ packaged specifically for use with the [Dockstore.org](https://dockstore.org/) f * [Supported input formats](#supported-input-formats) * [Options for customisation:](#options-for-customisation) * [Usable Cores](#usable-cores) -* [Other uses](#other-uses) - * [Native docker](#native-docker) - * [Singularity](#singularity) -* [Verifying your deployment](#verifying-your-deployment) -* [Development environment](#development-environment) -* [Release process](#release-process) +* [Run instructions](#run-instructions) +* [Development](#development) + * [Verifying your deployment](#verifying-your-deployment) + * [Development environment](#development-environment) + * [Release process](#release-process) * [LICENCE](#licence) ## Supported input formats @@ -31,73 +30,18 @@ packaged specifically for use with the [Dockstore.org](https://dockstore.org/) f ## Options for customisation: -* BWA specific mapping parameters (defaults are based on attempts at a global standard). +* BWA specific mapping parameters (defaults are based on attempts at a global standard) * Optionally output CRAM (scramble parameters can be modified) +* Optionally run with BWA-MEM2 +* Optionally run with bwa-kit post-processing (for calling on alternative contigs) -## Usable Cores +## Run instructions -When running outside of a docker container you can set the number of CPUs via: +The full documentation covering input files, optional parameters, and methods of running dockstore-cgpmap can be found in the [github wiki][github wiki]. -* `export CPU=N` -* `-threads|-t` option of `ds-cgpmap.pl` +## Development -If not set detects available cores on system. - -## Other uses - -### Native docker - -All of the tools installed as part of [PCAP-core][pcap-core] are available for direct use. - -``` -export CGPMAP_VER=X.X.X -docker pull quay.io/wtsicgp/dockstore-cgpmap:$CGPMAP_VER -# interactive session -docker --rm -ti [--volume ...] quay.io/wtsicgp/dockstore-cgpmap:$CGPMAP_VER bash -``` - -### Singularity - -The resulting docker container has been tested with Singularity. The command to exec is: - -``` -ds-cgpmap.pl -h -``` - -Expected use would be along the lines of: - -``` -export CGPMAP_VER=X.X.X -singularity pull docker://quay.io/wtsicgp/dockstore-cgpmap:$CGPMAP_VER - -singularity exec\ - --workdir /.../workspace \ - --home /.../workspace:/home \ - --bind /.../ref/human:/var/spool/ref:ro \ - --bind /.../example_data/cgpmap/insilico_21:/var/spool/data:ro \ - dockstore-cgpmap-${CGPMAP_VER}.simg \ - ds-cgpmap.pl \ - -r /var/spool/ref/core_ref_GRCh37d5.tar.gz \ - -i /var/spool/ref/bwa_idx_GRCh37d5.tar.gz \ - -s SOMENAME \ - -t 6 \ - /var/spool/data/\*.bam -``` - -For a system automatically attaching _all local mount points_ (not default singularity behaviour) -you need not specify any `exec` params (workdir, home, bind) but you should specify the `-outdir` -option for `ds-cgpmap.pl` to prevent data being written to your home directory. - -By default results are written to the home directory of the container so ensure you bind -a large volume and set the `-home` variable. As indicated above the location can be overridden -via the options of `ds-cgpmap.pl` - -## Verifying your deployment - -The `examples/` tree contains test json files populated with data that can be used to verify the -tool. More details can be found [here](examples/README.md). - -## Development environment +### Development environment This project uses git pre-commit hooks. Please enable them to prevent inappropriate large files being included. Any pull request found not to have adhered to this will be rejected and the branch @@ -109,7 +53,7 @@ Activate the hooks with git config core.hooksPath git-hooks ``` -## Release process +### Release process This project is maintained using HubFlow. @@ -158,6 +102,7 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008, [bwa-mem.pl]: https://github.com/cancerit/PCAP-core/blob/master/bin/bwa_mem.pl [cgpmap-expected]: ftp://ftp.sanger.ac.uk/pub/cancer/dockstore/expected [pcap-core]: https://github.com/cancerit/PCAP-core +[github-wiki]: https://github.com/cancerit/dockstore-cgpmap/wiki [travis-base]: https://travis-ci.org/cancerit/dockstore-cgpmap diff --git a/examples/cgpmap/bamBaiOut/bam_in.json b/examples/cgpmap/bamBaiOut/bam_in.json index 8382de2..7bc2a3b 100644 --- a/examples/cgpmap/bamBaiOut/bam_in.json +++ b/examples/cgpmap/bamBaiOut/bam_in.json @@ -16,6 +16,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, diff --git a/examples/cgpmap/bamBaiOut/cram_in.json b/examples/cgpmap/bamBaiOut/cram_in.json index c09f1f0..607780c 100644 --- a/examples/cgpmap/bamBaiOut/cram_in.json +++ b/examples/cgpmap/bamBaiOut/cram_in.json @@ -16,6 +16,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, diff --git a/examples/cgpmap/bamBaiOut/fq_gz_in.json b/examples/cgpmap/bamBaiOut/fq_gz_in.json index 23ee971..17deaf0 100644 --- a/examples/cgpmap/bamBaiOut/fq_gz_in.json +++ b/examples/cgpmap/bamBaiOut/fq_gz_in.json @@ -34,6 +34,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, diff --git a/examples/cgpmap/bamBaiOut/fq_gz_in_qcreads.json b/examples/cgpmap/bamBaiOut/fq_gz_in_qcreads.json index b7b04d8..7973130 100644 --- a/examples/cgpmap/bamBaiOut/fq_gz_in_qcreads.json +++ b/examples/cgpmap/bamBaiOut/fq_gz_in_qcreads.json @@ -34,6 +34,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, diff --git a/examples/cgpmap/bamCsiOut/fq_gz_in.json b/examples/cgpmap/bamCsiOut/fq_gz_in.json index 23ee971..17deaf0 100644 --- a/examples/cgpmap/bamCsiOut/fq_gz_in.json +++ b/examples/cgpmap/bamCsiOut/fq_gz_in.json @@ -34,6 +34,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, diff --git a/examples/cgpmap/cramOut/fq_gz_in.json b/examples/cgpmap/cramOut/fq_gz_in.json index e6d637f..d077b03 100644 --- a/examples/cgpmap/cramOut/fq_gz_in.json +++ b/examples/cgpmap/cramOut/fq_gz_in.json @@ -34,6 +34,7 @@ "mmqcfrag": 0.05, "threads": 0, "bwamem2": false, + "bwakit": false, "nomarkdup": false, "dupmode": "t", "legacy": false, From 973262fff5ca0cff884b0063a6588690dea18903 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 09:20:33 +0000 Subject: [PATCH 05/20] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49dbc04..975d350 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ packaged specifically for use with the [Dockstore.org](https://dockstore.org/) f ## Run instructions -The full documentation covering input files, optional parameters, and methods of running dockstore-cgpmap can be found in the [github wiki][github wiki]. +The full documentation covering input files, optional parameters, and methods of running dockstore-cgpmap can be found in the [github wiki][github-wiki]. ## Development From 9e6cacf05afe36954d8a2b74515a6a2e25e9880f Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 09:23:56 +0000 Subject: [PATCH 06/20] updated readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 975d350..89cbdb0 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,10 @@ The full documentation covering input files, optional parameters, and methods of ## Development +### Verifying your deployment + +The `examples/` tree contains test json files populated with data that can be used to verify the tool. More details on running Dockstore locally for testing purposes can be found in the [github wiki][github-wiki]. + ### Development environment This project uses git pre-commit hooks. Please enable them to prevent inappropriate large files From f4d78d3ca5982c97eb5f9d66e2c3ec607312dc9e Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 11:37:39 +0000 Subject: [PATCH 07/20] Updated readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 89cbdb0..a548e9b 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ packaged specifically for use with the [Dockstore.org](https://dockstore.org/) f ## Options for customisation: * BWA specific mapping parameters (defaults are based on attempts at a global standard) -* Optionally output CRAM (scramble parameters can be modified) +* Optionally output CRAM (see `seqslice` to for faster access, recommend 1000) + * applied to `seqs_per_slice` option of htslib/samtools. * Optionally run with BWA-MEM2 * Optionally run with bwa-kit post-processing (for calling on alternative contigs) From 3090bd3cb606d99f5fb1244567e19f035899598a Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 12:00:55 +0000 Subject: [PATCH 08/20] init circleci --- .circleci/config.yml | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..b56f9ca --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,62 @@ +# 1. Set your GitHub default branch to dev/develop +# 2. In circleci project settings->advanced set: +# - GitHub Status Updates (this should be on already) +# - Only build pull requests +# - Auto-cancel redundant builds +# - Free and Open Source (this should be on already) +# 3. If existing project with quay.io, remove the quay build triggers. +# - as circleci is pushing the image +# - reevaluate when/if quay.io adds ability to authenticate pulls + +# version: 2.1 +# jobs: +# build: +# environment: +# IMAGE_NAME: quay.io/wtsicgp/pcap-core +# docker: +# - image: circleci/buildpack-deps:stretch +# steps: +# - checkout +# - setup_remote_docker +# - run: +# name: Build Docker image +# command: | +# set -e +# BRANCH_OR_TAG="${CIRCLE_TAG:-$CIRCLE_BRANCH}" +# CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -) +# echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin +# echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io +# docker build -t $IMAGE_NAME:$CLEAN_BRANCH . +# echo 'Verify program(s) from base image' +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bedgraph --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwjoin --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bw --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwcat --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bwbases --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bg2bw --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH detectExtremeDepth --version +# echo 'Verify program(s) from this repo' +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH diff_bams --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH reheadSQ --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam_stats --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH mismatchQc --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwa_mem.pl --version +# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH merge_or_mark.pl --version +# if [ "$CIRCLE_TAG" = "$BRANCH_OR_TAG" ]; then +# echo 'Pushing image' +# docker push $IMAGE_NAME:$CLEAN_BRANCH +# else +# echo "I am not a tag so not pushing image" +# fi + +# workflows: +# version: 2.1 +# build_test: +# jobs: +# - build: +# context: +# - dockerhub-casmservice +# - quayio-casmservice +# filters: +# tags: +# only: /.+/ From db2247025c42489d209b2fa23ccd22e8350a83d2 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 15:32:15 +0000 Subject: [PATCH 09/20] Updated circle ci config --- .circleci/config.yml | 117 ++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 51 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b56f9ca..446ca10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,55 +8,70 @@ # - as circleci is pushing the image # - reevaluate when/if quay.io adds ability to authenticate pulls -# version: 2.1 -# jobs: -# build: -# environment: -# IMAGE_NAME: quay.io/wtsicgp/pcap-core -# docker: -# - image: circleci/buildpack-deps:stretch -# steps: -# - checkout -# - setup_remote_docker -# - run: -# name: Build Docker image -# command: | -# set -e -# BRANCH_OR_TAG="${CIRCLE_TAG:-$CIRCLE_BRANCH}" -# CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -) -# echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin -# echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io -# docker build -t $IMAGE_NAME:$CLEAN_BRANCH . -# echo 'Verify program(s) from base image' -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bedgraph --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwjoin --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bw --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwcat --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam2bwbases --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bg2bw --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH detectExtremeDepth --version -# echo 'Verify program(s) from this repo' -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH diff_bams --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH reheadSQ --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bam_stats --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH mismatchQc --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH bwa_mem.pl --version -# docker run -t --rm $IMAGE_NAME:$CLEAN_BRANCH merge_or_mark.pl --version -# if [ "$CIRCLE_TAG" = "$BRANCH_OR_TAG" ]; then -# echo 'Pushing image' -# docker push $IMAGE_NAME:$CLEAN_BRANCH -# else -# echo "I am not a tag so not pushing image" -# fi +jobs: + build-test-push-job: + docker: + - image: circleci/buildpack-deps:stretch + environment: + IMAGE_NAME: quay.io/wtsicgp/dockstore-cgpmap + steps: + - checkout + - setup_remote_docker: + - run: + name: Build Docker image + command: | + set -e + echo "Building Docker image" + docker build -t test:${CIRCLE_BUILD_NUM} . + - run: + name: Validate CWL files + command: | + set -e + echo "Validating CWL files" + cwltool --validate Dockstore.cwl + cwltool --validate cwls/cgpmap-bamBaiOut.cwl + cwltool --validate cwls/cgpmap-bamCsiOut.cwl + cwltool --validate cwls/cgpmap-cramOut.cwl + - run: + name: Verify inherited programs + command: | + set -e + echo 'Verifying programs inherited from cgpbigwig' + docker run -t --rm test:${CIRLCE_BUILD_NUM} bwjoin --version + echo 'Verifying programs inherited from pcap-core' + docker run -t --rm test:${CIRLCE_BUILD_NUM} bwa_mem.pl -version + docker run -t --rm test:${CIRLCE_BUILD_NUM} bammarkduplicates2 --version + docker run -t --rm test:${CIRLCE_BUILD_NUM} samtools --version + docker run -t --rm test:${CIRLCE_BUILD_NUM} bash -c 'bwa 2>&1 | grep Version' + - run: + name: Verify programs + command: | + set -e + echo 'Verify programs from dockstore-cgpmap' + docker run --rm -t test:${CIRCLE_BUILD_NUM} ds-cgpmap.pl -h + - run: + name: Push image to quay.io + command: | + set -e + BRANCH_OR_TAG="${CIRCLE_TAG:-$CIRCLE_BRANCH}" + if [ "$CIRCLE_TAG" = "$BRANCH_OR_TAG" ]; then + echo 'Pushing image' + echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin + echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io + CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -) + docker tag test:${CIRLCE_BUILD_NUM} ${IMAGE_NAME}:${CLEAN_BRANCH} + # docker push ${IMAGE_NAME}:${CLEAN_BRANCH} + else + echo "I am not a tag so not pushing image" + fi -# workflows: -# version: 2.1 -# build_test: -# jobs: -# - build: -# context: -# - dockerhub-casmservice -# - quayio-casmservice -# filters: -# tags: -# only: /.+/ +workflows: + build-test-push-workflow: + jobs: + - build-test-push-job + context: + - dockerhub-casmservice + - quayio-casmservice + filters: + tags: + only: /.+/ From 7fb577159b546bbcbd7115c23d7bcbbc3adc7741 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 15:34:56 +0000 Subject: [PATCH 10/20] circle ci fix --- .circleci/config.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 446ca10..0e6836f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,4 @@ -# 1. Set your GitHub default branch to dev/develop -# 2. In circleci project settings->advanced set: -# - GitHub Status Updates (this should be on already) -# - Only build pull requests -# - Auto-cancel redundant builds -# - Free and Open Source (this should be on already) -# 3. If existing project with quay.io, remove the quay build triggers. -# - as circleci is pushing the image -# - reevaluate when/if quay.io adds ability to authenticate pulls - +version: 2.1 jobs: build-test-push-job: docker: From 74c057e11caeac3a1c750fe1f97a325e5bb8a762 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 15:36:34 +0000 Subject: [PATCH 11/20] circle ci fix --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e6836f..cc327d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ jobs: workflows: build-test-push-workflow: jobs: - - build-test-push-job + - build-test-push-job: context: - dockerhub-casmservice - quayio-casmservice From 98b42637d053f9fd36b55dd5a1cab8a11cdf5ce7 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 15:46:13 +0000 Subject: [PATCH 12/20] circle ci fix --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cc327d8..1b2cb20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: IMAGE_NAME: quay.io/wtsicgp/dockstore-cgpmap steps: - checkout - - setup_remote_docker: + - setup_remote_docker - run: name: Build Docker image command: | @@ -57,6 +57,7 @@ jobs: fi workflows: + version: 2.1 build-test-push-workflow: jobs: - build-test-push-job: From 9b2d0cf6ca325b78e4a1ea828cee610e463a5960 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 15:48:00 +0000 Subject: [PATCH 13/20] circle ci fix --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b2cb20..96722b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,6 +19,9 @@ jobs: command: | set -e echo "Validating CWL files" + virtualenv -p python3 venv + source venv/bin/activate + pip install html5lib cwltool cwltool --validate Dockstore.cwl cwltool --validate cwls/cgpmap-bamBaiOut.cwl cwltool --validate cwls/cgpmap-bamCsiOut.cwl From ffca270ddb0cce20ce73ee9fbb32b3a88c7f624b Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:01:59 +0000 Subject: [PATCH 14/20] circle ci fix --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96722b0..13ddec7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,9 +19,8 @@ jobs: command: | set -e echo "Validating CWL files" - virtualenv -p python3 venv - source venv/bin/activate - pip install html5lib cwltool + sudo apt-get -y install python3-pip + pip3 install cwltool cwltool --validate Dockstore.cwl cwltool --validate cwls/cgpmap-bamBaiOut.cwl cwltool --validate cwls/cgpmap-bamCsiOut.cwl From c7ef48c054564db49844cb53da3f4f8fc3b01c37 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:04:42 +0000 Subject: [PATCH 15/20] circle ci fix --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13ddec7..08f1a26 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,12 +30,12 @@ jobs: command: | set -e echo 'Verifying programs inherited from cgpbigwig' - docker run -t --rm test:${CIRLCE_BUILD_NUM} bwjoin --version + docker run --rm -t test:${CIRLCE_BUILD_NUM} bwjoin --version echo 'Verifying programs inherited from pcap-core' - docker run -t --rm test:${CIRLCE_BUILD_NUM} bwa_mem.pl -version - docker run -t --rm test:${CIRLCE_BUILD_NUM} bammarkduplicates2 --version - docker run -t --rm test:${CIRLCE_BUILD_NUM} samtools --version - docker run -t --rm test:${CIRLCE_BUILD_NUM} bash -c 'bwa 2>&1 | grep Version' + docker run --rm -t test:${CIRLCE_BUILD_NUM} bwa_mem.pl -version + docker run --rm -t test:${CIRLCE_BUILD_NUM} bammarkduplicates2 --version + docker run --rm -t test:${CIRLCE_BUILD_NUM} samtools --version + docker run --rm -t test:${CIRLCE_BUILD_NUM} bash -c 'bwa 2>&1 | grep Version' - run: name: Verify programs command: | From 924651d73830e048f0bd6eb7c0ddcca7086909fd Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:11:25 +0000 Subject: [PATCH 16/20] circle ci fix --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08f1a26..59210d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: command: | set -e echo "Building Docker image" - docker build -t test:${CIRCLE_BUILD_NUM} . + docker build -t test . - run: name: Validate CWL files command: | @@ -41,7 +41,7 @@ jobs: command: | set -e echo 'Verify programs from dockstore-cgpmap' - docker run --rm -t test:${CIRCLE_BUILD_NUM} ds-cgpmap.pl -h + docker run --rm -t test ds-cgpmap.pl -h - run: name: Push image to quay.io command: | @@ -52,7 +52,7 @@ jobs: echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USER" --password-stdin echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -) - docker tag test:${CIRLCE_BUILD_NUM} ${IMAGE_NAME}:${CLEAN_BRANCH} + docker tag test ${IMAGE_NAME}:${CLEAN_BRANCH} # docker push ${IMAGE_NAME}:${CLEAN_BRANCH} else echo "I am not a tag so not pushing image" From 5be827d8e9c04e290a096c04386a2a0552f68781 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:13:49 +0000 Subject: [PATCH 17/20] circle ci fix --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59210d3..afe0478 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,12 +30,12 @@ jobs: command: | set -e echo 'Verifying programs inherited from cgpbigwig' - docker run --rm -t test:${CIRLCE_BUILD_NUM} bwjoin --version + docker run --rm -t test bwjoin --version echo 'Verifying programs inherited from pcap-core' - docker run --rm -t test:${CIRLCE_BUILD_NUM} bwa_mem.pl -version - docker run --rm -t test:${CIRLCE_BUILD_NUM} bammarkduplicates2 --version - docker run --rm -t test:${CIRLCE_BUILD_NUM} samtools --version - docker run --rm -t test:${CIRLCE_BUILD_NUM} bash -c 'bwa 2>&1 | grep Version' + docker run --rm -t test bwa_mem.pl -version + docker run --rm -t test bammarkduplicates2 --version + docker run --rm -t test samtools --version + docker run --rm -t test bash -c 'bwa 2>&1 | grep Version' - run: name: Verify programs command: | From d47bd533ecf4c2e36b80e0033946be6b6601c4bf Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:16:23 +0000 Subject: [PATCH 18/20] finalising circle ci config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afe0478..db4fdbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ jobs: echo "$QUAYIO_PASSWORD" | docker login -u "$QUAYIO_USER" --password-stdin quay.io CLEAN_BRANCH=$(echo $BRANCH_OR_TAG | tr / -) docker tag test ${IMAGE_NAME}:${CLEAN_BRANCH} - # docker push ${IMAGE_NAME}:${CLEAN_BRANCH} + docker push ${IMAGE_NAME}:${CLEAN_BRANCH} else echo "I am not a tag so not pushing image" fi From b37d2b238a3ad82100da4f7a1ba8f177402d7c0f Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:19:01 +0000 Subject: [PATCH 19/20] removed travis.yml --- .travis.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e78a2b3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -notifications: - slack: wtsi-cgpit:ptUMR1tkNyZJYd9TpGoss8WR - email: false - -sudo: false - -services: - - docker - -install: - - virtualenv -p python3 venv - - source venv/bin/activate - - pip install html5lib cwltool - -script: - - set -e - - echo 'Validate CWL file(s)' - - cwltool --validate Dockstore.cwl - - cwltool --validate cwls/cgpmap-bamBaiOut.cwl - - cwltool --validate cwls/cgpmap-bamCsiOut.cwl - - cwltool --validate cwls/cgpmap-cramOut.cwl - - echo 'Build and check docker image' - - docker build -t dockstore-cgpmap . - - docker images | grep -c dockstore-cgpmap - - echo 'Verify program(s) from each inherited package (cgpbigwig)' - - docker run -t --rm dockstore-cgpmap bwjoin --version - - echo 'Verify program(s) from each inherited package (pcap-core)' - - docker run -t --rm dockstore-cgpmap bwa_mem.pl -version - - docker run -t --rm dockstore-cgpmap bammarkduplicates2 --version - - docker run -t --rm dockstore-cgpmap samtools --version - - docker run -t --rm dockstore-cgpmap bash -c 'bwa 2>&1 | grep Version' - - echo 'Verify program(s) this layer are found (dockstore-cgpmap)' - - docker run -t --rm dockstore-cgpmap ds-cgpmap.pl -h From e95d146130ce9e6eed037359d45d8a97f6150bb3 Mon Sep 17 00:00:00 2001 From: Gregory Leeman Date: Wed, 3 Nov 2021 16:42:42 +0000 Subject: [PATCH 20/20] Updated readme to reference circleci (not Travis) --- README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a548e9b..ac2cd94 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ packaged specifically for use with the [Dockstore.org](https://dockstore.org/) f [![Quay Badge][quay-status]][quay-repo] -| Master | Develop | -| --------------------------------------------- | ----------------------------------------------- | -| [![Master Badge][travis-master]][travis-base] | [![Develop Badge][travis-develop]][travis-base] | +| Master | Develop | +| --- | --- | +| [![Master Badge][circleci-master-badge]][circleci-master] | [![Develop Badge][circleci-develop-badge]][circleci-develop] | * [Supported input formats](#supported-input-formats) * [Options for customisation:](#options-for-customisation) @@ -65,7 +65,7 @@ This project is maintained using HubFlow. 1. Make appropriate changes 2. Bump version in `Dockerfile` and `cwls/mixins/requirements.yml` 3. Push changes -4. Check state on Travis +4. Check state on CircleCi 5. Generate the release (add notes to GitHub) 6. Confirm that image has been built on [quay.io][quay-builds] 7. Update the [dockstore][dockstore-cgpmap] entry, see [their docs][dockstore-get-started]. @@ -109,10 +109,11 @@ identical to a statement that reads ‘Copyright (c) 2005, 2006, 2007, 2008, [pcap-core]: https://github.com/cancerit/PCAP-core [github-wiki]: https://github.com/cancerit/dockstore-cgpmap/wiki - -[travis-base]: https://travis-ci.org/cancerit/dockstore-cgpmap -[travis-master]: https://travis-ci.org/cancerit/dockstore-cgpmap.svg?branch=master -[travis-develop]: https://travis-ci.org/cancerit/dockstore-cgpmap.svg?branch=develop + +[circleci-master-badge]: https://circleci.com/gh/cancerit/dockstore-cgpmap/tree/master.svg?style=svg +[circleci-master]: https://circleci.com/gh/cancerit/dockstore-cgpmap/tree/master +[circleci-develop-badge]: https://circleci.com/gh/cancerit/dockstore-cgpmap/tree/develop.svg?style=svg +[circleci-develop]: https://circleci.com/gh/cancerit/dockstore-cgpmap/tree/master [gitter-svg]: https://badges.gitter.im/dockstore-cgp/Lobby.svg