Merge branch 'release/3.0.0'

.dockerignore

@@ -0,0 +1,5 @@
+/examples/
+/cwls/
+/git-hooks/
+/*.*
+/LICENSE

.gitignore

@@ -0,0 +1 @@
+/env/

.travis.yml

@@ -8,7 +8,25 @@ services:
   - docker
 
 install:
- - docker build -t dockstore-cgpmap .
+ - virtualenv -p python3 venv
+ - source venv/bin/activate
+ - pip install html5lib cwltool
 
 script:
+ - set -e
+ - echo 'Validate CWL file(s)'
+ - cwltool --validate Dockstore.cwl
+ - cwltool --validate cwls/cgpmap-bamBaiOut.cwl
+ - cwltool --validate cwls/cgpmap-bamCsiOut.cwl
+ - cwltool --validate cwls/cgpmap-cramOut.cwl
+ - echo 'Build and check docker image'
+ - docker build -t dockstore-cgpmap .
  - docker images | grep -c dockstore-cgpmap
+ - echo 'Verify program from each inherited package is found (dockstore-cgpbigwig)'
+ - docker run -ti --rm dockstore-cgpmap bwjoin --version
+ - echo 'Verify a program from each new package is found (dockstore-cgpmap)'
+ - docker run -ti --rm dockstore-cgpmap ds-cgpmap.pl -h
+ - docker run -ti --rm dockstore-cgpmap bwa_mem.pl -version
+ - docker run -ti --rm dockstore-cgpmap bammarkduplicates2 --version
+ - docker run -ti --rm dockstore-cgpmap samtools --version
+ - docker run -ti --rm dockstore-cgpmap bash -c 'bwa 2>&1 | grep Version'

CHANGES.md

@@ -1,35 +1,68 @@
-### 2.0.1
+# CHANGES
+
+## 3.0.0
+
+* Drop pre/post exec functions in `mapping.sh`
+* Add ability to pair a groups file with fastq inputs to add info to readgroups
+in final BAM/CRAM files (PCAP-core).
+* BWA 0.7.17 - bug fixes for future alpine building.
+* Biobambam2 2.0.84 - via pre-compiled versions
+* HTSlib + Samtools 1.7
+* PCAP-core 4.1.1
+  * mismatchQc options added
+* cgpBigWig 1.0.0 (via dockstore-cgpbigwig 2.0.0)
+* Examples moved to more useful naming, now have:
+  * `examples/bamOutput` and `examples/cramOutput` each containing:
+    * `bam_input.json`
+    * `fastq_gz_input.json` - with yaml groupinfo file example.
+* Multiple CWL descriptors one for BAM output, another for CRAM, legacy version retained.
+  * Ensure you use the correct `json` examples.
+* Now using secondaryFiles for outputs to reduce repetitive entries in cwl and json.
+  * Bugfix to dockstore tool needed for output provisioning, 1.3.6+
+
+## 2.0.1
+
 * Test data in `examples/sample_configs.local.json` moved to a non-expiring location.
 
-### 2.0.0
+## 2.0.0
+
 * PCAP-core forked to cancerit and all legacy PCAWG code removed.
 * Update to cgpBigWig/libBigWig to handle bug detected in ASCAT.
 * First layer of streamlined install process to reduce build time of dependant images.
 * Biobambam2 now building from source (previously picked up precompiled 'etch').
 
-### 1.0.8
+## 1.0.8
+
 Fix in PCAP-core to handle passing of sample name when input BAM has no SM tag in header.
 
-### 1.0.7
+## 1.0.7
+
 HTSlib upgades in toolset for consistency.
 
-### 1.0.6
+## 1.0.6
+
 Bump PCAP-core version to fix fastq input handling
 
-### 1.0.5
+## 1.0.5
+
 Adds travis-ci
 
-### 1.0.4
+## 1.0.4
+
 Bad versions in 1.0.3
 
-### 1.0.3
+## 1.0.3
+
 Base PCAP-core upgraded to improve mapping through-put and CPU use.
 
-### 1.0.2
+## 1.0.2
+
 Added build badges
 
-### 1.0.1
+## 1.0.1
+
 Fix CWL to get description to display on Dockstore.org
 
-### 1.0.0
+## 1.0.0
+
 Initial release.  Fully functional.

Dockerfile

@@ -1,36 +1,86 @@
-FROM  ubuntu:14.04
+FROM  quay.io/wtsicgp/dockstore-cgpbigwig:2.0.0 as builder
 
-MAINTAINER  [email protected]
+USER  root
 
-LABEL uk.ac.sanger.cgp="Cancer Genome Project, Wellcome Trust Sanger Institute" \
-      version="2.0.3" \
-      description="The CGP mapping pipeline for dockstore.org"
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends\
+  apt-transport-https\
+  locales\
+  curl\
+  ca-certificates\
+  libperlio-gzip-perl\
+  make\
+  bzip2\
+  gcc\
+  psmisc\
+  time\
+  zlib1g-dev\
+  libbz2-dev\
+  liblzma-dev\
+  libcurl4-gnutls-dev\
+  libncurses5-dev\
+  nettle-dev\
+  libp11-kit-dev\
+  libtasn1-dev\
+  libgnutls-dev
 
-USER  root
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
 
 ENV OPT /opt/wtsi-cgp
-ENV PATH $OPT/bin:$PATH
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
 ENV PERL5LIB $OPT/lib/perl5
 ENV LD_LIBRARY_PATH $OPT/lib
-
-## USER CONFIGURATION
-RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
 
 RUN mkdir -p $OPT/bin
 
-ADD scripts/mapping.sh $OPT/bin/mapping.sh
-ADD scripts/ds-wrapper.pl $OPT/bin/ds-wrapper.pl
-RUN chmod a+x $OPT/bin/mapping.sh $OPT/bin/ds-wrapper.pl
+ADD build/opt-build.sh build/
+RUN bash build/opt-build.sh $OPT
 
-ADD build/apt-build.sh build/
-RUN bash build/apt-build.sh
+FROM  ubuntu:16.04
 
-ADD build/perllib-build.sh build/
-RUN bash build/perllib-build.sh
+MAINTAINER  [email protected]
 
-ADD build/opt-build.sh build/
-ADD build/biobambam2-build.sh build/
-RUN bash build/opt-build.sh $OPT
+LABEL vendor="Cancer, Ageing and Somatic Mutation, Wellcome Trust Sanger Institute"
+LABEL uk.ac.sanger.cgp.description="PCAP-core for dockstore.org"
+LABEL uk.ac.sanger.cgp.version="3.0.0"
+
+RUN apt-get -yq update
+RUN apt-get install -yq --no-install-recommends\
+  apt-transport-https\
+  locales\
+  curl\
+  ca-certificates\
+  libperlio-gzip-perl\
+  bzip2\
+  psmisc\
+  time\
+  zlib1g\
+  liblzma5\
+  libncurses5\
+  p11-kit
+
+RUN locale-gen en_US.UTF-8
+RUN update-locale LANG=en_US.UTF-8
+
+ENV OPT /opt/wtsi-cgp
+ENV PATH $OPT/bin:$OPT/biobambam2/bin:$PATH
+ENV PERL5LIB $OPT/lib/perl5
+ENV LD_LIBRARY_PATH $OPT/lib
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+
+RUN mkdir -p $OPT
+COPY --from=builder $OPT $OPT
+
+ADD scripts/mapping.sh $OPT/bin/mapping.sh
+ADD scripts/ds-cgpmap.pl $OPT/bin/ds-cgpmap.pl
+RUN chmod a+x $OPT/bin/mapping.sh $OPT/bin/ds-cgpmap.pl
+
+## USER CONFIGURATION
+RUN adduser --disabled-password --gecos '' ubuntu && chsh -s /bin/bash && mkdir -p /home/ubuntu
 
 USER    ubuntu
 WORKDIR /home/ubuntu

Dockstore.cwl

@@ -9,23 +9,19 @@ label: "CGP BWA-mem mapping flow"
 cwlVersion: v1.0
 
 doc: |
-    ![build_status](https://quay.io/repository/wtsicgp/dockstore-cgpmap/status)
-    A Docker container for the CGP BWA-mem mapping flow. See the [dockstore-cgpmap](https://github.com/cancerit/dockstore-cgpmap) website for more information.
+  Please use one of the new tools for v3+:
 
-dct:creator:
-  "@id": "http://orcid.org/0000-0002-5634-1539"
-  foaf:name: Keiran M Raine
-  foaf:mbox: "keiranmraine@gmail.com"
+    * [dockstore-cgpmap/cgpmap-bamOut](https://dockstore.org/containers/quay.io%2Fwtsicgp%2Fdockstore-cgpmap%2Fcgpmap-bamOut)
+    * [dockstore-cgpmap/cgpmap-cramOut](https://dockstore.org/containers/quay.io%2Fwtsicgp%2Fdockstore-cgpmap%2Fcgpmap-cramOut)
+
+  ![build_status](https://quay.io/repository/wtsicgp/dockstore-cgpmap/status)
+  A Docker container for PCAP-core. See the [dockstore-cgpmap](https://github.com/cancerit/dockstore-cgpmap) website for more information.
 
 requirements:
-  - class: DockerRequirement
-    dockerPull: "quay.io/wtsicgp/dockstore-cgpmap:2.0.3"
+  - $mixin: cwls/mixins/requirements.yml
 
 hints:
-  - class: ResourceRequirement
-    coresMin: 1 # works but long, 8 recommended
-    ramMin: 15000 # good for WGS human ~30-60x
-    outdirMin: 5000000 # unlikely any BAM processing would be possible in less
+  - $mixin: cwls/mixins/hints.yml
 
 inputs:
   reference:
@@ -52,41 +48,48 @@ inputs:
       position: 3
       separate: true
 
-  scramble:
-    type: string?
-    doc: "Options to pass to scramble when generating CRAM output, see scramble docs"
-    default: ''
-    inputBinding:
-      prefix: -scramble
-      position: 4
-      separate: true
-      shellQuote: true
-
   bwa:
     type: string?
     default: ' -Y -K 100000000'
     doc: "Mapping and output parameters to pass to BWA-mem, see BWA docs, default ' -Y -K 100000000'"
     inputBinding:
       prefix: -bwa
-      position: 5
+      position: 4
       separate: true
       shellQuote: false
 
-  cram:
+  groupinfo:
+    type: File?
+    doc: "Readgroup metadata file for FASTQ inputs"
+    inputBinding:
+      prefix: -groupinfo
+      position: 5
+      separate: true
+
+  mmqc:
     type: boolean
-    doc: "Set if output should be in CRAM format instead of BAM, see 'scramble' for tuning parameters."
+    doc: "Apply mismatch QC to reads following duplicate marking."
     inputBinding:
-      prefix: -cram
+      prefix: -qc
       position: 6
 
+  mmqcfrac:
+    type: float?
+    default: 0.05
+    doc: "Mismatch fraction to set as max before failing a read [0.05]"
+    inputBinding:
+      prefix: -qcf
+      position: 7
+      separate: true
+
   bams_in:
     type:
     - 'null'
     - type: array
       items: File
     doc: "Can be BAM, CRAM, fastq (paired or interleaved), BAM/CRAM can be mixed together but not FASTQ."
     inputBinding:
-      position: 7
+      position: 8
 
 outputs:
   out_bam:
@@ -119,4 +122,19 @@ outputs:
     outputBinding:
       glob: $(inputs.sample).bam.maptime
 
-baseCommand: ["/opt/wtsi-cgp/bin/ds-wrapper.pl"]
+baseCommand: ["/opt/wtsi-cgp/bin/ds-cgpmap.pl"]
+
+$schemas:
+  - http://schema.org/docs/schema_org_rdfa.html
+
+$namespaces:
+  s: http://schema.org/
+
+s:codeRepository: https://github.com/cancerit/dockstore-cgpmap
+s:license: https://spdx.org/licenses/AGPL-3.0-only
+
+s:author:
+  - class: s:Person
+    s:identifier: https://orcid.org/0000-0002-5634-1539
+    s:email: mailto:[email protected]
+    s:name: Keiran Raine