How do the functions in dsl=2 work?

[anttitakalahti]

I have this main.nf workflow that tries to calculate Base 64 encoded sha512 for input

#!/usr/bin/env nextflow

nextflow.enable.dsl=2

import java.nio.charset.StandardCharsets
import java.util.Base64
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec

workflow {
    labIDs = Channel.of('19457_108635_21A', '19455_108626_21', '108296', '108641')
    printLabIDWithSignature(labIDs)
    println ""
    printLabIDWithSignature.out.verbiage.view()
}

process printLabIDWithSignature {
  input:
    val lab_id
  output:
    stdout emit: verbiage
  shell:
    signatureFromFunction = getSignature(lab_id)

    Mac mac = Mac.getInstance("HmacSHA512")
    SecretKeySpec secretKeySpec = new SecretKeySpec("acdcabbacd".getBytes(StandardCharsets.UTF_8), "HmacSHA512")
    mac.init(secretKeySpec)
    url = "/api/v1/file_paths/dynamic/" + lab_id + "/bam.bam"
    digest = mac.doFinal(url.getBytes(StandardCharsets.UTF_8))
    signature = Base64.getEncoder().encodeToString(digest)

    '''
    echo "for lab_id: !{lab_id} i have signature: !{signature} and the function has: !{signatureFromFunction}"
    '''
}

def getSignature(lab_id) {
  println "  getSignature got input: " + lab_id

  Mac mac = Mac.getInstance("HmacSHA512")

  SecretKeySpec secretKeySpec = new SecretKeySpec("acdcabbacd".getBytes(StandardCharsets.UTF_8), "HmacSHA512")
  mac.init(secretKeySpec)

  url = "/api/v1/file_paths/dynamic/" + lab_id + "/bam.bam"

  println "  getSignature lab_id: " + lab_id + " url: " + url

  digest = mac.doFinal(url.getBytes(StandardCharsets.UTF_8))
  signature = Base64.getEncoder().encodeToString(digest)

  println "  getSignature lab_id: " + lab_id + " url: " + url + " signature: " + signature

  return signature
}

but the output suggests that the function calls don't work as I would expect and the output of that program is :

(˘⊖˘) ~/foo [antti.takalahti@FVFY128YHV2H] > nextflow main.nf
N E X T F L O W  ~  version 21.04.3
Launching `main.nf` [infallible_rosalind] - revision: a2a6cbeba6
executor >  local (4)
[2c/b30205] process > printLabIDWithSignature (3) [100%] 4 of 4 ✔

  getSignature got input: 19457_108635_21A
  getSignature got input: 108641
  getSignature got input: 108296
  getSignature got input: 19455_108626_21
  getSignature lab_id: 19457_108635_21A url: /api/v1/file_paths/dynamic/19457_108635_21A/bam.bam
  getSignature lab_id: 108641 url: /api/v1/file_paths/dynamic/108641/bam.bam
  getSignature lab_id: 108296 url: /api/v1/file_paths/dynamic/108296/bam.bam
  getSignature lab_id: 19455_108626_21 url: /api/v1/file_paths/dynamic/19455_108626_21/bam.bam
  getSignature lab_id: 19455_108626_21 url: /api/v1/file_paths/dynamic/108641/bam.bam signature: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==
  getSignature lab_id: 108296 url: /api/v1/file_paths/dynamic/108641/bam.bam signature: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==
  getSignature lab_id: 108641 url: /api/v1/file_paths/dynamic/108641/bam.bam signature: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==
  getSignature lab_id: 19457_108635_21A url: /api/v1/file_paths/dynamic/108641/bam.bam signature: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==
for lab_id: 19457_108635_21A i have signature: akok7CP1yTmcEMO4fQC4mhSTZoRvZcJl3j1f2yOClADFS87ocMLk26eqIRbtcZdA5RkXGy/q11OoUWS79F5egg== and the function has: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==

for lab_id: 19455_108626_21 i have signature: ScWBRYxhJxu4KzrRdgoA+hiGffzYoirsec1j1DgfZxhHR25Mp9vf5btorS4PHa2sr0QJnBfjSC2sxuTFFS+HXA== and the function has: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==

for lab_id: 108641 i have signature: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg== and the function has: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==

for lab_id: 108296 i have signature: OIUuZAOXjKrezHovV84vCKYbyohOwJiPjr027LWplCqV3j8uNsdAevCI92mN8Jzyl6fD22/C1ivkSocfARIpig== and the function has: W1+6jqVc+pz5gE82Yg+x/Medt69DnvvvyCs/5hG0ZcFk+V27Ki2Q9lfrakxXRuE4u8gp39EeH2wH+0iz42QXgg==

What am I doing wrong? How should I use the functions

[1pakch]

I looked at your problem when learning nextflow and I can confirm that something weird is going on here. The output is non-deterministic and obviously wrong. I used nextflow version 21.08.0-edge.5609

One launch:

> rm -rf .nextflow* ./work/ && nextflow run wf3.nf -params-file params.json | grep 'sig...(' | sort
siggr1(A)=uCWnQH3/Y
siggr1(A)=uCWnQH3/Y
siggr1(B)=/UqksMFAZ
siggr2(A)=uCWnQH3/Y
siggr2(A)=uCWnQH3/Y
siggr2(B)=/UqksMFAZ
sigsh1(A)=uCWnQH3/Y
sigsh1(A)=uCWnQH3/Y
sigsh1(B)=/UqksMFAZ
sigsh2(A)=uCWnQH3/Y
sigsh2(A)=uCWnQH3/Y
sigsh2(B)=uCWnQH3/Y

Another launch:

> rm -rf .nextflow* ./work/ && nextflow run wf3.nf -params-file params.json | grep 'sig...(' | sort
siggr1(A)=uCWnQH3/Y
siggr1(A)=uCWnQH3/Y
siggr1(A)=uCWnQH3/Y
siggr1(B)=/UqksMFAZ
siggr2(A)=uCWnQH3/Y
siggr2(A)=uCWnQH3/Y
siggr2(B)=/UqksMFAZ
sigsh1(A)=uCWnQH3/Y
sigsh1(A)=uCWnQH3/Y
sigsh1(B)=/UqksMFAZ
sigsh2(A)=uCWnQH3/Y
sigsh2(A)=uCWnQH3/Y
sigsh2(B)=uCWnQH3/Y

The reworked Nextflow file:

nextflow.enable.dsl=2

import java.nio.charset.StandardCharsets
import java.util.Base64
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec


def getSecretKeySpec() {
    new SecretKeySpec("acdcabbacd".getBytes(StandardCharsets.UTF_8), "HmacSHA512")
}

def getMac() {
    Mac mac = Mac.getInstance("HmacSHA512")
    mac.init(getSecretKeySpec())
    return mac
}

def getUrl(id) {
    "/api/v1/file_paths/dynamic/" + id + "/bam.bam"
}

def getSignature(id) {
  url = getUrl(id)
  digest = getMac().doFinal(url.getBytes(StandardCharsets.UTF_8))
  return Base64.getEncoder().encodeToString(digest)
}

process siggr1 {
  input:
    val id
  output:
    val result
  exec:
    url = getUrl(id)
    digest = getMac().doFinal(url.getBytes(StandardCharsets.UTF_8))
    sig = Base64.getEncoder().encodeToString(digest)[0..8]
    result = "siggr1(${id})=${sig}"
}

process siggr2 {
  input:
    val id
  output:
    val result
  exec:
    sig = getSignature(id)[0..8]
    result = "siggr2(${id})=${sig}"
}

process sigsh1 {
  input:
    val id
  output:
    stdout emit: verbiage
  shell:
    url = getUrl(id)
    digest = getMac().doFinal(url.getBytes(StandardCharsets.UTF_8))
    sig = Base64.getEncoder().encodeToString(digest)[0..8]
  '''
  echo -n "sigsh1(!{id})=!{sig}"
  '''
}

process sigsh2 {
  input:
    val id
  output:
    stdout emit: verbiage
  shell:
    sig = getSignature(id)[0..8]
  '''
  echo -n "sigsh2(!{id})=!{sig}"
  '''
}

workflow {
    ids = Channel.of('A', 'A', 'B')
    siggr1(ids)
    siggr2(ids)
    sigsh1(ids)
    sigsh2(ids)
    siggr1.out.view()
    siggr2.out.view()
    sigsh1.out.verbiage.view()
    sigsh2.out.verbiage.view()
}

[anttitakalahti]

the issue is with the global name space of groovy. If you try to use types (Mac mac ...) it will be in the global scope where as def mac ... will be in local scope.

[1pakch]

Oh, I see, thanks! Very unintuitive...
The related issue if someone will be reading this: #1437

[anttitakalahti]

changing code to this:

#!/usr/bin/env nextflow

nextflow.enable.dsl=2

import java.nio.charset.StandardCharsets
import java.util.Base64
import javax.crypto.Mac
import javax.crypto.spec.SecretKeySpec

workflow {
    labIDs = Channel.of('19457_108635_21A', '19455_108626_21', '108296', '108641')
    printLabIDWithSignature(labIDs)
    println ""
    printLabIDWithSignature.out.verbiage.view()
}

process printLabIDWithSignature {
  input:
    val lab_id
  output:
    stdout emit: verbiage
  shell:
    signatureFromFunction = getSignature(lab_id)

    Mac mac = Mac.getInstance("HmacSHA512")
    SecretKeySpec secretKeySpec = new SecretKeySpec("acdcabbacd".getBytes(StandardCharsets.UTF_8), "HmacSHA512")
    mac.init(secretKeySpec)
    url = "/api/v1/file_paths/dynamic/" + lab_id + "/bam.bam"
    digest = mac.doFinal(url.getBytes(StandardCharsets.UTF_8))
    signature = Base64.getEncoder().encodeToString(digest)

    '''
    echo "for lab_id: !{lab_id} i have signature: !{signature} and the function has: !{signatureFromFunction}"
    '''
}

def getSignature(lab_id) {
  println "  getSignature got input: " + lab_id

  def mac = Mac.getInstance("HmacSHA512")

  def secretKeySpec = new SecretKeySpec("acdcabbacd".getBytes(StandardCharsets.UTF_8), "HmacSHA512")
  mac.init(secretKeySpec)

  url = "/api/v1/file_paths/dynamic/" + lab_id + "/bam.bam"

  println "  getSignature lab_id: " + lab_id + " url: " + url

  def digest = mac.doFinal(url.getBytes(StandardCharsets.UTF_8))
  def signature = Base64.getEncoder().encodeToString(digest)

  println "  getSignature lab_id: " + lab_id + " url: " + url + " signature: " + signature

  return signature
}

solves the issue