Memory resource definitions

workflow/snakemake_rules/export_for_nextstrain.smk

@@ -22,6 +22,7 @@
 import re
 import requests
 import json
+from math import ceil
 from workflow.lib.persistent_dict import PersistentDict, NoSuchEntryError
 
 ruleorder: dated_json > finalize
@@ -80,7 +81,7 @@ rule export_all_regions:
         # Memory use scales primarily with the size of the metadata file.
         # Compared to other rules, this rule loads metadata as a pandas
         # DataFrame instead of a dictionary, so it uses much less memory.
-        mem_mb=lambda wildcards, input: 5 * int(input.metadata.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(5 * (input.metadata.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """

workflow/snakemake_rules/main_workflow.smk

@@ -1,3 +1,5 @@
+from math import ceil
+
 rule sanitize_metadata:
     input:
         metadata=lambda wildcards: _get_path_for_input("metadata", wildcards.origin)
@@ -803,13 +805,13 @@ rule tree:
         # Multiple sequence alignments can use up to 40 times their disk size in
         # memory, especially for larger alignments.
         # Note that Snakemake >5.10.0 supports input.size_mb to avoid converting from bytes to MB.
-        mem_mb=lambda wildcards, input: 40 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(40 * (input.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """
         augur tree \
             --alignment {input.alignment} \
-            --tree-builder-args {params.args} \
+            --tree-builder-args {params.args}' --mem {resources.mem_mb}M' \
             {params.exclude_sites} \
             --output {output.tree} \
             --nthreads {threads} 2>&1 | tee {log}
@@ -839,7 +841,7 @@ rule refine:
         # Multiple sequence alignments can use up to 15 times their disk size in
         # memory.
         # Note that Snakemake >5.10.0 supports input.size_mb to avoid converting from bytes to MB.
-        mem_mb=lambda wildcards, input: 15 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(15 * (input.size / 1024 / 1024))
     params:
         root = config["refine"]["root"],
         clock_rate = config["refine"]["clock_rate"],
@@ -893,7 +895,7 @@ rule ancestral:
         # Multiple sequence alignments can use up to 15 times their disk size in
         # memory.
         # Note that Snakemake >5.10.0 supports input.size_mb to avoid converting from bytes to MB.
-        mem_mb=lambda wildcards, input: 15 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(15 * (input.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """
@@ -924,7 +926,7 @@ rule translate:
         # Multiple sequence alignments can use up to 15 times their disk size in
         # memory.
         # Note that Snakemake >5.10.0 supports input.size_mb to avoid converting from bytes to MB.
-        mem_mb=lambda wildcards, input: 15 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(15 * (input.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """
@@ -1055,7 +1057,7 @@ rule clades:
         "benchmarks/clades_{build_name}.txt"
     resources:
         # Memory use scales primarily with size of the node data.
-        mem_mb=lambda wildcards, input: 3 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(3 * (input.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """
@@ -1081,7 +1083,7 @@ rule emerging_lineages:
         "benchmarks/emerging_lineages_{build_name}.txt"
     resources:
         # Memory use scales primarily with size of the node data.
-        mem_mb=lambda wildcards, input: 3 * int(input.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(3 * (input.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """
@@ -1125,7 +1127,7 @@ rule colors:
         # Memory use scales primarily with the size of the metadata file.
         # Compared to other rules, this rule loads metadata as a pandas
         # DataFrame instead of a dictionary, so it uses much less memory.
-        mem_mb=lambda wildcards, input: 5 * int(input.metadata.size / 1024 / 1024)
+        mem_mb=lambda wildcards, input: ceil(5 * (input.metadata.size / 1024 / 1024))
     conda: config["conda_environment"]
     shell:
         """