Skip to content
Snippets Groups Projects
Commit f30daff1 authored by kdc715's avatar kdc715
Browse files

samtools multithreading scheduling fix

parent 021311d9
Branches
No related merge requests found
Hide whitespace changes
Inline Side-by-side
config/cluster_config.yaml
+ 3
− 0
View file @ f30daff1
...@@ -27,6 +27,9 @@ resources: ...@@ -27,6 +27,9 @@ resources:
samtools: samtools:
mem: 4096 mem: 4096
time: "12h" time: "12h"
samtools_multithread:
mem_per_thread: 4096
time: "12h"
samtools_25G: samtools_25G:
mem: 25600 mem: 25600
time: "12h" time: "12h"
......
snakemodules/contigmap.smk
+ 12
− 8
View file @ f30daff1
...@@ -96,9 +96,9 @@ rule name_sort_unsorted: ...@@ -96,9 +96,9 @@ rule name_sort_unsorted:
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -106,7 +106,9 @@ rule name_sort_unsorted: ...@@ -106,7 +106,9 @@ rule name_sort_unsorted:
benchmark: benchmark:
"benchmarks/contigmap/{sample}_name_sort_unsorted.txt" "benchmarks/contigmap/{sample}_name_sort_unsorted.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input} 2> {log.err}" """
samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input} 2> {log.err}
"""
################################################################################ ################################################################################
...@@ -171,9 +173,9 @@ rule coordinate_sort_unsorted: ...@@ -171,9 +173,9 @@ rule coordinate_sort_unsorted:
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools_25G"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools_25G"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -181,7 +183,9 @@ rule coordinate_sort_unsorted: ...@@ -181,7 +183,9 @@ rule coordinate_sort_unsorted:
benchmark: benchmark:
"benchmarks/contigmap/{sample}_coordinate_sort_unsorted.txt" "benchmarks/contigmap/{sample}_coordinate_sort_unsorted.txt"
shell: shell:
"samtools sort -@ {threads} -m {resources.mem_mb}M -o {output} {input} 2> {log.err}" """
samtools sort -@ {threads} -m {resources.mem_per_thread}M -o {output} {input} 2> {log.err}
"""
rule index_sorted: rule index_sorted:
......
snakemodules/crop_remapped.smk
+ 8
− 8
View file @ f30daff1
...@@ -60,9 +60,9 @@ if config["SICKLE"]=="no": ...@@ -60,9 +60,9 @@ if config["SICKLE"]=="no":
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -70,7 +70,7 @@ if config["SICKLE"]=="no": ...@@ -70,7 +70,7 @@ if config["SICKLE"]=="no":
benchmark: benchmark:
"benchmarks/crop_remapped/{sample}_sort.txt" "benchmarks/crop_remapped/{sample}_sort.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input.mates} 2> {log.err}" "samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input.mates} 2> {log.err}"
elif config["SICKLE"]=="yes": elif config["SICKLE"]=="yes":
...@@ -113,9 +113,9 @@ elif config["SICKLE"]=="yes": ...@@ -113,9 +113,9 @@ elif config["SICKLE"]=="yes":
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -123,7 +123,7 @@ elif config["SICKLE"]=="yes": ...@@ -123,7 +123,7 @@ elif config["SICKLE"]=="yes":
benchmark: benchmark:
"benchmarks/crop_remapped/{sample}_sort_sickle.txt" "benchmarks/crop_remapped/{sample}_sort_sickle.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input.mates} 2> {log.err}" "samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input.mates} 2> {log.err}"
rule popins2_sickle: rule popins2_sickle:
input: input:
......
snakemodules/crop_unmapped.smk
+ 8
− 8
View file @ f30daff1
...@@ -58,9 +58,9 @@ if config["SICKLE"]=="no": ...@@ -58,9 +58,9 @@ if config["SICKLE"]=="no":
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -68,7 +68,7 @@ if config["SICKLE"]=="no": ...@@ -68,7 +68,7 @@ if config["SICKLE"]=="no":
benchmark: benchmark:
"benchmarks/crop_unmapped/{sample}_sort.txt" "benchmarks/crop_unmapped/{sample}_sort.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input.mates} 2> {log.err}" "samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input.mates} 2> {log.err}"
elif config["SICKLE"]=="yes": elif config["SICKLE"]=="yes":
...@@ -109,9 +109,9 @@ elif config["SICKLE"]=="yes": ...@@ -109,9 +109,9 @@ elif config["SICKLE"]=="yes":
output: output:
WORK_DIR + "/{sample}/non_ref.bam" WORK_DIR + "/{sample}/non_ref.bam"
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
conda: conda:
...@@ -121,7 +121,7 @@ elif config["SICKLE"]=="yes": ...@@ -121,7 +121,7 @@ elif config["SICKLE"]=="yes":
benchmark: benchmark:
"benchmarks/crop_unmapped/{sample}_sort_sickle.txt" "benchmarks/crop_unmapped/{sample}_sort_sickle.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input.mates} 2> {log.err}" "samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input.mates} 2> {log.err}"
rule popins2_sickle: rule popins2_sickle:
input: input:
......
snakemodules/kraken.smk
+ 8
− 8
View file @ f30daff1
...@@ -150,9 +150,9 @@ rule samtools_remap_classified_human: ...@@ -150,9 +150,9 @@ rule samtools_remap_classified_human:
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -162,7 +162,7 @@ rule samtools_remap_classified_human: ...@@ -162,7 +162,7 @@ rule samtools_remap_classified_human:
shell: shell:
""" """
samtools view -Sb {input.sam} > {output.remapped_unsorted} 2> {log.err}; samtools view -Sb {input.sam} > {output.remapped_unsorted} 2> {log.err};
samtools sort -@ {threads} -m {resources.mem_mb}M -o {output.remapped_bam} {output.remapped_unsorted} 2>> {log.err} samtools sort -@ {threads} -m {resources.mem_per_thread}M -o {output.remapped_bam} {output.remapped_unsorted} 2>> {log.err}
""" """
rule index_reads: rule index_reads:
...@@ -224,9 +224,9 @@ rule remapping_samsort_mates: ...@@ -224,9 +224,9 @@ rule remapping_samsort_mates:
conda: conda:
os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml") os.path.join(WORKFLOW_PATH,"snakemodules/envs/samtools.yml")
resources: resources:
# mem_mb = resources["samtools"]["mem"], mem_per_thread = resources["samtools_multithread"]["mem_per_thread"],
mem_mb = lambda wildcards, input, threads, attempt: resources["samtools"]["mem"] * threads, mem_mb = lambda wildcards, input, threads, attempt: resources["samtools_multithread"]["mem_per_thread"] * threads,
runtime = resources["samtools"]["time"] runtime = resources["samtools_multithread"]["time"]
threads: threads:
threads["multi"]["samtools"] threads["multi"]["samtools"]
log: log:
...@@ -234,7 +234,7 @@ rule remapping_samsort_mates: ...@@ -234,7 +234,7 @@ rule remapping_samsort_mates:
benchmark: benchmark:
"benchmarks/kraken/{sample}_remapping_samsort_mates.txt" "benchmarks/kraken/{sample}_remapping_samsort_mates.txt"
shell: shell:
"samtools sort -n -@ {threads} -m {resources.mem_mb}M -o {output} {input} 2> {log.err}" "samtools sort -n -@ {threads} -m {resources.mem_per_thread}M -o {output} {input} 2> {log.err}"
rule merge_set_mate: rule merge_set_mate:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment