[BENCHMARK] Switch the reference genome used for E Coli

Author: GZHoffie

bucket_map/benchmark/benchmark.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-GENOME_FILE="/mnt/d/genome/Egu.v3.genome_f.fasta"
+GENOME_FILE="/mnt/d/genome/GCA_000005845.2.fasta"
 BENCHMARK_PATH="/home/zhenhao/bucket-map/bucket_map/benchmark"
-QUERY_FILE="/mnt/d/genome/TS1.81.90.001.fq"
-INDICATOR="egu"
+QUERY_FILE="/mnt/d/genome/DRR035999.fastq"
+INDICATOR="EColi"
 
 # initialize benchmark directory if it doesnt exist
 mkdir -p ${BENCHMARK_PATH}

bucket_map/benchmark/benchmark_index.sh

@@ -13,16 +13,16 @@ echo "Indexing using bowtie2"
 
 # run indexing for bwa
 echo "Indexing using bwa"
-#/usr/bin/time -o "${BENCHMARK_PATH}/log/bwa_index.time" -v bwa index -p "${INDEX_INDICATOR}_bwa" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/bwa_index.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/bwa_index.time" -v bwa index -p "${INDEX_INDICATOR}_bwa" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/bwa_index.log"
 
 # run indexing for subread
 echo "Indexing using subread"
-#/usr/bin/time -o "${BENCHMARK_PATH}/log/subread_index.time" -v subread-buildindex -o "${INDEX_INDICATOR}_subread" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/subread_index.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/subread_index.time" -v subread-buildindex -o "${INDEX_INDICATOR}_subread" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/subread_index.log"
 
 # run indexing for minimap2
 echo "Indexing using minimap2"
-#/usr/bin/time -o "${BENCHMARK_PATH}/log/minimap2_index.time" -v minimap2 -d "${INDEX_INDICATOR}_minimap.mmi" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/minimap2_index.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/minimap2_index.time" -v minimap2 -d "${INDEX_INDICATOR}_minimap.mmi" ${FASTA_PATH} &> "${BENCHMARK_PATH}/log/minimap2_index.log"
 
 # run indexing for BucketMap
 echo "Indexing using BucketMap"
-#/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_index.time" -v bucketmap -x -i "${INDEX_INDICATOR}_bucketmap" &> "${BENCHMARK_PATH}/log/bucketmap_index.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_index.time" -v bucketmap -x -i "${INDEX_INDICATOR}_bucketmap" &> "${BENCHMARK_PATH}/log/bucketmap_index.log"

bucket_map/benchmark/benchmark_map.sh

@@ -28,7 +28,7 @@ echo "Mapping using minimap2"
 
 # run bucketmap
 echo "Mapping using BucketMap"
-/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_map.time" -v bucketmap --version-check 0 -r 150 -i "${INDEX_INDICATOR}_bucketmap" -q ${FASTQ_PATH} -o "${BENCHMARK_PATH}/output/bucketmap_map.sam" &> "${BENCHMARK_PATH}/log/bucketmap_map.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_map.time" -v bucketmap --version-check 0 -r 310 -s 30 -e 0.6 -i "${INDEX_INDICATOR}_bucketmap" -q ${FASTQ_PATH} -o "${BENCHMARK_PATH}/output/bucketmap_map.sam" &> "${BENCHMARK_PATH}/log/bucketmap_map.log"
 
 echo "Mapping using BucketMap_align"
-/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_align_map.time" -v bucketmap_align --version-check 0 -r 150 -u 30 -i "${INDEX_INDICATOR}_bucketmap" -q ${FASTQ_PATH} -o "${BENCHMARK_PATH}/output/bucketmap_align_map.sam" &> "${BENCHMARK_PATH}/log/bucketmap_align_map.log"
+/usr/bin/time -o "${BENCHMARK_PATH}/log/bucketmap_align_map.time" -v bucketmap_align --version-check 0 -r 310 -s 30 -e 0.6 -u 30 -i "${INDEX_INDICATOR}_bucketmap" -q ${FASTQ_PATH} -o "${BENCHMARK_PATH}/output/bucketmap_align_map.sam" &> "${BENCHMARK_PATH}/log/bucketmap_align_map.log"

bucket_map/benchmark/log/bowtie2_index.log

@@ -16,7 +16,7 @@ Settings:
   Random seed: 0
   Sizeofs: void*:8, int:4, long:8, size_t:8
 Input files DNA, FASTA:
-  /mnt/d/genome/GCA_900166955.1.fasta
+  /mnt/d/genome/GCA_000005845.2.fasta
 Building a SMALL index
 Reading reference sizes
   Time reading reference sizes: 00:00:00
@@ -25,10 +25,10 @@ Writing header
 Reserving space for joined string
 Joining reference sequences
   Time to join reference sequences: 00:00:00
-bmax according to bmaxDivN setting: 1383039
-Using parameters --bmax 1037280 --dcv 1024
+bmax according to bmaxDivN setting: 1160413
+Using parameters --bmax 870310 --dcv 1024
   Doing ahead-of-time memory usage test
-  Passed!  Constructing with these parameters: --bmax 1037280 --dcv 1024
+  Passed!  Constructing with these parameters: --bmax 870310 --dcv 1024
 Constructing suffix-array element generator
 Building DifferenceCoverSample
   Building sPrime
@@ -52,33 +52,33 @@ Multikey QSorting 12 samples
 Calculating bucket sizes
 Splitting and merging
   Splitting and merging time: 00:00:00
-Avg bucket size: 5.53216e+06 (target: 1037279)
+Avg bucket size: 4.64165e+06 (target: 870309)
 Converting suffix-array elements to index image
 Allocating ftab, absorbFtab
 Entering Ebwt loop
 Getting block 1 of 1
   No samples; assembling all-inclusive block
-  Sorting block of length 5532158 for bucket 1
+  Sorting block of length 4641652 for bucket 1
   (Using difference cover)
   Sorting block time: 00:00:00
-Returning block of 5532159 for bucket 1
+Returning block of 4641653 for bucket 1
 Exited Ebwt loop
 fchr[A]: 0
-fchr[C]: 1368477
-fchr[G]: 2761687
-fchr[T]: 4159906
-fchr[$]: 5532158
+fchr[C]: 1142742
+fchr[G]: 2322833
+fchr[T]: 3500270
+fchr[$]: 4641652
 Exiting Ebwt::buildToDisk()
 Returning from initFromVector
-Wrote 6039197 bytes to primary EBWT file: EColi_bowtie.1.bt2
-Wrote 1383044 bytes to secondary EBWT file: EColi_bowtie.2.bt2
+Wrote 5741809 bytes to primary EBWT file: EColi_bowtie.1.bt2
+Wrote 1160420 bytes to secondary EBWT file: EColi_bowtie.2.bt2
 Re-opening _in1 and _in2 as input streams
 Returning from Ebwt constructor
 Headers:
-    len: 5532158
-    bwtLen: 5532159
-    sz: 1383040
-    bwtSz: 1383040
+    len: 4641652
+    bwtLen: 4641653
+    sz: 1160413
+    bwtSz: 1160414
     lineRate: 6
     offRate: 4
     offMask: 0xfffffff0
@@ -87,19 +87,19 @@ Headers:
     eftabSz: 80
     ftabLen: 1048577
     ftabSz: 4194308
-    offsLen: 345760
-    offsSz: 1383040
+    offsLen: 290104
+    offsSz: 1160416
     lineSz: 64
     sideSz: 64
     sideBwtSz: 48
     sideBwtLen: 192
-    numSides: 28814
-    numLines: 28814
-    ebwtTotLen: 1844096
-    ebwtTotSz: 1844096
+    numSides: 24176
+    numLines: 24176
+    ebwtTotLen: 1547264
+    ebwtTotSz: 1547264
     color: 0
     reverse: 0
-Total time for call to driver() for forward index: 00:00:01
+Total time for call to driver() for forward index: 00:00:00
 Reading reference sizes
   Time reading reference sizes: 00:00:00
 Calculating joined length
@@ -108,10 +108,10 @@ Reserving space for joined string
 Joining reference sequences
   Time to join reference sequences: 00:00:00
   Time to reverse reference sequence: 00:00:00
-bmax according to bmaxDivN setting: 1383039
-Using parameters --bmax 1037280 --dcv 1024
+bmax according to bmaxDivN setting: 1160413
+Using parameters --bmax 870310 --dcv 1024
   Doing ahead-of-time memory usage test
-  Passed!  Constructing with these parameters: --bmax 1037280 --dcv 1024
+  Passed!  Constructing with these parameters: --bmax 870310 --dcv 1024
 Constructing suffix-array element generator
 Building DifferenceCoverSample
   Building sPrime
@@ -135,33 +135,33 @@ Multikey QSorting 12 samples
 Calculating bucket sizes
 Splitting and merging
   Splitting and merging time: 00:00:00
-Avg bucket size: 5.53216e+06 (target: 1037279)
+Avg bucket size: 4.64165e+06 (target: 870309)
 Converting suffix-array elements to index image
 Allocating ftab, absorbFtab
 Entering Ebwt loop
 Getting block 1 of 1
   No samples; assembling all-inclusive block
-  Sorting block of length 5532158 for bucket 1
+  Sorting block of length 4641652 for bucket 1
   (Using difference cover)
-  Sorting block time: 00:00:00
-Returning block of 5532159 for bucket 1
+  Sorting block time: 00:00:01
+Returning block of 4641653 for bucket 1
 Exited Ebwt loop
 fchr[A]: 0
-fchr[C]: 1368477
-fchr[G]: 2761687
-fchr[T]: 4159906
-fchr[$]: 5532158
+fchr[C]: 1142742
+fchr[G]: 2322833
+fchr[T]: 3500270
+fchr[$]: 4641652
 Exiting Ebwt::buildToDisk()
 Returning from initFromVector
-Wrote 6039197 bytes to primary EBWT file: EColi_bowtie.rev.1.bt2
-Wrote 1383044 bytes to secondary EBWT file: EColi_bowtie.rev.2.bt2
+Wrote 5741809 bytes to primary EBWT file: EColi_bowtie.rev.1.bt2
+Wrote 1160420 bytes to secondary EBWT file: EColi_bowtie.rev.2.bt2
 Re-opening _in1 and _in2 as input streams
 Returning from Ebwt constructor
 Headers:
-    len: 5532158
-    bwtLen: 5532159
-    sz: 1383040
-    bwtSz: 1383040
+    len: 4641652
+    bwtLen: 4641653
+    sz: 1160413
+    bwtSz: 1160414
     lineRate: 6
     offRate: 4
     offMask: 0xfffffff0
@@ -170,16 +170,16 @@ Headers:
     eftabSz: 80
     ftabLen: 1048577
     ftabSz: 4194308
-    offsLen: 345760
-    offsSz: 1383040
+    offsLen: 290104
+    offsSz: 1160416
     lineSz: 64
     sideSz: 64
     sideBwtSz: 48
     sideBwtLen: 192
-    numSides: 28814
-    numLines: 28814
-    ebwtTotLen: 1844096
-    ebwtTotSz: 1844096
+    numSides: 24176
+    numLines: 24176
+    ebwtTotLen: 1547264
+    ebwtTotSz: 1547264
     color: 0
     reverse: 1
-Total time for backward call to driver() for mirror index: 00:00:00
+Total time for backward call to driver() for mirror index: 00:00:01

bucket_map/benchmark/log/bowtie2_index.time

@@ -1,21 +1,21 @@
-	Command being timed: "bowtie2-build /mnt/d/genome/GCA_900166955.1.fasta EColi_bowtie"
-	User time (seconds): 1.51
-	System time (seconds): 0.13
-	Percent of CPU this job got: 94%
-	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.74
+	Command being timed: "bowtie2-build /mnt/d/genome/GCA_000005845.2.fasta EColi_bowtie"
+	User time (seconds): 1.28
+	System time (seconds): 0.09
+	Percent of CPU this job got: 97%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 0:01.41
 	Average shared text size (kbytes): 0
 	Average unshared data size (kbytes): 0
 	Average stack size (kbytes): 0
 	Average total size (kbytes): 0
-	Maximum resident set size (kbytes): 103104
+	Maximum resident set size (kbytes): 101960
 	Average resident set size (kbytes): 0
-	Major (requiring I/O) page faults: 109
-	Minor (reclaiming a frame) page faults: 12988
-	Voluntary context switches: 418
-	Involuntary context switches: 3
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 15265
+	Voluntary context switches: 193
+	Involuntary context switches: 1
 	Swaps: 0
-	File system inputs: 25488
-	File system outputs: 31736
+	File system inputs: 0
+	File system outputs: 29272
 	Socket messages sent: 0
 	Socket messages received: 0
 	Signals delivered: 0

bucket_map/benchmark/log/bowtie2_map.log

@@ -1,6 +1,6 @@
-4922564 reads; of these:
-  4922564 (100.00%) were unpaired; of these:
-    462017 (9.39%) aligned 0 times
-    3478827 (70.67%) aligned exactly 1 time
-    981720 (19.94%) aligned >1 times
-90.61% overall alignment rate
+1302395 reads; of these:
+  1302395 (100.00%) were unpaired; of these:
+    238893 (18.34%) aligned 0 times
+    1032371 (79.27%) aligned exactly 1 time
+    31131 (2.39%) aligned >1 times
+81.66% overall alignment rate

bucket_map/benchmark/log/bowtie2_map.time

@@ -1,21 +1,21 @@
-	Command being timed: "bowtie2 -x egu_bowtie -U /mnt/d/genome/TS1.81.90.001.fq -S /home/zhenhao/bucket-map/bucket_map/benchmark/output/bowtie2_map.sam"
-	User time (seconds): 743.00
-	System time (seconds): 7.05
+	Command being timed: "bowtie2 -x EColi_bowtie -U /mnt/d/genome/DRR035999.fastq -S /home/zhenhao/bucket-map/bucket_map/benchmark/output/bowtie2_map.sam"
+	User time (seconds): 196.40
+	System time (seconds): 2.52
 	Percent of CPU this job got: 98%
-	Elapsed (wall clock) time (h:mm:ss or m:ss): 12:41.37
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 3:21.23
 	Average shared text size (kbytes): 0
 	Average unshared data size (kbytes): 0
 	Average stack size (kbytes): 0
 	Average total size (kbytes): 0
-	Maximum resident set size (kbytes): 1961608
+	Maximum resident set size (kbytes): 30520
 	Average resident set size (kbytes): 0
-	Major (requiring I/O) page faults: 115
-	Minor (reclaiming a frame) page faults: 8232
-	Voluntary context switches: 100365
-	Involuntary context switches: 797
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 6632
+	Voluntary context switches: 31347
+	Involuntary context switches: 223
 	Swaps: 0
-	File system inputs: 3912920
-	File system outputs: 3700552
+	File system inputs: 8
+	File system outputs: 1688680
 	Socket messages sent: 0
 	Socket messages received: 0
 	Signals delivered: 0

bucket_map/benchmark/log/bucketmap_align_map.log

@@ -1,15 +1,15 @@
 [INFO]		Allowing Smith-Waterman for alignment verifications.
-[INFO]		Initializing indexer and mapper with bucket length: 65536, and number of buckets: 26507.
+[INFO]		Initializing indexer and mapper with bucket length: 65536, and number of buckets: 71.
 [INFO]		Set q-gram shape to be: [1,1,1,1,1,1,1,1,1] with number of effective characters: 9
-[ERROR]		The specified file already exists in directory: "/home/zhenhao/bucket-map/bucket_map/benchmark/index/egu_bucketmap.qgram".
-[BENCHMARK]	Number of Q-grams with distinguishability >= 0.499981: 251645 (95.9949%).
-[BENCHMARK]	Elapsed time for loading index files: 16.253 s.
-[INFO]		Successfully loaded "/home/zhenhao/bucket-map/bucket_map/benchmark/index/egu_bucketmap.qgram".
-[BENCHMARK]	Elapsed time for bucket mapping: 269.231 s (54.6932 μs/seq).
-[BENCHMARK]	Number of reads that have at least one candidate bucket: 4841636  (98.356%).
-[BENCHMARK]	Average number of buckets an original read is mapped to: 2.04986.
-[BENCHMARK]	Average number of buckets a reverse complement of the read is mapped to: 2.07859.
-[BENCHMARK]	Total time used for building k-mer index for each bucket: 204.802 s.
-[BENCHMARK]	Total time used for finding exact location of the sequences: 92.906 s (18.8735 μs/seq).
-[BENCHMARK]	Total mapped locations: 5422347 (1.10153 per sequence).
-[BENCHMARK]	Total time used for alignment verification and output: 231.415 s (42.678 μs per pairwise alignment).
+[ERROR]		The specified file already exists in directory: "/home/zhenhao/bucket-map/bucket_map/benchmark/index/EColi_bucketmap.qgram".
+[BENCHMARK]	Number of Q-grams with distinguishability >= 0.492958: 249872 (95.3186%).
+[BENCHMARK]	Elapsed time for loading index files: 0.057 s.
+[INFO]		Successfully loaded "/home/zhenhao/bucket-map/bucket_map/benchmark/index/EColi_bucketmap.qgram".
+[BENCHMARK]	Elapsed time for bucket mapping: 14.789 s (11.3552 μs/seq).
+[BENCHMARK]	Number of reads that have at least one candidate bucket: 1254472  (96.3204%).
+[BENCHMARK]	Average number of buckets an original read is mapped to: 1.12067.
+[BENCHMARK]	Average number of buckets a reverse complement of the read is mapped to: 1.13216.
+[BENCHMARK]	Total time used for building k-mer index for each bucket: 0.245 s.
+[BENCHMARK]	Total time used for finding exact location of the sequences: 6.599 s (5.06682 μs/seq).
+[BENCHMARK]	Total mapped locations: 1181663 (0.9073 per sequence).
+[BENCHMARK]	Total time used for alignment verification and output: 63.521 s (53.7556 μs per pairwise alignment).

bucket_map/benchmark/log/bucketmap_align_map.time

@@ -1,21 +1,21 @@
-	Command being timed: "bucketmap_align --version-check 0 -r 150 -u 30 -i egu_bucketmap -q /mnt/d/genome/TS1.81.90.001.fq -o /home/zhenhao/bucket-map/bucket_map/benchmark/output/bucketmap_align_map.sam"
-	User time (seconds): 852.20
-	System time (seconds): 3.88
-	Percent of CPU this job got: 99%
-	Elapsed (wall clock) time (h:mm:ss or m:ss): 14:21.91
+	Command being timed: "bucketmap_align --version-check 0 -r 310 -s 30 -e 0.6 -u 30 -i EColi_bucketmap -q /mnt/d/genome/DRR035999.fastq -o /home/zhenhao/bucket-map/bucket_map/benchmark/output/bucketmap_align_map.sam"
+	User time (seconds): 87.01
+	System time (seconds): 1.73
+	Percent of CPU this job got: 97%
+	Elapsed (wall clock) time (h:mm:ss or m:ss): 1:31.01
 	Average shared text size (kbytes): 0
 	Average unshared data size (kbytes): 0
 	Average stack size (kbytes): 0
 	Average total size (kbytes): 0
-	Maximum resident set size (kbytes): 1081252
+	Maximum resident set size (kbytes): 169976
 	Average resident set size (kbytes): 0
-	Major (requiring I/O) page faults: 2
-	Minor (reclaiming a frame) page faults: 192708
-	Voluntary context switches: 24820
-	Involuntary context switches: 596
+	Major (requiring I/O) page faults: 0
+	Minor (reclaiming a frame) page faults: 17050
+	Voluntary context switches: 9248
+	Involuntary context switches: 132
 	Swaps: 0
-	File system inputs: 16
-	File system outputs: 3484984
+	File system inputs: 0
+	File system outputs: 1445960
 	Socket messages sent: 0
 	Socket messages received: 0
 	Signals delivered: 0

bucket_map/benchmark/log/bucketmap_index.log

@@ -1,4 +1,7 @@
 [INFO]		Not using Smith-Waterman for alignment verifications.
-[INFO]		Initializing indexer and mapper with bucket length: 32768, and number of buckets: 52426.
+[INFO]		Initializing indexer and mapper with bucket length: 65536, and number of buckets: 71.
 [INFO]		Set q-gram shape to be: [1,1,1,1,1,1,1,1,1] with number of effective characters: 9
-[ERROR]		The specified file already exists in directory: "/home/zhenhao/bucket-map/bucket_map/benchmark/index/egu_bucketmap_2.qgram".
+[INFO]		The bucket q-gram index is stored in: "/home/zhenhao/bucket-map/bucket_map/benchmark/index/EColi_bucketmap.qgram".
+[INFO]		The number of buckets: 71.
+[INFO]		The bucket ids are stored in: "/home/zhenhao/bucket-map/bucket_map/benchmark/index/EColi_bucketmap.bucket_id".
+[BENCHMARK]	Elapsed time for creating and storing index files: 0.101 s.