Skip to content

Commit c3cbe30

Browse files
jrobinsojrobinso
authored
Update reference (#1999)
* Update deprecated genome reference urls -- s3 urls will be removed at some point in the future, this update translates them to long-term stable UCSC urls
1 parent 92c9839 commit c3cbe30

File tree

8 files changed

+249
-25
lines changed

8 files changed

+249
-25
lines changed

‎js/bigwig/bwReader.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -559,7 +559,7 @@ class BWReader {
559559
class ZoomLevelHeader {
560560
constructor(index, byteBuffer) {
561561
this.index = index
562-
this.reductionLevel = byteBuffer.getInt()
562+
this.reductionLevel = byteBuffer.getUInt()
563563
this.reserved = byteBuffer.getInt()
564564
this.dataOffset = byteBuffer.getLong()
565565
this.indexOffset = byteBuffer.getLong()

‎js/browser.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -610,7 +610,7 @@ class Browser {
610610
}
611611

612612
/**
613-
* Load a reference genome object. This includes the fasta, and optional cytoband, but no tracks. This method
613+
* Load a reference genome object. This includes the sequence, and optional cytoband, but no tracks. This method
614614
* is used by loadGenome and loadSession.
615615
*
616616
* @param genomeConfig

‎js/genome/genome.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {loadChromSizes} from "./chromSizes.js"
1010
import ChromAliasDefaults from "./chromAliasDefaults.js"
1111
import {igvxhr} from "../../node_modules/igv-utils/src/index.js"
1212
import {loadHub} from "../ucsc/hub/hubParser.js"
13+
import {updateReference} from "./updateReference.js"
1314

1415
const ucsdIDMap = new Map([
1516
["1kg_ref", "hg18"],
@@ -32,6 +33,7 @@ class Genome {
3233

3334
static async createGenome(options, browser) {
3435

36+
updateReference(options)
3537
const genome = new Genome(options, browser)
3638
await genome.init()
3739
return genome

‎js/genome/updateReference.js

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
/**
2+
* Update deprecated fasta & index urls in the reference object.
3+
*/
4+
5+
const isString = (x) => {
6+
return (x && typeof x === "string") || x instanceof String
7+
}
8+
9+
/**
10+
* Replaces deprecated s3 fasta URLs with twoBitURLs. The purpose here is to rescue references from saved sessions
11+
* that contain pointers to deprectated IGV s3 buckets. These buckets will eventually be deleted
12+
*
13+
* @param reference
14+
*/
15+
function updateReference(reference) {
16+
17+
if (!(requiresUpdate(reference) && updates[reference.id])) {
18+
return
19+
}
20+
21+
const updatedReference = updates[reference.id]
22+
if (updatedReference) {
23+
delete reference.fastaURL
24+
if (reference.indexURL) delete reference.indexURL
25+
reference.twoBitURL = updatedReference.twoBitURL
26+
if (updatedReference.twoBitBptURL) reference.twoBitBptURL = updatedReference.twoBitBptURL
27+
if (updatedReference.chromSizesURL) reference.chromSizesURL = updatedReference.chromSizesURL
28+
}
29+
}
30+
31+
function requiresUpdate(reference) {
32+
return isString(reference.fastaURL) &&
33+
(reference.fastaURL.startsWith("https://igv.org") ||
34+
["igv.org.genomes", "igv.broadinstitute.org", "igv.genepattern.org", "igvdata.broadinstitute.org",
35+
"igv-genepattern-org"].some(bucket => reference.fastaURL.includes(bucket)))
36+
}
37+
38+
const updates = {
39+
"hs1": {
40+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hs1/bigZips/hs1.2bit",
41+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hs1/bigZips/hs1.2bit.bpt",
42+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hs1/bigZips/hs1.chrom.sizes.txt"
43+
},
44+
"hg38": {
45+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.2bit",
46+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.chrom.sizes"
47+
},
48+
"hg19": {
49+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/bigZips/hg19.2bit",
50+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/bigZips/hg19.chrom.sizes"
51+
},
52+
"hg18": {
53+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg18/bigZips/hg18.2bit",
54+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg18/bigZips/hg18.chrom.sizes"
55+
},
56+
"mm39": {
57+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm39/bigZips/mm39.2bit",
58+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm39/bigZips/mm39.chrom.sizes"
59+
},
60+
"mm10": {
61+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm10/bigZips/mm10.2bit",
62+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm10/bigZips/mm10.chrom.sizes"
63+
},
64+
"mm9": {
65+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm9/bigZips/mm9.2bit",
66+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/mm9/bigZips/mm9.chrom.sizes"
67+
},
68+
"rn7": {
69+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/015/227/675/GCF_015227675.2/GCF_015227675.2.2bit",
70+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/015/227/675/GCF_015227675.2/GCF_015227675.2.2bit.bpt",
71+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/015/227/675/GCF_015227675.2/GCF_015227675.2.chrom.sizes.txt"
72+
},
73+
"rn6": {
74+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/001/895/GCF_000001895.5/GCF_000001895.5.2bit",
75+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/001/895/GCF_000001895.5/GCF_000001895.5.2bit.bpt",
76+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/001/895/GCF_000001895.5/GCF_000001895.5.chrom.sizes.txt"
77+
},
78+
"gorGor6": {
79+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/gorGor6/bigZips/gorGor6.2bit",
80+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/gorGor6/bigZips/gorGor6.chrom.sizes"
81+
},
82+
"gorGor4": {
83+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/gorGor4/bigZips/gorGor4.2bit",
84+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/gorGor4/bigZips/gorGor4.chrom.sizes"
85+
},
86+
"panTro6": {
87+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro6/bigZips/panTro6.2bit",
88+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro6/bigZips/panTro6.chrom.sizes"
89+
},
90+
"panTro5": {
91+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro5/bigZips/panTro5.2bit",
92+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro5/bigZips/panTro5.chrom.sizes"
93+
},
94+
"panTro4": {
95+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro4/bigZips/panTro4.2bit",
96+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panTro4/bigZips/panTro4.chrom.sizes"
97+
},
98+
"macFas5": {
99+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/macFas5/bigZips/macFas5.2bit",
100+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/macFas5/bigZips/macFas5.chrom.sizes"
101+
},
102+
"panPan2": {
103+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panPan2/bigZips/panPan2.2bit",
104+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/panPan2/bigZips/panPan2.chrom.sizes"
105+
},
106+
"canFam6": {
107+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam6/bigZips/canFam6.2bit",
108+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam6/bigZips/canFam6.chrom.sizes"
109+
},
110+
"canFam5": {
111+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam5/bigZips/canFam5.2bit",
112+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam5/bigZips/canFam5.chrom.sizes"
113+
},
114+
"canFam4": {
115+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam4/bigZips/canFam4.2bit",
116+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam4/bigZips/canFam4.chrom.sizes"
117+
},
118+
"canFam3": {
119+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam3/bigZips/canFam3.2bit",
120+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/canFam3/bigZips/canFam3.chrom.sizes"
121+
},
122+
"bosTau9": {
123+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/bosTau9/bigZips/bosTau9.2bit",
124+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/bosTau9/bigZips/bosTau9.chrom.sizes"
125+
},
126+
"bosTau8": {
127+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/bosTau8/bigZips/bosTau8.2bit",
128+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/bosTau8/bigZips/bosTau8.chrom.sizes"
129+
},
130+
"susScr11": {
131+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/susScr11/bigZips/susScr11.2bit",
132+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/susScr11/bigZips/susScr11.chrom.sizes"
133+
},
134+
"galGal6": {
135+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/galGal6/bigZips/galGal6.2bit",
136+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/galGal6/bigZips/galGal6.chrom.sizes"
137+
},
138+
"danRer11": {
139+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/danRer11/bigZips/danRer11.2bit",
140+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/danRer11/bigZips/danRer11.chrom.sizes"
141+
},
142+
"danRer10": {
143+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/danRer10/bigZips/danRer10.2bit",
144+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/danRer10/bigZips/danRer10.chrom.sizes"
145+
},
146+
"ce11": {
147+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/ce11/bigZips/ce11.2bit",
148+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/ce11/bigZips/ce11.chrom.sizes"
149+
},
150+
"dm6": {
151+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/dm6/bigZips/dm6.2bit",
152+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/dm6/bigZips/dm6.chrom.sizes"
153+
},
154+
"dm3": {
155+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/goldenPath/dm3/bigZips/dm3.2bit",
156+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/goldenPath/dm3/bigZips/dm3.chrom.sizes"
157+
},
158+
"sacCer3": {
159+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/146/045/GCF_000146045.2/GCF_000146045.2.2bit",
160+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/146/045/GCF_000146045.2/GCF_000146045.2.2bit.bpt",
161+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/146/045/GCF_000146045.2/GCF_000146045.2.chrom.sizes.txt"
162+
},
163+
"GCF_000002945.1": {
164+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/945/GCF_000002945.1/GCF_000002945.1.2bit",
165+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/945/GCF_000002945.1/GCF_000002945.1.2bit.bpt",
166+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/002/945/GCF_000002945.1/GCF_000002945.1.chrom.sizes.txt"
167+
},
168+
"GCF_009858895.2": {
169+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/009/858/895/GCF_009858895.2/GCF_009858895.2.2bit",
170+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/009/858/895/GCF_009858895.2/GCF_009858895.2.2bit.bpt",
171+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/009/858/895/GCF_009858895.2/GCF_009858895.2.chrom.sizes.txt"
172+
},
173+
"tair10": {
174+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/001/735/GCF_000001735.3/GCF_000001735.3.2bit",
175+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCF/000/001/735/GCF_000001735.3/GCF_000001735.3.chrom.sizes.txt"
176+
},
177+
"GCA_000022165.1": {
178+
"twoBitURL": "https://hgdownload.soe.ucsc.edu/hubs/GCA/000/022/165/GCA_000022165.1/GCA_000022165.1.2bit",
179+
"twoBitBptURL": "https://hgdownload.soe.ucsc.edu/hubs/GCA/000/022/165/GCA_000022165.1/GCA_000022165.1.2bit.bpt",
180+
"chromSizesURL": "https://hgdownload.soe.ucsc.edu/hubs/GCA/000/022/165/GCA_000022165.1/GCA_000022165.1.chrom.sizes.txt"
181+
}
182+
}
183+
184+
export {updateReference}
185+

‎test/data/bb/bigtools.bigWig

1.73 KB
Binary file not shown.

‎test/data/bb/nozooms.bw

544 Bytes
Binary file not shown.

‎test/testBigwig.js

Lines changed: 24 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -87,23 +87,30 @@ suite("testBigWig", function () {
8787
}
8888
})
8989

90-
//
91-
// test("Compressed bigwig", async function () {
92-
//
93-
// this.timeout(10000)
94-
//
95-
// //chr21:19,146,376-19,193,466
96-
// const url = "https://hgdownload.soe.ucsc.edu/hubs/GCA/009/914/755/GCA_009914755.4/bbi/GCA_009914755.4_T2T-CHM13v2.0.gc5Base.bw",
97-
// chr = "CP068275.2",
98-
// start = 26490012,
99-
// end = 26490012 + 1,
100-
// bpPerPixel = 1
101-
//
102-
// const bwReader = new BWReader({url: url})
103-
// const features = await bwReader.readFeatures(chr, start, chr, end, bpPerPixel)
104-
// assert.equal(features.length, 1) // Verified in iPad app
105-
//
106-
// })
90+
/**
91+
* Test a bigwig with a very large reduction level value. This value overflows a 32-bit signed integer,
92+
*/
93+
test("bigwig - big reductionLevel", async function () {
94+
95+
const genome = createGenome("ncbi")
96+
const url = "test/data/bb/bigtools.bigWig"
97+
const bwReader = new BWReader({url: url}, genome)
98+
const zoomLevelHeaders = await bwReader.getZoomHeaders()
99+
assert.equal(zoomLevelHeaders[0].reductionLevel, 2684354560)
100+
})
101+
102+
/**
103+
* Test a bigwig with no zoom levels.,
104+
*/
105+
test("bigwig - no zooms", async function () {
106+
107+
this.timeout(10000)
107108

109+
const genome = createGenome("ncbi")
110+
const url = "test/data/bb/nozooms.bw"
111+
const bwReader = new BWReader({url: url}, genome)
112+
const zoomLevelHeaders = await bwReader.getZoomHeaders()
113+
assert.equal(zoomLevelHeaders.length, 0)
114+
})
108115

109116
})

‎test/testGenome.js

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import "./utils/mockObjects.js"
22
import Genome from "../js/genome/genome.js"
3+
import {updateReference} from "../js/genome/updateReference.js"
34
import {assert} from 'chai'
45
import {shortenChromsomeName} from "../js/rulerTrack.js"
56

@@ -11,10 +12,11 @@ suite("testGenome", function () {
1112
this.timeout(200000)
1213

1314
const reference = {
14-
id: "hg19",
15-
fastaURL: "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/1kg_v37/human_g1k_v37_decoy.fasta",
16-
indexURL: "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/1kg_v37/human_g1k_v37_decoy.fasta.fai",
17-
cytobandURL: "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/b37/b37_cytoband.txt",
15+
"id": "b37_1kg",
16+
"name": "Human (1kg, b37+decoy)",
17+
"fastaURL": "https://storage.googleapis.com/genomics-public-data/references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta",
18+
"indexURL": "https://storage.googleapis.com/genomics-public-data/references/Homo_sapiens_assembly19_1000genomes_decoy/Homo_sapiens_assembly19_1000genomes_decoy.fasta.fai",
19+
"cytobandURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/database/cytoBand.txt.gz",
1820
wholeGenomeView: true
1921
}
2022

@@ -44,16 +46,44 @@ suite("testGenome", function () {
4446

4547

4648
test("Shorten name", function() {
47-
4849
const names = ["chr1", "chromosome_1"]
4950
const expected = ["1", "chromosome_1"]
5051

5152
for(let i=0; i<names.length; i++) {
5253
const shortened = shortenChromsomeName(names[i])
5354
assert.equal(shortened, expected[i])
5455
}
55-
5656
})
5757

58+
test("update reference", function() {
59+
60+
const reference = {
61+
"id": "hg18",
62+
"name": "Human (hg18)",
63+
"fastaURL": "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg18/hg18.fasta",
64+
"indexURL": "https://s3.amazonaws.com/igv.broadinstitute.org/genomes/seq/hg18/hg18.fasta.fai",
65+
"cytobandURL": "https://hgdownload.soe.ucsc.edu/goldenPath/hg18/database/cytoBandIdeo.txt.gz",
66+
"tracks": [
67+
{
68+
"name": "Refseq Genes",
69+
"format": "refgene",
70+
"url": "https://hgdownload.soe.ucsc.edu/goldenPath/hg18/database/refGene.txt.gz",
71+
"indexed": false,
72+
"visibilityWindow": -1,
73+
"order": 1000000,
74+
"searchable": true
75+
}
76+
],
77+
"chromosomeOrder": "chr1,chr2,chr3,chr4,chr5,chr6,chr7,chr8,chr9,chr10,chr11,chr12,chr13,chr14,chr15,chr16,chr17,chr18,chr19,chr20,chr21,chr22,chrX,chrY"
78+
}
79+
80+
updateReference(reference)
81+
assert.isNotOk(reference.fastaURL)
82+
assert.isNotOk(reference.indexURL)
83+
assert.isOk(reference.twoBitURL)
84+
assert.isOk(reference.chromSizesURL)
85+
86+
87+
})
5888

5989
})

0 commit comments

Comments
 (0)