diff --git a/modules/io.mjs b/modules/io.mjs index df02485d1..43a0df620 100644 --- a/modules/io.mjs +++ b/modules/io.mjs @@ -46,7 +46,10 @@ const clTStreamerElement = 'TStreamerElement', clTStreamerObject = 'TStreamerObj StlNames = ['', 'vector', 'list', 'deque', 'map', 'multimap', 'set', 'multiset', 'bitset'], // TObject bits - kIsReferenced = BIT(4), kHasUUID = BIT(5); + kIsReferenced = BIT(4), kHasUUID = BIT(5), + + // gap in http which can be merged into single http request + kMinimalHttpGap = 128; /** @summary Custom streamers for root classes @@ -3039,6 +3042,67 @@ class TFile { * @private */ async _open() { return this.readKeys(); } + /** @summary check if requested segments can be reordered or merged + * @private */ + #checkNeedReorder(place) { + let res = false, resort = false; + for (let n = 0; n < place.length - 2; n += 2) { + if (place[n] > place[n + 2]) { + res = resort = true; + } + if (place[n] + place[n + 1] > place[n + 2] - kMinimalHttpGap) { + res = true; + } + } + if (!res) { + return { + place, + blobs: [], + expectedSize: function(indx) { + return this.place[indx + 1]; + }, + addBuffer: function(indx, buf, o) { + this.blobs[indx/2] = new DataView(buf, o, this.place[indx + 1]); + } + }; + } + + res = { place, reorder: [], place_new: [], blobs: [] }; + + for (let n = 0; n < place.length; n += 2) + res.reorder.push({ pos: place[n], len: place[n + 1], indx: [n] }); + + if (resort) + res.reorder.sort((a, b) => { return a.pos - b.pos; }); + + for(let n = 0; n < res.reorder.length - 1; n++) { + const curr = res.reorder[n], + next = res.reorder[n + 1]; + if (curr.pos + curr.len + kMinimalHttpGap > next.pos) { + curr.indx.push(...next.indx); + curr.len = next.pos + next.len - curr.pos; + res.reorder.splice(n + 1, 1); // remove segment + n--; + } + } + + res.reorder.forEach(elem => res.place_new.push(elem.pos, elem.len)); + + res.expectedSize = function(indx) { + return this.reorder[indx / 2].len; + }; + + res.addBuffer = function(indx, buf, o) { + const elem = this.reorder[indx / 2], + pos0 = elem.pos; + elem.indx.forEach(indx0 => { + this.blobs[indx0/2] = new DataView(buf, o + this.place[indx0] - pos0, this.place[indx0 + 1]); + }); + }; + + return res; + } + /** @summary read buffer(s) from the file * @return {Promise} with read buffers * @private */ @@ -3046,10 +3110,13 @@ class TFile { if ((this.fFileContent !== null) && !filename && (!this.fAcceptRanges || this.fFileContent.canExtract(place))) return this.fFileContent.extract(place); + const reorder = this.#checkNeedReorder(place); + if (reorder?.place_new) + place = reorder?.place_new; + let resolveFunc, rejectFunc; const file = this, first_block = (place[0] === 0) && (place.length === 2), - blobs = [], // array of requested segments promise = new Promise((resolve, reject) => { resolveFunc = resolve; rejectFunc = reject; @@ -3075,12 +3142,15 @@ class TFile { } } - function send_new_request(increment) { - if (increment) { + function send_new_request(arg) { + if (arg === 'noranges') { + file.fMaxRanges = 1; + last = Math.min(last, first + file.fMaxRanges * 2); + } else if (arg) { first = last; last = Math.min(first + file.fMaxRanges * 2, place.length); if (first >= place.length) - return resolveFunc(blobs); + return resolveFunc(reorder.blobs.length === 1 ? reorder.blobs[0] : reorder.blobs); } let fullurl = fileurl, ranges = 'bytes', totalsz = 0; @@ -3097,7 +3167,7 @@ class TFile { // when read first block, allow to read more - maybe ranges are not supported and full file content will be returned if (file.fAcceptRanges && first_block) - totalsz = Math.max(totalsz, 1e7); + totalsz = Math.max(totalsz, 1e5); return createHttpRequest(fullurl, 'buf', read_callback, undefined, true).then(xhr => { if (file.fAcceptRanges) { @@ -3215,70 +3285,33 @@ class TFile { // if only single segment requested, return result as is if (last - first === 2) { - const b = new DataView(res); - if (place.length === 2) - return resolveFunc(b); - blobs.push(b); + reorder.addBuffer(first, res, 0); return send_new_request(true); } // object to access response data - const hdr = this.getResponseHeader('Content-Type'), - ismulti = isStr(hdr) && (hdr.indexOf('multipart') >= 0), - view = new DataView(res); - - if (!ismulti) { - // server may returns simple buffer, which combines all segments together - - const hdr_range = this.getResponseHeader('Content-Range'); - let segm_start = 0, segm_last = -1; - - if (isStr(hdr_range) && hdr_range.indexOf('bytes') >= 0) { - const parts = hdr_range.slice(hdr_range.indexOf('bytes') + 6).split(/[\s-/]+/); - if (parts.length === 3) { - segm_start = Number.parseInt(parts[0]); - segm_last = Number.parseInt(parts[1]); - if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) { - segm_start = 0; - segm_last = -1; - } - } - } - - let canbe_single_segment = (segm_start <= segm_last); - for (let n = first; n < last; n += 2) { - if ((place[n] < segm_start) || (place[n] + place[n + 1] - 1 > segm_last)) - canbe_single_segment = false; - } - - if (canbe_single_segment) { - for (let n = first; n < last; n += 2) - blobs.push(new DataView(res, place[n] - segm_start, place[n + 1])); - return send_new_request(true); - } - - if ((file.fMaxRanges === 1) || first) - return rejectFunc(Error('Server returns normal response when multipart was requested, disable multirange support')); + const hdr = this.getResponseHeader('Content-Type'); - file.fMaxRanges = 1; - last = Math.min(last, file.fMaxRanges * 2); - - return send_new_request(); + if (!isStr(hdr) || (hdr.indexOf('multipart') < 0)) { + console.error('Did not found multipart in content-type - fallback to single range request'); + return send_new_request('noranges'); } // multipart messages requires special handling - const indx = hdr.indexOf('boundary='); - let boundary = '', n = first, o = 0, normal_order = true; + const indx = hdr.indexOf('boundary='), view = new DataView(res); + let boundary = ''; if (indx > 0) { boundary = hdr.slice(indx + 9); if ((boundary[0] === '"') && (boundary.at(-1) === '"')) boundary = boundary.slice(1, boundary.length - 1); boundary = '--' + boundary; - } else - console.error('Did not found boundary id in the response header'); + } else { + console.error('Did not found boundary id in the response header - fallback to single range request'); + return send_new_request('noranges'); + } - while (n < last) { + for(let n = first, o = 0; n < last; n += 2) { let code1, code2 = view.getUint8(o), nline = 0, line = '', finish_header = false, segm_start = 0, segm_last = -1; @@ -3297,6 +3330,7 @@ class TFile { if (parts.length === 3) { segm_start = Number.parseInt(parts[0]); segm_last = Number.parseInt(parts[1]); + // TODO: check for consistency if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) { segm_start = 0; segm_last = -1; @@ -3319,44 +3353,16 @@ class TFile { o++; } - if (!finish_header) - return rejectFunc(Error('Cannot decode header in multipart message')); - - if (segm_start > segm_last) { - // fall-back solution, believe that segments same as requested - blobs.push(new DataView(res, o, place[n + 1])); - o += place[n + 1]; - n += 2; - } else if (normal_order) { - const n0 = n; - while ((n < last) && (place[n] >= segm_start) && (place[n] + place[n + 1] - 1 <= segm_last)) { - blobs.push(new DataView(res, o + place[n] - segm_start, place[n + 1])); - n += 2; - } + const segm_size = segm_last - segm_start + 1; - if (n > n0) - o += (segm_last - segm_start + 1); - else - normal_order = false; + if (!finish_header || (segm_size <= 0) || (reorder.expectedSize(n) !== segm_size)) { + console.error('Failure decoding multirange header - fallback to single range request'); + return send_new_request('noranges'); } - if (!normal_order) { - // special situation when server reorder segments in the reply - let isany = false; - for (let n1 = n; n1 < last; n1 += 2) { - if ((place[n1] >= segm_start) && (place[n1] + place[n1 + 1] - 1 <= segm_last)) { - blobs[n1 / 2] = new DataView(res, o + place[n1] - segm_start, place[n1 + 1]); - isany = true; - } - } - if (!isany) - return rejectFunc(Error(`Provided fragment ${segm_start} - ${segm_last} out of requested multi-range request`)); + reorder.addBuffer(n, res, o); - while (blobs[n / 2]) - n += 2; - - o += (segm_last - segm_start + 1); - } + o += segm_size; } send_new_request(true);