Implement reorder of segments in http requests

modules/io.mjs

@@ -46,7 +46,10 @@
       StlNames = ['', 'vector', 'list', 'deque', 'map', 'multimap', 'set', 'multiset', 'bitset'],
 
       // TObject bits
-      kIsReferenced = BIT(4), kHasUUID = BIT(5);
+      kIsReferenced = BIT(4), kHasUUID = BIT(5),
+
+      // gap in http which can be merged into single http request
+      kMinimalHttpGap = 128;
 
 
 /** @summary Custom streamers for root classes
@@ -3039,17 +3042,81 @@
      * @private */
    async _open() { return this.readKeys(); }
 
+   /** @summary check if requested segments can be reordered or merged
+    * @private */
+   #checkNeedReorder(place) {
+      let res = false, resort = false;
+      for (let n = 0; n < place.length - 2; n += 2) {
+         if (place[n] > place[n + 2]) {
+            res = resort = true;
+         }
+         if (place[n] + place[n + 1] > place[n + 2] - kMinimalHttpGap) {
+            res = true;
+         }
+      }
+      if (!res) {
+         return {
+            place,
+            blobs: [],
+            expectedSize: function(indx) {
+               return this.place[indx + 1];
+            },
+            addBuffer: function(indx, buf, o) {
+               this.blobs[indx/2] = new DataView(buf, o, this.place[indx + 1]);
+            }
+         };
+      }
+
+      res = { place, reorder: [], place_new: [], blobs: [] };
+
+      for (let n = 0; n < place.length; n += 2)
+         res.reorder.push({ pos: place[n], len: place[n + 1], indx: [n] });
+
+      if (resort)
+         res.reorder.sort((a, b) => { return a.pos - b.pos; });
+
+      for(let n = 0; n < res.reorder.length - 1; n++) {
+         const curr = res.reorder[n],
+               next = res.reorder[n + 1];
+         if (curr.pos + curr.len + kMinimalHttpGap > next.pos) {
+            curr.indx.push(...next.indx);
+            curr.len = next.pos + next.len - curr.pos;
+            res.reorder.splice(n + 1, 1); // remove segment
+            n--;
+         }
+      }
+
+      res.reorder.forEach(elem => res.place_new.push(elem.pos, elem.len));
+
+      res.expectedSize = function(indx) {
+         return this.reorder[indx / 2].len;
+      };
+
+      res.addBuffer = function(indx, buf, o) {
+         const elem = this.reorder[indx / 2],
+               pos0 = elem.pos;
+         elem.indx.forEach(indx0 => {
+            this.blobs[indx0/2] = new DataView(buf, o + this.place[indx0] - pos0, this.place[indx0 + 1]);
+         });
+      };
+
+      return res;
+   }
+
    /** @summary read buffer(s) from the file
     * @return {Promise} with read buffers
     * @private */
    async readBuffer(place, filename, progress_callback) {
       if ((this.fFileContent !== null) && !filename && (!this.fAcceptRanges || this.fFileContent.canExtract(place)))
          return this.fFileContent.extract(place);
 
+      const reorder = this.#checkNeedReorder(place);
+      if (reorder?.place_new)
+         place = reorder?.place_new;
+
       let resolveFunc, rejectFunc;
 
       const file = this, first_block = (place[0] === 0) && (place.length === 2),
-            blobs = [], // array of requested segments
             promise = new Promise((resolve, reject) => {
                resolveFunc = resolve;
                rejectFunc = reject;
@@ -3075,12 +3142,15 @@
          }
       }
 
-      function send_new_request(increment) {
-         if (increment) {
+      function send_new_request(arg) {
+         if (arg === 'noranges') {
+            file.fMaxRanges = 1;
+            last = Math.min(last, first + file.fMaxRanges * 2);
+         } else if (arg) {
             first = last;
             last = Math.min(first + file.fMaxRanges * 2, place.length);
             if (first >= place.length)
-               return resolveFunc(blobs);
+               return resolveFunc(reorder.blobs.length === 1 ? reorder.blobs[0] : reorder.blobs);
          }
 
          let fullurl = fileurl, ranges = 'bytes', totalsz = 0;
@@ -3097,7 +3167,7 @@
 
          // when read first block, allow to read more - maybe ranges are not supported and full file content will be returned
          if (file.fAcceptRanges && first_block)
-            totalsz = Math.max(totalsz, 1e7);
+            totalsz = Math.max(totalsz, 1e5);
 
          return createHttpRequest(fullurl, 'buf', read_callback, undefined, true).then(xhr => {
             if (file.fAcceptRanges) {
@@ -3215,70 +3285,33 @@
 
          // if only single segment requested, return result as is
          if (last - first === 2) {
-            const b = new DataView(res);
-            if (place.length === 2)
-               return resolveFunc(b);
-            blobs.push(b);
+            reorder.addBuffer(first, res, 0);
             return send_new_request(true);
          }
 
          // object to access response data
-         const hdr = this.getResponseHeader('Content-Type'),
-               ismulti = isStr(hdr) && (hdr.indexOf('multipart') >= 0),
-               view = new DataView(res);
-
-         if (!ismulti) {
-            // server may returns simple buffer, which combines all segments together
-
-            const hdr_range = this.getResponseHeader('Content-Range');
-            let segm_start = 0, segm_last = -1;
-
-            if (isStr(hdr_range) && hdr_range.indexOf('bytes') >= 0) {
-               const parts = hdr_range.slice(hdr_range.indexOf('bytes') + 6).split(/[\s-/]+/);
-               if (parts.length === 3) {
-                  segm_start = Number.parseInt(parts[0]);
-                  segm_last = Number.parseInt(parts[1]);
-                  if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) {
-                     segm_start = 0;
-                     segm_last = -1;
-                  }
-               }
-            }
-
-            let canbe_single_segment = (segm_start <= segm_last);
-            for (let n = first; n < last; n += 2) {
-               if ((place[n] < segm_start) || (place[n] + place[n + 1] - 1 > segm_last))
-                  canbe_single_segment = false;
-            }
-
-            if (canbe_single_segment) {
-               for (let n = first; n < last; n += 2)
-                  blobs.push(new DataView(res, place[n] - segm_start, place[n + 1]));
-               return send_new_request(true);
-            }
-
-            if ((file.fMaxRanges === 1) || first)
-               return rejectFunc(Error('Server returns normal response when multipart was requested, disable multirange support'));
+         const hdr = this.getResponseHeader('Content-Type');
 
-            file.fMaxRanges = 1;
-            last = Math.min(last, file.fMaxRanges * 2);
-
-            return send_new_request();
+         if (!isStr(hdr) || (hdr.indexOf('multipart') < 0)) {
+            console.error('Did not found multipart in content-type - fallback to single range request');
+            return send_new_request('noranges');
          }
 
          // multipart messages requires special handling
 
-         const indx = hdr.indexOf('boundary=');
-         let boundary = '', n = first, o = 0, normal_order = true;
+         const indx = hdr.indexOf('boundary='), view = new DataView(res);
+         let boundary = '';
          if (indx > 0) {
             boundary = hdr.slice(indx + 9);
             if ((boundary[0] === '"') && (boundary.at(-1) === '"'))
                boundary = boundary.slice(1, boundary.length - 1);
             boundary = '--' + boundary;
-         } else
-            console.error('Did not found boundary id in the response header');
+         } else {
+            console.error('Did not found boundary id in the response header - fallback to single range request');
+            return send_new_request('noranges');
+         }
 
-         while (n < last) {
+         for(let n = first, o = 0; n < last; n += 2) {
             let code1, code2 = view.getUint8(o), nline = 0, line = '',
                 finish_header = false, segm_start = 0, segm_last = -1;
 
@@ -3297,6 +3330,7 @@
                      if (parts.length === 3) {
                         segm_start = Number.parseInt(parts[0]);
                         segm_last = Number.parseInt(parts[1]);
+                        // TODO: check for consistency
                         if (!Number.isInteger(segm_start) || !Number.isInteger(segm_last) || (segm_start > segm_last)) {
                            segm_start = 0;
                            segm_last = -1;
@@ -3319,44 +3353,16 @@
                o++;
             }
 
-            if (!finish_header)
-               return rejectFunc(Error('Cannot decode header in multipart message'));
-
-            if (segm_start > segm_last) {
-               // fall-back solution, believe that segments same as requested
-               blobs.push(new DataView(res, o, place[n + 1]));
-               o += place[n + 1];
-               n += 2;
-            } else if (normal_order) {
-               const n0 = n;
-               while ((n < last) && (place[n] >= segm_start) && (place[n] + place[n + 1] - 1 <= segm_last)) {
-                  blobs.push(new DataView(res, o + place[n] - segm_start, place[n + 1]));
-                  n += 2;
-               }
+            const segm_size = segm_last - segm_start + 1;
 
-               if (n > n0)
-                  o += (segm_last - segm_start + 1);
-               else
-                  normal_order = false;
+            if (!finish_header || (segm_size <= 0) || (reorder.expectedSize(n) !== segm_size)) {
+               console.error('Failure decoding multirange header - fallback to single range request');
+               return send_new_request('noranges');
             }
 
-            if (!normal_order) {
-               // special situation when server reorder segments in the reply
-               let isany = false;
-               for (let n1 = n; n1 < last; n1 += 2) {
-                  if ((place[n1] >= segm_start) && (place[n1] + place[n1 + 1] - 1 <= segm_last)) {
-                     blobs[n1 / 2] = new DataView(res, o + place[n1] - segm_start, place[n1 + 1]);
-                     isany = true;
-                  }
-               }
-               if (!isany)
-                  return rejectFunc(Error(`Provided fragment ${segm_start} - ${segm_last} out of requested multi-range request`));
+            reorder.addBuffer(n, res, o);
 
-               while (blobs[n / 2])
-                  n += 2;
-
-               o += (segm_last - segm_start + 1);
-            }
+            o += segm_size;
          }
 
          send_new_request(true);