Basic understanding of tab characters

- We can now correctly indent with tabs instead of spaces if the editor
  settings desire it.
- We now understand how tab characters that appear in the middle of a
  paragraph affect the paragraph's visual flow.

Still to come: dealing with a mix of tab and space indentation.

Author: josh-berry

lib/magic-reflow.js

@@ -60,7 +60,15 @@ function last_in_range(array, start, end, cb) {
     return start - 1;
 }
 
-function vlen(text, start_col, tab_len) {
+function vlen_of_tab_at(start_col, tab_vlen) {
+    // Returns the visual length of a tab character, if that tab character
+    // appears at the specified /start_col/.  The /tab_vlen/ parameter indicates
+    // the visual length of tab characters generally (e.g. 1 tab = 8 spaces).
+
+    return tab_vlen - (start_col % tab_vlen);
+}
+
+function vlen(text, start_col, tab_vlen) {
     // Compute the visual length of /text/, assuming (for the purposes of tab
     // alignment) it starts at column /start_col/.  The /text/ should not
     // contain any newlines.
@@ -69,7 +77,7 @@ function vlen(text, start_col, tab_len) {
     for (let t of text) {
         switch (t) {
             case '\t':
-                vl = (((vl / tab_len)|0) + 1) * tab_len;
+                vl += vlen_of_tab_at(vl, tab_vlen);
                 break;
             case '\n':
             case '\r':
@@ -82,6 +90,24 @@ function vlen(text, start_col, tab_len) {
     return vl - start_col;
 }
 
+function indent_with_tabs(vlen, start_col, tab_vlen) {
+    // Return a sequence of tab and space characters long enough to visually
+    // indent from /start_col/ to /start_col + vlen/, assuming tab characters
+    // are /tab_vlen/ columns wide.
+
+    let i = '';
+    let tv = vlen_of_tab_at(start_col, tab_vlen);
+    if (tv <= vlen) {
+        i = '\t';
+        vlen -= tv;
+    }
+    if (vlen >= tab_vlen) {
+        tv = (vlen/tab_vlen)|0;
+        i = i.concat('\t'.repeat(tv));
+        vlen -= tv * tab_vlen;
+    }
+    return i.concat(' '.repeat(vlen));
+}
 
 
 // Match a line with LINE_PATT and build an object describing what's in it.
@@ -198,6 +224,20 @@ export default {
             // How long should a line be, overall?
             line_vlen: atom.config.get('editor.preferredLineLength',
                 {scope: editor.getRootScopeDescriptor()}),
+
+            // What is the starting (visual) column?  We will assume that there
+            // is a prefix that is this many columns wide, so we'll wrap the
+            // text to fit in (line_vlen - start_col).
+            //
+            // We need both start_col and line_vlen so we can tell how much
+            // visual space a leading tab takes up.  If we simply subtracted
+            // from line_vlen as we process indented text, we wouldn't be able
+            // to compute the visual length of tab characters correctly.
+            start_col: 0,
+
+            // When adding leading whitespace, should we use spaces or tabs?
+            soft_tab: atom.config.get('editor.softTabs',
+                {scope: editor.getRootScopeDescriptor()}),
         };
 
         let text = this.reflow(editor.getTextInRange(range), state);
@@ -212,6 +252,8 @@ export default {
 
         if (! state.line_vlen) state.line_vlen = 80;
         if (! state.tab_vlen) state.tab_vlen = 8;
+        if (! state.start_col) state.start_col = 0;
+        if (state.soft_tab === undefined) state.soft_tab = true;
 
         let [head, body, tail] = this.head_body_tail(text);
         let useTabs = body.indexOf('\t') != -1;
@@ -261,7 +303,10 @@ export default {
         if (debug) debug(`retrying without leading: ${leading}`);
 
         let reflowed = this.remove_and_reflow(leading, lines, state);
-        let indent = ' '.repeat(leading.length);
+        let ilen = vlen(leading, state.start_col, state.tab_vlen);
+        let indent = state.soft_tab
+                   ? ' '.repeat(ilen)
+                   : indent_with_tabs(ilen, state.start_col, state.tab_vlen);
 
         let first = true;
         for (let line of reflowed) {
@@ -274,8 +319,11 @@ export default {
     remove_and_reflow(prefix, lines, state) {
         return this.reflow_lines(Array.from(
             this.remove_prefix_and_ws(prefix, lines)), {
-                line_vlen: state.line_vlen - prefix.length,
+                line_vlen: state.line_vlen,
                 tab_vlen: state.tab_vlen,
+                start_col: state.start_col
+                    + vlen(prefix, state.start_col, state.tab_vlen),
+                soft_tab: state.soft_tab,
             });
     },
 
@@ -490,7 +538,7 @@ export default {
         }
 
         // This is a garden-variety paragraph.  Wrap it accordingly.
-        return [end, this.wrap_para(para.join('\n'), state.line_vlen)];
+        return [end, this.wrap_para(para.join('\n'), state)];
     },
 
     consume_single_line(lines, start, startm, state) {
@@ -506,17 +554,17 @@ export default {
 
         } else {
             // Garden-variety paragraph, no whitespace; reflow it.
-            gen = this.wrap_para(lines[start], state.line_vlen);
+            gen = this.wrap_para(lines[start], state);
         }
 
         return [start + 1, gen];
     },
 
-    wrap_para: function*(text, fill, firstFill) {
+    wrap_para: function*(text, state) {
         // Wrap /text/ as a single paragraph, yielding lines of wrapped text
         // that are no longer than /fill/.
 
-        if (debug) debug('wrapping paragraph:', text);
+        if (debug) debug('wrapping paragraph:', text, state);
 
         // XXX firstFill isn't used anymore, with the support for leading
         // indents gone.
@@ -529,34 +577,35 @@ export default {
         let res = [];
         let start = 0;
         let end = start;
-        let curFill = firstFill ? firstFill : fill;
+        let vl = state.start_col;
 
-        for (let [ws, word] of this.segments_of(text)) {
+        for (let [run, ws, word] of this.segments_of(text)) {
             let newend = end + ws.length + word.length;
+            let newvl = vl + vlen(run, vl, state.tab_vlen);
 
             if (end == start) {
                 // Current line is empty. Skip over leading whitespace that
                 // would otherwise get put at the beginning of the line.
                 start += ws.length;
+                newvl = vl + vlen(word, vl, state.tab_vlen);
 
-            } else if (newend > start + curFill) {
+            } else if (newvl > state.line_vlen) {
                 // Current line is full.  Emit what we have and start the next
                 // line.
-                let res = text.substr(start, end - start);
-                yield res;
-                curFill = fill; // no longer on the first line
+                yield text.substr(start, end - start);
 
                 // Skip leading whitespace on the next line, and start the line
                 // with the current word.
                 start = end + ws.length;
+                newvl = state.start_col + vlen(word, vl, state.tab_vlen);
             }
 
             end = newend;
+            vl = newvl;
         }
 
         if (start != end) {
-            let res = text.substr(start, end - start);
-            yield res;
+            yield text.substr(start, end - start);
         }
     },
 
@@ -599,15 +648,18 @@ export default {
     },
 
     segments_of: function*(text) {
-        // Split /text/ at word boundaries, yielding [ws, word] pairs where /ws/
-        // is the whitespace that appears before /word/.
+        // Split /text/ at word boundaries, yielding [run, ws, word] tuples
+        // where /ws/ is the whitespace that appears before /word/, and /run/ is
+        // ws.concat(word).
 
         let span = /(\s*)(\S+)/gm;
         let m;
         while (m = span.exec(text)) {
-            yield [m[1], m[2]];
+            yield m;
         }
     },
 
-    vlen: vlen, // for testing purposes
+    // Exports for testing purposes, used in the spec file
+    vlen: vlen,
+    indent_with_tabs: indent_with_tabs,
 };

spec/magic-reflow-spec.js

@@ -7,68 +7,96 @@ import MagicReflow from '../lib/magic-reflow';
 // To run a specific `it` or `describe` block add an `f` to the front (e.g.
 // `fit` or `fdescribe`).  Remove the `f` to unfocus the block.
 
-function test([input, line_vlen, tab_vlen], expected) {
+function test([input, line_vlen, tab_vlen, soft_tab], expected) {
     console.log(`BEGIN TEST:\n${input}`);
     let actual = MagicReflow.reflow(
-        input, {line_vlen: line_vlen, tab_vlen: tab_vlen});
+        input, {line_vlen: line_vlen, tab_vlen: tab_vlen, soft_tab: soft_tab});
     console.log(`TEST EXPECTED:\n${expected}`);
     console.log(`TEST ACTUAL:\n${[actual]}`);
     expect(actual).toBe(expected);
 };
 
-function test_vlen(args, expected) {
+function qcheck(what, args, expected) {
     it(`returns ${expected} for ${args}`, () => {
-        let actual = MagicReflow.vlen(...args);
+        let actual = what(...args);
         expect(actual).toBe(expected);
     });
 }
 
 describe('MagicReflow', () => {
     describe('vlen', () => {
+        let vlen = MagicReflow.vlen;
         describe('empty strings', () => {
-            test_vlen(['', 0, 8], 0);
-            test_vlen(['', 4, 8], 0);
+            qcheck(vlen, ['', 0, 8], 0);
+            qcheck(vlen, ['', 4, 8], 0);
         });
 
         describe('non-tab chars', () => {
-            test_vlen(['  ', 0, 4], 2);
-            test_vlen(['foo', 0, 4], 3);
+            qcheck(vlen, ['  ', 0, 4], 2);
+            qcheck(vlen, ['foo', 0, 4], 3);
         });
 
         describe('simple tabs', () => {
-            test_vlen(['\t', 0, 8], 8);
-            test_vlen(['\t', 4, 8], 4);
-            test_vlen(['\t', 8, 8], 8);
+            qcheck(vlen, ['\t', 0, 8], 8);
+            qcheck(vlen, ['\t', 4, 8], 4);
+            qcheck(vlen, ['\t', 8, 8], 8);
 
-            test_vlen(['\t', 0, 4], 4);
-            test_vlen(['\t', 4, 4], 4);
+            qcheck(vlen, ['\t', 0, 4], 4);
+            qcheck(vlen, ['\t', 4, 4], 4);
         });
 
         describe('short text followed by tab', () => {
-            test_vlen(['foo\t', 0, 8], 8);
-            test_vlen(['foo\t', 4, 8], 4);
-            test_vlen(['foo\t', 8, 8], 8);
+            qcheck(vlen, ['foo\t', 0, 8], 8);
+            qcheck(vlen, ['foo\t', 4, 8], 4);
+            qcheck(vlen, ['foo\t', 8, 8], 8);
 
-            test_vlen(['foo\t', 0, 4], 4);
-            test_vlen(['foo\t', 4, 4], 4);
+            qcheck(vlen, ['foo\t', 0, 4], 4);
+            qcheck(vlen, ['foo\t', 4, 4], 4);
         });
 
         describe('tab in middle', () => {
-            test_vlen(['foo\tbar', 0, 8], 8+3);
-            test_vlen(['foo\tbar', 4, 8], 4+3);
-            test_vlen(['foo\tbar', 8, 8], 8+3);
+            qcheck(vlen, ['foo\tbar', 0, 8], 8+3);
+            qcheck(vlen, ['foo\tbar', 4, 8], 4+3);
+            qcheck(vlen, ['foo\tbar', 8, 8], 8+3);
 
-            test_vlen(['foo\tbar', 0, 4], 4+3);
-            test_vlen(['foo\tbar', 4, 4], 4+3);
+            qcheck(vlen, ['foo\tbar', 0, 4], 4+3);
+            qcheck(vlen, ['foo\tbar', 4, 4], 4+3);
         });
 
         describe('multiple tabs', () => {
-            test_vlen(['foo\t\t', 0, 8], 8+8);
-            test_vlen(['foo\t\t', 4, 8], 4+8);
-            test_vlen(['foo\t\t', 8, 8], 8+8);
+            qcheck(vlen, ['foo\t\t', 0, 8], 8+8);
+            qcheck(vlen, ['foo\t\t', 4, 8], 4+8);
+            qcheck(vlen, ['foo\t\t', 8, 8], 8+8);
 
-            test_vlen(['foo\t\t', 0, 4], 4+4);
-            test_vlen(['foo\t\t', 4, 4], 4+4);
+            qcheck(vlen, ['foo\t\t', 0, 4], 4+4);
+            qcheck(vlen, ['foo\t\t', 4, 4], 4+4);
+        });
+    });
+
+    describe('indent_with_tabs', () => {
+        let iwt = MagicReflow.indent_with_tabs;
+
+        describe('empty strings', () => {
+            qcheck(iwt, [0, 0, 4], '');
+            qcheck(iwt, [0, 1, 4], '');
+        });
+
+        describe('first tab', () => {
+            qcheck(iwt, [4, 0, 4], '\t');
+            qcheck(iwt, [3, 1, 4], '\t');
+        });
+
+        describe('multiple tabs', () => {
+            qcheck(iwt, [8, 0, 4], '\t\t');
+            qcheck(iwt, [6, 2, 4], '\t\t');
+        });
+
+        describe('trailing spaces', () => {
+            qcheck(iwt, [2, 0, 4], '  ');
+            qcheck(iwt, [2, 2, 4], '\t');
+            qcheck(iwt, [1, 2, 4], ' ');
+            qcheck(iwt, [5, 2, 4], '\t   ');
+            qcheck(iwt, [5, 6, 4], '\t   ');
         });
     });
 
@@ -184,24 +212,34 @@ describe('MagicReflow', () => {
         ));
     });
 
-    // XXX Need to implement visual-width calculations to handle tabs... :/
-    xdescribe('when dealing with leading tab characters', () => {
-        it('preserves leading indents with tabs (one line)', () => test(
-            ['\tLeading tab.\nSecond line.', 40],
-            '\tLeading tab.  Second line.'
-        ));
+    describe('when dealing with leading tab characters', () => {
         it('preserves block style with tabs (one line)', () => test(
             ['\tLeading tab.\n\tSecond line.', 40],
             '\tLeading tab.  Second line.'
         ));
-        it('preserves leading indents with tabs (multi-line)', () => test(
-            ['\tLeading tab.\nSecond line.', 40],
-            '\tLeading tab.  Second line.'
-        ));
         it('preserves block style with tabs (multi-line)', () => test(
             ['\tLeading tab that is a long line.\n\tSecond line.', 24, 8],
             '\tLeading tab that\n\tis a long line.\n\tSecond line.'
         ));
+        it('uses the correct tab width for indentation', () => test(
+            ['\tLeading tab that is a long line.  Should be 24 cols.', 24, 4],
+            '\tLeading tab that is\n\ta long line.  Should\n\tbe 24 cols.',
+        ));
+        it('handles tabs after leading sigils', () => test(
+            ['1.\tThis is a numbered list item that needs wrapping.', 24, 4, false],
+            '1.\tThis is a numbered\n\tlist item that needs\n\twrapping.'
+        ));
+
+        // BEGIN: Leading indents are not supported currently.
+        xit('preserves leading indents with tabs (one line)', () => test(
+            ['\tLeading tab.\nSecond line.', 40],
+            '\tLeading tab.  Second line.'
+        ));
+        xit('preserves leading indents with tabs (multi-line)', () => test(
+            ['\tLeading tab.\nSecond line.', 40],
+            '\tLeading tab.  Second line.'
+        ));
+        // END: leading indents
     });
 
     for (let sigil of ['#', '##', '//', '///', '--', ';', ';;', ';;;']) {