Handle inconsistent leading tabs/spaces

Recognize and handle visually-equivalent mixes of tabs and spaces when
used for leading indentation, and replace with the appropriate
whitespace when reflowing (according to editor settings).

Author: josh-berry

lib/magic-reflow.js

@@ -74,41 +74,71 @@ function vlen(text, start_col, tab_vlen) {
     // contain any newlines.
 
     let vl = start_col;
-    for (let t of text) {
-        switch (t) {
+
+    let re = /([^\t\r\n]+|[\t\r\n])/gm;
+    let m;
+    while (m = re.exec(text)) {
+        switch (m[1]) {
             case '\t':
                 vl += vlen_of_tab_at(vl, tab_vlen);
                 break;
             case '\n':
             case '\r':
                 throw "Text contains newlines";
             default:
-                ++vl;
+                vl += m[1].length;
                 break;
         }
     }
+
     return vl - start_col;
 }
 
-function indent_with_tabs(vlen, start_col, tab_vlen) {
+function indent_with_tabs(vlen, tab_vlen) {
     // Return a sequence of tab and space characters long enough to visually
-    // indent from /start_col/ to /start_col + vlen/, assuming tab characters
-    // are /tab_vlen/ columns wide.
+    // indent from column 0 to /vlen/, assuming tab characters are /tab_vlen/
+    // columns wide.
 
     let i = '';
-    let tv = vlen_of_tab_at(start_col, tab_vlen);
-    if (tv <= vlen) {
-        i = '\t';
-        vlen -= tv;
-    }
     if (vlen >= tab_vlen) {
-        tv = (vlen/tab_vlen)|0;
-        i = i.concat('\t'.repeat(tv));
-        vlen -= tv * tab_vlen;
+        let tabs = (vlen/tab_vlen)|0;
+        i = i.concat('\t'.repeat(tabs));
+        vlen -= tabs * tab_vlen;
     }
     return i.concat(' '.repeat(vlen));
 }
 
+function* leading_tabs_to_spaces(lines, tab_vlen) {
+    // Replaces leading tab characters in each line of /lines/ with a
+    // visually-appropriate number of spaces.  Assumes that tab characters are
+    // typically /tab_vlen/ columns wide.
+
+    // This is O(n^2) becuase of the vlen() call, but that should be fine since
+    // we're only operating on a single line.
+    for (let line of lines) {
+        // Fixup all leading whitespace, to catch weird corner cases like
+        // " \t \tThe rest of the line"
+        yield line.replace(/^\s+/, (ws) => {
+            return ws.replace(/\t/g, (m, off, t) => {
+                let vl = vlen(t.substr(0, off), 0, tab_vlen);
+                return ' '.repeat(vlen_of_tab_at(vl, tab_vlen));
+            });
+        });
+    }
+}
+
+function* leading_spaces_to_tabs(lines, tab_vlen) {
+    // Undoes what leading_tabs_to_spaces does -- replaces runs of space
+    // characters with an appropriate number of tabs.
+
+    for (let line of lines) {
+        yield line.replace(/^\s+/, (ws) => {
+            return indent_with_tabs(vlen(ws, 0, tab_vlen), tab_vlen);
+        });
+    }
+}
+
+
 
 // Match a line with LINE_PATT and build an object describing what's in it.
 function match_line(text) {
@@ -259,12 +289,20 @@ export default {
         let useTabs = body.indexOf('\t') != -1;
 
         let lines = body.split(/\r\n|\r|\n/);
-        let out = this.reflow_lines(lines, state);
+        let out = this.reflow_lines(
+            leading_tabs_to_spaces(lines, state.tab_vlen),
+            state);
+
+        if (! state.soft_tab) {
+            out = leading_spaces_to_tabs(out, state.tab_vlen);
+        }
 
         return head.concat(Array.from(out).join('\n'), tail);
     },
 
     reflow_lines: function*(lines, state) {
+        lines = Array.from(lines);
+
         if (debug) debug('reflowing:', state, lines);
 
         // Break the text into blocks, and reflow each block separately.
@@ -304,9 +342,7 @@ export default {
 
         let reflowed = this.remove_and_reflow(leading, lines, state);
         let ilen = vlen(leading, state.start_col, state.tab_vlen);
-        let indent = state.soft_tab
-                   ? ' '.repeat(ilen)
-                   : indent_with_tabs(ilen, state.start_col, state.tab_vlen);
+        let indent = ' '.repeat(ilen);
 
         let first = true;
         for (let line of reflowed) {
@@ -662,4 +698,6 @@ export default {
     // Exports for testing purposes, used in the spec file
     vlen: vlen,
     indent_with_tabs: indent_with_tabs,
+    leading_tabs_to_spaces: leading_tabs_to_spaces,
+    leading_spaces_to_tabs: leading_spaces_to_tabs,
 };

spec/magic-reflow-spec.js

@@ -23,6 +23,16 @@ function qcheck(what, args, expected) {
     });
 }
 
+function qcheckgen(gen, what, args, expected) {
+    it(`${what}: returns ${expected} for ${args}`, () => {
+        console.log(`BEGIN TEST ${what}:`, args);
+        let actual = Array.from(gen(...args));
+        console.log(`ACTUAL:  `, actual);
+        console.log(`EXPECTED:`, [expected]);
+        expect(actual[0]).toBe(expected);
+    });
+}
+
 describe('MagicReflow', () => {
     describe('vlen', () => {
         let vlen = MagicReflow.vlen;
@@ -76,28 +86,30 @@ describe('MagicReflow', () => {
     describe('indent_with_tabs', () => {
         let iwt = MagicReflow.indent_with_tabs;
 
-        describe('empty strings', () => {
-            qcheck(iwt, [0, 0, 4], '');
-            qcheck(iwt, [0, 1, 4], '');
-        });
+        qcheck(iwt, [0, 4], '');
+        qcheck(iwt, [1, 4], ' ');
+        qcheck(iwt, [4, 4], '\t');
+        qcheck(iwt, [5, 4], '\t ');
+        qcheck(iwt, [8, 4], '\t\t');
+        qcheck(iwt, [10, 4], '\t\t  ');
+    });
 
-        describe('first tab', () => {
-            qcheck(iwt, [4, 0, 4], '\t');
-            qcheck(iwt, [3, 1, 4], '\t');
-        });
+    describe('leading_tabs_to_spaces', () => {
+        let gen = MagicReflow.leading_tabs_to_spaces;
 
-        describe('multiple tabs', () => {
-            qcheck(iwt, [8, 0, 4], '\t\t');
-            qcheck(iwt, [6, 2, 4], '\t\t');
-        });
+        qcheckgen(gen, 'tab', [['\tfoo'], 4], '    foo');
+        qcheckgen(gen, 'tab+spc', [['\t  foo'], 4], '      foo');
+        qcheckgen(gen, 'spc+tab+spc', [[' \t  foo'], 4], '      foo');
+        qcheckgen(gen, 'spc+tab+spc+tab', [[' \t \tfoo'], 4], '        foo');
+    });
 
-        describe('trailing spaces', () => {
-            qcheck(iwt, [2, 0, 4], '  ');
-            qcheck(iwt, [2, 2, 4], '\t');
-            qcheck(iwt, [1, 2, 4], ' ');
-            qcheck(iwt, [5, 2, 4], '\t   ');
-            qcheck(iwt, [5, 6, 4], '\t   ');
-        });
+    describe('leading_spaces_to_tabs', () => {
+        let gen = MagicReflow.leading_spaces_to_tabs;
+
+        qcheckgen(gen, 'tab', [['    foo'], 4], '\tfoo');
+        qcheckgen(gen, 'tab+spc', [['      foo'], 4], '\t  foo');
+        qcheckgen(gen, 'spc+tab+spc', [['      foo'], 4], '\t  foo');
+        qcheckgen(gen, 'spc+tab+spc+tab', [['        foo'], 4], '\t\tfoo');
     });
 
     describe('when reflowing a single paragraph', () => {
@@ -194,6 +206,17 @@ describe('MagicReflow', () => {
             '    This is\n  the first\n  line.  This\n  is the\n  second line.'
         ));
 
+        it('converts spaces to tabs if appropriate', () => test(
+            ['    This is the first line.', 24, 4, false],
+            '\tThis is the first line.'
+        ));
+
+        it('handles inconsistent use of tabs and spaces', () => test(
+            ['\tLeading tab.\n        Second line.', 40, 8, true],
+            '        Leading tab.  Second line.'
+        ));
+
+
         it('fixes inconsistent indentation', () => test(
             [`
     This is the first line.
@@ -214,22 +237,32 @@ describe('MagicReflow', () => {
 
     describe('when dealing with leading tab characters', () => {
         it('preserves block style with tabs (one line)', () => test(
-            ['\tLeading tab.\n\tSecond line.', 40],
+            ['\tLeading tab.\n\tSecond line.', 40, 8, false],
             '\tLeading tab.  Second line.'
         ));
         it('preserves block style with tabs (multi-line)', () => test(
-            ['\tLeading tab that is a long line.\n\tSecond line.', 24, 8],
+            ['\tLeading tab that is a long line.\n\tSecond line.', 24, 8, false],
             '\tLeading tab that\n\tis a long line.\n\tSecond line.'
         ));
         it('uses the correct tab width for indentation', () => test(
-            ['\tLeading tab that is a long line.  Should be 24 cols.', 24, 4],
+            ['\tLeading tab that is a long line.  Should be 24 cols.', 24, 4, false],
             '\tLeading tab that is\n\ta long line.  Should\n\tbe 24 cols.',
         ));
         it('handles tabs after leading sigils', () => test(
             ['1.\tThis is a numbered list item that needs wrapping.', 24, 4, false],
             '1.\tThis is a numbered\n\tlist item that needs\n\twrapping.'
         ));
 
+        it('converts tabs to spaces if appropriate', () => test(
+            ['\tLeading tab.\n\tSecond line.', 40, 8, true],
+            '        Leading tab.  Second line.'
+        ));
+
+        it('handles inconsistent use of tabs and spaces', () => test(
+            ['\tLeading tab.\n        Second line.', 40, 8, false],
+            '\tLeading tab.  Second line.'
+        ));
+
         // BEGIN: Leading indents are not supported currently.
         xit('preserves leading indents with tabs (one line)', () => test(
             ['\tLeading tab.\nSecond line.', 40],