Merge pull request #1664 from UziTech/fix-highlight-async

docs/USING_ADVANCED.md

@@ -58,6 +58,7 @@ console.log(marked(markdownString));
 |smartLists  |`boolean` |`false`  |v0.2.8   |If true, use smarter list behavior than those found in `markdown.pl`.|
 |smartypants |`boolean` |`false`  |v0.2.9   |If true, use "smart" typographic punctuation for things like quotes and dashes.|
 |tokenizer    |`object`  |`new Tokenizer()`|v1.0.0|An object containing functions to create tokens from markdown. See [extensibility](/#/USING_PRO.md) for more details.|
+|walkTokens   |`function`  |`null`|v1.1.0|A function which is called for every token. See [extensibility](/#/USING_PRO.md) for more details.|
 |xhtml       |`boolean` |`false`  |v0.3.2   |If true, emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.|
 
 <h2 id="highlight">Asynchronous highlighting</h2>

docs/USING_PRO.md

@@ -10,6 +10,8 @@ The `renderer` and `tokenizer` options can be an object with functions that will
 
 The `renderer` and `tokenizer` functions can return false to fallback to the previous function.
 
+The `walkTokens` option can be a function that will be called with every token before rendering. When calling `use` multiple times with different `walkTokens` functions each function will be called in the **reverse** order in which they were assigned.
+
 All other options will overwrite previously set options.
 
 <h2 id="renderer">The renderer</h2>
@@ -188,6 +190,35 @@ smartypants('"this ... string"')
 // "“this … string”"
 ```
 
+<h2 id="walk-tokens">Walk Tokens</h2>
+
+The walkTokens function gets called with every token. Child tokens are called before moving on to sibling tokens. Each token is passed by reference so updates are persisted when passed to the parser. The return value of the function is ignored.
+
+**Example:** Overriding heading tokens to start at h2.
+
+```js
+const marked = require('marked');
+
+// Override function
+const walkTokens = (token) => {
+  if (token.type === 'heading') {
+    token.depth += 1;
+  }
+};
+
+marked.use({ walkTokens });
+
+// Run marked
+console.log(marked('# heading 2\n\n## heading 3'));
+```
+
+**Output:**
+
+```html
+<h2 id="heading-2">heading 2</h2>
+<h3 id="heading-3">heading 3</h3>
+```
+
 <h2 id="lexer">The lexer</h2>
 
 The lexer takes a markdown string and calls the tokenizer functions.

docs/index.html

@@ -157,6 +157,7 @@
                             <li><a href="#/USING_PRO.md#use">marked.use()</a></li>
                             <li><a href="#/USING_PRO.md#renderer">Renderer</a></li>
                             <li><a href="#/USING_PRO.md#tokenizer">Tokenizer</a></li>
+                            <li><a href="#/USING_PRO.md#walk-tokens">Walk Tokens</a></li>
                             <li><a href="#/USING_PRO.md#lexer">Lexer</a></li>
                             <li><a href="#/USING_PRO.md#parser">Parser</a></li>
                         </ul>

src/Tokenizer.js

@@ -269,6 +269,7 @@ module.exports = class Tokenizer {
         }
 
         list.items.push({
+          type: 'list_item',
           raw,
           task: istask,
           checked: ischecked,

src/defaults.js

@@ -16,6 +16,7 @@ function getDefaults() {
     smartLists: false,
     smartypants: false,
     tokenizer: null,
+    walkTokens: null,
     xhtml: false
   };
 }

src/marked.js

@@ -28,18 +28,17 @@ function marked(src, opt, callback) {
       + Object.prototype.toString.call(src) + ', string expected');
   }
 
-  if (callback || typeof opt === 'function') {
-    if (!callback) {
+  if (typeof opt === 'function') {
     callback = opt;
     opt = null;
   }
 
   opt = merge({}, marked.defaults, opt || {});
   checkSanitizeDeprecation(opt);
+
+  if (callback) {
     const highlight = opt.highlight;
-    let tokens,
-      pending,
-      i = 0;
+    let tokens;
 
     try {
       tokens = Lexer.lex(src, opt);
@@ -47,21 +46,16 @@ function marked(src, opt, callback) {
       return callback(e);
     }
 
-    pending = tokens.length;
-
     const done = function(err) {
-      if (err) {
-        opt.highlight = highlight;
-        return callback(err);
-      }
-
       let out;
 
+      if (!err) {
         try {
           out = Parser.parse(tokens, opt);
         } catch (e) {
           err = e;
         }
+      }
 
       opt.highlight = highlight;
 
@@ -76,34 +70,45 @@ function marked(src, opt, callback) {
 
     delete opt.highlight;
 
-    if (!pending) return done();
+    if (!tokens.length) return done();
 
-    for (; i < tokens.length; i++) {
-      (function(token) {
-        if (token.type !== 'code') {
-          return --pending || done();
-        }
-        return highlight(token.text, token.lang, function(err, code) {
-          if (err) return done(err);
-          if (code == null || code === token.text) {
-            return --pending || done();
+    let pending = 0;
+    marked.walkTokens(tokens, function(token) {
+      if (token.type === 'code') {
+        pending++;
+        highlight(token.text, token.lang, function(err, code) {
+          if (err) {
+            return done(err);
           }
+          if (code != null && code !== token.text) {
             token.text = code;
             token.escaped = true;
-          --pending || done();
+          }
+
+          pending--;
+          if (pending === 0) {
+            done();
+          }
         });
-      })(tokens[i]);
+      }
+    });
+
+    if (pending === 0) {
+      done();
     }
 
     return;
   }
+
   try {
-    opt = merge({}, marked.defaults, opt || {});
-    checkSanitizeDeprecation(opt);
-    return Parser.parse(Lexer.lex(src, opt), opt);
+    const tokens = Lexer.lex(src, opt);
+    if (opt.walkTokens) {
+      marked.walkTokens(tokens, opt.walkTokens);
+    }
+    return Parser.parse(tokens, opt);
   } catch (e) {
     e.message += '\nPlease report this to https://github.com/markedjs/marked.';
-    if ((opt || marked.defaults).silent) {
+    if (opt.silent) {
       return '<p>An error occurred:</p><pre>'
         + escape(e.message + '', true)
         + '</pre>';
@@ -161,9 +166,50 @@ marked.use = function(extension) {
     }
     opts.tokenizer = tokenizer;
   }
+  if (extension.walkTokens) {
+    const walkTokens = marked.defaults.walkTokens;
+    opts.walkTokens = (token) => {
+      extension.walkTokens(token);
+      if (walkTokens) {
+        walkTokens(token);
+      }
+    };
+  }
   marked.setOptions(opts);
 };
 
+/**
+ * Run callback for every token
+ */
+
+marked.walkTokens = function(tokens, callback) {
+  for (const token of tokens) {
+    callback(token);
+    switch (token.type) {
+      case 'table': {
+        for (const cell of token.tokens.header) {
+          marked.walkTokens(cell, callback);
+        }
+        for (const row of token.tokens.cells) {
+          for (const cell of row) {
+            marked.walkTokens(cell, callback);
+          }
+        }
+        break;
+      }
+      case 'list': {
+        marked.walkTokens(token.items, callback);
+        break;
+      }
+      default: {
+        if (token.tokens) {
+          marked.walkTokens(token.tokens, callback);
+        }
+      }
+    }
+  }
+};
+
 /**
  * Expose
  */

test/unit/Lexer-spec.js

@@ -307,6 +307,7 @@ a | b
             loose: false,
             items: [
               {
+                type: 'list_item',
                 raw: '- item 1',
                 task: false,
                 checked: undefined,
@@ -320,6 +321,7 @@ a | b
                 }]
               },
               {
+                type: 'list_item',
                 raw: '- item 2\n',
                 task: false,
                 checked: undefined,

test/unit/marked-spec.js

@@ -132,6 +132,18 @@ describe('use extension', () => {
     expect(html).toBe('<p>extension</p>\n');
   });
 
+  it('should use walkTokens', () => {
+    let walked = 0;
+    const extension = {
+      walkTokens(token) {
+        walked++;
+      }
+    };
+    marked.use(extension);
+    marked('text');
+    expect(walked).toBe(2);
+  });
+
   it('should use options from extension', () => {
     const extension = {
       headerIds: false
@@ -141,6 +153,29 @@ describe('use extension', () => {
     expect(html).toBe('<h1>heading</h1>\n');
   });
 
+  it('should call all walkTokens in reverse order', () => {
+    let walkedOnce = 0;
+    let walkedTwice = 0;
+    const extension1 = {
+      walkTokens(token) {
+        if (token.walkedOnce) {
+          walkedTwice++;
+        }
+      }
+    };
+    const extension2 = {
+      walkTokens(token) {
+        walkedOnce++;
+        token.walkedOnce = true;
+      }
+    };
+    marked.use(extension1);
+    marked.use(extension2);
+    marked('text');
+    expect(walkedOnce).toBe(2);
+    expect(walkedTwice).toBe(2);
+  });
+
   it('should use last extension function and not override others', () => {
     const extension1 = {
       renderer: {
@@ -229,3 +264,167 @@ paragraph
     expect(html).toBe('arrow no options\nfunction options\nshorthand options\n');
   });
 });
+
+describe('async highlight', () => {
+  let highlight, markdown;
+  beforeEach(() => {
+    highlight = jasmine.createSpy('highlight', (text, lang, callback) => {
+      setImmediate(() => {
+        callback(null, `async ${text || ''}`);
+      });
+    });
+    markdown = `
+\`\`\`lang1
+text 1
+\`\`\`
+
+> \`\`\`lang2
+> text 2
+> \`\`\`
+
+- \`\`\`lang3
+  text 3
+  \`\`\`
+`;
+  });
+
+  it('should highlight codeblocks async', (done) => {
+    highlight.and.callThrough();
+
+    marked(markdown, { highlight }, (err, html) => {
+      if (err) {
+        fail(err);
+      }
+
+      expect(html).toBe(`<pre><code class="language-lang1">async text 1</code></pre>
+<blockquote>
+<pre><code class="language-lang2">async text 2</code></pre>
+</blockquote>
+<ul>
+<li><pre><code class="language-lang3">async text 3</code></pre>
+</li>
+</ul>
+`);
+      done();
+    });
+  });
+
+  it('should call callback for each error in highlight', (done) => {
+    highlight.and.callFake((lang, text, callback) => {
+      callback(new Error('highlight error'));
+    });
+
+    let numErrors = 0;
+    marked(markdown, { highlight }, (err, html) => {
+      expect(err).toBeTruthy();
+      expect(html).toBeUndefined();
+
+      if (err) {
+        numErrors++;
+      }
+
+      if (numErrors === 3) {
+        done();
+      }
+    });
+  });
+});
+
+describe('walkTokens', () => {
+  it('should walk over every token', () => {
+    const markdown = `
+paragraph
+
+---
+
+# heading
+
+\`\`\`
+code
+\`\`\`
+
+| a | b |
+|---|---|
+| 1 | 2 |
+| 3 | 4 |
+
+> blockquote
+
+- list
+
+<div>html</div>
+
+[link](https://example.com)
+
+![image](https://example.com/image.jpg)
+
+**strong**
+
+*em*
+
+\`codespan\`
+
+~~del~~
+
+br
+br
+`;
+    const tokens = marked.lexer(markdown, { ...marked.getDefaults(), breaks: true });
+    const tokensSeen = [];
+    marked.walkTokens(tokens, (token) => {
+      tokensSeen.push([token.type, (token.raw || '').replace(/\n/g, '')]);
+    });
+
+    expect(tokensSeen).toEqual([
+      ['paragraph', 'paragraph'],
+      ['text', 'paragraph'],
+      ['space', ''],
+      ['hr', '---'],
+      ['heading', '# heading'],
+      ['text', 'heading'],
+      ['code', '```code```'],
+      ['table', '| a | b ||---|---|| 1 | 2 || 3 | 4 |'],
+      ['text', 'a'],
+      ['text', 'b'],
+      ['text', '1'],
+      ['text', '2'],
+      ['text', '3'],
+      ['text', '4'],
+      ['blockquote', '> blockquote'],
+      ['paragraph', 'blockquote'],
+      ['text', 'blockquote'],
+      ['list', '- list'],
+      ['list_item', '- list'],
+      ['text', 'list'],
+      ['text', 'list'],
+      ['space', ''],
+      ['html', '<div>html</div>'],
+      ['paragraph', '[link](https://example.com)'],
+      ['link', '[link](https://example.com)'],
+      ['text', 'link'],
+      ['space', ''],
+      ['paragraph', '![image](https://example.com/image.jpg)'],
+      ['image', '![image](https://example.com/image.jpg)'],
+      ['space', ''],
+      ['paragraph', '**strong**'],
+      ['strong', '**strong**'],
+      ['text', 'strong'],
+      ['space', ''],
+      ['paragraph', '*em*'],
+      ['em', '*em*'],
+      ['text', 'em'],
+      ['space', ''],
+      ['paragraph', '`codespan`'],
+      ['codespan', '`codespan`'],
+      ['space', ''],
+      ['paragraph', '~~del~~'],
+      ['del', '~~del~~'],
+      ['text', 'del'],
+      ['space', ''],
+      ['paragraph', 'brbr'],
+      ['text', 'br'],
+      ['br', ''],
+      ['text', 'br']
+    ]);
+  });
+});