diff --git a/docs/USING_ADVANCED.md b/docs/USING_ADVANCED.md index 3abf6427..23224d16 100644 --- a/docs/USING_ADVANCED.md +++ b/docs/USING_ADVANCED.md @@ -58,6 +58,7 @@ console.log(marked(markdownString)); |smartLists |`boolean` |`false` |v0.2.8 |If true, use smarter list behavior than those found in `markdown.pl`.| |smartypants |`boolean` |`false` |v0.2.9 |If true, use "smart" typographic punctuation for things like quotes and dashes.| |tokenizer |`object` |`new Tokenizer()`|v1.0.0|An object containing functions to create tokens from markdown. See [extensibility](/#/USING_PRO.md) for more details.| +|walkTokens |`function` |`null`|v1.1.0|A function which is called for every token. See [extensibility](/#/USING_PRO.md) for more details.| |xhtml |`boolean` |`false` |v0.3.2 |If true, emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.|

Asynchronous highlighting

diff --git a/docs/USING_PRO.md b/docs/USING_PRO.md index 9b6bd7ac..5912d2b5 100644 --- a/docs/USING_PRO.md +++ b/docs/USING_PRO.md @@ -10,6 +10,8 @@ The `renderer` and `tokenizer` options can be an object with functions that will The `renderer` and `tokenizer` functions can return false to fallback to the previous function. +The `walkTokens` option can be a function that will be called with every token before rendering. When calling `use` multiple times with different `walkTokens` functions each function will be called in the **reverse** order in which they were assigned. + All other options will overwrite previously set options.

The renderer

@@ -188,6 +190,35 @@ smartypants('"this ... string"') // "“this … string”" ``` +

Walk Tokens

+ +The walkTokens function gets called with every token. Child tokens are called before moving on to sibling tokens. Each token is passed by reference so updates are persisted when passed to the parser. The return value of the function is ignored. + +**Example:** Overriding heading tokens to start at h2. + +```js +const marked = require('marked'); + +// Override function +const walkTokens = (token) => { + if (token.type === 'heading') { + token.depth += 1; + } +}; + +marked.use({ walkTokens }); + +// Run marked +console.log(marked('# heading 2\n\n## heading 3')); +``` + +**Output:** + +```html +

heading 2

+

heading 3

+``` +

The lexer

The lexer takes a markdown string and calls the tokenizer functions. diff --git a/docs/index.html b/docs/index.html index a7de1818..a6c29308 100644 --- a/docs/index.html +++ b/docs/index.html @@ -157,6 +157,7 @@
  • marked.use()
  • Renderer
  • Tokenizer
    • +
  • Walk Tokens
  • Lexer
  • Parser
    • diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 512b45dc..e3ff002b 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -269,6 +269,7 @@ module.exports = class Tokenizer { } list.items.push({ + type: 'list_item', raw, task: istask, checked: ischecked, diff --git a/src/defaults.js b/src/defaults.js index 0153bb43..fe376563 100644 --- a/src/defaults.js +++ b/src/defaults.js @@ -16,6 +16,7 @@ function getDefaults() { smartLists: false, smartypants: false, tokenizer: null, + walkTokens: null, xhtml: false }; } diff --git a/src/marked.js b/src/marked.js index 8488e79f..83ffc003 100644 --- a/src/marked.js +++ b/src/marked.js @@ -28,18 +28,17 @@ function marked(src, opt, callback) { + Object.prototype.toString.call(src) + ', string expected'); } - if (callback || typeof opt === 'function') { - if (!callback) { - callback = opt; - opt = null; - } + if (typeof opt === 'function') { + callback = opt; + opt = null; + } - opt = merge({}, marked.defaults, opt || {}); - checkSanitizeDeprecation(opt); + opt = merge({}, marked.defaults, opt || {}); + checkSanitizeDeprecation(opt); + + if (callback) { const highlight = opt.highlight; - let tokens, - pending, - i = 0; + let tokens; try { tokens = Lexer.lex(src, opt); @@ -47,20 +46,15 @@ function marked(src, opt, callback) { return callback(e); } - pending = tokens.length; - const done = function(err) { - if (err) { - opt.highlight = highlight; - return callback(err); - } - let out; - try { - out = Parser.parse(tokens, opt); - } catch (e) { - err = e; + if (!err) { + try { + out = Parser.parse(tokens, opt); + } catch (e) { + err = e; + } } opt.highlight = highlight; @@ -76,34 +70,45 @@ function marked(src, opt, callback) { delete opt.highlight; - if (!pending) return done(); + if (!tokens.length) return done(); - for (; i < tokens.length; i++) { - (function(token) { - if (token.type !== 'code') { - return --pending || done(); - } - return highlight(token.text, token.lang, function(err, code) { - if (err) return done(err); - if (code == null || code === token.text) { - return --pending || done(); + let pending = 0; + marked.walkTokens(tokens, function(token) { + if (token.type === 'code') { + pending++; + highlight(token.text, token.lang, function(err, code) { + if (err) { + return done(err); + } + if (code != null && code !== token.text) { + token.text = code; + token.escaped = true; + } + + pending--; + if (pending === 0) { + done(); } - token.text = code; - token.escaped = true; - --pending || done(); }); - })(tokens[i]); + } + }); + + if (pending === 0) { + done(); } return; } + try { - opt = merge({}, marked.defaults, opt || {}); - checkSanitizeDeprecation(opt); - return Parser.parse(Lexer.lex(src, opt), opt); + const tokens = Lexer.lex(src, opt); + if (opt.walkTokens) { + marked.walkTokens(tokens, opt.walkTokens); + } + return Parser.parse(tokens, opt); } catch (e) { e.message += '\nPlease report this to https://github.com/markedjs/marked.'; - if ((opt || marked.defaults).silent) { + if (opt.silent) { return '

    An error occurred:

    '
         + escape(e.message + '', true)
         + '
    '; @@ -161,9 +166,50 @@ marked.use = function(extension) { } opts.tokenizer = tokenizer; } + if (extension.walkTokens) { + const walkTokens = marked.defaults.walkTokens; + opts.walkTokens = (token) => { + extension.walkTokens(token); + if (walkTokens) { + walkTokens(token); + } + }; + } marked.setOptions(opts); }; +/** + * Run callback for every token + */ + +marked.walkTokens = function(tokens, callback) { + for (const token of tokens) { + callback(token); + switch (token.type) { + case 'table': { + for (const cell of token.tokens.header) { + marked.walkTokens(cell, callback); + } + for (const row of token.tokens.cells) { + for (const cell of row) { + marked.walkTokens(cell, callback); + } + } + break; + } + case 'list': { + marked.walkTokens(token.items, callback); + break; + } + default: { + if (token.tokens) { + marked.walkTokens(token.tokens, callback); + } + } + } + } +}; + /** * Expose */ diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js index 9b2b5d76..fea19c9f 100644 --- a/test/unit/Lexer-spec.js +++ b/test/unit/Lexer-spec.js @@ -307,6 +307,7 @@ a | b loose: false, items: [ { + type: 'list_item', raw: '- item 1', task: false, checked: undefined, @@ -320,6 +321,7 @@ a | b }] }, { + type: 'list_item', raw: '- item 2\n', task: false, checked: undefined, diff --git a/test/unit/marked-spec.js b/test/unit/marked-spec.js index 1b6beafd..f1253b49 100644 --- a/test/unit/marked-spec.js +++ b/test/unit/marked-spec.js @@ -132,6 +132,18 @@ describe('use extension', () => { expect(html).toBe('

    extension

    \n'); }); + it('should use walkTokens', () => { + let walked = 0; + const extension = { + walkTokens(token) { + walked++; + } + }; + marked.use(extension); + marked('text'); + expect(walked).toBe(2); + }); + it('should use options from extension', () => { const extension = { headerIds: false @@ -141,6 +153,29 @@ describe('use extension', () => { expect(html).toBe('

    heading

    \n'); }); + it('should call all walkTokens in reverse order', () => { + let walkedOnce = 0; + let walkedTwice = 0; + const extension1 = { + walkTokens(token) { + if (token.walkedOnce) { + walkedTwice++; + } + } + }; + const extension2 = { + walkTokens(token) { + walkedOnce++; + token.walkedOnce = true; + } + }; + marked.use(extension1); + marked.use(extension2); + marked('text'); + expect(walkedOnce).toBe(2); + expect(walkedTwice).toBe(2); + }); + it('should use last extension function and not override others', () => { const extension1 = { renderer: { @@ -229,3 +264,167 @@ paragraph expect(html).toBe('arrow no options\nfunction options\nshorthand options\n'); }); }); + +describe('async highlight', () => { + let highlight, markdown; + beforeEach(() => { + highlight = jasmine.createSpy('highlight', (text, lang, callback) => { + setImmediate(() => { + callback(null, `async ${text || ''}`); + }); + }); + markdown = ` +\`\`\`lang1 +text 1 +\`\`\` + +> \`\`\`lang2 +> text 2 +> \`\`\` + +- \`\`\`lang3 + text 3 + \`\`\` +`; + }); + + it('should highlight codeblocks async', (done) => { + highlight.and.callThrough(); + + marked(markdown, { highlight }, (err, html) => { + if (err) { + fail(err); + } + + expect(html).toBe(`
    async text 1
    +
    +
    async text 2
    +
    + +`); + done(); + }); + }); + + it('should call callback for each error in highlight', (done) => { + highlight.and.callFake((lang, text, callback) => { + callback(new Error('highlight error')); + }); + + let numErrors = 0; + marked(markdown, { highlight }, (err, html) => { + expect(err).toBeTruthy(); + expect(html).toBeUndefined(); + + if (err) { + numErrors++; + } + + if (numErrors === 3) { + done(); + } + }); + }); +}); + +describe('walkTokens', () => { + it('should walk over every token', () => { + const markdown = ` +paragraph + +--- + +# heading + +\`\`\` +code +\`\`\` + +| a | b | +|---|---| +| 1 | 2 | +| 3 | 4 | + +> blockquote + +- list + +
    html
    + +[link](https://example.com) + +![image](https://example.com/image.jpg) + +**strong** + +*em* + +\`codespan\` + +~~del~~ + +br +br +`; + const tokens = marked.lexer(markdown, { ...marked.getDefaults(), breaks: true }); + const tokensSeen = []; + marked.walkTokens(tokens, (token) => { + tokensSeen.push([token.type, (token.raw || '').replace(/\n/g, '')]); + }); + + expect(tokensSeen).toEqual([ + ['paragraph', 'paragraph'], + ['text', 'paragraph'], + ['space', ''], + ['hr', '---'], + ['heading', '# heading'], + ['text', 'heading'], + ['code', '```code```'], + ['table', '| a | b ||---|---|| 1 | 2 || 3 | 4 |'], + ['text', 'a'], + ['text', 'b'], + ['text', '1'], + ['text', '2'], + ['text', '3'], + ['text', '4'], + ['blockquote', '> blockquote'], + ['paragraph', 'blockquote'], + ['text', 'blockquote'], + ['list', '- list'], + ['list_item', '- list'], + ['text', 'list'], + ['text', 'list'], + ['space', ''], + ['html', '
    html
    '], + ['paragraph', '[link](https://example.com)'], + ['link', '[link](https://example.com)'], + ['text', 'link'], + ['space', ''], + ['paragraph', '![image](https://example.com/image.jpg)'], + ['image', '![image](https://example.com/image.jpg)'], + ['space', ''], + ['paragraph', '**strong**'], + ['strong', '**strong**'], + ['text', 'strong'], + ['space', ''], + ['paragraph', '*em*'], + ['em', '*em*'], + ['text', 'em'], + ['space', ''], + ['paragraph', '`codespan`'], + ['codespan', '`codespan`'], + ['space', ''], + ['paragraph', '~~del~~'], + ['del', '~~del~~'], + ['text', 'del'], + ['space', ''], + ['paragraph', 'brbr'], + ['text', 'br'], + ['br', ''], + ['text', 'br'] + ]); + }); +});