import { Marked, Renderer, lexer, parseInline, getDefaults, walkTokens, defaults, setOptions } from '../../lib/marked.esm.js';
import { timeout } from './utils.js';
import assert from 'node:assert';
import { describe, it, beforeEach, mock } from 'node:test';
describe('marked unit', () => {
let marked;
beforeEach(() => {
marked = new Marked();
setOptions(getDefaults());
});
describe('Test paragraph token type', () => {
it('should use the "paragraph" type on top level', () => {
const md = 'A Paragraph.\n\n> A blockquote\n\n- list item\n';
const tokens = lexer(md);
assert.strictEqual(tokens[0].type, 'paragraph');
assert.strictEqual(tokens[2].tokens[0].type, 'paragraph');
assert.strictEqual(tokens[3].items[0].tokens[0].type, 'text');
});
});
describe('changeDefaults', () => {
it('should change global defaults', async() => {
const { defaults, setOptions } = await import('../../lib/marked.esm.js');
assert.ok(!defaults.test);
setOptions({ test: true });
assert.ok((await import('../../lib/marked.esm.js')).defaults.test);
});
});
describe('inlineLexer', () => {
it('should send html to renderer.html', () => {
const renderer = new Renderer();
mock.method(renderer, 'html');
const md = 'HTML Image:
';
marked.parse(md, { renderer });
assert.strictEqual(renderer.html.mock.calls[0].arguments[0], '
');
});
});
describe('task', () => {
it('space after checkbox', () => {
const html = marked.parse('- [ ] item');
assert.strictEqual(html, '
\n');
});
it('space after loose checkbox', () => {
const html = marked.parse('- [ ] item 1\n\n- [ ] item 2');
assert.strictEqual(html, '\n');
});
});
describe('parseInline', () => {
it('should parse inline tokens', () => {
const md = '**strong** _em_';
const html = parseInline(md);
assert.strictEqual(html, 'strong em');
});
it('should not parse block tokens', () => {
const md = '# header\n\n_em_';
const html = parseInline(md);
assert.strictEqual(html, '# header\n\nem');
});
});
describe('use extension', () => {
it('should use custom block tokenizer + renderer extensions', () => {
const underline = {
name: 'underline',
level: 'block',
tokenizer(src) {
const rule = /^:([^\n]*)(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'underline',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
},
renderer(token) {
return `${token.text}\n`;
}
};
marked.use({ extensions: [underline] });
let html = marked.parse('Not Underlined\n:Underlined\nNot Underlined');
assert.strictEqual(html, 'Not Underlined\n:Underlined\nNot Underlined
\n');
html = marked.parse('Not Underlined\n\n:Underlined\n\nNot Underlined');
assert.strictEqual(html, 'Not Underlined
\nUnderlined\nNot Underlined
\n');
});
it('should interrupt paragraphs if using "start" property', () => {
const underline = {
extensions: [{
name: 'underline',
level: 'block',
start(src) { return src.indexOf(':'); },
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'underline',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
},
renderer(token) {
return `${token.text}\n`;
}
}]
};
marked.use(underline);
const html = marked.parse('Not Underlined A\n:Underlined B:\nNot Underlined C\n:Not Underlined D');
assert.strictEqual(html, 'Not Underlined A
\nUnderlined B\nNot Underlined C\n:Not Underlined D
\n');
});
it('should use custom inline tokenizer + renderer extensions', () => {
const underline = {
name: 'underline',
level: 'inline',
start(src) { return src.indexOf('='); },
tokenizer(src) {
const rule = /^=([^=]+)=/;
const match = rule.exec(src);
if (match) {
return {
type: 'underline',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
},
renderer(token) {
return `${token.text}`;
}
};
marked.use({ extensions: [underline] });
const html = marked.parse('Not Underlined =Underlined= Not Underlined');
assert.strictEqual(html, 'Not Underlined Underlined Not Underlined
\n');
});
it('should handle interacting block and inline extensions', () => {
const descriptionlist = {
name: 'descriptionList',
level: 'block',
start(src) {
const match = src.match(/:[^:\n]/);
if (match) {
return match.index;
}
},
tokenizer(src, tokens) {
const rule = /^(?::[^:\n]+:[^:\n]*(?:\n|$))+/;
const match = rule.exec(src);
if (match) {
const token = {
type: 'descriptionList',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[0].trim(), // You can add additional properties to your tokens to pass along to the renderer
tokens: []
};
this.lexer.inlineTokens(token.text, token.tokens);
return token;
}
},
renderer(token) {
return `${this.parser.parseInline(token.tokens)}\n
`;
}
};
const description = {
name: 'description',
level: 'inline',
start(src) { return src.indexOf(':'); },
tokenizer(src, tokens) {
const rule = /^:([^:\n]+):([^:\n]*)(?:\n|$)/;
const match = rule.exec(src);
if (match) {
const token = {
type: 'description',
raw: match[0],
dt: [],
dd: []
};
this.lexer.inline(match[1].trim(), token.dt);
this.lexer.inline(match[2].trim(), token.dd);
return token;
}
},
renderer(token) {
return `\n${this.parser.parseInline(token.dt)}${this.parser.parseInline(token.dd)}`;
}
};
marked.use({ extensions: [descriptionlist, description] });
const html = marked.parse('A Description List with One Description:\n'
+ ': Topic 1 : Description 1\n'
+ ': **Topic 2** : *Description 2*');
assert.strictEqual(html, 'A Description List with One Description:
\n'
+ ''
+ '\n- Topic 1
- Description 1
'
+ '\n- Topic 2
- Description 2
'
+ '\n
');
});
it('should allow other options mixed into the extension', () => {
const extension = {
name: 'underline',
level: 'block',
start(src) { return src.indexOf(':'); },
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'underline',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
},
renderer(token) {
return `${token.text}\n`;
}
};
marked.use({ silent: true, extensions: [extension] });
const html = marked.parse(':test:\ntest\n');
assert.strictEqual(html, 'test\ntest
\n');
});
it('should handle renderers that return false', () => {
const extension = {
name: 'test',
level: 'block',
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'test',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
},
renderer(token) {
if (token.text === 'test') {
return 'test';
}
return false;
}
};
const fallbackRenderer = {
name: 'test',
level: 'block',
renderer(token) {
if (token.text === 'Test') {
return 'fallback';
}
return false;
}
};
marked.use({ extensions: [fallbackRenderer, extension] });
const html = marked.parse(':Test:\n\n:test:\n\n:none:');
assert.strictEqual(html, 'fallbacktest');
});
it('should fall back when tokenizers return false', () => {
const extension = {
name: 'test',
level: 'block',
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'test',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1].trim() // You can add additional properties to your tokens to pass along to the renderer
};
}
return false;
},
renderer(token) {
return token.text;
}
};
const extension2 = {
name: 'test',
level: 'block',
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
if (match[1].match(/^[A-Z]/)) {
return {
type: 'test',
raw: match[0],
text: match[1].trim().toUpperCase()
};
}
}
return false;
}
};
marked.use({ extensions: [extension, extension2] });
const html = marked.parse(':Test:\n\n:test:');
assert.strictEqual(html, 'TESTtest');
});
it('should override original tokenizer/renderer with same name, but fall back if returns false', () => {
const extension = {
extensions: [{
name: 'heading',
level: 'block',
tokenizer(src) {
return false; // fall back to default `heading` tokenizer
},
renderer(token) {
return '' + token.text + ' RENDERER EXTENSION\n';
}
},
{
name: 'code',
level: 'block',
tokenizer(src) {
const rule = /^:([^\n]*):(?:\n|$)/;
const match = rule.exec(src);
if (match) {
return {
type: 'code',
raw: match[0],
text: match[1].trim() + ' TOKENIZER EXTENSION'
};
}
},
renderer(token) {
return false; // fall back to default `code` renderer
}
}]
};
marked.use(extension);
const html = marked.parse('# extension1\n:extension2:');
assert.strictEqual(html, 'extension1 RENDERER EXTENSION
\nextension2 TOKENIZER EXTENSION\n
\n');
});
it('should walk only specified child tokens', () => {
const walkableDescription = {
extensions: [{
name: 'walkableDescription',
level: 'inline',
start(src) { return src.indexOf(':'); },
tokenizer(src, tokens) {
const rule = /^:([^:\n]+):([^:\n]*)(?:\n|$)/;
const match = rule.exec(src);
if (match) {
const token = {
type: 'walkableDescription',
raw: match[0],
dt: this.lexer.inline(match[1].trim()),
dd: [],
tokens: []
};
this.lexer.inline(match[2].trim(), token.dd);
this.lexer.inline('unwalked', token.tokens);
return token;
}
},
renderer(token) {
return `\n${this.parser.parseInline(token.dt)} - ${this.parser.parseInline(token.tokens)}${this.parser.parseInline(token.dd)}`;
},
childTokens: ['dd', 'dt']
}],
walkTokens(token) {
if (token.type === 'text') {
token.text += ' walked';
}
}
};
marked.use(walkableDescription);
const html = marked.parse(': Topic 1 : Description 1\n'
+ ': **Topic 2** : *Description 2*');
assert.strictEqual(html, '\n
Topic 1 walked - unwalkedDescription 1 walked'
+ '\nTopic 2 walked - unwalkedDescription 2 walked\n');
});
it('should walk child token arrays', () => {
const walkableDescription = {
extensions: [{
name: 'walkableDescription',
level: 'inline',
start(src) { return src.indexOf(':'); },
tokenizer(src, tokens) {
const rule = /^:([^:\n]+):([^:\n]*)(?:\n|$)/;
const match = rule.exec(src);
if (match) {
const token = {
type: 'walkableDescription',
raw: match[0],
dt: [this.lexer.inline(match[1].trim())],
dd: [[this.lexer.inline(match[2].trim())]]
};
return token;
}
},
renderer(token) {
return `\n${this.parser.parseInline(token.dt[0])}${this.parser.parseInline(token.dd[0][0])}`;
},
childTokens: ['dd', 'dt']
}],
walkTokens(token) {
if (token.type === 'text') {
token.text += ' walked';
}
}
};
marked.use(walkableDescription);
const html = marked.parse(': Topic 1 : Description 1\n'
+ ': **Topic 2** : *Description 2*');
assert.strictEqual(html, '\n
Topic 1 walkedDescription 1 walked'
+ '\nTopic 2 walkedDescription 2 walked\n');
});
describe('multiple extensions', () => {
function createExtension(name) {
return {
extensions: [{
name: `block-${name}`,
level: 'block',
start(src) { return src.indexOf('::'); },
tokenizer(src, tokens) {
if (src.startsWith(`::${name}\n`)) {
const text = `:${name}`;
const token = {
type: `block-${name}`,
raw: `::${name}\n`,
text,
tokens: []
};
this.lexer.inline(token.text, token.tokens);
return token;
}
},
renderer(token) {
return `<${token.type}>${this.parser.parseInline(token.tokens)}${token.type}>\n`;
}
}, {
name: `inline-${name}`,
level: 'inline',
start(src) { return src.indexOf(':'); },
tokenizer(src, tokens) {
if (src.startsWith(`:${name}`)) {
return {
type: `inline-${name}`,
raw: `:${name}`,
text: `used ${name}`
};
}
},
renderer(token) {
return token.text;
}
}],
tokenizer: {
heading(src) {
if (src.startsWith(`# ${name}`)) {
const token = {
type: 'heading',
raw: `# ${name}`,
text: `used ${name}`,
depth: 1,
tokens: []
};
this.lexer.inline(token.text, token.tokens);
return token;
}
return false;
}
},
renderer: {
heading(text, depth, raw) {
if (text === name) {
return `${text}\n`;
}
return false;
}
},
walkTokens(token) {
if (token.text === `used ${name}`) {
token.text += ' walked';
}
}
};
}
function createFalseExtension(name) {
return {
extensions: [{
name: `block-${name}`,
level: 'block',
start(src) { return src.indexOf('::'); },
tokenizer(src, tokens) {
return false;
},
renderer(token) {
return false;
}
}, {
name: `inline-${name}`,
level: 'inline',
start(src) { return src.indexOf(':'); },
tokenizer(src, tokens) {
return false;
},
renderer(token) {
return false;
}
}]
};
}
function runTest() {
const html = marked.parse(`
::extension1
::extension2
:extension1
:extension2
# extension1
# extension2
# no extension
`);
assert.strictEqual(`\n${html}\n`.replace(/\n+/g, '\n'), `
used extension1 walked
used extension2 walked
used extension1 walked
used extension2 walked
used extension1 walked
used extension2 walked
no extension
`);
}
it('should merge extensions when calling marked.use multiple times', () => {
marked.use(createExtension('extension1'));
marked.use(createExtension('extension2'));
runTest();
});
it('should merge extensions when calling marked.use with multiple extensions', () => {
marked.use(
createExtension('extension1'),
createExtension('extension2')
);
runTest();
});
it('should fall back to any extensions with the same name if the first returns false', () => {
marked.use(
createExtension('extension1'),
createExtension('extension2'),
createFalseExtension('extension1'),
createFalseExtension('extension2')
);
runTest();
});
it('should merge extensions correctly', () => {
marked.use(
{},
{ tokenizer: {} },
{ renderer: {} },
{ walkTokens: () => {} },
{ extensions: [] }
);
// should not throw
marked.parse('# test');
});
});
it('should be async if any extension in use args is async', () => {
marked.use(
{ async: true },
{ async: false }
);
assert.ok(marked.defaults.async);
});
it('should be async if any extension in use is async', () => {
marked.use({ async: true });
marked.use({ async: false });
assert.ok(marked.defaults.async);
});
it('should reset async with setOptions', () => {
marked.use({ async: true });
setOptions({ async: false });
assert.ok(!defaults.async);
});
it('should return Promise if async', () => {
assert.ok(marked.parse('test', { async: true }) instanceof Promise);
});
it('should return string if not async', () => {
assert.strictEqual(typeof marked.parse('test', { async: false }), 'string');
});
it('should return Promise if async is set by extension', () => {
marked.use({ async: true });
assert.ok(marked.parse('test', { async: false }) instanceof Promise);
});
it('should allow deleting/editing tokens', () => {
const styleTags = {
extensions: [{
name: 'inlineStyleTag',
level: 'inline',
start(src) {
const match = src.match(/ *{[^\{]/);
if (match) {
return match.index;
}
},
tokenizer(src, tokens) {
const rule = /^ *{([^\{\}\n]+)}$/;
const match = rule.exec(src);
if (match) {
return {
type: 'inlineStyleTag',
raw: match[0], // This is the text that you want your token to consume from the source
text: match[1]
};
}
}
},
{
name: 'styled',
renderer(token) {
token.type = token.originalType;
const text = this.parser.parse([token]);
const openingTag = /(<[^\s<>]+)([^\n<>]*>.*)/s.exec(text);
if (openingTag) {
return `${openingTag[1]} ${token.style}${openingTag[2]}`;
}
return text;
}
}],
walkTokens(token) {
if (token.tokens) {
const finalChildToken = token.tokens[token.tokens.length - 1];
if (finalChildToken && finalChildToken.type === 'inlineStyleTag') {
token.originalType = token.type;
token.type = 'styled';
token.style = `style="color:${finalChildToken.text};"`;
token.tokens.pop();
}
}
}
};
marked.use(styleTags);
const html = marked.parse('This is a *paragraph* with blue text. {blue}\n'
+ '# This is a *header* with red text {red}');
assert.strictEqual(html, 'This is a paragraph with blue text.
\n'
+ 'This is a header with red text
\n');
});
it('should use renderer', () => {
const extension = {
renderer: {
paragraph(text) {
return 'extension';
}
}
};
mock.method(extension.renderer, 'paragraph');
marked.use(extension);
const html = marked.parse('text');
assert.strictEqual(extension.renderer.paragraph.mock.calls[0].arguments[0], 'text');
assert.strictEqual(html, 'extension');
});
it('should use tokenizer', () => {
const extension = {
tokenizer: {
paragraph(text) {
const token = {
type: 'paragraph',
raw: text,
text: 'extension',
tokens: []
};
this.lexer.inline(token.text, token.tokens);
return token;
}
}
};
mock.method(extension.tokenizer, 'paragraph');
marked.use(extension);
const html = marked.parse('text');
assert.strictEqual(extension.tokenizer.paragraph.mock.calls[0].arguments[0], 'text');
assert.strictEqual(html, 'extension
\n');
});
it('should use walkTokens', () => {
let walked = 0;
const extension = {
walkTokens(token) {
walked++;
}
};
marked.use(extension);
marked.parse('text');
assert.strictEqual(walked, 2);
});
it('should use options from extension', () => {
const extension = {
breaks: true
};
marked.use(extension);
const html = marked.parse('line1\nline2');
assert.strictEqual(html, 'line1
line2
\n');
});
it('should call all walkTokens in reverse order', () => {
let walkedOnce = 0;
let walkedTwice = 0;
const extension1 = {
walkTokens(token) {
if (token.walkedOnce) {
walkedTwice++;
}
}
};
const extension2 = {
walkTokens(token) {
walkedOnce++;
token.walkedOnce = true;
}
};
marked.use(extension1);
marked.use(extension2);
marked.parse('text');
assert.strictEqual(walkedOnce, 2);
assert.strictEqual(walkedTwice, 2);
});
it('should use last extension function and not override others', () => {
const extension1 = {
renderer: {
paragraph(text) {
return 'extension1 paragraph\n';
},
html(html) {
return 'extension1 html\n';
}
}
};
const extension2 = {
renderer: {
paragraph(text) {
return 'extension2 paragraph\n';
}
}
};
marked.use(extension1);
marked.use(extension2);
const html = marked.parse(`
paragraph