Merge pull request #1652 from UziTech/codespan-newline

Codespan newline

docs/USING_PRO.md

@@ -155,7 +155,7 @@ console.log(marked('$ latex code $\n\n` other code `'));
 - table(*string* src)
 - lheading(*string* src)
 - paragraph(*string* src)
-- text(*string* src)
+- text(*string* src, *array* tokens)
 
 ### Inline level tokenizer methods
 

src/Lexer.js

@@ -112,7 +112,7 @@ module.exports = class Lexer {
    */
   blockTokens(src, tokens = [], top = true) {
     src = src.replace(/^ +$/gm, '');
-    let token, i, l;
+    let token, i, l, lastToken;
 
     while (src) {
       // newline
@@ -127,7 +127,13 @@ module.exports = class Lexer {
       // code
       if (token = this.tokenizer.code(src, tokens)) {
         src = src.substring(token.raw.length);
-        tokens.push(token);
+        if (token.type) {
+          tokens.push(token);
+        } else {
+          lastToken = tokens[tokens.length - 1];
+          lastToken.raw += '\n' + token.raw;
+          lastToken.text += '\n' + token.text;
+        }
         continue;
       }
 
@@ -219,9 +225,15 @@ module.exports = class Lexer {
       }
 
       // text
-      if (token = this.tokenizer.text(src)) {
+      if (token = this.tokenizer.text(src, tokens)) {
         src = src.substring(token.raw.length);
-        tokens.push(token);
+        if (token.type) {
+          tokens.push(token);
+        } else {
+          lastToken = tokens[tokens.length - 1];
+          lastToken.raw += '\n' + token.raw;
+          lastToken.text += '\n' + token.text;
+        }
         continue;
       }
 

src/Tokenizer.js

@@ -84,21 +84,21 @@ module.exports = class Tokenizer {
       const lastToken = tokens[tokens.length - 1];
       // An indented code block cannot interrupt a paragraph.
       if (lastToken && lastToken.type === 'paragraph') {
-        tokens.pop();
-        lastToken.text += '\n' + cap[0].trimRight();
-        lastToken.raw += '\n' + cap[0];
-        return lastToken;
-      } else {
-        const text = cap[0].replace(/^ {4}/gm, '');
         return {
-          type: 'code',
           raw: cap[0],
-          codeBlockStyle: 'indented',
+          text: cap[0].trimRight()
-          text: !this.options.pedantic
-            ? rtrim(text, '\n')
-            : text
         };
       }
+
+      const text = cap[0].replace(/^ {4}/gm, '');
+      return {
+        type: 'code',
+        raw: cap[0],
+        codeBlockStyle: 'indented',
+        text: !this.options.pedantic
+          ? rtrim(text, '\n')
+          : text
+      };
     }
   }
 
@@ -374,9 +374,17 @@ module.exports = class Tokenizer {
     }
   }
 
-  text(src) {
+  text(src, tokens) {
     const cap = this.rules.block.text.exec(src);
     if (cap) {
+      const lastToken = tokens[tokens.length - 1];
+      if (lastToken && lastToken.type === 'text') {
+        return {
+          raw: cap[0],
+          text: cap[0]
+        };
+      }
+
       return {
         type: 'text',
         raw: cap[0],
@@ -504,10 +512,17 @@ module.exports = class Tokenizer {
   codespan(src) {
     const cap = this.rules.inline.code.exec(src);
     if (cap) {
+      let text = cap[2].replace(/\n/g, ' ');
+      const hasNonSpaceChars = /[^ ]/.test(text);
+      const hasSpaceCharsOnBothEnds = text.startsWith(' ') && text.endsWith(' ');
+      if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) {
+        text = text.substring(1, text.length - 1);
+      }
+      text = escape(text, true);
       return {
         type: 'codespan',
         raw: cap[0],
-        text: escape(cap[2].trim(), true)
+        text
       };
     }
   }

test/specs/new/codespan_newline.html

@@ -0,0 +1,5 @@
+<p><code>code code</code></p>
+
+<ul>
+<li><code>code code</code></li>
+</ul>

test/specs/new/codespan_newline.md

@@ -0,0 +1,5 @@
+`code
+code`
+
+- `code
+code`

test/unit/Lexer-spec.js

@@ -752,12 +752,95 @@ a | b
         });
       });
 
-      it('code', () => {
+      describe('codespan', () => {
-        expectInlineTokens({
+        it('code', () => {
-          md: '`code`',
+          expectInlineTokens({
-          tokens: [
+            md: '`code`',
-            { type: 'codespan', raw: '`code`', text: 'code' }
+            tokens: [
-          ]
+              { type: 'codespan', raw: '`code`', text: 'code' }
+            ]
+          });
+        });
+
+        it('only spaces not stripped', () => {
+          expectInlineTokens({
+            md: '`   `',
+            tokens: [
+              { type: 'codespan', raw: '`   `', text: '   ' }
+            ]
+          });
+        });
+
+        it('beginning space only not stripped', () => {
+          expectInlineTokens({
+            md: '` a`',
+            tokens: [
+              { type: 'codespan', raw: '` a`', text: ' a' }
+            ]
+          });
+        });
+
+        it('end space only not stripped', () => {
+          expectInlineTokens({
+            md: '`a `',
+            tokens: [
+              { type: 'codespan', raw: '`a `', text: 'a ' }
+            ]
+          });
+        });
+
+        it('begin and end spaces are stripped', () => {
+          expectInlineTokens({
+            md: '` a `',
+            tokens: [
+              { type: 'codespan', raw: '` a `', text: 'a' }
+            ]
+          });
+        });
+
+        it('begin and end newlines are stripped', () => {
+          expectInlineTokens({
+            md: '`\na\n`',
+            tokens: [
+              { type: 'codespan', raw: '`\na\n`', text: 'a' }
+            ]
+          });
+        });
+
+        it('begin and end tabs are not stripped', () => {
+          expectInlineTokens({
+            md: '`\ta\t`',
+            tokens: [
+              { type: 'codespan', raw: '`\ta\t`', text: '\ta\t' }
+            ]
+          });
+        });
+
+        it('begin and end newlines', () => {
+          expectInlineTokens({
+            md: '`\na\n`',
+            tokens: [
+              { type: 'codespan', raw: '`\na\n`', text: 'a' }
+            ]
+          });
+        });
+
+        it('begin and end multiple spaces only one stripped', () => {
+          expectInlineTokens({
+            md: '`  a  `',
+            tokens: [
+              { type: 'codespan', raw: '`  a  `', text: ' a ' }
+            ]
+          });
+        });
+
+        it('newline to space', () => {
+          expectInlineTokens({
+            md: '`a\nb`',
+            tokens: [
+              { type: 'codespan', raw: '`a\nb`', text: 'a b' }
+            ]
+          });
         });
       });