markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "unicode/utf8"
24)
25
26const VERSION = "1.1"
27
28// These are the supported markdown parsing extensions.
29// OR these values together to select multiple extensions.
30const (
31 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota // ignore emphasis markers inside words
32 EXTENSION_TABLES // render tables
33 EXTENSION_FENCED_CODE // render fenced code blocks
34 EXTENSION_AUTOLINK // detect embedded URLs that are not explicitly marked
35 EXTENSION_STRIKETHROUGH // strikethrough text using ~~test~~
36 EXTENSION_LAX_HTML_BLOCKS // loosen up HTML block parsing rules
37 EXTENSION_SPACE_HEADERS // be strict about prefix header rules
38 EXTENSION_HARD_LINE_BREAK // translate newlines into line breaks
39 EXTENSION_TAB_SIZE_EIGHT // expand tabs to eight spaces instead of four
40 EXTENSION_FOOTNOTES // Pandoc-style footnotes
41 EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK // No need to insert an empty line to start a (code, quote, order list, unorder list)block
42)
43
44// These are the possible flag values for the link renderer.
45// Only a single one of these values will be used; they are not ORed together.
46// These are mostly of interest if you are writing a new output format.
47const (
48 LINK_TYPE_NOT_AUTOLINK = iota
49 LINK_TYPE_NORMAL
50 LINK_TYPE_EMAIL
51)
52
53// These are the possible flag values for the ListItem renderer.
54// Multiple flag values may be ORed together.
55// These are mostly of interest if you are writing a new output format.
56const (
57 LIST_TYPE_ORDERED = 1 << iota
58 LIST_ITEM_CONTAINS_BLOCK
59 LIST_ITEM_BEGINNING_OF_LIST
60 LIST_ITEM_END_OF_LIST
61)
62
63// These are the possible flag values for the table cell renderer.
64// Only a single one of these values will be used; they are not ORed together.
65// These are mostly of interest if you are writing a new output format.
66const (
67 TABLE_ALIGNMENT_LEFT = 1 << iota
68 TABLE_ALIGNMENT_RIGHT
69 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
70)
71
72// The size of a tab stop.
73const (
74 TAB_SIZE_DEFAULT = 4
75 TAB_SIZE_EIGHT = 8
76)
77
78// These are the tags that are recognized as HTML block tags.
79// Any of these can be included in markdown text without special escaping.
80var blockTags = map[string]bool{
81 "p": true,
82 "dl": true,
83 "h1": true,
84 "h2": true,
85 "h3": true,
86 "h4": true,
87 "h5": true,
88 "h6": true,
89 "ol": true,
90 "ul": true,
91 "del": true,
92 "div": true,
93 "ins": true,
94 "pre": true,
95 "form": true,
96 "math": true,
97 "table": true,
98 "iframe": true,
99 "script": true,
100 "fieldset": true,
101 "noscript": true,
102 "blockquote": true,
103
104 // HTML5
105 "video": true,
106 "aside": true,
107 "canvas": true,
108 "figure": true,
109 "footer": true,
110 "header": true,
111 "hgroup": true,
112 "output": true,
113 "article": true,
114 "section": true,
115 "progress": true,
116 "figcaption": true,
117}
118
119// Renderer is the rendering interface.
120// This is mostly of interest if you are implementing a new rendering format.
121//
122// When a byte slice is provided, it contains the (rendered) contents of the
123// element.
124//
125// When a callback is provided instead, it will write the contents of the
126// respective element directly to the output buffer and return true on success.
127// If the callback returns false, the rendering function should reset the
128// output buffer as though it had never been called.
129//
130// Currently Html and Latex implementations are provided
131type Renderer interface {
132 // block-level callbacks
133 BlockCode(out *bytes.Buffer, text []byte, lang string)
134 BlockQuote(out *bytes.Buffer, text []byte)
135 BlockHtml(out *bytes.Buffer, text []byte)
136 Header(out *bytes.Buffer, text func() bool, level int)
137 HRule(out *bytes.Buffer)
138 List(out *bytes.Buffer, text func() bool, flags int)
139 ListItem(out *bytes.Buffer, text []byte, flags int)
140 Paragraph(out *bytes.Buffer, text func() bool)
141 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
142 TableRow(out *bytes.Buffer, text []byte)
143 TableHeaderCell(out *bytes.Buffer, text []byte, flags int)
144 TableCell(out *bytes.Buffer, text []byte, flags int)
145 Footnotes(out *bytes.Buffer, text func() bool)
146 FootnoteItem(out *bytes.Buffer, name, text []byte, flags int)
147
148 // Span-level callbacks
149 AutoLink(out *bytes.Buffer, link []byte, kind int)
150 CodeSpan(out *bytes.Buffer, text []byte)
151 DoubleEmphasis(out *bytes.Buffer, text []byte)
152 Emphasis(out *bytes.Buffer, text []byte)
153 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
154 LineBreak(out *bytes.Buffer)
155 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
156 RawHtmlTag(out *bytes.Buffer, tag []byte)
157 TripleEmphasis(out *bytes.Buffer, text []byte)
158 StrikeThrough(out *bytes.Buffer, text []byte)
159 FootnoteRef(out *bytes.Buffer, ref []byte, id int)
160
161 // Low-level callbacks
162 Entity(out *bytes.Buffer, entity []byte)
163 NormalText(out *bytes.Buffer, text []byte)
164
165 // Header and footer
166 DocumentHeader(out *bytes.Buffer)
167 DocumentFooter(out *bytes.Buffer)
168}
169
170// Callback functions for inline parsing. One such function is defined
171// for each character that triggers a response when parsing inline data.
172type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
173
174// Parser holds runtime state used by the parser.
175// This is constructed by the Markdown function.
176type parser struct {
177 r Renderer
178 refs map[string]*reference
179 inlineCallback [256]inlineParser
180 flags int
181 nesting int
182 maxNesting int
183 insideLink bool
184
185 // Footnotes need to be ordered as well as available to quickly check for
186 // presence. If a ref is also a footnote, it's stored both in refs and here
187 // in notes. Slice is nil if footnotes not enabled.
188 notes []*reference
189}
190
191//
192//
193// Public interface
194//
195//
196
197// MarkdownBasic is a convenience function for simple rendering.
198// It processes markdown input with no extensions enabled.
199func MarkdownBasic(input []byte) []byte {
200 // set up the HTML renderer
201 htmlFlags := HTML_USE_XHTML
202 renderer := HtmlRenderer(htmlFlags, "", "")
203
204 // set up the parser
205 extensions := 0
206
207 return Markdown(input, renderer, extensions)
208}
209
210// Call Markdown with most useful extensions enabled
211// MarkdownCommon is a convenience function for simple rendering.
212// It processes markdown input with common extensions enabled, including:
213//
214// * Smartypants processing with smart fractions and LaTeX dashes
215//
216// * Intra-word emphasis suppression
217//
218// * Tables
219//
220// * Fenced code blocks
221//
222// * Autolinking
223//
224// * Strikethrough support
225//
226// * Strict header parsing
227func MarkdownCommon(input []byte) []byte {
228 // set up the HTML renderer
229 htmlFlags := 0
230 htmlFlags |= HTML_USE_XHTML
231 htmlFlags |= HTML_USE_SMARTYPANTS
232 htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
233 htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
234 htmlFlags |= HTML_SKIP_SCRIPT
235 renderer := HtmlRenderer(htmlFlags, "", "")
236
237 // set up the parser
238 extensions := 0
239 extensions |= EXTENSION_NO_INTRA_EMPHASIS
240 extensions |= EXTENSION_TABLES
241 extensions |= EXTENSION_FENCED_CODE
242 extensions |= EXTENSION_AUTOLINK
243 extensions |= EXTENSION_STRIKETHROUGH
244 extensions |= EXTENSION_SPACE_HEADERS
245
246 return Markdown(input, renderer, extensions)
247}
248
249// Markdown is the main rendering function.
250// It parses and renders a block of markdown-encoded text.
251// The supplied Renderer is used to format the output, and extensions dictates
252// which non-standard extensions are enabled.
253//
254// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
255// LatexRenderer, respectively.
256func Markdown(input []byte, renderer Renderer, extensions int) []byte {
257 // no point in parsing if we can't render
258 if renderer == nil {
259 return nil
260 }
261
262 // fill in the render structure
263 p := new(parser)
264 p.r = renderer
265 p.flags = extensions
266 p.refs = make(map[string]*reference)
267 p.maxNesting = 16
268 p.insideLink = false
269
270 // register inline parsers
271 p.inlineCallback['*'] = emphasis
272 p.inlineCallback['_'] = emphasis
273 if extensions&EXTENSION_STRIKETHROUGH != 0 {
274 p.inlineCallback['~'] = emphasis
275 }
276 p.inlineCallback['`'] = codeSpan
277 p.inlineCallback['\n'] = lineBreak
278 p.inlineCallback['['] = link
279 p.inlineCallback['<'] = leftAngle
280 p.inlineCallback['\\'] = escape
281 p.inlineCallback['&'] = entity
282
283 if extensions&EXTENSION_AUTOLINK != 0 {
284 p.inlineCallback[':'] = autoLink
285 }
286
287 if extensions&EXTENSION_FOOTNOTES != 0 {
288 p.notes = make([]*reference, 0)
289 }
290
291 first := firstPass(p, input)
292 second := secondPass(p, first)
293
294 return second
295}
296
297// first pass:
298// - extract references
299// - expand tabs
300// - normalize newlines
301// - copy everything else
302func firstPass(p *parser, input []byte) []byte {
303 var out bytes.Buffer
304 tabSize := TAB_SIZE_DEFAULT
305 if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
306 tabSize = TAB_SIZE_EIGHT
307 }
308 beg, end := 0, 0
309 for beg < len(input) { // iterate over lines
310 if end = isReference(p, input[beg:], tabSize); end > 0 {
311 beg += end
312 } else { // skip to the next line
313 end = beg
314 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
315 end++
316 }
317
318 // add the line body if present
319 if end > beg {
320 expandTabs(&out, input[beg:end], tabSize)
321 }
322 out.WriteByte('\n')
323
324 if end < len(input) && input[end] == '\r' {
325 end++
326 }
327 if end < len(input) && input[end] == '\n' {
328 end++
329 }
330
331 beg = end
332 }
333 }
334
335 // empty input?
336 if out.Len() == 0 {
337 out.WriteByte('\n')
338 }
339
340 return out.Bytes()
341}
342
343// second pass: actual rendering
344func secondPass(p *parser, input []byte) []byte {
345 var output bytes.Buffer
346
347 p.r.DocumentHeader(&output)
348 p.block(&output, input)
349
350 if p.flags&EXTENSION_FOOTNOTES != 0 && len(p.notes) > 0 {
351 p.r.Footnotes(&output, func() bool {
352 flags := LIST_ITEM_BEGINNING_OF_LIST
353 for _, ref := range p.notes {
354 var buf bytes.Buffer
355 if ref.hasBlock {
356 flags |= LIST_ITEM_CONTAINS_BLOCK
357 p.block(&buf, ref.title)
358 } else {
359 p.inline(&buf, ref.title)
360 }
361 p.r.FootnoteItem(&output, ref.link, buf.Bytes(), flags)
362 flags &^= LIST_ITEM_BEGINNING_OF_LIST | LIST_ITEM_CONTAINS_BLOCK
363 }
364
365 return true
366 })
367 }
368
369 p.r.DocumentFooter(&output)
370
371 if p.nesting != 0 {
372 panic("Nesting level did not end at zero")
373 }
374
375 return output.Bytes()
376}
377
378//
379// Link references
380//
381// This section implements support for references that (usually) appear
382// as footnotes in a document, and can be referenced anywhere in the document.
383// The basic format is:
384//
385// [1]: http://www.google.com/ "Google"
386// [2]: http://www.github.com/ "Github"
387//
388// Anywhere in the document, the reference can be linked by referring to its
389// label, i.e., 1 and 2 in this example, as in:
390//
391// This library is hosted on [Github][2], a git hosting site.
392//
393// Actual footnotes as specified in Pandoc and supported by some other Markdown
394// libraries such as php-markdown are also taken care of. They look like this:
395//
396// This sentence needs a bit of further explanation.[^note]
397//
398// [^note]: This is the explanation.
399//
400// Footnotes should be placed at the end of the document in an ordered list.
401// Inline footnotes such as:
402//
403// Inline footnotes^[Not supported.] also exist.
404//
405// are not yet supported.
406
407// References are parsed and stored in this struct.
408type reference struct {
409 link []byte
410 title []byte
411 noteId int // 0 if not a footnote ref
412 hasBlock bool
413}
414
415// Check whether or not data starts with a reference link.
416// If so, it is parsed and stored in the list of references
417// (in the render struct).
418// Returns the number of bytes to skip to move past it,
419// or zero if the first line is not a reference.
420func isReference(p *parser, data []byte, tabSize int) int {
421 // up to 3 optional leading spaces
422 if len(data) < 4 {
423 return 0
424 }
425 i := 0
426 for i < 3 && data[i] == ' ' {
427 i++
428 }
429
430 noteId := 0
431
432 // id part: anything but a newline between brackets
433 if data[i] != '[' {
434 return 0
435 }
436 i++
437 if p.flags&EXTENSION_FOOTNOTES != 0 {
438 if data[i] == '^' {
439 // we can set it to anything here because the proper noteIds will
440 // be assigned later during the second pass. It just has to be != 0
441 noteId = 1
442 i++
443 }
444 }
445 idOffset := i
446 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
447 i++
448 }
449 if i >= len(data) || data[i] != ']' {
450 return 0
451 }
452 idEnd := i
453
454 // spacer: colon (space | tab)* newline? (space | tab)*
455 i++
456 if i >= len(data) || data[i] != ':' {
457 return 0
458 }
459 i++
460 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
461 i++
462 }
463 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
464 i++
465 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
466 i++
467 }
468 }
469 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
470 i++
471 }
472 if i >= len(data) {
473 return 0
474 }
475
476 var (
477 linkOffset, linkEnd int
478 titleOffset, titleEnd int
479 lineEnd int
480 raw []byte
481 hasBlock bool
482 )
483
484 if p.flags&EXTENSION_FOOTNOTES != 0 && noteId != 0 {
485 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
486 lineEnd = linkEnd
487 } else {
488 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
489 }
490 if lineEnd == 0 {
491 return 0
492 }
493
494 // a valid ref has been found
495
496 ref := &reference{
497 noteId: noteId,
498 hasBlock: hasBlock,
499 }
500
501 if noteId > 0 {
502 // reusing the link field for the id since footnotes don't have links
503 ref.link = data[idOffset:idEnd]
504 // if footnote, it's not really a title, it's the contained text
505 ref.title = raw
506 } else {
507 ref.link = data[linkOffset:linkEnd]
508 ref.title = data[titleOffset:titleEnd]
509 }
510
511 // id matches are case-insensitive
512 id := string(bytes.ToLower(data[idOffset:idEnd]))
513
514 p.refs[id] = ref
515
516 return lineEnd
517}
518
519func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
520 // link: whitespace-free sequence, optionally between angle brackets
521 if data[i] == '<' {
522 i++
523 }
524 linkOffset = i
525 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
526 i++
527 }
528 linkEnd = i
529 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
530 linkOffset++
531 linkEnd--
532 }
533
534 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
535 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
536 i++
537 }
538 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
539 return
540 }
541
542 // compute end-of-line
543 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
544 lineEnd = i
545 }
546 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
547 lineEnd++
548 }
549
550 // optional (space|tab)* spacer after a newline
551 if lineEnd > 0 {
552 i = lineEnd + 1
553 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
554 i++
555 }
556 }
557
558 // optional title: any non-newline sequence enclosed in '"() alone on its line
559 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
560 i++
561 titleOffset = i
562
563 // look for EOL
564 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
565 i++
566 }
567 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
568 titleEnd = i + 1
569 } else {
570 titleEnd = i
571 }
572
573 // step back
574 i--
575 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
576 i--
577 }
578 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
579 lineEnd = titleEnd
580 titleEnd = i
581 }
582 }
583
584 return
585}
586
587// The first bit of this logic is the same as (*parser).listItem, but the rest
588// is much simpler. This function simply finds the entire block and shifts it
589// over by one tab if it is indeed a block (just returns the line if it's not).
590// blockEnd is the end of the section in the input buffer, and contents is the
591// extracted text that was shifted over one tab. It will need to be rendered at
592// the end of the document.
593func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
594 if i == 0 || len(data) == 0 {
595 return
596 }
597
598 // skip leading whitespace on first line
599 for i < len(data) && data[i] == ' ' {
600 i++
601 }
602
603 blockStart = i
604
605 // find the end of the line
606 blockEnd = i
607 for i < len(data) && data[i-1] != '\n' {
608 i++
609 }
610
611 // get working buffer
612 var raw bytes.Buffer
613
614 // put the first line into the working buffer
615 raw.Write(data[blockEnd:i])
616 blockEnd = i
617
618 // process the following lines
619 containsBlankLine := false
620
621gatherLines:
622 for blockEnd < len(data) {
623 i++
624
625 // find the end of this line
626 for i < len(data) && data[i-1] != '\n' {
627 i++
628 }
629
630 // if it is an empty line, guess that it is part of this item
631 // and move on to the next line
632 if p.isEmpty(data[blockEnd:i]) > 0 {
633 containsBlankLine = true
634 blockEnd = i
635 continue
636 }
637
638 n := 0
639 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
640 // this is the end of the block.
641 // we don't want to include this last line in the index.
642 break gatherLines
643 }
644
645 // if there were blank lines before this one, insert a new one now
646 if containsBlankLine {
647 raw.WriteByte('\n')
648 containsBlankLine = false
649 }
650
651 // get rid of that first tab, write to buffer
652 raw.Write(data[blockEnd+n : i])
653 hasBlock = true
654
655 blockEnd = i
656 }
657
658 if data[blockEnd-1] != '\n' {
659 raw.WriteByte('\n')
660 }
661
662 contents = raw.Bytes()
663
664 return
665}
666
667//
668//
669// Miscellaneous helper functions
670//
671//
672
673// Test if a character is a punctuation symbol.
674// Taken from a private function in regexp in the stdlib.
675func ispunct(c byte) bool {
676 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
677 if c == r {
678 return true
679 }
680 }
681 return false
682}
683
684// Test if a character is a whitespace character.
685func isspace(c byte) bool {
686 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
687}
688
689// Test if a character is letter.
690func isletter(c byte) bool {
691 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
692}
693
694// Test if a character is a letter or a digit.
695// TODO: check when this is looking for ASCII alnum and when it should use unicode
696func isalnum(c byte) bool {
697 return (c >= '0' && c <= '9') || isletter(c)
698}
699
700// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
701// always ends output with a newline
702func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
703 // first, check for common cases: no tabs, or only tabs at beginning of line
704 i, prefix := 0, 0
705 slowcase := false
706 for i = 0; i < len(line); i++ {
707 if line[i] == '\t' {
708 if prefix == i {
709 prefix++
710 } else {
711 slowcase = true
712 break
713 }
714 }
715 }
716
717 // no need to decode runes if all tabs are at the beginning of the line
718 if !slowcase {
719 for i = 0; i < prefix*tabSize; i++ {
720 out.WriteByte(' ')
721 }
722 out.Write(line[prefix:])
723 return
724 }
725
726 // the slow case: we need to count runes to figure out how
727 // many spaces to insert for each tab
728 column := 0
729 i = 0
730 for i < len(line) {
731 start := i
732 for i < len(line) && line[i] != '\t' {
733 _, size := utf8.DecodeRune(line[i:])
734 i += size
735 column++
736 }
737
738 if i > start {
739 out.Write(line[start:i])
740 }
741
742 if i >= len(line) {
743 break
744 }
745
746 for {
747 out.WriteByte(' ')
748 column++
749 if column%tabSize == 0 {
750 break
751 }
752 }
753
754 i++
755 }
756}
757
758// Find if a line counts as indented or not.
759// Returns number of characters the indent is (0 = not indented).
760func isIndented(data []byte, indentSize int) int {
761 if len(data) == 0 {
762 return 0
763 }
764 if data[0] == '\t' {
765 return 1
766 }
767 if len(data) < indentSize {
768 return 0
769 }
770 for i := 0; i < indentSize; i++ {
771 if data[i] != ' ' {
772 return 0
773 }
774 }
775 return indentSize
776}
777
778// Create a url-safe slug for fragments
779func slugify(in []byte) []byte {
780 if len(in) == 0 {
781 return in
782 }
783 out := make([]byte, 0, len(in))
784 sym := false
785
786 for _, ch := range in {
787 if isalnum(ch) {
788 sym = false
789 out = append(out, ch)
790 } else if sym {
791 continue
792 } else {
793 out = append(out, '-')
794 sym = true
795 }
796 }
797 var a, b int
798 var ch byte
799 for a, ch = range out {
800 if ch != '-' {
801 break
802 }
803 }
804 for b = len(out) - 1; b > 0; b-- {
805 if out[b] != '-' {
806 break
807 }
808 }
809 return out[a : b+1]
810}