markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "unicode/utf8"
24)
25
26const VERSION = "1.1"
27
28// These are the supported markdown parsing extensions.
29// OR these values together to select multiple extensions.
30const (
31 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota // ignore emphasis markers inside words
32 EXTENSION_TABLES // render tables
33 EXTENSION_FENCED_CODE // render fenced code blocks
34 EXTENSION_AUTOLINK // detect embedded URLs that are not explicitly marked
35 EXTENSION_STRIKETHROUGH // strikethrough text using ~~test~~
36 EXTENSION_LAX_HTML_BLOCKS // loosen up HTML block parsing rules
37 EXTENSION_SPACE_HEADERS // be strict about prefix header rules
38 EXTENSION_HARD_LINE_BREAK // translate newlines into line breaks
39 EXTENSION_TAB_SIZE_EIGHT // expand tabs to eight spaces instead of four
40 EXTENSION_FOOTNOTES // Pandoc-style footnotes
41 EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK // No need to insert an empty line to start a (code, quote, order list, unorder list)block
42)
43
44// These are the possible flag values for the link renderer.
45// Only a single one of these values will be used; they are not ORed together.
46// These are mostly of interest if you are writing a new output format.
47const (
48 LINK_TYPE_NOT_AUTOLINK = iota
49 LINK_TYPE_NORMAL
50 LINK_TYPE_EMAIL
51)
52
53// These are the possible flag values for the ListItem renderer.
54// Multiple flag values may be ORed together.
55// These are mostly of interest if you are writing a new output format.
56const (
57 LIST_TYPE_ORDERED = 1 << iota
58 LIST_ITEM_CONTAINS_BLOCK
59 LIST_ITEM_BEGINNING_OF_LIST
60 LIST_ITEM_END_OF_LIST
61)
62
63// These are the possible flag values for the table cell renderer.
64// Only a single one of these values will be used; they are not ORed together.
65// These are mostly of interest if you are writing a new output format.
66const (
67 TABLE_ALIGNMENT_LEFT = 1 << iota
68 TABLE_ALIGNMENT_RIGHT
69 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
70)
71
72// The size of a tab stop.
73const (
74 TAB_SIZE_DEFAULT = 4
75 TAB_SIZE_EIGHT = 8
76)
77
78// These are the tags that are recognized as HTML block tags.
79// Any of these can be included in markdown text without special escaping.
80var blockTags = map[string]bool{
81 "p": true,
82 "dl": true,
83 "h1": true,
84 "h2": true,
85 "h3": true,
86 "h4": true,
87 "h5": true,
88 "h6": true,
89 "ol": true,
90 "ul": true,
91 "del": true,
92 "div": true,
93 "ins": true,
94 "pre": true,
95 "form": true,
96 "math": true,
97 "table": true,
98 "iframe": true,
99 "script": true,
100 "fieldset": true,
101 "noscript": true,
102 "blockquote": true,
103
104 // HTML5
105 "video": true,
106 "aside": true,
107 "canvas": true,
108 "figure": true,
109 "footer": true,
110 "header": true,
111 "hgroup": true,
112 "output": true,
113 "article": true,
114 "section": true,
115 "progress": true,
116 "figcaption": true,
117}
118
119// Renderer is the rendering interface.
120// This is mostly of interest if you are implementing a new rendering format.
121//
122// When a byte slice is provided, it contains the (rendered) contents of the
123// element.
124//
125// When a callback is provided instead, it will write the contents of the
126// respective element directly to the output buffer and return true on success.
127// If the callback returns false, the rendering function should reset the
128// output buffer as though it had never been called.
129//
130// Currently Html and Latex implementations are provided
131type Renderer interface {
132 // block-level callbacks
133 BlockCode(out *bytes.Buffer, text []byte, lang string)
134 BlockQuote(out *bytes.Buffer, text []byte)
135 BlockHtml(out *bytes.Buffer, text []byte)
136 Header(out *bytes.Buffer, text func() bool, level int)
137 HRule(out *bytes.Buffer)
138 List(out *bytes.Buffer, text func() bool, flags int)
139 ListItem(out *bytes.Buffer, text []byte, flags int)
140 Paragraph(out *bytes.Buffer, text func() bool)
141 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
142 TableRow(out *bytes.Buffer, text []byte)
143 TableCell(out *bytes.Buffer, text []byte, flags int)
144 Footnotes(out *bytes.Buffer, text func() bool)
145 FootnoteItem(out *bytes.Buffer, name, text []byte, flags int)
146
147 // Span-level callbacks
148 AutoLink(out *bytes.Buffer, link []byte, kind int)
149 CodeSpan(out *bytes.Buffer, text []byte)
150 DoubleEmphasis(out *bytes.Buffer, text []byte)
151 Emphasis(out *bytes.Buffer, text []byte)
152 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
153 LineBreak(out *bytes.Buffer)
154 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
155 RawHtmlTag(out *bytes.Buffer, tag []byte)
156 TripleEmphasis(out *bytes.Buffer, text []byte)
157 StrikeThrough(out *bytes.Buffer, text []byte)
158 FootnoteRef(out *bytes.Buffer, ref []byte, id int)
159
160 // Low-level callbacks
161 Entity(out *bytes.Buffer, entity []byte)
162 NormalText(out *bytes.Buffer, text []byte)
163
164 // Header and footer
165 DocumentHeader(out *bytes.Buffer)
166 DocumentFooter(out *bytes.Buffer)
167}
168
169// Callback functions for inline parsing. One such function is defined
170// for each character that triggers a response when parsing inline data.
171type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
172
173// Parser holds runtime state used by the parser.
174// This is constructed by the Markdown function.
175type parser struct {
176 r Renderer
177 refs map[string]*reference
178 inlineCallback [256]inlineParser
179 flags int
180 nesting int
181 maxNesting int
182 insideLink bool
183
184 // Footnotes need to be ordered as well as available to quickly check for
185 // presence. If a ref is also a footnote, it's stored both in refs and here
186 // in notes. Slice is nil if footnotes not enabled.
187 notes []*reference
188}
189
190//
191//
192// Public interface
193//
194//
195
196// MarkdownBasic is a convenience function for simple rendering.
197// It processes markdown input with no extensions enabled.
198func MarkdownBasic(input []byte) []byte {
199 // set up the HTML renderer
200 htmlFlags := HTML_USE_XHTML
201 renderer := HtmlRenderer(htmlFlags, "", "")
202
203 // set up the parser
204 extensions := 0
205
206 return Markdown(input, renderer, extensions)
207}
208
209// Call Markdown with most useful extensions enabled
210// MarkdownCommon is a convenience function for simple rendering.
211// It processes markdown input with common extensions enabled, including:
212//
213// * Smartypants processing with smart fractions and LaTeX dashes
214//
215// * Intra-word emphasis suppression
216//
217// * Tables
218//
219// * Fenced code blocks
220//
221// * Autolinking
222//
223// * Strikethrough support
224//
225// * Strict header parsing
226func MarkdownCommon(input []byte) []byte {
227 // set up the HTML renderer
228 htmlFlags := 0
229 htmlFlags |= HTML_USE_XHTML
230 htmlFlags |= HTML_USE_SMARTYPANTS
231 htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
232 htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
233 htmlFlags |= HTML_SKIP_SCRIPT
234 renderer := HtmlRenderer(htmlFlags, "", "")
235
236 // set up the parser
237 extensions := 0
238 extensions |= EXTENSION_NO_INTRA_EMPHASIS
239 extensions |= EXTENSION_TABLES
240 extensions |= EXTENSION_FENCED_CODE
241 extensions |= EXTENSION_AUTOLINK
242 extensions |= EXTENSION_STRIKETHROUGH
243 extensions |= EXTENSION_SPACE_HEADERS
244
245 return Markdown(input, renderer, extensions)
246}
247
248// Markdown is the main rendering function.
249// It parses and renders a block of markdown-encoded text.
250// The supplied Renderer is used to format the output, and extensions dictates
251// which non-standard extensions are enabled.
252//
253// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
254// LatexRenderer, respectively.
255func Markdown(input []byte, renderer Renderer, extensions int) []byte {
256 // no point in parsing if we can't render
257 if renderer == nil {
258 return nil
259 }
260
261 // fill in the render structure
262 p := new(parser)
263 p.r = renderer
264 p.flags = extensions
265 p.refs = make(map[string]*reference)
266 p.maxNesting = 16
267 p.insideLink = false
268
269 // register inline parsers
270 p.inlineCallback['*'] = emphasis
271 p.inlineCallback['_'] = emphasis
272 if extensions&EXTENSION_STRIKETHROUGH != 0 {
273 p.inlineCallback['~'] = emphasis
274 }
275 p.inlineCallback['`'] = codeSpan
276 p.inlineCallback['\n'] = lineBreak
277 p.inlineCallback['['] = link
278 p.inlineCallback['<'] = leftAngle
279 p.inlineCallback['\\'] = escape
280 p.inlineCallback['&'] = entity
281
282 if extensions&EXTENSION_AUTOLINK != 0 {
283 p.inlineCallback[':'] = autoLink
284 }
285
286 if extensions&EXTENSION_FOOTNOTES != 0 {
287 p.notes = make([]*reference, 0)
288 }
289
290 first := firstPass(p, input)
291 second := secondPass(p, first)
292
293 return second
294}
295
296// first pass:
297// - extract references
298// - expand tabs
299// - normalize newlines
300// - copy everything else
301func firstPass(p *parser, input []byte) []byte {
302 var out bytes.Buffer
303 tabSize := TAB_SIZE_DEFAULT
304 if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
305 tabSize = TAB_SIZE_EIGHT
306 }
307 beg, end := 0, 0
308 for beg < len(input) { // iterate over lines
309 if end = isReference(p, input[beg:], tabSize); end > 0 {
310 beg += end
311 } else { // skip to the next line
312 end = beg
313 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
314 end++
315 }
316
317 // add the line body if present
318 if end > beg {
319 expandTabs(&out, input[beg:end], tabSize)
320 }
321 out.WriteByte('\n')
322
323 if end < len(input) && input[end] == '\r' {
324 end++
325 }
326 if end < len(input) && input[end] == '\n' {
327 end++
328 }
329
330 beg = end
331 }
332 }
333
334 // empty input?
335 if out.Len() == 0 {
336 out.WriteByte('\n')
337 }
338
339 return out.Bytes()
340}
341
342// second pass: actual rendering
343func secondPass(p *parser, input []byte) []byte {
344 var output bytes.Buffer
345
346 p.r.DocumentHeader(&output)
347 p.block(&output, input)
348
349 if p.flags&EXTENSION_FOOTNOTES != 0 && len(p.notes) > 0 {
350 p.r.Footnotes(&output, func() bool {
351 flags := LIST_ITEM_BEGINNING_OF_LIST
352 for _, ref := range p.notes {
353 var buf bytes.Buffer
354 if ref.hasBlock {
355 flags |= LIST_ITEM_CONTAINS_BLOCK
356 p.block(&buf, ref.title)
357 } else {
358 p.inline(&buf, ref.title)
359 }
360 p.r.FootnoteItem(&output, ref.link, buf.Bytes(), flags)
361 flags &^= LIST_ITEM_BEGINNING_OF_LIST | LIST_ITEM_CONTAINS_BLOCK
362 }
363
364 return true
365 })
366 }
367
368 p.r.DocumentFooter(&output)
369
370 if p.nesting != 0 {
371 panic("Nesting level did not end at zero")
372 }
373
374 return output.Bytes()
375}
376
377//
378// Link references
379//
380// This section implements support for references that (usually) appear
381// as footnotes in a document, and can be referenced anywhere in the document.
382// The basic format is:
383//
384// [1]: http://www.google.com/ "Google"
385// [2]: http://www.github.com/ "Github"
386//
387// Anywhere in the document, the reference can be linked by referring to its
388// label, i.e., 1 and 2 in this example, as in:
389//
390// This library is hosted on [Github][2], a git hosting site.
391//
392// Actual footnotes as specified in Pandoc and supported by some other Markdown
393// libraries such as php-markdown are also taken care of. They look like this:
394//
395// This sentence needs a bit of further explanation.[^note]
396//
397// [^note]: This is the explanation.
398//
399// Footnotes should be placed at the end of the document in an ordered list.
400// Inline footnotes such as:
401//
402// Inline footnotes^[Not supported.] also exist.
403//
404// are not yet supported.
405
406// References are parsed and stored in this struct.
407type reference struct {
408 link []byte
409 title []byte
410 noteId int // 0 if not a footnote ref
411 hasBlock bool
412}
413
414// Check whether or not data starts with a reference link.
415// If so, it is parsed and stored in the list of references
416// (in the render struct).
417// Returns the number of bytes to skip to move past it,
418// or zero if the first line is not a reference.
419func isReference(p *parser, data []byte, tabSize int) int {
420 // up to 3 optional leading spaces
421 if len(data) < 4 {
422 return 0
423 }
424 i := 0
425 for i < 3 && data[i] == ' ' {
426 i++
427 }
428
429 noteId := 0
430
431 // id part: anything but a newline between brackets
432 if data[i] != '[' {
433 return 0
434 }
435 i++
436 if p.flags&EXTENSION_FOOTNOTES != 0 {
437 if data[i] == '^' {
438 // we can set it to anything here because the proper noteIds will
439 // be assigned later during the second pass. It just has to be != 0
440 noteId = 1
441 i++
442 }
443 }
444 idOffset := i
445 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
446 i++
447 }
448 if i >= len(data) || data[i] != ']' {
449 return 0
450 }
451 idEnd := i
452
453 // spacer: colon (space | tab)* newline? (space | tab)*
454 i++
455 if i >= len(data) || data[i] != ':' {
456 return 0
457 }
458 i++
459 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
460 i++
461 }
462 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
463 i++
464 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
465 i++
466 }
467 }
468 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
469 i++
470 }
471 if i >= len(data) {
472 return 0
473 }
474
475 var (
476 linkOffset, linkEnd int
477 titleOffset, titleEnd int
478 lineEnd int
479 raw []byte
480 hasBlock bool
481 )
482
483 if p.flags&EXTENSION_FOOTNOTES != 0 && noteId != 0 {
484 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
485 lineEnd = linkEnd
486 } else {
487 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
488 }
489 if lineEnd == 0 {
490 return 0
491 }
492
493 // a valid ref has been found
494
495 ref := &reference{
496 noteId: noteId,
497 hasBlock: hasBlock,
498 }
499
500 if noteId > 0 {
501 // reusing the link field for the id since footnotes don't have links
502 ref.link = data[idOffset:idEnd]
503 // if footnote, it's not really a title, it's the contained text
504 ref.title = raw
505 } else {
506 ref.link = data[linkOffset:linkEnd]
507 ref.title = data[titleOffset:titleEnd]
508 }
509
510 // id matches are case-insensitive
511 id := string(bytes.ToLower(data[idOffset:idEnd]))
512
513 p.refs[id] = ref
514
515 return lineEnd
516}
517
518func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
519 // link: whitespace-free sequence, optionally between angle brackets
520 if data[i] == '<' {
521 i++
522 }
523 linkOffset = i
524 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
525 i++
526 }
527 linkEnd = i
528 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
529 linkOffset++
530 linkEnd--
531 }
532
533 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
534 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
535 i++
536 }
537 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
538 return
539 }
540
541 // compute end-of-line
542 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
543 lineEnd = i
544 }
545 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
546 lineEnd++
547 }
548
549 // optional (space|tab)* spacer after a newline
550 if lineEnd > 0 {
551 i = lineEnd + 1
552 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
553 i++
554 }
555 }
556
557 // optional title: any non-newline sequence enclosed in '"() alone on its line
558 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
559 i++
560 titleOffset = i
561
562 // look for EOL
563 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
564 i++
565 }
566 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
567 titleEnd = i + 1
568 } else {
569 titleEnd = i
570 }
571
572 // step back
573 i--
574 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
575 i--
576 }
577 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
578 lineEnd = titleEnd
579 titleEnd = i
580 }
581 }
582
583 return
584}
585
586// The first bit of this logic is the same as (*parser).listItem, but the rest
587// is much simpler. This function simply finds the entire block and shifts it
588// over by one tab if it is indeed a block (just returns the line if it's not).
589// blockEnd is the end of the section in the input buffer, and contents is the
590// extracted text that was shifted over one tab. It will need to be rendered at
591// the end of the document.
592func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
593 if i == 0 || len(data) == 0 {
594 return
595 }
596
597 // skip leading whitespace on first line
598 for i < len(data) && data[i] == ' ' {
599 i++
600 }
601
602 blockStart = i
603
604 // find the end of the line
605 blockEnd = i
606 for i < len(data) && data[i-1] != '\n' {
607 i++
608 }
609
610 // get working buffer
611 var raw bytes.Buffer
612
613 // put the first line into the working buffer
614 raw.Write(data[blockEnd:i])
615 blockEnd = i
616
617 // process the following lines
618 containsBlankLine := false
619
620gatherLines:
621 for blockEnd < len(data) {
622 i++
623
624 // find the end of this line
625 for i < len(data) && data[i-1] != '\n' {
626 i++
627 }
628
629 // if it is an empty line, guess that it is part of this item
630 // and move on to the next line
631 if p.isEmpty(data[blockEnd:i]) > 0 {
632 containsBlankLine = true
633 blockEnd = i
634 continue
635 }
636
637 n := 0
638 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
639 // this is the end of the block.
640 // we don't want to include this last line in the index.
641 break gatherLines
642 }
643
644 // if there were blank lines before this one, insert a new one now
645 if containsBlankLine {
646 raw.WriteByte('\n')
647 containsBlankLine = false
648 }
649
650 // get rid of that first tab, write to buffer
651 raw.Write(data[blockEnd+n : i])
652 hasBlock = true
653
654 blockEnd = i
655 }
656
657 if data[blockEnd-1] != '\n' {
658 raw.WriteByte('\n')
659 }
660
661 contents = raw.Bytes()
662
663 return
664}
665
666//
667//
668// Miscellaneous helper functions
669//
670//
671
672// Test if a character is a punctuation symbol.
673// Taken from a private function in regexp in the stdlib.
674func ispunct(c byte) bool {
675 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
676 if c == r {
677 return true
678 }
679 }
680 return false
681}
682
683// Test if a character is a whitespace character.
684func isspace(c byte) bool {
685 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
686}
687
688// Test if a character is letter.
689func isletter(c byte) bool {
690 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
691}
692
693// Test if a character is a letter or a digit.
694// TODO: check when this is looking for ASCII alnum and when it should use unicode
695func isalnum(c byte) bool {
696 return (c >= '0' && c <= '9') || isletter(c)
697}
698
699// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
700// always ends output with a newline
701func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
702 // first, check for common cases: no tabs, or only tabs at beginning of line
703 i, prefix := 0, 0
704 slowcase := false
705 for i = 0; i < len(line); i++ {
706 if line[i] == '\t' {
707 if prefix == i {
708 prefix++
709 } else {
710 slowcase = true
711 break
712 }
713 }
714 }
715
716 // no need to decode runes if all tabs are at the beginning of the line
717 if !slowcase {
718 for i = 0; i < prefix*tabSize; i++ {
719 out.WriteByte(' ')
720 }
721 out.Write(line[prefix:])
722 return
723 }
724
725 // the slow case: we need to count runes to figure out how
726 // many spaces to insert for each tab
727 column := 0
728 i = 0
729 for i < len(line) {
730 start := i
731 for i < len(line) && line[i] != '\t' {
732 _, size := utf8.DecodeRune(line[i:])
733 i += size
734 column++
735 }
736
737 if i > start {
738 out.Write(line[start:i])
739 }
740
741 if i >= len(line) {
742 break
743 }
744
745 for {
746 out.WriteByte(' ')
747 column++
748 if column%tabSize == 0 {
749 break
750 }
751 }
752
753 i++
754 }
755}
756
757// Find if a line counts as indented or not.
758// Returns number of characters the indent is (0 = not indented).
759func isIndented(data []byte, indentSize int) int {
760 if len(data) == 0 {
761 return 0
762 }
763 if data[0] == '\t' {
764 return 1
765 }
766 if len(data) < indentSize {
767 return 0
768 }
769 for i := 0; i < indentSize; i++ {
770 if data[i] != ' ' {
771 return 0
772 }
773 }
774 return indentSize
775}
776
777// Create a url-safe slug for fragments
778func slugify(in []byte) []byte {
779 if len(in) == 0 {
780 return in
781 }
782 out := make([]byte, 0, len(in))
783 sym := false
784
785 for _, ch := range in {
786 if isalnum(ch) {
787 sym = false
788 out = append(out, ch)
789 } else if sym {
790 continue
791 } else {
792 out = append(out, '-')
793 sym = true
794 }
795 }
796 var a, b int
797 var ch byte
798 for a, ch = range out {
799 if ch != '-' {
800 break
801 }
802 }
803 for b = len(out) - 1; b > 0; b-- {
804 if out[b] != '-' {
805 break
806 }
807 }
808 return out[a : b+1]
809}