markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "unicode/utf8"
24)
25
26const VERSION = "1.1"
27
28// These are the supported markdown parsing extensions.
29// OR these values together to select multiple extensions.
30const (
31 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota // ignore emphasis markers inside words
32 EXTENSION_TABLES // render tables
33 EXTENSION_FENCED_CODE // render fenced code blocks
34 EXTENSION_AUTOLINK // detect embedded URLs that are not explicitly marked
35 EXTENSION_STRIKETHROUGH // strikethrough text using ~~test~~
36 EXTENSION_LAX_HTML_BLOCKS // loosen up HTML block parsing rules
37 EXTENSION_SPACE_HEADERS // be strict about prefix header rules
38 EXTENSION_HARD_LINE_BREAK // translate newlines into line breaks
39 EXTENSION_TAB_SIZE_EIGHT // expand tabs to eight spaces instead of four
40)
41
42// These are the possible flag values for the link renderer.
43// Only a single one of these values will be used; they are not ORed together.
44// These are mostly of interest if you are writing a new output format.
45const (
46 LINK_TYPE_NOT_AUTOLINK = iota
47 LINK_TYPE_NORMAL
48 LINK_TYPE_EMAIL
49)
50
51// These are the possible flag values for the ListItem renderer.
52// Multiple flag values may be ORed together.
53// These are mostly of interest if you are writing a new output format.
54const (
55 LIST_TYPE_ORDERED = 1 << iota
56 LIST_ITEM_CONTAINS_BLOCK
57 LIST_ITEM_BEGINNING_OF_LIST
58 LIST_ITEM_END_OF_LIST
59)
60
61// These are the possible flag values for the table cell renderer.
62// Only a single one of these values will be used; they are not ORed together.
63// These are mostly of interest if you are writing a new output format.
64const (
65 TABLE_ALIGNMENT_LEFT = 1 << iota
66 TABLE_ALIGNMENT_RIGHT
67 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
68)
69
70// The size of a tab stop.
71const (
72 TAB_SIZE_DEFAULT = 4
73 TAB_SIZE_EIGHT = 8
74)
75
76// These are the tags that are recognized as HTML block tags.
77// Any of these can be included in markdown text without special escaping.
78var blockTags = map[string]bool{
79 "p": true,
80 "dl": true,
81 "h1": true,
82 "h2": true,
83 "h3": true,
84 "h4": true,
85 "h5": true,
86 "h6": true,
87 "ol": true,
88 "ul": true,
89 "del": true,
90 "div": true,
91 "ins": true,
92 "pre": true,
93 "form": true,
94 "math": true,
95 "table": true,
96 "iframe": true,
97 "script": true,
98 "fieldset": true,
99 "noscript": true,
100 "blockquote": true,
101
102 // HTML5
103 "video": true,
104 "aside": true,
105 "canvas": true,
106 "figure": true,
107 "footer": true,
108 "header": true,
109 "hgroup": true,
110 "output": true,
111 "article": true,
112 "section": true,
113 "progress": true,
114 "figcaption": true,
115}
116
117// Renderer is the rendering interface.
118// This is mostly of interest if you are implementing a new rendering format.
119//
120// When a byte slice is provided, it contains the (rendered) contents of the
121// element.
122//
123// When a callback is provided instead, it will write the contents of the
124// respective element directly to the output buffer and return true on success.
125// If the callback returns false, the rendering function should reset the
126// output buffer as though it had never been called.
127//
128// Currently Html and Latex implementations are provided
129type Renderer interface {
130 // block-level callbacks
131 BlockCode(out *bytes.Buffer, text []byte, lang string)
132 BlockQuote(out *bytes.Buffer, text []byte)
133 BlockHtml(out *bytes.Buffer, text []byte)
134 Header(out *bytes.Buffer, text func() bool, level int)
135 HRule(out *bytes.Buffer)
136 List(out *bytes.Buffer, text func() bool, flags int)
137 ListItem(out *bytes.Buffer, text []byte, flags int)
138 Paragraph(out *bytes.Buffer, text func() bool)
139 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
140 TableRow(out *bytes.Buffer, text []byte)
141 TableCell(out *bytes.Buffer, text []byte, flags int)
142
143 // Span-level callbacks
144 AutoLink(out *bytes.Buffer, link []byte, kind int)
145 CodeSpan(out *bytes.Buffer, text []byte)
146 DoubleEmphasis(out *bytes.Buffer, text []byte)
147 Emphasis(out *bytes.Buffer, text []byte)
148 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
149 LineBreak(out *bytes.Buffer)
150 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
151 RawHtmlTag(out *bytes.Buffer, tag []byte)
152 TripleEmphasis(out *bytes.Buffer, text []byte)
153 StrikeThrough(out *bytes.Buffer, text []byte)
154
155 // Low-level callbacks
156 Entity(out *bytes.Buffer, entity []byte)
157 NormalText(out *bytes.Buffer, text []byte)
158
159 // Header and footer
160 DocumentHeader(out *bytes.Buffer)
161 DocumentFooter(out *bytes.Buffer)
162}
163
164// Callback functions for inline parsing. One such function is defined
165// for each character that triggers a response when parsing inline data.
166type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
167
168// Parser holds runtime state used by the parser.
169// This is constructed by the Markdown function.
170type parser struct {
171 r Renderer
172 refs map[string]*reference
173 inlineCallback [256]inlineParser
174 flags int
175 nesting int
176 maxNesting int
177 insideLink bool
178}
179
180//
181//
182// Public interface
183//
184//
185
186// MarkdownBasic is a convenience function for simple rendering.
187// It processes markdown input with no extensions enabled.
188func MarkdownBasic(input []byte) []byte {
189 // set up the HTML renderer
190 htmlFlags := HTML_USE_XHTML
191 renderer := HtmlRenderer(htmlFlags, "", "")
192
193 // set up the parser
194 extensions := 0
195
196 return Markdown(input, renderer, extensions)
197}
198
199// Call Markdown with most useful extensions enabled
200// MarkdownCommon is a convenience function for simple rendering.
201// It processes markdown input with common extensions enabled, including:
202//
203// * Smartypants processing with smart fractions and LaTeX dashes
204//
205// * Intra-word emphasis supression
206//
207// * Tables
208//
209// * Fenced code blocks
210//
211// * Autolinking
212//
213// * Strikethrough support
214//
215// * Strict header parsing
216func MarkdownCommon(input []byte) []byte {
217 // set up the HTML renderer
218 htmlFlags := 0
219 htmlFlags |= HTML_USE_XHTML
220 htmlFlags |= HTML_USE_SMARTYPANTS
221 htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
222 htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
223 renderer := HtmlRenderer(htmlFlags, "", "")
224
225 // set up the parser
226 extensions := 0
227 extensions |= EXTENSION_NO_INTRA_EMPHASIS
228 extensions |= EXTENSION_TABLES
229 extensions |= EXTENSION_FENCED_CODE
230 extensions |= EXTENSION_AUTOLINK
231 extensions |= EXTENSION_STRIKETHROUGH
232 extensions |= EXTENSION_SPACE_HEADERS
233
234 return Markdown(input, renderer, extensions)
235}
236
237// Markdown is the main rendering function.
238// It parses and renders a block of markdown-encoded text.
239// The supplied Renderer is used to format the output, and extensions dictates
240// which non-standard extensions are enabled.
241//
242// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
243// LatexRenderer, respectively.
244func Markdown(input []byte, renderer Renderer, extensions int) []byte {
245 // no point in parsing if we can't render
246 if renderer == nil {
247 return nil
248 }
249
250 // fill in the render structure
251 p := new(parser)
252 p.r = renderer
253 p.flags = extensions
254 p.refs = make(map[string]*reference)
255 p.maxNesting = 16
256 p.insideLink = false
257
258 // register inline parsers
259 p.inlineCallback['*'] = emphasis
260 p.inlineCallback['_'] = emphasis
261 if extensions&EXTENSION_STRIKETHROUGH != 0 {
262 p.inlineCallback['~'] = emphasis
263 }
264 p.inlineCallback['`'] = codeSpan
265 p.inlineCallback['\n'] = lineBreak
266 p.inlineCallback['['] = link
267 p.inlineCallback['<'] = leftAngle
268 p.inlineCallback['\\'] = escape
269 p.inlineCallback['&'] = entity
270
271 if extensions&EXTENSION_AUTOLINK != 0 {
272 p.inlineCallback[':'] = autoLink
273 }
274
275 first := firstPass(p, input)
276 second := secondPass(p, first)
277
278 return second
279}
280
281// first pass:
282// - extract references
283// - expand tabs
284// - normalize newlines
285// - copy everything else
286func firstPass(p *parser, input []byte) []byte {
287 var out bytes.Buffer
288 tabSize := TAB_SIZE_DEFAULT
289 if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
290 tabSize = TAB_SIZE_EIGHT
291 }
292 beg, end := 0, 0
293 for beg < len(input) { // iterate over lines
294 if end = isReference(p, input[beg:]); end > 0 {
295 beg += end
296 } else { // skip to the next line
297 end = beg
298 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
299 end++
300 }
301
302 // add the line body if present
303 if end > beg {
304 expandTabs(&out, input[beg:end], tabSize)
305 }
306 out.WriteByte('\n')
307
308 if end < len(input) && input[end] == '\r' {
309 end++
310 }
311 if end < len(input) && input[end] == '\n' {
312 end++
313 }
314
315 beg = end
316 }
317 }
318
319 // empty input?
320 if out.Len() == 0 {
321 out.WriteByte('\n')
322 }
323
324 return out.Bytes()
325}
326
327// second pass: actual rendering
328func secondPass(p *parser, input []byte) []byte {
329 var output bytes.Buffer
330
331 p.r.DocumentHeader(&output)
332 p.block(&output, input)
333 p.r.DocumentFooter(&output)
334
335 if p.nesting != 0 {
336 panic("Nesting level did not end at zero")
337 }
338
339 return output.Bytes()
340}
341
342//
343// Link references
344//
345// This section implements support for references that (usually) appear
346// as footnotes in a document, and can be referenced anywhere in the document.
347// The basic format is:
348//
349// [1]: http://www.google.com/ "Google"
350// [2]: http://www.github.com/ "Github"
351//
352// Anywhere in the document, the reference can be linked by referring to its
353// label, i.e., 1 and 2 in this example, as in:
354//
355// This library is hosted on [Github][2], a git hosting site.
356
357// References are parsed and stored in this struct.
358type reference struct {
359 link []byte
360 title []byte
361}
362
363// Check whether or not data starts with a reference link.
364// If so, it is parsed and stored in the list of references
365// (in the render struct).
366// Returns the number of bytes to skip to move past it,
367// or zero if the first line is not a reference.
368func isReference(p *parser, data []byte) int {
369 // up to 3 optional leading spaces
370 if len(data) < 4 {
371 return 0
372 }
373 i := 0
374 for i < 3 && data[i] == ' ' {
375 i++
376 }
377
378 // id part: anything but a newline between brackets
379 if data[i] != '[' {
380 return 0
381 }
382 i++
383 idOffset := i
384 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
385 i++
386 }
387 if i >= len(data) || data[i] != ']' {
388 return 0
389 }
390 idEnd := i
391
392 // spacer: colon (space | tab)* newline? (space | tab)*
393 i++
394 if i >= len(data) || data[i] != ':' {
395 return 0
396 }
397 i++
398 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
399 i++
400 }
401 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
402 i++
403 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
404 i++
405 }
406 }
407 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
408 i++
409 }
410 if i >= len(data) {
411 return 0
412 }
413
414 // link: whitespace-free sequence, optionally between angle brackets
415 if data[i] == '<' {
416 i++
417 }
418 linkOffset := i
419 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
420 i++
421 }
422 linkEnd := i
423 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
424 linkOffset++
425 linkEnd--
426 }
427
428 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
429 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
430 i++
431 }
432 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
433 return 0
434 }
435
436 // compute end-of-line
437 lineEnd := 0
438 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
439 lineEnd = i
440 }
441 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
442 lineEnd++
443 }
444
445 // optional (space|tab)* spacer after a newline
446 if lineEnd > 0 {
447 i = lineEnd + 1
448 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
449 i++
450 }
451 }
452
453 // optional title: any non-newline sequence enclosed in '"() alone on its line
454 titleOffset, titleEnd := 0, 0
455 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
456 i++
457 titleOffset = i
458
459 // look for EOL
460 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
461 i++
462 }
463 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
464 titleEnd = i + 1
465 } else {
466 titleEnd = i
467 }
468
469 // step back
470 i--
471 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
472 i--
473 }
474 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
475 lineEnd = titleEnd
476 titleEnd = i
477 }
478 }
479 if lineEnd == 0 { // garbage after the link
480 return 0
481 }
482
483 // a valid ref has been found
484
485 // id matches are case-insensitive
486 id := string(bytes.ToLower(data[idOffset:idEnd]))
487 p.refs[id] = &reference{
488 link: data[linkOffset:linkEnd],
489 title: data[titleOffset:titleEnd],
490 }
491
492 return lineEnd
493}
494
495//
496//
497// Miscellaneous helper functions
498//
499//
500
501// Test if a character is a punctuation symbol.
502// Taken from a private function in regexp in the stdlib.
503func ispunct(c byte) bool {
504 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
505 if c == r {
506 return true
507 }
508 }
509 return false
510}
511
512// Test if a character is a whitespace character.
513func isspace(c byte) bool {
514 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
515}
516
517// Test if a character is a letter or a digit.
518// TODO: check when this is looking for ASCII alnum and when it should use unicode
519func isalnum(c byte) bool {
520 return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
521}
522
523// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
524// always ends output with a newline
525func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
526 // first, check for common cases: no tabs, or only tabs at beginning of line
527 i, prefix := 0, 0
528 slowcase := false
529 for i = 0; i < len(line); i++ {
530 if line[i] == '\t' {
531 if prefix == i {
532 prefix++
533 } else {
534 slowcase = true
535 break
536 }
537 }
538 }
539
540 // no need to decode runes if all tabs are at the beginning of the line
541 if !slowcase {
542 for i = 0; i < prefix*tabSize; i++ {
543 out.WriteByte(' ')
544 }
545 out.Write(line[prefix:])
546 return
547 }
548
549 // the slow case: we need to count runes to figure out how
550 // many spaces to insert for each tab
551 column := 0
552 i = 0
553 for i < len(line) {
554 start := i
555 for i < len(line) && line[i] != '\t' {
556 _, size := utf8.DecodeRune(line[i:])
557 i += size
558 column++
559 }
560
561 if i > start {
562 out.Write(line[start:i])
563 }
564
565 if i >= len(line) {
566 break
567 }
568
569 for {
570 out.WriteByte(' ')
571 column++
572 if column%tabSize == 0 {
573 break
574 }
575 }
576
577 i++
578 }
579}