markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "utf8"
24)
25
26const VERSION = "1.0"
27
28// These are the supported markdown parsing extensions.
29// OR these values together to select multiple extensions.
30const (
31 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota // ignore emphasis markers inside words
32 EXTENSION_TABLES // render tables
33 EXTENSION_FENCED_CODE // render fenced code blocks
34 EXTENSION_AUTOLINK // detect embedded URLs that are not explicitly marked
35 EXTENSION_STRIKETHROUGH // strikethrough text using ~~test~~
36 EXTENSION_LAX_HTML_BLOCKS // loosen up HTML block parsing rules
37 EXTENSION_SPACE_HEADERS // be strict about prefix header rules
38 EXTENSION_HARD_LINE_BREAK // translate newlines into line breaks
39 EXTENSION_TAB_SIZE_EIGHT // expand tabs to eight spaces instead of four
40)
41
42// These are the possible flag values for the link renderer.
43// Only a single one of these values will be used; they are not ORed together.
44// These are mostly of interest if you are writing a new output format.
45const (
46 LINK_TYPE_NOT_AUTOLINK = iota
47 LINK_TYPE_NORMAL
48 LINK_TYPE_EMAIL
49)
50
51// These are the possible flag values for the ListItem renderer.
52// Multiple flag values may be ORed together.
53// These are mostly of interest if you are writing a new output format.
54const (
55 LIST_TYPE_ORDERED = 1 << iota
56 LIST_ITEM_CONTAINS_BLOCK
57 LIST_ITEM_BEGINNING_OF_LIST
58 LIST_ITEM_END_OF_LIST
59)
60
61// These are the possible flag values for the table cell renderer.
62// Only a single one of these values will be used; they are not ORed together.
63// These are mostly of interest if you are writing a new output format.
64const (
65 TABLE_ALIGNMENT_LEFT = 1 << iota
66 TABLE_ALIGNMENT_RIGHT
67 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
68)
69
70// The size of a tab stop.
71const (
72 TAB_SIZE_DEFAULT = 4
73 TAB_SIZE_EIGHT = 8
74)
75
76// These are the tags that are recognized as HTML block tags.
77// Any of these can be included in markdown text without special escaping.
78var blockTags = map[string]bool{
79 "p": true,
80 "dl": true,
81 "h1": true,
82 "h2": true,
83 "h3": true,
84 "h4": true,
85 "h5": true,
86 "h6": true,
87 "ol": true,
88 "ul": true,
89 "del": true,
90 "div": true,
91 "ins": true,
92 "pre": true,
93 "form": true,
94 "math": true,
95 "table": true,
96 "iframe": true,
97 "script": true,
98 "fieldset": true,
99 "noscript": true,
100 "blockquote": true,
101}
102
103// Renderer is the rendering interface.
104// This is mostly of interest if you are implementing a new rendering format.
105//
106// When a byte slice is provided, it contains the (rendered) contents of the
107// element.
108//
109// When a callback is provided instead, it will write the contents of the
110// respective element directly to the output buffer and return true on success.
111// If the callback returns false, the rendering function should reset the
112// output buffer as though it had never been called.
113//
114// Currently Html and Latex implementations are provided
115type Renderer interface {
116 // block-level callbacks
117 BlockCode(out *bytes.Buffer, text []byte, lang string)
118 BlockQuote(out *bytes.Buffer, text []byte)
119 BlockHtml(out *bytes.Buffer, text []byte)
120 Header(out *bytes.Buffer, text func() bool, level int)
121 HRule(out *bytes.Buffer)
122 List(out *bytes.Buffer, text func() bool, flags int)
123 ListItem(out *bytes.Buffer, text []byte, flags int)
124 Paragraph(out *bytes.Buffer, text func() bool)
125 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
126 TableRow(out *bytes.Buffer, text []byte)
127 TableCell(out *bytes.Buffer, text []byte, flags int)
128
129 // Span-level callbacks
130 AutoLink(out *bytes.Buffer, link []byte, kind int)
131 CodeSpan(out *bytes.Buffer, text []byte)
132 DoubleEmphasis(out *bytes.Buffer, text []byte)
133 Emphasis(out *bytes.Buffer, text []byte)
134 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
135 LineBreak(out *bytes.Buffer)
136 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
137 RawHtmlTag(out *bytes.Buffer, tag []byte)
138 TripleEmphasis(out *bytes.Buffer, text []byte)
139 StrikeThrough(out *bytes.Buffer, text []byte)
140
141 // Low-level callbacks
142 Entity(out *bytes.Buffer, entity []byte)
143 NormalText(out *bytes.Buffer, text []byte)
144
145 // Header and footer
146 DocumentHeader(out *bytes.Buffer)
147 DocumentFooter(out *bytes.Buffer)
148}
149
150// Callback functions for inline parsing. One such function is defined
151// for each character that triggers a response when parsing inline data.
152type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
153
154// Parser holds runtime state used by the parser.
155// This is constructed by the Markdown function.
156type parser struct {
157 r Renderer
158 refs map[string]*reference
159 inlineCallback [256]inlineParser
160 flags int
161 nesting int
162 maxNesting int
163 insideLink bool
164}
165
166
167//
168//
169// Public interface
170//
171//
172
173// MarkdownBasic is a convenience function for simple rendering.
174// It processes markdown input with no extensions enabled.
175func MarkdownBasic(input []byte) []byte {
176 // set up the HTML renderer
177 htmlFlags := HTML_USE_XHTML
178 renderer := HtmlRenderer(htmlFlags, "", "")
179
180 // set up the parser
181 extensions := 0
182
183 return Markdown(input, renderer, extensions)
184}
185
186// Call Markdown with most useful extensions enabled
187// MarkdownCommon is a convenience function for simple rendering.
188// It processes markdown input with common extensions enabled, including:
189//
190// * Smartypants processing with smart fractions and LaTeX dashes
191//
192// * Intra-word emphasis supression
193//
194// * Tables
195//
196// * Fenced code blocks
197//
198// * Autolinking
199//
200// * Strikethrough support
201//
202// * Strict header parsing
203func MarkdownCommon(input []byte) []byte {
204 // set up the HTML renderer
205 htmlFlags := 0
206 htmlFlags |= HTML_USE_XHTML
207 htmlFlags |= HTML_USE_SMARTYPANTS
208 htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
209 htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
210 renderer := HtmlRenderer(htmlFlags, "", "")
211
212 // set up the parser
213 extensions := 0
214 extensions |= EXTENSION_NO_INTRA_EMPHASIS
215 extensions |= EXTENSION_TABLES
216 extensions |= EXTENSION_FENCED_CODE
217 extensions |= EXTENSION_AUTOLINK
218 extensions |= EXTENSION_STRIKETHROUGH
219 extensions |= EXTENSION_SPACE_HEADERS
220
221 return Markdown(input, renderer, extensions)
222}
223
224// Markdown is the main rendering function.
225// It parses and renders a block of markdown-encoded text.
226// The supplied Renderer is used to format the output, and extensions dictates
227// which non-standard extensions are enabled.
228//
229// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
230// LatexRenderer, respectively.
231func Markdown(input []byte, renderer Renderer, extensions int) []byte {
232 // no point in parsing if we can't render
233 if renderer == nil {
234 return nil
235 }
236
237 // fill in the render structure
238 p := new(parser)
239 p.r = renderer
240 p.flags = extensions
241 p.refs = make(map[string]*reference)
242 p.maxNesting = 16
243 p.insideLink = false
244
245 // register inline parsers
246 p.inlineCallback['*'] = emphasis
247 p.inlineCallback['_'] = emphasis
248 if extensions&EXTENSION_STRIKETHROUGH != 0 {
249 p.inlineCallback['~'] = emphasis
250 }
251 p.inlineCallback['`'] = codeSpan
252 p.inlineCallback['\n'] = lineBreak
253 p.inlineCallback['['] = link
254 p.inlineCallback['<'] = leftAngle
255 p.inlineCallback['\\'] = escape
256 p.inlineCallback['&'] = entity
257
258 if extensions&EXTENSION_AUTOLINK != 0 {
259 p.inlineCallback[':'] = autoLink
260 }
261
262 first := firstPass(p, input)
263 second := secondPass(p, first)
264
265 return second
266}
267
268// first pass:
269// - extract references
270// - expand tabs
271// - normalize newlines
272// - copy everything else
273func firstPass(p *parser, input []byte) []byte {
274 var out bytes.Buffer
275 tabSize := TAB_SIZE_DEFAULT
276 if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
277 tabSize = TAB_SIZE_EIGHT
278 }
279 beg, end := 0, 0
280 for beg < len(input) { // iterate over lines
281 if end = isReference(p, input[beg:]); end > 0 {
282 beg += end
283 } else { // skip to the next line
284 end = beg
285 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
286 end++
287 }
288
289 // add the line body if present
290 if end > beg {
291 expandTabs(&out, input[beg:end], tabSize)
292 }
293 out.WriteByte('\n')
294
295 if end < len(input) && input[end] == '\r' {
296 end++
297 }
298 if end < len(input) && input[end] == '\n' {
299 end++
300 }
301
302 beg = end
303 }
304 }
305
306 // empty input?
307 if out.Len() == 0 {
308 out.WriteByte('\n')
309 }
310
311 return out.Bytes()
312}
313
314// second pass: actual rendering
315func secondPass(p *parser, input []byte) []byte {
316 var output bytes.Buffer
317
318 p.r.DocumentHeader(&output)
319 p.block(&output, input)
320 p.r.DocumentFooter(&output)
321
322 if p.nesting != 0 {
323 panic("Nesting level did not end at zero")
324 }
325
326 return output.Bytes()
327}
328
329
330//
331// Link references
332//
333// This section implements support for references that (usually) appear
334// as footnotes in a document, and can be referenced anywhere in the document.
335// The basic format is:
336//
337// [1]: http://www.google.com/ "Google"
338// [2]: http://www.github.com/ "Github"
339//
340// Anywhere in the document, the reference can be linked by referring to its
341// label, i.e., 1 and 2 in this example, as in:
342//
343// This library is hosted on [Github][2], a git hosting site.
344
345// References are parsed and stored in this struct.
346type reference struct {
347 link []byte
348 title []byte
349}
350
351// Check whether or not data starts with a reference link.
352// If so, it is parsed and stored in the list of references
353// (in the render struct).
354// Returns the number of bytes to skip to move past it,
355// or zero if the first line is not a reference.
356func isReference(p *parser, data []byte) int {
357 // up to 3 optional leading spaces
358 if len(data) < 4 {
359 return 0
360 }
361 i := 0
362 for i < 3 && data[i] == ' ' {
363 i++
364 }
365
366 // id part: anything but a newline between brackets
367 if data[i] != '[' {
368 return 0
369 }
370 i++
371 idOffset := i
372 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
373 i++
374 }
375 if i >= len(data) || data[i] != ']' {
376 return 0
377 }
378 idEnd := i
379
380 // spacer: colon (space | tab)* newline? (space | tab)*
381 i++
382 if i >= len(data) || data[i] != ':' {
383 return 0
384 }
385 i++
386 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
387 i++
388 }
389 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
390 i++
391 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
392 i++
393 }
394 }
395 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
396 i++
397 }
398 if i >= len(data) {
399 return 0
400 }
401
402 // link: whitespace-free sequence, optionally between angle brackets
403 if data[i] == '<' {
404 i++
405 }
406 linkOffset := i
407 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
408 i++
409 }
410 linkEnd := i
411 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
412 linkOffset++
413 linkEnd--
414 }
415
416 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
417 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
418 i++
419 }
420 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
421 return 0
422 }
423
424 // compute end-of-line
425 lineEnd := 0
426 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
427 lineEnd = i
428 }
429 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
430 lineEnd++
431 }
432
433 // optional (space|tab)* spacer after a newline
434 if lineEnd > 0 {
435 i = lineEnd + 1
436 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
437 i++
438 }
439 }
440
441 // optional title: any non-newline sequence enclosed in '"() alone on its line
442 titleOffset, titleEnd := 0, 0
443 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
444 i++
445 titleOffset = i
446
447 // look for EOL
448 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
449 i++
450 }
451 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
452 titleEnd = i + 1
453 } else {
454 titleEnd = i
455 }
456
457 // step back
458 i--
459 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
460 i--
461 }
462 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
463 lineEnd = titleEnd
464 titleEnd = i
465 }
466 }
467 if lineEnd == 0 { // garbage after the link
468 return 0
469 }
470
471 // a valid ref has been found
472
473 // id matches are case-insensitive
474 id := string(bytes.ToLower(data[idOffset:idEnd]))
475 p.refs[id] = &reference{
476 link: data[linkOffset:linkEnd],
477 title: data[titleOffset:titleEnd],
478 }
479
480 return lineEnd
481}
482
483
484//
485//
486// Miscellaneous helper functions
487//
488//
489
490
491// Test if a character is a punctuation symbol.
492// Taken from a private function in regexp in the stdlib.
493func ispunct(c byte) bool {
494 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
495 if c == r {
496 return true
497 }
498 }
499 return false
500}
501
502// Test if a character is a whitespace character.
503func isspace(c byte) bool {
504 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
505}
506
507// Test if a character is a letter or a digit.
508// TODO: check when this is looking for ASCII alnum and when it should use unicode
509func isalnum(c byte) bool {
510 return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
511}
512
513// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
514// always ends output with a newline
515func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
516 // first, check for common cases: no tabs, or only tabs at beginning of line
517 i, prefix := 0, 0
518 slowcase := false
519 for i = 0; i < len(line); i++ {
520 if line[i] == '\t' {
521 if prefix == i {
522 prefix++
523 } else {
524 slowcase = true
525 break
526 }
527 }
528 }
529
530 // no need to decode runes if all tabs are at the beginning of the line
531 if !slowcase {
532 for i = 0; i < prefix*tabSize; i++ {
533 out.WriteByte(' ')
534 }
535 out.Write(line[prefix:])
536 return
537 }
538
539 // the slow case: we need to count runes to figure out how
540 // many spaces to insert for each tab
541 column := 0
542 i = 0
543 for i < len(line) {
544 start := i
545 for i < len(line) && line[i] != '\t' {
546 _, size := utf8.DecodeRune(line[i:])
547 i += size
548 column++
549 }
550
551 if i > start {
552 out.Write(line[start:i])
553 }
554
555 if i >= len(line) {
556 break
557 }
558
559 for {
560 out.WriteByte(' ')
561 column++
562 if column%tabSize == 0 {
563 break
564 }
565 }
566
567 i++
568 }
569}