markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16package blackfriday
17
18import (
19 "bytes"
20 "utf8"
21)
22
23const VERSION = "0.5"
24
25// These are the supported markdown parsing extensions.
26// OR these values together to select multiple extensions.
27const (
28 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota
29 EXTENSION_TABLES
30 EXTENSION_FENCED_CODE
31 EXTENSION_AUTOLINK
32 EXTENSION_STRIKETHROUGH
33 EXTENSION_LAX_HTML_BLOCKS
34 EXTENSION_SPACE_HEADERS
35 EXTENSION_HARD_LINE_BREAK
36 EXTENSION_NO_EXPAND_TABS
37 EXTENSION_TAB_SIZE_EIGHT
38)
39
40// These are the possible flag values for the link renderer.
41// Only a single one of these values will be used; they are not ORed together.
42// These are mostly of interest if you are writing a new output format.
43const (
44 LINK_TYPE_NOT_AUTOLINK = iota
45 LINK_TYPE_NORMAL
46 LINK_TYPE_EMAIL
47)
48
49// These are the possible flag values for the listitem renderer.
50// Multiple flag values may be ORed together.
51// These are mostly of interest if you are writing a new output format.
52const (
53 LIST_TYPE_ORDERED = 1 << iota
54 LIST_ITEM_CONTAINS_BLOCK
55 LIST_ITEM_END_OF_LIST
56)
57
58// These are the possible flag values for the table cell renderer.
59// Only a single one of these values will be used; they are not ORed together.
60// These are mostly of interest if you are writing a new output format.
61const (
62 TABLE_ALIGNMENT_LEFT = 1 << iota
63 TABLE_ALIGNMENT_RIGHT
64 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
65)
66
67// The size of a tab stop.
68const (
69 TAB_SIZE_DEFAULT = 4
70 TAB_SIZE_EIGHT = 8
71)
72
73// These are the tags that are recognized as HTML block tags.
74// Any of these can be included in markdown text without special escaping.
75var blockTags = map[string]bool{
76 "p": true,
77 "dl": true,
78 "h1": true,
79 "h2": true,
80 "h3": true,
81 "h4": true,
82 "h5": true,
83 "h6": true,
84 "ol": true,
85 "ul": true,
86 "del": true,
87 "div": true,
88 "ins": true,
89 "pre": true,
90 "form": true,
91 "math": true,
92 "table": true,
93 "iframe": true,
94 "script": true,
95 "fieldset": true,
96 "noscript": true,
97 "blockquote": true,
98}
99
100// This struct defines the rendering interface.
101// A series of callback functions are registered to form a complete renderer.
102// A single interface{} value field is provided, and that value is handed to
103// each callback. Leaving a field blank suppresses rendering that type of output
104// except where noted.
105//
106// This is mostly of interest if you are implementing a new rendering format.
107// Most users will use the convenience functions to fill in this structure.
108type Renderer struct {
109 // block-level callbacks---nil skips the block
110 BlockCode func(out *bytes.Buffer, text []byte, lang string, opaque interface{})
111 BlockQuote func(out *bytes.Buffer, text []byte, opaque interface{})
112 BlockHtml func(out *bytes.Buffer, text []byte, opaque interface{})
113 Header func(out *bytes.Buffer, text func() bool, level int, opaque interface{})
114 HRule func(out *bytes.Buffer, opaque interface{})
115 List func(out *bytes.Buffer, text func() bool, flags int, opaque interface{})
116 ListItem func(out *bytes.Buffer, text []byte, flags int, opaque interface{})
117 Paragraph func(out *bytes.Buffer, text func() bool, opaque interface{})
118 Table func(out *bytes.Buffer, header []byte, body []byte, columnData []int, opaque interface{})
119 TableRow func(out *bytes.Buffer, text []byte, opaque interface{})
120 TableCell func(out *bytes.Buffer, text []byte, flags int, opaque interface{})
121
122 // Span-level callbacks---nil or return false prints the span verbatim
123 AutoLink func(out *bytes.Buffer, link []byte, kind int, opaque interface{}) bool
124 CodeSpan func(out *bytes.Buffer, text []byte, opaque interface{}) bool
125 DoubleEmphasis func(out *bytes.Buffer, text []byte, opaque interface{}) bool
126 Emphasis func(out *bytes.Buffer, text []byte, opaque interface{}) bool
127 Image func(out *bytes.Buffer, link []byte, title []byte, alt []byte, opaque interface{}) bool
128 LineBreak func(out *bytes.Buffer, opaque interface{}) bool
129 Link func(out *bytes.Buffer, link []byte, title []byte, content []byte, opaque interface{}) bool
130 RawHtmlTag func(out *bytes.Buffer, tag []byte, opaque interface{}) bool
131 TripleEmphasis func(out *bytes.Buffer, text []byte, opaque interface{}) bool
132 StrikeThrough func(out *bytes.Buffer, text []byte, opaque interface{}) bool
133
134 // Low-level callbacks---nil copies input directly into the output
135 Entity func(out *bytes.Buffer, entity []byte, opaque interface{})
136 NormalText func(out *bytes.Buffer, text []byte, opaque interface{})
137
138 // Header and footer
139 DocumentHeader func(out *bytes.Buffer, opaque interface{})
140 DocumentFooter func(out *bytes.Buffer, opaque interface{})
141
142 // User data---passed back to every callback
143 Opaque interface{}
144}
145
146type inlineParser func(out *bytes.Buffer, parser *Parser, data []byte, offset int) int
147
148type Parser struct {
149 r *Renderer
150 refs map[string]*reference
151 inline [256]inlineParser
152 flags int
153 nesting int
154 maxNesting int
155 insideLink bool
156}
157
158
159//
160//
161// Public interface
162//
163//
164
165// Call Markdown with no extensions
166func MarkdownBasic(input []byte) []byte {
167 // set up the HTML renderer
168 htmlFlags := HTML_USE_XHTML
169 renderer := HtmlRenderer(htmlFlags, "", "")
170
171 // set up the parser
172 extensions := 0
173
174 return Markdown(input, renderer, extensions)
175}
176
177// Call Markdown with most useful extensions enabled
178func MarkdownCommon(input []byte) []byte {
179 // set up the HTML renderer
180 htmlFlags := 0
181 htmlFlags |= HTML_USE_XHTML
182 htmlFlags |= HTML_USE_SMARTYPANTS
183 htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
184 htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
185 renderer := HtmlRenderer(htmlFlags, "", "")
186
187 // set up the parser
188 extensions := 0
189 extensions |= EXTENSION_NO_INTRA_EMPHASIS
190 extensions |= EXTENSION_TABLES
191 extensions |= EXTENSION_FENCED_CODE
192 extensions |= EXTENSION_AUTOLINK
193 extensions |= EXTENSION_STRIKETHROUGH
194 extensions |= EXTENSION_SPACE_HEADERS
195
196 return Markdown(input, renderer, extensions)
197}
198
199// Parse and render a block of markdown-encoded text.
200// The renderer is used to format the output, and extensions dictates which
201// non-standard extensions are enabled.
202func Markdown(input []byte, renderer *Renderer, extensions int) []byte {
203 // no point in parsing if we can't render
204 if renderer == nil {
205 return nil
206 }
207
208 // fill in the render structure
209 parser := new(Parser)
210 parser.r = renderer
211 parser.flags = extensions
212 parser.refs = make(map[string]*reference)
213 parser.maxNesting = 16
214 parser.insideLink = false
215
216 // register inline parsers
217 if parser.r.Emphasis != nil || parser.r.DoubleEmphasis != nil || parser.r.TripleEmphasis != nil {
218 parser.inline['*'] = inlineEmphasis
219 parser.inline['_'] = inlineEmphasis
220 if extensions&EXTENSION_STRIKETHROUGH != 0 {
221 parser.inline['~'] = inlineEmphasis
222 }
223 }
224 if parser.r.CodeSpan != nil {
225 parser.inline['`'] = inlineCodeSpan
226 }
227 if parser.r.LineBreak != nil {
228 parser.inline['\n'] = inlineLineBreak
229 }
230 if parser.r.Image != nil || parser.r.Link != nil {
231 parser.inline['['] = inlineLink
232 }
233 parser.inline['<'] = inlineLAngle
234 parser.inline['\\'] = inlineEscape
235 parser.inline['&'] = inlineEntity
236
237 if extensions&EXTENSION_AUTOLINK != 0 {
238 parser.inline[':'] = inlineAutoLink
239 }
240
241 first := firstPass(parser, input)
242 second := secondPass(parser, first)
243
244 return second
245}
246
247// first pass:
248// - extract references
249// - expand tabs
250// - normalize newlines
251// - copy everything else
252func firstPass(parser *Parser, input []byte) []byte {
253 var out bytes.Buffer
254 tabSize := TAB_SIZE_DEFAULT
255 if parser.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
256 tabSize = TAB_SIZE_EIGHT
257 }
258 beg, end := 0, 0
259 for beg < len(input) { // iterate over lines
260 if end = isReference(parser, input[beg:]); end > 0 {
261 beg += end
262 } else { // skip to the next line
263 end = beg
264 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
265 end++
266 }
267
268 // add the line body if present
269 if end > beg {
270 if parser.flags&EXTENSION_NO_EXPAND_TABS == 0 {
271 expandTabs(&out, input[beg:end], tabSize)
272 } else {
273 out.Write(input[beg:end])
274 }
275 }
276 out.WriteByte('\n')
277
278 if end < len(input) && input[end] == '\r' {
279 end++
280 }
281 if end < len(input) && input[end] == '\n' {
282 end++
283 }
284
285 beg = end
286 }
287 }
288 return out.Bytes()
289}
290
291// second pass: actual rendering
292func secondPass(parser *Parser, input []byte) []byte {
293 var output bytes.Buffer
294 if parser.r.DocumentHeader != nil {
295 parser.r.DocumentHeader(&output, parser.r.Opaque)
296 }
297
298 parser.parseBlock(&output, input)
299
300 if parser.r.DocumentFooter != nil {
301 parser.r.DocumentFooter(&output, parser.r.Opaque)
302 }
303
304 if parser.nesting != 0 {
305 panic("Nesting level did not end at zero")
306 }
307
308 return output.Bytes()
309}
310
311
312//
313// Link references
314//
315// This section implements support for references that (usually) appear
316// as footnotes in a document, and can be referenced anywhere in the document.
317// The basic format is:
318//
319// [1]: http://www.google.com/ "Google"
320// [2]: http://www.github.com/ "Github"
321//
322// Anywhere in the document, the reference can be linked by referring to its
323// label, i.e., 1 and 2 in this example, as in:
324//
325// This library is hosted on [Github][2], a git hosting site.
326
327// References are parsed and stored in this struct.
328type reference struct {
329 link []byte
330 title []byte
331}
332
333// Check whether or not data starts with a reference link.
334// If so, it is parsed and stored in the list of references
335// (in the render struct).
336// Returns the number of bytes to skip to move past it,
337// or zero if the first line is not a reference.
338func isReference(parser *Parser, data []byte) int {
339 // up to 3 optional leading spaces
340 if len(data) < 4 {
341 return 0
342 }
343 i := 0
344 for i < 3 && data[i] == ' ' {
345 i++
346 }
347
348 // id part: anything but a newline between brackets
349 if data[i] != '[' {
350 return 0
351 }
352 i++
353 idOffset := i
354 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
355 i++
356 }
357 if i >= len(data) || data[i] != ']' {
358 return 0
359 }
360 idEnd := i
361
362 // spacer: colon (space | tab)* newline? (space | tab)*
363 i++
364 if i >= len(data) || data[i] != ':' {
365 return 0
366 }
367 i++
368 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
369 i++
370 }
371 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
372 i++
373 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
374 i++
375 }
376 }
377 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
378 i++
379 }
380 if i >= len(data) {
381 return 0
382 }
383
384 // link: whitespace-free sequence, optionally between angle brackets
385 if data[i] == '<' {
386 i++
387 }
388 linkOffset := i
389 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
390 i++
391 }
392 linkEnd := i
393 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
394 linkOffset++
395 linkEnd--
396 }
397
398 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
399 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
400 i++
401 }
402 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
403 return 0
404 }
405
406 // compute end-of-line
407 lineEnd := 0
408 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
409 lineEnd = i
410 }
411 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
412 lineEnd++
413 }
414
415 // optional (space|tab)* spacer after a newline
416 if lineEnd > 0 {
417 i = lineEnd + 1
418 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
419 i++
420 }
421 }
422
423 // optional title: any non-newline sequence enclosed in '"() alone on its line
424 titleOffset, titleEnd := 0, 0
425 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
426 i++
427 titleOffset = i
428
429 // look for EOL
430 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
431 i++
432 }
433 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
434 titleEnd = i + 1
435 } else {
436 titleEnd = i
437 }
438
439 // step back
440 i--
441 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
442 i--
443 }
444 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
445 lineEnd = titleEnd
446 titleEnd = i
447 }
448 }
449 if lineEnd == 0 { // garbage after the link
450 return 0
451 }
452
453 // a valid ref has been found
454 if parser == nil {
455 return lineEnd
456 }
457
458 // id matches are case-insensitive
459 id := string(bytes.ToLower(data[idOffset:idEnd]))
460 parser.refs[id] = &reference{
461 link: data[linkOffset:linkEnd],
462 title: data[titleOffset:titleEnd],
463 }
464
465 return lineEnd
466}
467
468
469//
470//
471// Miscellaneous helper functions
472//
473//
474
475
476// Test if a character is a punctuation symbol.
477// Taken from a private function in regexp in the stdlib.
478func ispunct(c byte) bool {
479 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
480 if c == r {
481 return true
482 }
483 }
484 return false
485}
486
487// Test if a character is a whitespace character.
488func isspace(c byte) bool {
489 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
490}
491
492// Test if a character is a letter or a digit.
493// TODO: check when this is looking for ASCII alnum and when it should use unicode
494func isalnum(c byte) bool {
495 return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
496}
497
498// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
499// always ends output with a newline
500func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
501 // first, check for common cases: no tabs, or only tabs at beginning of line
502 i, prefix := 0, 0
503 slowcase := false
504 for i = 0; i < len(line); i++ {
505 if line[i] == '\t' {
506 if prefix == i {
507 prefix++
508 } else {
509 slowcase = true
510 break
511 }
512 }
513 }
514
515 // no need to decode runes if all tabs are at the beginning of the line
516 if !slowcase {
517 for i = 0; i < prefix*tabSize; i++ {
518 out.WriteByte(' ')
519 }
520 out.Write(line[prefix:])
521 return
522 }
523
524 // the slow case: we need to count runes to figure out how
525 // many spaces to insert for each tab
526 column := 0
527 i = 0
528 for i < len(line) {
529 start := i
530 for i < len(line) && line[i] != '\t' {
531 _, size := utf8.DecodeRune(line[i:])
532 i += size
533 column++
534 }
535
536 if i > start {
537 out.Write(line[start:i])
538 }
539
540 if i >= len(line) {
541 break
542 }
543
544 for {
545 out.WriteByte(' ')
546 column++
547 if column%tabSize == 0 {
548 break
549 }
550 }
551
552 i++
553 }
554}