1//
  2// Blackfriday Markdown Processor
  3// Available at http://github.com/russross/blackfriday
  4//
  5// Copyright © 2011 Russ Ross <russ@russross.com>.
  6// Distributed under the Simplified BSD License.
  7// See README.md for details.
  8//
  9
 10//
 11//
 12// Markdown parsing and processing
 13//
 14//
 15
 16package blackfriday
 17
 18import (
 19	"bytes"
 20	"utf8"
 21)
 22
 23const VERSION = "0.6"
 24
 25// These are the supported markdown parsing extensions.
 26// OR these values together to select multiple extensions.
 27const (
 28	EXTENSION_NO_INTRA_EMPHASIS = 1 << iota
 29	EXTENSION_TABLES
 30	EXTENSION_FENCED_CODE
 31	EXTENSION_AUTOLINK
 32	EXTENSION_STRIKETHROUGH
 33	EXTENSION_LAX_HTML_BLOCKS
 34	EXTENSION_SPACE_HEADERS
 35	EXTENSION_HARD_LINE_BREAK
 36	EXTENSION_NO_EXPAND_TABS
 37	EXTENSION_TAB_SIZE_EIGHT
 38)
 39
 40// These are the possible flag values for the link renderer.
 41// Only a single one of these values will be used; they are not ORed together.
 42// These are mostly of interest if you are writing a new output format.
 43const (
 44	LINK_TYPE_NOT_AUTOLINK = iota
 45	LINK_TYPE_NORMAL
 46	LINK_TYPE_EMAIL
 47)
 48
 49// These are the possible flag values for the listitem renderer.
 50// Multiple flag values may be ORed together.
 51// These are mostly of interest if you are writing a new output format.
 52const (
 53	LIST_TYPE_ORDERED = 1 << iota
 54	LIST_ITEM_CONTAINS_BLOCK
 55	LIST_ITEM_END_OF_LIST
 56)
 57
 58// These are the possible flag values for the table cell renderer.
 59// Only a single one of these values will be used; they are not ORed together.
 60// These are mostly of interest if you are writing a new output format.
 61const (
 62	TABLE_ALIGNMENT_LEFT = 1 << iota
 63	TABLE_ALIGNMENT_RIGHT
 64	TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
 65)
 66
 67// The size of a tab stop.
 68const (
 69	TAB_SIZE_DEFAULT = 4
 70	TAB_SIZE_EIGHT   = 8
 71)
 72
 73// These are the tags that are recognized as HTML block tags.
 74// Any of these can be included in markdown text without special escaping.
 75var blockTags = map[string]bool{
 76	"p":          true,
 77	"dl":         true,
 78	"h1":         true,
 79	"h2":         true,
 80	"h3":         true,
 81	"h4":         true,
 82	"h5":         true,
 83	"h6":         true,
 84	"ol":         true,
 85	"ul":         true,
 86	"del":        true,
 87	"div":        true,
 88	"ins":        true,
 89	"pre":        true,
 90	"form":       true,
 91	"math":       true,
 92	"table":      true,
 93	"iframe":     true,
 94	"script":     true,
 95	"fieldset":   true,
 96	"noscript":   true,
 97	"blockquote": true,
 98}
 99
100// This interface defines the rendering interface.
101// This is mostly of interest if you are implementing a new rendering format.
102// Currently Html and Latex implementations are provided
103type Renderer interface {
104	// block-level callbacks
105	BlockCode(out *bytes.Buffer, text []byte, lang string)
106	BlockQuote(out *bytes.Buffer, text []byte)
107	BlockHtml(out *bytes.Buffer, text []byte)
108	Header(out *bytes.Buffer, text func() bool, level int)
109	HRule(out *bytes.Buffer)
110	List(out *bytes.Buffer, text func() bool, flags int)
111	ListItem(out *bytes.Buffer, text []byte, flags int)
112	Paragraph(out *bytes.Buffer, text func() bool)
113	Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
114	TableRow(out *bytes.Buffer, text []byte)
115	TableCell(out *bytes.Buffer, text []byte, flags int)
116
117	// Span-level callbacks
118	AutoLink(out *bytes.Buffer, link []byte, kind int)
119	CodeSpan(out *bytes.Buffer, text []byte)
120	DoubleEmphasis(out *bytes.Buffer, text []byte)
121	Emphasis(out *bytes.Buffer, text []byte)
122	Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
123	LineBreak(out *bytes.Buffer)
124	Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
125	RawHtmlTag(out *bytes.Buffer, tag []byte)
126	TripleEmphasis(out *bytes.Buffer, text []byte)
127	StrikeThrough(out *bytes.Buffer, text []byte)
128
129	// Low-level callbacks
130	Entity(out *bytes.Buffer, entity []byte)
131	NormalText(out *bytes.Buffer, text []byte)
132
133	// Header and footer
134	DocumentHeader(out *bytes.Buffer)
135	DocumentFooter(out *bytes.Buffer)
136}
137
138// Callback functions for inline parsing. One such function is defined
139// for each character that triggers a response when parsing inline data.
140type inlineParser func(parser *Parser, out *bytes.Buffer, data []byte, offset int) int
141
142// The main parser object.
143// This is constructed by the Markdown function and
144// contains state used during the parsing process.
145type Parser struct {
146	r          Renderer
147	refs       map[string]*reference
148	inline     [256]inlineParser
149	flags      int
150	nesting    int
151	maxNesting int
152	insideLink bool
153}
154
155
156//
157//
158// Public interface
159//
160//
161
162// Call Markdown with no extensions
163func MarkdownBasic(input []byte) []byte {
164	// set up the HTML renderer
165	htmlFlags := HTML_USE_XHTML
166	renderer := HtmlRenderer(htmlFlags, "", "")
167
168	// set up the parser
169	extensions := 0
170
171	return Markdown(input, renderer, extensions)
172}
173
174// Call Markdown with most useful extensions enabled
175func MarkdownCommon(input []byte) []byte {
176	// set up the HTML renderer
177	htmlFlags := 0
178	htmlFlags |= HTML_USE_XHTML
179	htmlFlags |= HTML_USE_SMARTYPANTS
180	htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
181	htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
182	renderer := HtmlRenderer(htmlFlags, "", "")
183
184	// set up the parser
185	extensions := 0
186	extensions |= EXTENSION_NO_INTRA_EMPHASIS
187	extensions |= EXTENSION_TABLES
188	extensions |= EXTENSION_FENCED_CODE
189	extensions |= EXTENSION_AUTOLINK
190	extensions |= EXTENSION_STRIKETHROUGH
191	extensions |= EXTENSION_SPACE_HEADERS
192
193	return Markdown(input, renderer, extensions)
194}
195
196// Parse and render a block of markdown-encoded text.
197// The renderer is used to format the output, and extensions dictates which
198// non-standard extensions are enabled.
199func Markdown(input []byte, renderer Renderer, extensions int) []byte {
200	// no point in parsing if we can't render
201	if renderer == nil {
202		return nil
203	}
204
205	// fill in the render structure
206	parser := new(Parser)
207	parser.r = renderer
208	parser.flags = extensions
209	parser.refs = make(map[string]*reference)
210	parser.maxNesting = 16
211	parser.insideLink = false
212
213	// register inline parsers
214	parser.inline['*'] = inlineEmphasis
215	parser.inline['_'] = inlineEmphasis
216	if extensions&EXTENSION_STRIKETHROUGH != 0 {
217		parser.inline['~'] = inlineEmphasis
218	}
219	parser.inline['`'] = inlineCodeSpan
220	parser.inline['\n'] = inlineLineBreak
221	parser.inline['['] = inlineLink
222	parser.inline['<'] = inlineLAngle
223	parser.inline['\\'] = inlineEscape
224	parser.inline['&'] = inlineEntity
225
226	if extensions&EXTENSION_AUTOLINK != 0 {
227		parser.inline[':'] = inlineAutoLink
228	}
229
230	first := firstPass(parser, input)
231	second := secondPass(parser, first)
232
233	return second
234}
235
236// first pass:
237// - extract references
238// - expand tabs
239// - normalize newlines
240// - copy everything else
241func firstPass(parser *Parser, input []byte) []byte {
242	var out bytes.Buffer
243	tabSize := TAB_SIZE_DEFAULT
244	if parser.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
245		tabSize = TAB_SIZE_EIGHT
246	}
247	beg, end := 0, 0
248	for beg < len(input) { // iterate over lines
249		if end = isReference(parser, input[beg:]); end > 0 {
250			beg += end
251		} else { // skip to the next line
252			end = beg
253			for end < len(input) && input[end] != '\n' && input[end] != '\r' {
254				end++
255			}
256
257			// add the line body if present
258			if end > beg {
259				if parser.flags&EXTENSION_NO_EXPAND_TABS == 0 {
260					expandTabs(&out, input[beg:end], tabSize)
261				} else {
262					out.Write(input[beg:end])
263				}
264			}
265			out.WriteByte('\n')
266
267			if end < len(input) && input[end] == '\r' {
268				end++
269			}
270			if end < len(input) && input[end] == '\n' {
271				end++
272			}
273
274			beg = end
275		}
276	}
277	return out.Bytes()
278}
279
280// second pass: actual rendering
281func secondPass(parser *Parser, input []byte) []byte {
282	var output bytes.Buffer
283
284	parser.r.DocumentHeader(&output)
285	parser.parseBlock(&output, input)
286	parser.r.DocumentFooter(&output)
287
288	if parser.nesting != 0 {
289		panic("Nesting level did not end at zero")
290	}
291
292	return output.Bytes()
293}
294
295
296//
297// Link references
298//
299// This section implements support for references that (usually) appear
300// as footnotes in a document, and can be referenced anywhere in the document.
301// The basic format is:
302//
303//    [1]: http://www.google.com/ "Google"
304//    [2]: http://www.github.com/ "Github"
305//
306// Anywhere in the document, the reference can be linked by referring to its
307// label, i.e., 1 and 2 in this example, as in:
308//
309//    This library is hosted on [Github][2], a git hosting site.
310
311// References are parsed and stored in this struct.
312type reference struct {
313	link  []byte
314	title []byte
315}
316
317// Check whether or not data starts with a reference link.
318// If so, it is parsed and stored in the list of references
319// (in the render struct).
320// Returns the number of bytes to skip to move past it,
321// or zero if the first line is not a reference.
322func isReference(parser *Parser, data []byte) int {
323	// up to 3 optional leading spaces
324	if len(data) < 4 {
325		return 0
326	}
327	i := 0
328	for i < 3 && data[i] == ' ' {
329		i++
330	}
331
332	// id part: anything but a newline between brackets
333	if data[i] != '[' {
334		return 0
335	}
336	i++
337	idOffset := i
338	for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
339		i++
340	}
341	if i >= len(data) || data[i] != ']' {
342		return 0
343	}
344	idEnd := i
345
346	// spacer: colon (space | tab)* newline? (space | tab)*
347	i++
348	if i >= len(data) || data[i] != ':' {
349		return 0
350	}
351	i++
352	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
353		i++
354	}
355	if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
356		i++
357		if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
358			i++
359		}
360	}
361	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
362		i++
363	}
364	if i >= len(data) {
365		return 0
366	}
367
368	// link: whitespace-free sequence, optionally between angle brackets
369	if data[i] == '<' {
370		i++
371	}
372	linkOffset := i
373	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
374		i++
375	}
376	linkEnd := i
377	if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
378		linkOffset++
379		linkEnd--
380	}
381
382	// optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
383	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
384		i++
385	}
386	if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
387		return 0
388	}
389
390	// compute end-of-line
391	lineEnd := 0
392	if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
393		lineEnd = i
394	}
395	if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
396		lineEnd++
397	}
398
399	// optional (space|tab)* spacer after a newline
400	if lineEnd > 0 {
401		i = lineEnd + 1
402		for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
403			i++
404		}
405	}
406
407	// optional title: any non-newline sequence enclosed in '"() alone on its line
408	titleOffset, titleEnd := 0, 0
409	if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
410		i++
411		titleOffset = i
412
413		// look for EOL
414		for i < len(data) && data[i] != '\n' && data[i] != '\r' {
415			i++
416		}
417		if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
418			titleEnd = i + 1
419		} else {
420			titleEnd = i
421		}
422
423		// step back
424		i--
425		for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
426			i--
427		}
428		if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
429			lineEnd = titleEnd
430			titleEnd = i
431		}
432	}
433	if lineEnd == 0 { // garbage after the link
434		return 0
435	}
436
437	// a valid ref has been found
438	if parser == nil {
439		return lineEnd
440	}
441
442	// id matches are case-insensitive
443	id := string(bytes.ToLower(data[idOffset:idEnd]))
444	parser.refs[id] = &reference{
445		link:  data[linkOffset:linkEnd],
446		title: data[titleOffset:titleEnd],
447	}
448
449	return lineEnd
450}
451
452
453//
454//
455// Miscellaneous helper functions
456//
457//
458
459
460// Test if a character is a punctuation symbol.
461// Taken from a private function in regexp in the stdlib.
462func ispunct(c byte) bool {
463	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
464		if c == r {
465			return true
466		}
467	}
468	return false
469}
470
471// Test if a character is a whitespace character.
472func isspace(c byte) bool {
473	return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
474}
475
476// Test if a character is a letter or a digit.
477// TODO: check when this is looking for ASCII alnum and when it should use unicode
478func isalnum(c byte) bool {
479	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
480}
481
482// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
483// always ends output with a newline
484func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
485	// first, check for common cases: no tabs, or only tabs at beginning of line
486	i, prefix := 0, 0
487	slowcase := false
488	for i = 0; i < len(line); i++ {
489		if line[i] == '\t' {
490			if prefix == i {
491				prefix++
492			} else {
493				slowcase = true
494				break
495			}
496		}
497	}
498
499	// no need to decode runes if all tabs are at the beginning of the line
500	if !slowcase {
501		for i = 0; i < prefix*tabSize; i++ {
502			out.WriteByte(' ')
503		}
504		out.Write(line[prefix:])
505		return
506	}
507
508	// the slow case: we need to count runes to figure out how
509	// many spaces to insert for each tab
510	column := 0
511	i = 0
512	for i < len(line) {
513		start := i
514		for i < len(line) && line[i] != '\t' {
515			_, size := utf8.DecodeRune(line[i:])
516			i += size
517			column++
518		}
519
520		if i > start {
521			out.Write(line[start:i])
522		}
523
524		if i >= len(line) {
525			break
526		}
527
528		for {
529			out.WriteByte(' ')
530			column++
531			if column%tabSize == 0 {
532				break
533			}
534		}
535
536		i++
537	}
538}