all repos — grayfriday @ 9c7cf8b1b7ce43e76f0cf9bc1ebf3fb7430bc5ea

blackfriday fork with a few changes

markdown.go (view raw)

  1//
  2// Blackfriday Markdown Processor
  3// Available at http://github.com/russross/blackfriday
  4//
  5// Copyright © 2011 Russ Ross <russ@russross.com>.
  6// Distributed under the Simplified BSD License.
  7// See README.md for details.
  8//
  9
 10//
 11//
 12// Markdown parsing and processing
 13//
 14//
 15
 16// Blackfriday markdown processor.
 17//
 18// Translates plain text with simple formatting rules into HTML or LaTeX.
 19package blackfriday
 20
 21import (
 22	"bytes"
 23	"unicode/utf8"
 24)
 25
 26const VERSION = "1.1"
 27
 28// These are the supported markdown parsing extensions.
 29// OR these values together to select multiple extensions.
 30const (
 31	EXTENSION_NO_INTRA_EMPHASIS          = 1 << iota // ignore emphasis markers inside words
 32	EXTENSION_TABLES                                 // render tables
 33	EXTENSION_FENCED_CODE                            // render fenced code blocks
 34	EXTENSION_AUTOLINK                               // detect embedded URLs that are not explicitly marked
 35	EXTENSION_STRIKETHROUGH                          // strikethrough text using ~~test~~
 36	EXTENSION_LAX_HTML_BLOCKS                        // loosen up HTML block parsing rules
 37	EXTENSION_SPACE_HEADERS                          // be strict about prefix header rules
 38	EXTENSION_HARD_LINE_BREAK                        // translate newlines into line breaks
 39	EXTENSION_TAB_SIZE_EIGHT                         // expand tabs to eight spaces instead of four
 40	EXTENSION_FOOTNOTES                              // Pandoc-style footnotes
 41	EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK             // No need to insert an empty line to start a (code, quote, order list, unorder list)block
 42	EXTENSION_HEADER_IDS                             // specify header IDs  with {#id}
 43)
 44
 45// These are the possible flag values for the link renderer.
 46// Only a single one of these values will be used; they are not ORed together.
 47// These are mostly of interest if you are writing a new output format.
 48const (
 49	LINK_TYPE_NOT_AUTOLINK = iota
 50	LINK_TYPE_NORMAL
 51	LINK_TYPE_EMAIL
 52)
 53
 54// These are the possible flag values for the ListItem renderer.
 55// Multiple flag values may be ORed together.
 56// These are mostly of interest if you are writing a new output format.
 57const (
 58	LIST_TYPE_ORDERED = 1 << iota
 59	LIST_ITEM_CONTAINS_BLOCK
 60	LIST_ITEM_BEGINNING_OF_LIST
 61	LIST_ITEM_END_OF_LIST
 62)
 63
 64// These are the possible flag values for the table cell renderer.
 65// Only a single one of these values will be used; they are not ORed together.
 66// These are mostly of interest if you are writing a new output format.
 67const (
 68	TABLE_ALIGNMENT_LEFT = 1 << iota
 69	TABLE_ALIGNMENT_RIGHT
 70	TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
 71)
 72
 73// The size of a tab stop.
 74const (
 75	TAB_SIZE_DEFAULT = 4
 76	TAB_SIZE_EIGHT   = 8
 77)
 78
 79// These are the tags that are recognized as HTML block tags.
 80// Any of these can be included in markdown text without special escaping.
 81var blockTags = map[string]bool{
 82	"p":          true,
 83	"dl":         true,
 84	"h1":         true,
 85	"h2":         true,
 86	"h3":         true,
 87	"h4":         true,
 88	"h5":         true,
 89	"h6":         true,
 90	"ol":         true,
 91	"ul":         true,
 92	"del":        true,
 93	"div":        true,
 94	"ins":        true,
 95	"pre":        true,
 96	"form":       true,
 97	"math":       true,
 98	"table":      true,
 99	"iframe":     true,
100	"script":     true,
101	"fieldset":   true,
102	"noscript":   true,
103	"blockquote": true,
104
105	// HTML5
106	"video":      true,
107	"aside":      true,
108	"canvas":     true,
109	"figure":     true,
110	"footer":     true,
111	"header":     true,
112	"hgroup":     true,
113	"output":     true,
114	"article":    true,
115	"section":    true,
116	"progress":   true,
117	"figcaption": true,
118}
119
120// Renderer is the rendering interface.
121// This is mostly of interest if you are implementing a new rendering format.
122//
123// When a byte slice is provided, it contains the (rendered) contents of the
124// element.
125//
126// When a callback is provided instead, it will write the contents of the
127// respective element directly to the output buffer and return true on success.
128// If the callback returns false, the rendering function should reset the
129// output buffer as though it had never been called.
130//
131// Currently Html and Latex implementations are provided
132type Renderer interface {
133	// block-level callbacks
134	BlockCode(out *bytes.Buffer, text []byte, lang string)
135	BlockQuote(out *bytes.Buffer, text []byte)
136	BlockHtml(out *bytes.Buffer, text []byte)
137	Header(out *bytes.Buffer, text func() bool, level int, id string)
138	HRule(out *bytes.Buffer)
139	List(out *bytes.Buffer, text func() bool, flags int)
140	ListItem(out *bytes.Buffer, text []byte, flags int)
141	Paragraph(out *bytes.Buffer, text func() bool)
142	Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
143	TableRow(out *bytes.Buffer, text []byte)
144	TableHeaderCell(out *bytes.Buffer, text []byte, flags int)
145	TableCell(out *bytes.Buffer, text []byte, flags int)
146	Footnotes(out *bytes.Buffer, text func() bool)
147	FootnoteItem(out *bytes.Buffer, name, text []byte, flags int)
148
149	// Span-level callbacks
150	AutoLink(out *bytes.Buffer, link []byte, kind int)
151	CodeSpan(out *bytes.Buffer, text []byte)
152	DoubleEmphasis(out *bytes.Buffer, text []byte)
153	Emphasis(out *bytes.Buffer, text []byte)
154	Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
155	LineBreak(out *bytes.Buffer)
156	Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
157	RawHtmlTag(out *bytes.Buffer, tag []byte)
158	TripleEmphasis(out *bytes.Buffer, text []byte)
159	StrikeThrough(out *bytes.Buffer, text []byte)
160	FootnoteRef(out *bytes.Buffer, ref []byte, id int)
161
162	// Low-level callbacks
163	Entity(out *bytes.Buffer, entity []byte)
164	NormalText(out *bytes.Buffer, text []byte)
165
166	// Header and footer
167	DocumentHeader(out *bytes.Buffer)
168	DocumentFooter(out *bytes.Buffer)
169
170	GetFlags() int
171}
172
173// Callback functions for inline parsing. One such function is defined
174// for each character that triggers a response when parsing inline data.
175type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
176
177// Parser holds runtime state used by the parser.
178// This is constructed by the Markdown function.
179type parser struct {
180	r              Renderer
181	refs           map[string]*reference
182	inlineCallback [256]inlineParser
183	flags          int
184	nesting        int
185	maxNesting     int
186	insideLink     bool
187
188	// Footnotes need to be ordered as well as available to quickly check for
189	// presence. If a ref is also a footnote, it's stored both in refs and here
190	// in notes. Slice is nil if footnotes not enabled.
191	notes []*reference
192}
193
194//
195//
196// Public interface
197//
198//
199
200// MarkdownBasic is a convenience function for simple rendering.
201// It processes markdown input with no extensions enabled.
202func MarkdownBasic(input []byte) []byte {
203	// set up the HTML renderer
204	htmlFlags := HTML_USE_XHTML
205	renderer := HtmlRenderer(htmlFlags, "", "")
206
207	// set up the parser
208	extensions := 0
209
210	return Markdown(input, renderer, extensions)
211}
212
213// Call Markdown with most useful extensions enabled
214// MarkdownCommon is a convenience function for simple rendering.
215// It processes markdown input with common extensions enabled, including:
216//
217// * Smartypants processing with smart fractions and LaTeX dashes
218//
219// * Intra-word emphasis suppression
220//
221// * Tables
222//
223// * Fenced code blocks
224//
225// * Autolinking
226//
227// * Strikethrough support
228//
229// * Strict header parsing
230//
231// * Custom Header IDs
232func MarkdownCommon(input []byte) []byte {
233	// set up the HTML renderer
234	htmlFlags := 0
235	htmlFlags |= HTML_USE_XHTML
236	htmlFlags |= HTML_USE_SMARTYPANTS
237	htmlFlags |= HTML_SMARTYPANTS_FRACTIONS
238	htmlFlags |= HTML_SMARTYPANTS_LATEX_DASHES
239	htmlFlags |= HTML_SANITIZE_OUTPUT
240	renderer := HtmlRenderer(htmlFlags, "", "")
241
242	// set up the parser
243	extensions := 0
244	extensions |= EXTENSION_NO_INTRA_EMPHASIS
245	extensions |= EXTENSION_TABLES
246	extensions |= EXTENSION_FENCED_CODE
247	extensions |= EXTENSION_AUTOLINK
248	extensions |= EXTENSION_STRIKETHROUGH
249	extensions |= EXTENSION_SPACE_HEADERS
250	extensions |= EXTENSION_HEADER_IDS
251
252	return Markdown(input, renderer, extensions)
253}
254
255// Markdown is the main rendering function.
256// It parses and renders a block of markdown-encoded text.
257// The supplied Renderer is used to format the output, and extensions dictates
258// which non-standard extensions are enabled.
259//
260// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
261// LatexRenderer, respectively.
262func Markdown(input []byte, renderer Renderer, extensions int) []byte {
263	// no point in parsing if we can't render
264	if renderer == nil {
265		return nil
266	}
267
268	// fill in the render structure
269	p := new(parser)
270	p.r = renderer
271	p.flags = extensions
272	p.refs = make(map[string]*reference)
273	p.maxNesting = 16
274	p.insideLink = false
275
276	// register inline parsers
277	p.inlineCallback['*'] = emphasis
278	p.inlineCallback['_'] = emphasis
279	if extensions&EXTENSION_STRIKETHROUGH != 0 {
280		p.inlineCallback['~'] = emphasis
281	}
282	p.inlineCallback['`'] = codeSpan
283	p.inlineCallback['\n'] = lineBreak
284	p.inlineCallback['['] = link
285	p.inlineCallback['<'] = leftAngle
286	p.inlineCallback['\\'] = escape
287	p.inlineCallback['&'] = entity
288
289	if extensions&EXTENSION_AUTOLINK != 0 {
290		p.inlineCallback[':'] = autoLink
291	}
292
293	if extensions&EXTENSION_FOOTNOTES != 0 {
294		p.notes = make([]*reference, 0)
295	}
296
297	first := firstPass(p, input)
298	second := secondPass(p, first)
299
300	if renderer.GetFlags()&HTML_SANITIZE_OUTPUT != 0 {
301		second = sanitizeHtml(second)
302	}
303
304	return second
305}
306
307// first pass:
308// - extract references
309// - expand tabs
310// - normalize newlines
311// - copy everything else
312// - add missing newlines before fenced code blocks
313func firstPass(p *parser, input []byte) []byte {
314	var out bytes.Buffer
315	tabSize := TAB_SIZE_DEFAULT
316	if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
317		tabSize = TAB_SIZE_EIGHT
318	}
319	beg, end := 0, 0
320	lastLineWasBlank := false
321	lastFencedCodeBlockEnd := 0
322	for beg < len(input) { // iterate over lines
323		if end = isReference(p, input[beg:], tabSize); end > 0 {
324			beg += end
325		} else { // skip to the next line
326			end = beg
327			for end < len(input) && input[end] != '\n' && input[end] != '\r' {
328				end++
329			}
330
331			if p.flags&EXTENSION_FENCED_CODE != 0 {
332				// when last line was none blank and a fenced code block comes after
333				if beg >= lastFencedCodeBlockEnd {
334					if i := p.fencedCode(&out, append(input[beg:], '\n'), false); i > 0 {
335						if !lastLineWasBlank {
336							out.WriteByte('\n') // need to inject additional linebreak
337						}
338						lastFencedCodeBlockEnd = beg + i
339					}
340				}
341				lastLineWasBlank = end == beg
342			}
343
344			// add the line body if present
345			if end > beg {
346				if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
347					out.Write(input[beg:end])
348				} else {
349					expandTabs(&out, input[beg:end], tabSize)
350				}
351			}
352			out.WriteByte('\n')
353
354			if end < len(input) && input[end] == '\r' {
355				end++
356			}
357			if end < len(input) && input[end] == '\n' {
358				end++
359			}
360
361			beg = end
362		}
363	}
364
365	// empty input?
366	if out.Len() == 0 {
367		out.WriteByte('\n')
368	}
369
370	return out.Bytes()
371}
372
373// second pass: actual rendering
374func secondPass(p *parser, input []byte) []byte {
375	var output bytes.Buffer
376
377	p.r.DocumentHeader(&output)
378	p.block(&output, input)
379
380	if p.flags&EXTENSION_FOOTNOTES != 0 && len(p.notes) > 0 {
381		p.r.Footnotes(&output, func() bool {
382			flags := LIST_ITEM_BEGINNING_OF_LIST
383			for _, ref := range p.notes {
384				var buf bytes.Buffer
385				if ref.hasBlock {
386					flags |= LIST_ITEM_CONTAINS_BLOCK
387					p.block(&buf, ref.title)
388				} else {
389					p.inline(&buf, ref.title)
390				}
391				p.r.FootnoteItem(&output, ref.link, buf.Bytes(), flags)
392				flags &^= LIST_ITEM_BEGINNING_OF_LIST | LIST_ITEM_CONTAINS_BLOCK
393			}
394
395			return true
396		})
397	}
398
399	p.r.DocumentFooter(&output)
400
401	if p.nesting != 0 {
402		panic("Nesting level did not end at zero")
403	}
404
405	return output.Bytes()
406}
407
408//
409// Link references
410//
411// This section implements support for references that (usually) appear
412// as footnotes in a document, and can be referenced anywhere in the document.
413// The basic format is:
414//
415//    [1]: http://www.google.com/ "Google"
416//    [2]: http://www.github.com/ "Github"
417//
418// Anywhere in the document, the reference can be linked by referring to its
419// label, i.e., 1 and 2 in this example, as in:
420//
421//    This library is hosted on [Github][2], a git hosting site.
422//
423// Actual footnotes as specified in Pandoc and supported by some other Markdown
424// libraries such as php-markdown are also taken care of. They look like this:
425//
426//    This sentence needs a bit of further explanation.[^note]
427//
428//    [^note]: This is the explanation.
429//
430// Footnotes should be placed at the end of the document in an ordered list.
431// Inline footnotes such as:
432//
433//    Inline footnotes^[Not supported.] also exist.
434//
435// are not yet supported.
436
437// References are parsed and stored in this struct.
438type reference struct {
439	link     []byte
440	title    []byte
441	noteId   int // 0 if not a footnote ref
442	hasBlock bool
443}
444
445// Check whether or not data starts with a reference link.
446// If so, it is parsed and stored in the list of references
447// (in the render struct).
448// Returns the number of bytes to skip to move past it,
449// or zero if the first line is not a reference.
450func isReference(p *parser, data []byte, tabSize int) int {
451	// up to 3 optional leading spaces
452	if len(data) < 4 {
453		return 0
454	}
455	i := 0
456	for i < 3 && data[i] == ' ' {
457		i++
458	}
459
460	noteId := 0
461
462	// id part: anything but a newline between brackets
463	if data[i] != '[' {
464		return 0
465	}
466	i++
467	if p.flags&EXTENSION_FOOTNOTES != 0 {
468		if data[i] == '^' {
469			// we can set it to anything here because the proper noteIds will
470			// be assigned later during the second pass. It just has to be != 0
471			noteId = 1
472			i++
473		}
474	}
475	idOffset := i
476	for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
477		i++
478	}
479	if i >= len(data) || data[i] != ']' {
480		return 0
481	}
482	idEnd := i
483
484	// spacer: colon (space | tab)* newline? (space | tab)*
485	i++
486	if i >= len(data) || data[i] != ':' {
487		return 0
488	}
489	i++
490	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
491		i++
492	}
493	if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
494		i++
495		if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
496			i++
497		}
498	}
499	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
500		i++
501	}
502	if i >= len(data) {
503		return 0
504	}
505
506	var (
507		linkOffset, linkEnd   int
508		titleOffset, titleEnd int
509		lineEnd               int
510		raw                   []byte
511		hasBlock              bool
512	)
513
514	if p.flags&EXTENSION_FOOTNOTES != 0 && noteId != 0 {
515		linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
516		lineEnd = linkEnd
517	} else {
518		linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
519	}
520	if lineEnd == 0 {
521		return 0
522	}
523
524	// a valid ref has been found
525
526	ref := &reference{
527		noteId:   noteId,
528		hasBlock: hasBlock,
529	}
530
531	if noteId > 0 {
532		// reusing the link field for the id since footnotes don't have links
533		ref.link = data[idOffset:idEnd]
534		// if footnote, it's not really a title, it's the contained text
535		ref.title = raw
536	} else {
537		ref.link = data[linkOffset:linkEnd]
538		ref.title = data[titleOffset:titleEnd]
539	}
540
541	// id matches are case-insensitive
542	id := string(bytes.ToLower(data[idOffset:idEnd]))
543
544	p.refs[id] = ref
545
546	return lineEnd
547}
548
549func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
550	// link: whitespace-free sequence, optionally between angle brackets
551	if data[i] == '<' {
552		i++
553	}
554	linkOffset = i
555	for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
556		i++
557	}
558	linkEnd = i
559	if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
560		linkOffset++
561		linkEnd--
562	}
563
564	// optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
565	for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
566		i++
567	}
568	if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
569		return
570	}
571
572	// compute end-of-line
573	if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
574		lineEnd = i
575	}
576	if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
577		lineEnd++
578	}
579
580	// optional (space|tab)* spacer after a newline
581	if lineEnd > 0 {
582		i = lineEnd + 1
583		for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
584			i++
585		}
586	}
587
588	// optional title: any non-newline sequence enclosed in '"() alone on its line
589	if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
590		i++
591		titleOffset = i
592
593		// look for EOL
594		for i < len(data) && data[i] != '\n' && data[i] != '\r' {
595			i++
596		}
597		if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
598			titleEnd = i + 1
599		} else {
600			titleEnd = i
601		}
602
603		// step back
604		i--
605		for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
606			i--
607		}
608		if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
609			lineEnd = titleEnd
610			titleEnd = i
611		}
612	}
613
614	return
615}
616
617// The first bit of this logic is the same as (*parser).listItem, but the rest
618// is much simpler. This function simply finds the entire block and shifts it
619// over by one tab if it is indeed a block (just returns the line if it's not).
620// blockEnd is the end of the section in the input buffer, and contents is the
621// extracted text that was shifted over one tab. It will need to be rendered at
622// the end of the document.
623func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
624	if i == 0 || len(data) == 0 {
625		return
626	}
627
628	// skip leading whitespace on first line
629	for i < len(data) && data[i] == ' ' {
630		i++
631	}
632
633	blockStart = i
634
635	// find the end of the line
636	blockEnd = i
637	for i < len(data) && data[i-1] != '\n' {
638		i++
639	}
640
641	// get working buffer
642	var raw bytes.Buffer
643
644	// put the first line into the working buffer
645	raw.Write(data[blockEnd:i])
646	blockEnd = i
647
648	// process the following lines
649	containsBlankLine := false
650
651gatherLines:
652	for blockEnd < len(data) {
653		i++
654
655		// find the end of this line
656		for i < len(data) && data[i-1] != '\n' {
657			i++
658		}
659
660		// if it is an empty line, guess that it is part of this item
661		// and move on to the next line
662		if p.isEmpty(data[blockEnd:i]) > 0 {
663			containsBlankLine = true
664			blockEnd = i
665			continue
666		}
667
668		n := 0
669		if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
670			// this is the end of the block.
671			// we don't want to include this last line in the index.
672			break gatherLines
673		}
674
675		// if there were blank lines before this one, insert a new one now
676		if containsBlankLine {
677			raw.WriteByte('\n')
678			containsBlankLine = false
679		}
680
681		// get rid of that first tab, write to buffer
682		raw.Write(data[blockEnd+n : i])
683		hasBlock = true
684
685		blockEnd = i
686	}
687
688	if data[blockEnd-1] != '\n' {
689		raw.WriteByte('\n')
690	}
691
692	contents = raw.Bytes()
693
694	return
695}
696
697//
698//
699// Miscellaneous helper functions
700//
701//
702
703// Test if a character is a punctuation symbol.
704// Taken from a private function in regexp in the stdlib.
705func ispunct(c byte) bool {
706	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
707		if c == r {
708			return true
709		}
710	}
711	return false
712}
713
714// Test if a character is a whitespace character.
715func isspace(c byte) bool {
716	return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
717}
718
719// Test if a character is letter.
720func isletter(c byte) bool {
721	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
722}
723
724// Test if a character is a letter or a digit.
725// TODO: check when this is looking for ASCII alnum and when it should use unicode
726func isalnum(c byte) bool {
727	return (c >= '0' && c <= '9') || isletter(c)
728}
729
730// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
731// always ends output with a newline
732func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
733	// first, check for common cases: no tabs, or only tabs at beginning of line
734	i, prefix := 0, 0
735	slowcase := false
736	for i = 0; i < len(line); i++ {
737		if line[i] == '\t' {
738			if prefix == i {
739				prefix++
740			} else {
741				slowcase = true
742				break
743			}
744		}
745	}
746
747	// no need to decode runes if all tabs are at the beginning of the line
748	if !slowcase {
749		for i = 0; i < prefix*tabSize; i++ {
750			out.WriteByte(' ')
751		}
752		out.Write(line[prefix:])
753		return
754	}
755
756	// the slow case: we need to count runes to figure out how
757	// many spaces to insert for each tab
758	column := 0
759	i = 0
760	for i < len(line) {
761		start := i
762		for i < len(line) && line[i] != '\t' {
763			_, size := utf8.DecodeRune(line[i:])
764			i += size
765			column++
766		}
767
768		if i > start {
769			out.Write(line[start:i])
770		}
771
772		if i >= len(line) {
773			break
774		}
775
776		for {
777			out.WriteByte(' ')
778			column++
779			if column%tabSize == 0 {
780				break
781			}
782		}
783
784		i++
785	}
786}
787
788// Find if a line counts as indented or not.
789// Returns number of characters the indent is (0 = not indented).
790func isIndented(data []byte, indentSize int) int {
791	if len(data) == 0 {
792		return 0
793	}
794	if data[0] == '\t' {
795		return 1
796	}
797	if len(data) < indentSize {
798		return 0
799	}
800	for i := 0; i < indentSize; i++ {
801		if data[i] != ' ' {
802			return 0
803		}
804	}
805	return indentSize
806}
807
808// Create a url-safe slug for fragments
809func slugify(in []byte) []byte {
810	if len(in) == 0 {
811		return in
812	}
813	out := make([]byte, 0, len(in))
814	sym := false
815
816	for _, ch := range in {
817		if isalnum(ch) {
818			sym = false
819			out = append(out, ch)
820		} else if sym {
821			continue
822		} else {
823			out = append(out, '-')
824			sym = true
825		}
826	}
827	var a, b int
828	var ch byte
829	for a, ch = range out {
830		if ch != '-' {
831			break
832		}
833	}
834	for b = len(out) - 1; b > 0; b-- {
835		if out[b] != '-' {
836			break
837		}
838	}
839	return out[a : b+1]
840}