markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "fmt"
24 "strings"
25 "unicode/utf8"
26)
27
28const VERSION = "1.4"
29
30type Extensions int
31
32// These are the supported markdown parsing extensions.
33// OR these values together to select multiple extensions.
34const (
35 NoExtensions Extensions = 0
36 NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
37 Tables // Render tables
38 FencedCode // Render fenced code blocks
39 Autolink // Detect embedded URLs that are not explicitly marked
40 Strikethrough // Strikethrough text using ~~test~~
41 LaxHTMLBlocks // Loosen up HTML block parsing rules
42 SpaceHeaders // Be strict about prefix header rules
43 HardLineBreak // Translate newlines into line breaks
44 TabSizeEight // Expand tabs to eight spaces instead of four
45 Footnotes // Pandoc-style footnotes
46 NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
47 HeaderIDs // specify header IDs with {#id}
48 Titleblock // Titleblock ala pandoc
49 AutoHeaderIDs // Create the header ID from the text
50 BackslashLineBreak // Translate trailing backslashes into line breaks
51 DefinitionLists // Render definition lists
52
53 commonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants |
54 SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
55
56 commonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
57 Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
58 BackslashLineBreak | DefinitionLists
59)
60
61type LinkType int
62
63// These are the possible flag values for the link renderer.
64// Only a single one of these values will be used; they are not ORed together.
65// These are mostly of interest if you are writing a new output format.
66const (
67 LinkTypeNotAutolink LinkType = iota
68 LinkTypeNormal
69 LinkTypeEmail
70)
71
72type ListType int
73
74// These are the possible flag values for the ListItem renderer.
75// Multiple flag values may be ORed together.
76// These are mostly of interest if you are writing a new output format.
77const (
78 ListTypeOrdered ListType = 1 << iota
79 ListTypeDefinition
80 ListTypeTerm
81
82 ListItemContainsBlock
83 ListItemBeginningOfList
84 ListItemEndOfList
85)
86
87type TableFlags int
88
89// These are the possible flag values for the table cell renderer.
90// Only a single one of these values will be used; they are not ORed together.
91// These are mostly of interest if you are writing a new output format.
92const (
93 TableAlignmentLeft = 1 << iota
94 TableAlignmentRight
95 TableAlignmentCenter = (TableAlignmentLeft | TableAlignmentRight)
96)
97
98// The size of a tab stop.
99const (
100 TabSizeDefault = 4
101 TabSizeDouble = 8
102)
103
104// blockTags is a set of tags that are recognized as HTML block tags.
105// Any of these can be included in markdown text without special escaping.
106var blockTags = map[string]struct{}{
107 "blockquote": struct{}{},
108 "del": struct{}{},
109 "div": struct{}{},
110 "dl": struct{}{},
111 "fieldset": struct{}{},
112 "form": struct{}{},
113 "h1": struct{}{},
114 "h2": struct{}{},
115 "h3": struct{}{},
116 "h4": struct{}{},
117 "h5": struct{}{},
118 "h6": struct{}{},
119 "iframe": struct{}{},
120 "ins": struct{}{},
121 "math": struct{}{},
122 "noscript": struct{}{},
123 "ol": struct{}{},
124 "pre": struct{}{},
125 "p": struct{}{},
126 "script": struct{}{},
127 "style": struct{}{},
128 "table": struct{}{},
129 "ul": struct{}{},
130
131 // HTML5
132 "address": struct{}{},
133 "article": struct{}{},
134 "aside": struct{}{},
135 "canvas": struct{}{},
136 "figcaption": struct{}{},
137 "figure": struct{}{},
138 "footer": struct{}{},
139 "header": struct{}{},
140 "hgroup": struct{}{},
141 "main": struct{}{},
142 "nav": struct{}{},
143 "output": struct{}{},
144 "progress": struct{}{},
145 "section": struct{}{},
146 "video": struct{}{},
147}
148
149// Renderer is the rendering interface.
150// This is mostly of interest if you are implementing a new rendering format.
151//
152// When a byte slice is provided, it contains the (rendered) contents of the
153// element.
154//
155// When a callback is provided instead, it will write the contents of the
156// respective element directly to the output buffer and return true on success.
157// If the callback returns false, the rendering function should reset the
158// output buffer as though it had never been called.
159//
160// Currently Html and Latex implementations are provided
161type Renderer interface {
162 // block-level callbacks
163 BlockCode(out *bytes.Buffer, text []byte, lang string)
164 BlockQuote(out *bytes.Buffer, text []byte)
165 BlockHtml(out *bytes.Buffer, text []byte)
166 BeginHeader(out *bytes.Buffer, level int, id string) int
167 EndHeader(out *bytes.Buffer, level int, id string, tocMarker int)
168 HRule(out *bytes.Buffer)
169 BeginList(out *bytes.Buffer, flags ListType)
170 EndList(out *bytes.Buffer, flags ListType)
171 ListItem(out *bytes.Buffer, text []byte, flags ListType)
172 BeginParagraph(out *bytes.Buffer)
173 EndParagraph(out *bytes.Buffer)
174 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
175 TableRow(out *bytes.Buffer, text []byte)
176 TableHeaderCell(out *bytes.Buffer, text []byte, flags int)
177 TableCell(out *bytes.Buffer, text []byte, flags int)
178 BeginFootnotes(out *bytes.Buffer)
179 EndFootnotes(out *bytes.Buffer)
180 FootnoteItem(out *bytes.Buffer, name, text []byte, flags ListType)
181 TitleBlock(out *bytes.Buffer, text []byte)
182
183 // Span-level callbacks
184 AutoLink(out *bytes.Buffer, link []byte, kind LinkType)
185 CodeSpan(out *bytes.Buffer, text []byte)
186 DoubleEmphasis(out *bytes.Buffer, text []byte)
187 Emphasis(out *bytes.Buffer, text []byte)
188 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
189 LineBreak(out *bytes.Buffer)
190 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
191 RawHtmlTag(out *bytes.Buffer, tag []byte)
192 TripleEmphasis(out *bytes.Buffer, text []byte)
193 StrikeThrough(out *bytes.Buffer, text []byte)
194 FootnoteRef(out *bytes.Buffer, ref []byte, id int)
195
196 // Low-level callbacks
197 Entity(out *bytes.Buffer, entity []byte)
198 NormalText(out *bytes.Buffer, text []byte)
199
200 // Header and footer
201 DocumentHeader(out *bytes.Buffer)
202 DocumentFooter(out *bytes.Buffer)
203
204 GetFlags() HtmlFlags
205}
206
207// Callback functions for inline parsing. One such function is defined
208// for each character that triggers a response when parsing inline data.
209type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
210
211// Parser holds runtime state used by the parser.
212// This is constructed by the Markdown function.
213type parser struct {
214 r Renderer
215 refOverride ReferenceOverrideFunc
216 refs map[string]*reference
217 inlineCallback [256]inlineParser
218 flags Extensions
219 nesting int
220 maxNesting int
221 insideLink bool
222
223 // Footnotes need to be ordered as well as available to quickly check for
224 // presence. If a ref is also a footnote, it's stored both in refs and here
225 // in notes. Slice is nil if footnotes not enabled.
226 notes []*reference
227}
228
229func (p *parser) getRef(refid string) (ref *reference, found bool) {
230 if p.refOverride != nil {
231 r, overridden := p.refOverride(refid)
232 if overridden {
233 if r == nil {
234 return nil, false
235 }
236 return &reference{
237 link: []byte(r.Link),
238 title: []byte(r.Title),
239 noteId: 0,
240 hasBlock: false,
241 text: []byte(r.Text)}, true
242 }
243 }
244 // refs are case insensitive
245 ref, found = p.refs[strings.ToLower(refid)]
246 return ref, found
247}
248
249//
250//
251// Public interface
252//
253//
254
255// Reference represents the details of a link.
256// See the documentation in Options for more details on use-case.
257type Reference struct {
258 // Link is usually the URL the reference points to.
259 Link string
260 // Title is the alternate text describing the link in more detail.
261 Title string
262 // Text is the optional text to override the ref with if the syntax used was
263 // [refid][]
264 Text string
265}
266
267// ReferenceOverrideFunc is expected to be called with a reference string and
268// return either a valid Reference type that the reference string maps to or
269// nil. If overridden is false, the default reference logic will be executed.
270// See the documentation in Options for more details on use-case.
271type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
272
273// Options represents configurable overrides and callbacks (in addition to the
274// extension flag set) for configuring a Markdown parse.
275type Options struct {
276 // Extensions is a flag set of bit-wise ORed extension bits. See the
277 // Extensions flags defined in this package.
278 Extensions Extensions
279
280 // ReferenceOverride is an optional function callback that is called every
281 // time a reference is resolved.
282 //
283 // In Markdown, the link reference syntax can be made to resolve a link to
284 // a reference instead of an inline URL, in one of the following ways:
285 //
286 // * [link text][refid]
287 // * [refid][]
288 //
289 // Usually, the refid is defined at the bottom of the Markdown document. If
290 // this override function is provided, the refid is passed to the override
291 // function first, before consulting the defined refids at the bottom. If
292 // the override function indicates an override did not occur, the refids at
293 // the bottom will be used to fill in the link details.
294 ReferenceOverride ReferenceOverrideFunc
295}
296
297// MarkdownBasic is a convenience function for simple rendering.
298// It processes markdown input with no extensions enabled.
299func MarkdownBasic(input []byte) []byte {
300 // set up the HTML renderer
301 htmlFlags := UseXHTML
302 renderer := HtmlRenderer(htmlFlags, "", "")
303
304 // set up the parser
305 return MarkdownOptions(input, renderer, Options{Extensions: 0})
306}
307
308// Call Markdown with most useful extensions enabled
309// MarkdownCommon is a convenience function for simple rendering.
310// It processes markdown input with common extensions enabled, including:
311//
312// * Smartypants processing with smart fractions and LaTeX dashes
313//
314// * Intra-word emphasis suppression
315//
316// * Tables
317//
318// * Fenced code blocks
319//
320// * Autolinking
321//
322// * Strikethrough support
323//
324// * Strict header parsing
325//
326// * Custom Header IDs
327func MarkdownCommon(input []byte) []byte {
328 // set up the HTML renderer
329 renderer := HtmlRenderer(commonHtmlFlags, "", "")
330 return MarkdownOptions(input, renderer, Options{
331 Extensions: commonExtensions})
332}
333
334// Markdown is the main rendering function.
335// It parses and renders a block of markdown-encoded text.
336// The supplied Renderer is used to format the output, and extensions dictates
337// which non-standard extensions are enabled.
338//
339// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
340// LatexRenderer, respectively.
341func Markdown(input []byte, renderer Renderer, extensions Extensions) []byte {
342 return MarkdownOptions(input, renderer, Options{
343 Extensions: extensions})
344}
345
346// MarkdownOptions is just like Markdown but takes additional options through
347// the Options struct.
348func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
349 // no point in parsing if we can't render
350 if renderer == nil {
351 return nil
352 }
353
354 extensions := opts.Extensions
355
356 // fill in the render structure
357 p := new(parser)
358 p.r = renderer
359 p.flags = extensions
360 p.refOverride = opts.ReferenceOverride
361 p.refs = make(map[string]*reference)
362 p.maxNesting = 16
363 p.insideLink = false
364
365 // register inline parsers
366 p.inlineCallback['*'] = emphasis
367 p.inlineCallback['_'] = emphasis
368 if extensions&Strikethrough != 0 {
369 p.inlineCallback['~'] = emphasis
370 }
371 p.inlineCallback['`'] = codeSpan
372 p.inlineCallback['\n'] = lineBreak
373 p.inlineCallback['['] = link
374 p.inlineCallback['<'] = leftAngle
375 p.inlineCallback['\\'] = escape
376 p.inlineCallback['&'] = entity
377 p.inlineCallback['!'] = maybeImage
378 p.inlineCallback['^'] = maybeInlineFootnote
379
380 if extensions&Autolink != 0 {
381 p.inlineCallback['h'] = maybeAutoLink
382 p.inlineCallback['m'] = maybeAutoLink
383 p.inlineCallback['f'] = maybeAutoLink
384 p.inlineCallback['H'] = maybeAutoLink
385 p.inlineCallback['M'] = maybeAutoLink
386 p.inlineCallback['F'] = maybeAutoLink
387 }
388
389 if extensions&Footnotes != 0 {
390 p.notes = make([]*reference, 0)
391 }
392
393 first := firstPass(p, input)
394 second := secondPass(p, first)
395 return second
396}
397
398// first pass:
399// - extract references
400// - expand tabs
401// - normalize newlines
402// - copy everything else
403func firstPass(p *parser, input []byte) []byte {
404 var out bytes.Buffer
405 tabSize := TabSizeDefault
406 if p.flags&TabSizeEight != 0 {
407 tabSize = TabSizeDouble
408 }
409 beg, end := 0, 0
410 lastFencedCodeBlockEnd := 0
411 for beg < len(input) { // iterate over lines
412 if end = isReference(p, input[beg:], tabSize); end > 0 {
413 beg += end
414 } else { // skip to the next line
415 end = beg
416 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
417 end++
418 }
419
420 if p.flags&FencedCode != 0 {
421 // track fenced code block boundaries to suppress tab expansion
422 // inside them:
423 if beg >= lastFencedCodeBlockEnd {
424 if i := p.fencedCode(&out, input[beg:], false); i > 0 {
425 lastFencedCodeBlockEnd = beg + i
426 }
427 }
428 }
429
430 // add the line body if present
431 if end > beg {
432 if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
433 out.Write(input[beg:end])
434 } else {
435 expandTabs(&out, input[beg:end], tabSize)
436 }
437 }
438 out.WriteByte('\n')
439
440 if end < len(input) && input[end] == '\r' {
441 end++
442 }
443 if end < len(input) && input[end] == '\n' {
444 end++
445 }
446
447 beg = end
448 }
449 }
450
451 // empty input?
452 if out.Len() == 0 {
453 out.WriteByte('\n')
454 }
455
456 return out.Bytes()
457}
458
459// second pass: actual rendering
460func secondPass(p *parser, input []byte) []byte {
461 var output bytes.Buffer
462
463 p.r.DocumentHeader(&output)
464 p.block(&output, input)
465
466 if p.flags&Footnotes != 0 && len(p.notes) > 0 {
467 p.r.BeginFootnotes(&output)
468 flags := ListItemBeginningOfList
469 for i := 0; i < len(p.notes); i += 1 {
470 ref := p.notes[i]
471 var buf bytes.Buffer
472 if ref.hasBlock {
473 flags |= ListItemContainsBlock
474 p.block(&buf, ref.title)
475 } else {
476 p.inline(&buf, ref.title)
477 }
478 p.r.FootnoteItem(&output, ref.link, buf.Bytes(), flags)
479 flags &^= ListItemBeginningOfList | ListItemContainsBlock
480 }
481 p.r.EndFootnotes(&output)
482 }
483
484 p.r.DocumentFooter(&output)
485
486 if p.nesting != 0 {
487 panic("Nesting level did not end at zero")
488 }
489
490 return output.Bytes()
491}
492
493//
494// Link references
495//
496// This section implements support for references that (usually) appear
497// as footnotes in a document, and can be referenced anywhere in the document.
498// The basic format is:
499//
500// [1]: http://www.google.com/ "Google"
501// [2]: http://www.github.com/ "Github"
502//
503// Anywhere in the document, the reference can be linked by referring to its
504// label, i.e., 1 and 2 in this example, as in:
505//
506// This library is hosted on [Github][2], a git hosting site.
507//
508// Actual footnotes as specified in Pandoc and supported by some other Markdown
509// libraries such as php-markdown are also taken care of. They look like this:
510//
511// This sentence needs a bit of further explanation.[^note]
512//
513// [^note]: This is the explanation.
514//
515// Footnotes should be placed at the end of the document in an ordered list.
516// Inline footnotes such as:
517//
518// Inline footnotes^[Not supported.] also exist.
519//
520// are not yet supported.
521
522// References are parsed and stored in this struct.
523type reference struct {
524 link []byte
525 title []byte
526 noteId int // 0 if not a footnote ref
527 hasBlock bool
528 text []byte
529}
530
531func (r *reference) String() string {
532 return fmt.Sprintf("{link: %q, title: %q, text: %q, noteId: %d, hasBlock: %v}",
533 r.link, r.title, r.text, r.noteId, r.hasBlock)
534}
535
536// Check whether or not data starts with a reference link.
537// If so, it is parsed and stored in the list of references
538// (in the render struct).
539// Returns the number of bytes to skip to move past it,
540// or zero if the first line is not a reference.
541func isReference(p *parser, data []byte, tabSize int) int {
542 // up to 3 optional leading spaces
543 if len(data) < 4 {
544 return 0
545 }
546 i := 0
547 for i < 3 && data[i] == ' ' {
548 i++
549 }
550
551 noteId := 0
552
553 // id part: anything but a newline between brackets
554 if data[i] != '[' {
555 return 0
556 }
557 i++
558 if p.flags&Footnotes != 0 {
559 if i < len(data) && data[i] == '^' {
560 // we can set it to anything here because the proper noteIds will
561 // be assigned later during the second pass. It just has to be != 0
562 noteId = 1
563 i++
564 }
565 }
566 idOffset := i
567 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
568 i++
569 }
570 if i >= len(data) || data[i] != ']' {
571 return 0
572 }
573 idEnd := i
574
575 // spacer: colon (space | tab)* newline? (space | tab)*
576 i++
577 if i >= len(data) || data[i] != ':' {
578 return 0
579 }
580 i++
581 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
582 i++
583 }
584 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
585 i++
586 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
587 i++
588 }
589 }
590 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
591 i++
592 }
593 if i >= len(data) {
594 return 0
595 }
596
597 var (
598 linkOffset, linkEnd int
599 titleOffset, titleEnd int
600 lineEnd int
601 raw []byte
602 hasBlock bool
603 )
604
605 if p.flags&Footnotes != 0 && noteId != 0 {
606 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
607 lineEnd = linkEnd
608 } else {
609 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
610 }
611 if lineEnd == 0 {
612 return 0
613 }
614
615 // a valid ref has been found
616
617 ref := &reference{
618 noteId: noteId,
619 hasBlock: hasBlock,
620 }
621
622 if noteId > 0 {
623 // reusing the link field for the id since footnotes don't have links
624 ref.link = data[idOffset:idEnd]
625 // if footnote, it's not really a title, it's the contained text
626 ref.title = raw
627 } else {
628 ref.link = data[linkOffset:linkEnd]
629 ref.title = data[titleOffset:titleEnd]
630 }
631
632 // id matches are case-insensitive
633 id := string(bytes.ToLower(data[idOffset:idEnd]))
634
635 p.refs[id] = ref
636
637 return lineEnd
638}
639
640func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
641 // link: whitespace-free sequence, optionally between angle brackets
642 if data[i] == '<' {
643 i++
644 }
645 linkOffset = i
646 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
647 i++
648 }
649 if i == len(data) {
650 return
651 }
652 linkEnd = i
653 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
654 linkOffset++
655 linkEnd--
656 }
657
658 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
659 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
660 i++
661 }
662 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
663 return
664 }
665
666 // compute end-of-line
667 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
668 lineEnd = i
669 }
670 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
671 lineEnd++
672 }
673
674 // optional (space|tab)* spacer after a newline
675 if lineEnd > 0 {
676 i = lineEnd + 1
677 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
678 i++
679 }
680 }
681
682 // optional title: any non-newline sequence enclosed in '"() alone on its line
683 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
684 i++
685 titleOffset = i
686
687 // look for EOL
688 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
689 i++
690 }
691 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
692 titleEnd = i + 1
693 } else {
694 titleEnd = i
695 }
696
697 // step back
698 i--
699 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
700 i--
701 }
702 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
703 lineEnd = titleEnd
704 titleEnd = i
705 }
706 }
707
708 return
709}
710
711// The first bit of this logic is the same as (*parser).listItem, but the rest
712// is much simpler. This function simply finds the entire block and shifts it
713// over by one tab if it is indeed a block (just returns the line if it's not).
714// blockEnd is the end of the section in the input buffer, and contents is the
715// extracted text that was shifted over one tab. It will need to be rendered at
716// the end of the document.
717func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
718 if i == 0 || len(data) == 0 {
719 return
720 }
721
722 // skip leading whitespace on first line
723 for i < len(data) && data[i] == ' ' {
724 i++
725 }
726
727 blockStart = i
728
729 // find the end of the line
730 blockEnd = i
731 for i < len(data) && data[i-1] != '\n' {
732 i++
733 }
734
735 // get working buffer
736 var raw bytes.Buffer
737
738 // put the first line into the working buffer
739 raw.Write(data[blockEnd:i])
740 blockEnd = i
741
742 // process the following lines
743 containsBlankLine := false
744
745gatherLines:
746 for blockEnd < len(data) {
747 i++
748
749 // find the end of this line
750 for i < len(data) && data[i-1] != '\n' {
751 i++
752 }
753
754 // if it is an empty line, guess that it is part of this item
755 // and move on to the next line
756 if p.isEmpty(data[blockEnd:i]) > 0 {
757 containsBlankLine = true
758 blockEnd = i
759 continue
760 }
761
762 n := 0
763 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
764 // this is the end of the block.
765 // we don't want to include this last line in the index.
766 break gatherLines
767 }
768
769 // if there were blank lines before this one, insert a new one now
770 if containsBlankLine {
771 raw.WriteByte('\n')
772 containsBlankLine = false
773 }
774
775 // get rid of that first tab, write to buffer
776 raw.Write(data[blockEnd+n : i])
777 hasBlock = true
778
779 blockEnd = i
780 }
781
782 if data[blockEnd-1] != '\n' {
783 raw.WriteByte('\n')
784 }
785
786 contents = raw.Bytes()
787
788 return
789}
790
791//
792//
793// Miscellaneous helper functions
794//
795//
796
797// Test if a character is a punctuation symbol.
798// Taken from a private function in regexp in the stdlib.
799func ispunct(c byte) bool {
800 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
801 if c == r {
802 return true
803 }
804 }
805 return false
806}
807
808// Test if a character is a whitespace character.
809func isspace(c byte) bool {
810 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
811}
812
813// Test if a character is letter.
814func isletter(c byte) bool {
815 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
816}
817
818// Test if a character is a letter or a digit.
819// TODO: check when this is looking for ASCII alnum and when it should use unicode
820func isalnum(c byte) bool {
821 return (c >= '0' && c <= '9') || isletter(c)
822}
823
824// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
825// always ends output with a newline
826func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
827 // first, check for common cases: no tabs, or only tabs at beginning of line
828 i, prefix := 0, 0
829 slowcase := false
830 for i = 0; i < len(line); i++ {
831 if line[i] == '\t' {
832 if prefix == i {
833 prefix++
834 } else {
835 slowcase = true
836 break
837 }
838 }
839 }
840
841 // no need to decode runes if all tabs are at the beginning of the line
842 if !slowcase {
843 for i = 0; i < prefix*tabSize; i++ {
844 out.WriteByte(' ')
845 }
846 out.Write(line[prefix:])
847 return
848 }
849
850 // the slow case: we need to count runes to figure out how
851 // many spaces to insert for each tab
852 column := 0
853 i = 0
854 for i < len(line) {
855 start := i
856 for i < len(line) && line[i] != '\t' {
857 _, size := utf8.DecodeRune(line[i:])
858 i += size
859 column++
860 }
861
862 if i > start {
863 out.Write(line[start:i])
864 }
865
866 if i >= len(line) {
867 break
868 }
869
870 for {
871 out.WriteByte(' ')
872 column++
873 if column%tabSize == 0 {
874 break
875 }
876 }
877
878 i++
879 }
880}
881
882// Find if a line counts as indented or not.
883// Returns number of characters the indent is (0 = not indented).
884func isIndented(data []byte, indentSize int) int {
885 if len(data) == 0 {
886 return 0
887 }
888 if data[0] == '\t' {
889 return 1
890 }
891 if len(data) < indentSize {
892 return 0
893 }
894 for i := 0; i < indentSize; i++ {
895 if data[i] != ' ' {
896 return 0
897 }
898 }
899 return indentSize
900}
901
902// Create a url-safe slug for fragments
903func slugify(in []byte) []byte {
904 if len(in) == 0 {
905 return in
906 }
907 out := make([]byte, 0, len(in))
908 sym := false
909
910 for _, ch := range in {
911 if isalnum(ch) {
912 sym = false
913 out = append(out, ch)
914 } else if sym {
915 continue
916 } else {
917 out = append(out, '-')
918 sym = true
919 }
920 }
921 var a, b int
922 var ch byte
923 for a, ch = range out {
924 if ch != '-' {
925 break
926 }
927 }
928 for b = len(out) - 1; b > 0; b-- {
929 if out[b] != '-' {
930 break
931 }
932 }
933 return out[a : b+1]
934}