markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "fmt"
24 "strings"
25 "unicode/utf8"
26)
27
28const VERSION = "1.4"
29
30type Extensions int
31
32// These are the supported markdown parsing extensions.
33// OR these values together to select multiple extensions.
34const (
35 NoExtensions Extensions = 0
36 NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
37 Tables // Render tables
38 FencedCode // Render fenced code blocks
39 Autolink // Detect embedded URLs that are not explicitly marked
40 Strikethrough // Strikethrough text using ~~test~~
41 LaxHTMLBlocks // Loosen up HTML block parsing rules
42 SpaceHeaders // Be strict about prefix header rules
43 HardLineBreak // Translate newlines into line breaks
44 TabSizeEight // Expand tabs to eight spaces instead of four
45 Footnotes // Pandoc-style footnotes
46 NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
47 HeaderIDs // specify header IDs with {#id}
48 Titleblock // Titleblock ala pandoc
49 AutoHeaderIDs // Create the header ID from the text
50 BackslashLineBreak // Translate trailing backslashes into line breaks
51 DefinitionLists // Render definition lists
52
53 commonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants |
54 SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
55
56 commonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
57 Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
58 BackslashLineBreak | DefinitionLists
59)
60
61type LinkType int
62
63// These are the possible flag values for the link renderer.
64// Only a single one of these values will be used; they are not ORed together.
65// These are mostly of interest if you are writing a new output format.
66const (
67 LinkTypeNotAutolink LinkType = iota
68 LinkTypeNormal
69 LinkTypeEmail
70)
71
72type ListType int
73
74// These are the possible flag values for the ListItem renderer.
75// Multiple flag values may be ORed together.
76// These are mostly of interest if you are writing a new output format.
77const (
78 ListTypeOrdered ListType = 1 << iota
79 ListTypeDefinition
80 ListTypeTerm
81
82 ListItemContainsBlock
83 ListItemBeginningOfList
84 ListItemEndOfList
85)
86
87type TableFlags int
88
89// These are the possible flag values for the table cell renderer.
90// Only a single one of these values will be used; they are not ORed together.
91// These are mostly of interest if you are writing a new output format.
92const (
93 TableAlignmentLeft = 1 << iota
94 TableAlignmentRight
95 TableAlignmentCenter = (TableAlignmentLeft | TableAlignmentRight)
96)
97
98// The size of a tab stop.
99const (
100 TabSizeDefault = 4
101 TabSizeDouble = 8
102)
103
104// blockTags is a set of tags that are recognized as HTML block tags.
105// Any of these can be included in markdown text without special escaping.
106var blockTags = map[string]struct{}{
107 "blockquote": struct{}{},
108 "del": struct{}{},
109 "div": struct{}{},
110 "dl": struct{}{},
111 "fieldset": struct{}{},
112 "form": struct{}{},
113 "h1": struct{}{},
114 "h2": struct{}{},
115 "h3": struct{}{},
116 "h4": struct{}{},
117 "h5": struct{}{},
118 "h6": struct{}{},
119 "iframe": struct{}{},
120 "ins": struct{}{},
121 "math": struct{}{},
122 "noscript": struct{}{},
123 "ol": struct{}{},
124 "pre": struct{}{},
125 "p": struct{}{},
126 "script": struct{}{},
127 "style": struct{}{},
128 "table": struct{}{},
129 "ul": struct{}{},
130
131 // HTML5
132 "address": struct{}{},
133 "article": struct{}{},
134 "aside": struct{}{},
135 "canvas": struct{}{},
136 "figcaption": struct{}{},
137 "figure": struct{}{},
138 "footer": struct{}{},
139 "header": struct{}{},
140 "hgroup": struct{}{},
141 "main": struct{}{},
142 "nav": struct{}{},
143 "output": struct{}{},
144 "progress": struct{}{},
145 "section": struct{}{},
146 "video": struct{}{},
147}
148
149// Renderer is the rendering interface.
150// This is mostly of interest if you are implementing a new rendering format.
151//
152// When a byte slice is provided, it contains the (rendered) contents of the
153// element.
154//
155// When a callback is provided instead, it will write the contents of the
156// respective element directly to the output buffer and return true on success.
157// If the callback returns false, the rendering function should reset the
158// output buffer as though it had never been called.
159//
160// Currently Html and Latex implementations are provided
161type Renderer interface {
162 // block-level callbacks
163 BlockCode(text []byte, lang string)
164 BlockQuote(text []byte)
165 BlockHtml(text []byte)
166 BeginHeader(level int, id string)
167 EndHeader(level int, id string, header []byte)
168 HRule()
169 BeginList(flags ListType)
170 EndList(flags ListType)
171 ListItem(text []byte, flags ListType)
172 BeginParagraph()
173 EndParagraph()
174 Table(header []byte, body []byte, columnData []int)
175 TableRow(text []byte)
176 TableHeaderCell(out *bytes.Buffer, text []byte, flags int)
177 TableCell(out *bytes.Buffer, text []byte, flags int)
178 BeginFootnotes()
179 EndFootnotes()
180 FootnoteItem(name, text []byte, flags ListType)
181 TitleBlock(text []byte)
182
183 // Span-level callbacks
184 AutoLink(link []byte, kind LinkType)
185 CodeSpan(text []byte)
186 DoubleEmphasis(text []byte)
187 Emphasis(text []byte)
188 Image(link []byte, title []byte, alt []byte)
189 LineBreak()
190 Link(link []byte, title []byte, content []byte)
191 RawHtmlTag(tag []byte)
192 TripleEmphasis(text []byte)
193 StrikeThrough(text []byte)
194 FootnoteRef(ref []byte, id int)
195
196 // Low-level callbacks
197 Entity(entity []byte)
198 NormalText(text []byte)
199
200 // Header and footer
201 DocumentHeader()
202 DocumentFooter()
203
204 GetFlags() HtmlFlags
205 CaptureWrites(processor func()) []byte
206 CopyWrites(processor func()) []byte
207 Write(b []byte) (int, error)
208 GetResult() []byte
209}
210
211// Callback functions for inline parsing. One such function is defined
212// for each character that triggers a response when parsing inline data.
213type inlineParser func(p *parser, data []byte, offset int) int
214
215// Parser holds runtime state used by the parser.
216// This is constructed by the Markdown function.
217type parser struct {
218 r Renderer
219 refOverride ReferenceOverrideFunc
220 refs map[string]*reference
221 inlineCallback [256]inlineParser
222 flags Extensions
223 nesting int
224 maxNesting int
225 insideLink bool
226
227 // Footnotes need to be ordered as well as available to quickly check for
228 // presence. If a ref is also a footnote, it's stored both in refs and here
229 // in notes. Slice is nil if footnotes not enabled.
230 notes []*reference
231
232 doc *Node
233 tip *Node // = doc
234 oldTip *Node
235 lastMatchedContainer *Node // = doc
236 allClosed bool
237}
238
239func (p *parser) getRef(refid string) (ref *reference, found bool) {
240 if p.refOverride != nil {
241 r, overridden := p.refOverride(refid)
242 if overridden {
243 if r == nil {
244 return nil, false
245 }
246 return &reference{
247 link: []byte(r.Link),
248 title: []byte(r.Title),
249 noteId: 0,
250 hasBlock: false,
251 text: []byte(r.Text)}, true
252 }
253 }
254 // refs are case insensitive
255 ref, found = p.refs[strings.ToLower(refid)]
256 return ref, found
257}
258
259func (p *parser) finalize(block *Node) {
260 above := block.Parent
261 block.open = false
262 p.tip = above
263}
264
265func (p *parser) addChild(node NodeType, offset uint32) *Node {
266 for !p.tip.canContain(node) {
267 p.finalize(p.tip)
268 }
269 newNode := NewNode(node)
270 newNode.content = []byte{}
271 p.tip.appendChild(newNode)
272 p.tip = newNode
273 return newNode
274}
275
276func (p *parser) closeUnmatchedBlocks() {
277 if !p.allClosed {
278 for p.oldTip != p.lastMatchedContainer {
279 parent := p.oldTip.Parent
280 p.finalize(p.oldTip)
281 p.oldTip = parent
282 }
283 p.allClosed = true
284 }
285}
286
287//
288//
289// Public interface
290//
291//
292
293// Reference represents the details of a link.
294// See the documentation in Options for more details on use-case.
295type Reference struct {
296 // Link is usually the URL the reference points to.
297 Link string
298 // Title is the alternate text describing the link in more detail.
299 Title string
300 // Text is the optional text to override the ref with if the syntax used was
301 // [refid][]
302 Text string
303}
304
305// ReferenceOverrideFunc is expected to be called with a reference string and
306// return either a valid Reference type that the reference string maps to or
307// nil. If overridden is false, the default reference logic will be executed.
308// See the documentation in Options for more details on use-case.
309type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
310
311// Options represents configurable overrides and callbacks (in addition to the
312// extension flag set) for configuring a Markdown parse.
313type Options struct {
314 // Extensions is a flag set of bit-wise ORed extension bits. See the
315 // Extensions flags defined in this package.
316 Extensions Extensions
317
318 // ReferenceOverride is an optional function callback that is called every
319 // time a reference is resolved.
320 //
321 // In Markdown, the link reference syntax can be made to resolve a link to
322 // a reference instead of an inline URL, in one of the following ways:
323 //
324 // * [link text][refid]
325 // * [refid][]
326 //
327 // Usually, the refid is defined at the bottom of the Markdown document. If
328 // this override function is provided, the refid is passed to the override
329 // function first, before consulting the defined refids at the bottom. If
330 // the override function indicates an override did not occur, the refids at
331 // the bottom will be used to fill in the link details.
332 ReferenceOverride ReferenceOverrideFunc
333}
334
335// MarkdownBasic is a convenience function for simple rendering.
336// It processes markdown input with no extensions enabled.
337func MarkdownBasic(input []byte) []byte {
338 // set up the HTML renderer
339 htmlFlags := UseXHTML
340 renderer := HtmlRenderer(htmlFlags, "", "")
341
342 // set up the parser
343 return MarkdownOptions(input, renderer, Options{Extensions: 0})
344}
345
346// Call Markdown with most useful extensions enabled
347// MarkdownCommon is a convenience function for simple rendering.
348// It processes markdown input with common extensions enabled, including:
349//
350// * Smartypants processing with smart fractions and LaTeX dashes
351//
352// * Intra-word emphasis suppression
353//
354// * Tables
355//
356// * Fenced code blocks
357//
358// * Autolinking
359//
360// * Strikethrough support
361//
362// * Strict header parsing
363//
364// * Custom Header IDs
365func MarkdownCommon(input []byte) []byte {
366 // set up the HTML renderer
367 renderer := HtmlRenderer(commonHtmlFlags, "", "")
368 return MarkdownOptions(input, renderer, Options{
369 Extensions: commonExtensions})
370}
371
372// Markdown is the main rendering function.
373// It parses and renders a block of markdown-encoded text.
374// The supplied Renderer is used to format the output, and extensions dictates
375// which non-standard extensions are enabled.
376//
377// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
378// LatexRenderer, respectively.
379func Markdown(input []byte, renderer Renderer, extensions Extensions) []byte {
380 return MarkdownOptions(input, renderer, Options{
381 Extensions: extensions})
382}
383
384// MarkdownOptions is just like Markdown but takes additional options through
385// the Options struct.
386func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
387 // no point in parsing if we can't render
388 if renderer == nil {
389 return nil
390 }
391
392 extensions := opts.Extensions
393
394 // fill in the render structure
395 p := new(parser)
396 p.r = renderer
397 p.flags = extensions
398 p.refOverride = opts.ReferenceOverride
399 p.refs = make(map[string]*reference)
400 p.maxNesting = 16
401 p.insideLink = false
402
403 docNode := NewNode(Document)
404 p.doc = docNode
405 p.tip = docNode
406 p.oldTip = docNode
407 p.lastMatchedContainer = docNode
408 p.allClosed = true
409
410 // register inline parsers
411 p.inlineCallback['*'] = emphasis
412 p.inlineCallback['_'] = emphasis
413 if extensions&Strikethrough != 0 {
414 p.inlineCallback['~'] = emphasis
415 }
416 p.inlineCallback['`'] = codeSpan
417 p.inlineCallback['\n'] = lineBreak
418 p.inlineCallback['['] = link
419 p.inlineCallback['<'] = leftAngle
420 p.inlineCallback['\\'] = escape
421 p.inlineCallback['&'] = entity
422 p.inlineCallback['!'] = maybeImage
423 p.inlineCallback['^'] = maybeInlineFootnote
424
425 if extensions&Autolink != 0 {
426 p.inlineCallback['h'] = maybeAutoLink
427 p.inlineCallback['m'] = maybeAutoLink
428 p.inlineCallback['f'] = maybeAutoLink
429 p.inlineCallback['H'] = maybeAutoLink
430 p.inlineCallback['M'] = maybeAutoLink
431 p.inlineCallback['F'] = maybeAutoLink
432 }
433
434 if extensions&Footnotes != 0 {
435 p.notes = make([]*reference, 0)
436 }
437
438 first := firstPass(p, input)
439 second := secondPass(p, first)
440 return second
441}
442
443// first pass:
444// - extract references
445// - expand tabs
446// - normalize newlines
447// - copy everything else
448func firstPass(p *parser, input []byte) []byte {
449 var out bytes.Buffer
450 tabSize := TabSizeDefault
451 if p.flags&TabSizeEight != 0 {
452 tabSize = TabSizeDouble
453 }
454 beg, end := 0, 0
455 lastFencedCodeBlockEnd := 0
456 for beg < len(input) { // iterate over lines
457 if end = isReference(p, input[beg:], tabSize); end > 0 {
458 beg += end
459 } else { // skip to the next line
460 end = beg
461 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
462 end++
463 }
464
465 if p.flags&FencedCode != 0 {
466 // track fenced code block boundaries to suppress tab expansion
467 // inside them:
468 if beg >= lastFencedCodeBlockEnd {
469 if i := p.fencedCode(input[beg:], false); i > 0 {
470 lastFencedCodeBlockEnd = beg + i
471 }
472 }
473 }
474
475 // add the line body if present
476 if end > beg {
477 if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
478 out.Write(input[beg:end])
479 } else {
480 expandTabs(&out, input[beg:end], tabSize)
481 }
482 }
483 out.WriteByte('\n')
484
485 if end < len(input) && input[end] == '\r' {
486 end++
487 }
488 if end < len(input) && input[end] == '\n' {
489 end++
490 }
491
492 beg = end
493 }
494 }
495
496 // empty input?
497 if out.Len() == 0 {
498 out.WriteByte('\n')
499 }
500
501 return out.Bytes()
502}
503
504// second pass: actual rendering
505func secondPass(p *parser, input []byte) []byte {
506 p.r.DocumentHeader()
507 p.block(input)
508
509 if p.flags&Footnotes != 0 && len(p.notes) > 0 {
510 p.r.BeginFootnotes()
511 flags := ListItemBeginningOfList
512 for i := 0; i < len(p.notes); i += 1 {
513 ref := p.notes[i]
514 var buf bytes.Buffer
515 if ref.hasBlock {
516 flags |= ListItemContainsBlock
517 buf.Write(p.r.CaptureWrites(func() {
518 p.block(ref.title)
519 }))
520 } else {
521 buf.Write(p.r.CaptureWrites(func() {
522 p.inline(ref.title)
523 }))
524 }
525 p.r.FootnoteItem(ref.link, buf.Bytes(), flags)
526 flags &^= ListItemBeginningOfList | ListItemContainsBlock
527 }
528 p.r.EndFootnotes()
529 }
530
531 p.r.DocumentFooter()
532
533 if p.nesting != 0 {
534 panic("Nesting level did not end at zero")
535 }
536
537 return p.r.GetResult()
538}
539
540//
541// Link references
542//
543// This section implements support for references that (usually) appear
544// as footnotes in a document, and can be referenced anywhere in the document.
545// The basic format is:
546//
547// [1]: http://www.google.com/ "Google"
548// [2]: http://www.github.com/ "Github"
549//
550// Anywhere in the document, the reference can be linked by referring to its
551// label, i.e., 1 and 2 in this example, as in:
552//
553// This library is hosted on [Github][2], a git hosting site.
554//
555// Actual footnotes as specified in Pandoc and supported by some other Markdown
556// libraries such as php-markdown are also taken care of. They look like this:
557//
558// This sentence needs a bit of further explanation.[^note]
559//
560// [^note]: This is the explanation.
561//
562// Footnotes should be placed at the end of the document in an ordered list.
563// Inline footnotes such as:
564//
565// Inline footnotes^[Not supported.] also exist.
566//
567// are not yet supported.
568
569// References are parsed and stored in this struct.
570type reference struct {
571 link []byte
572 title []byte
573 noteId int // 0 if not a footnote ref
574 hasBlock bool
575 text []byte
576}
577
578func (r *reference) String() string {
579 return fmt.Sprintf("{link: %q, title: %q, text: %q, noteId: %d, hasBlock: %v}",
580 r.link, r.title, r.text, r.noteId, r.hasBlock)
581}
582
583// Check whether or not data starts with a reference link.
584// If so, it is parsed and stored in the list of references
585// (in the render struct).
586// Returns the number of bytes to skip to move past it,
587// or zero if the first line is not a reference.
588func isReference(p *parser, data []byte, tabSize int) int {
589 // up to 3 optional leading spaces
590 if len(data) < 4 {
591 return 0
592 }
593 i := 0
594 for i < 3 && data[i] == ' ' {
595 i++
596 }
597
598 noteId := 0
599
600 // id part: anything but a newline between brackets
601 if data[i] != '[' {
602 return 0
603 }
604 i++
605 if p.flags&Footnotes != 0 {
606 if i < len(data) && data[i] == '^' {
607 // we can set it to anything here because the proper noteIds will
608 // be assigned later during the second pass. It just has to be != 0
609 noteId = 1
610 i++
611 }
612 }
613 idOffset := i
614 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
615 i++
616 }
617 if i >= len(data) || data[i] != ']' {
618 return 0
619 }
620 idEnd := i
621
622 // spacer: colon (space | tab)* newline? (space | tab)*
623 i++
624 if i >= len(data) || data[i] != ':' {
625 return 0
626 }
627 i++
628 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
629 i++
630 }
631 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
632 i++
633 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
634 i++
635 }
636 }
637 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
638 i++
639 }
640 if i >= len(data) {
641 return 0
642 }
643
644 var (
645 linkOffset, linkEnd int
646 titleOffset, titleEnd int
647 lineEnd int
648 raw []byte
649 hasBlock bool
650 )
651
652 if p.flags&Footnotes != 0 && noteId != 0 {
653 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
654 lineEnd = linkEnd
655 } else {
656 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
657 }
658 if lineEnd == 0 {
659 return 0
660 }
661
662 // a valid ref has been found
663
664 ref := &reference{
665 noteId: noteId,
666 hasBlock: hasBlock,
667 }
668
669 if noteId > 0 {
670 // reusing the link field for the id since footnotes don't have links
671 ref.link = data[idOffset:idEnd]
672 // if footnote, it's not really a title, it's the contained text
673 ref.title = raw
674 } else {
675 ref.link = data[linkOffset:linkEnd]
676 ref.title = data[titleOffset:titleEnd]
677 }
678
679 // id matches are case-insensitive
680 id := string(bytes.ToLower(data[idOffset:idEnd]))
681
682 p.refs[id] = ref
683
684 return lineEnd
685}
686
687func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
688 // link: whitespace-free sequence, optionally between angle brackets
689 if data[i] == '<' {
690 i++
691 }
692 linkOffset = i
693 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
694 i++
695 }
696 if i == len(data) {
697 return
698 }
699 linkEnd = i
700 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
701 linkOffset++
702 linkEnd--
703 }
704
705 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
706 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
707 i++
708 }
709 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
710 return
711 }
712
713 // compute end-of-line
714 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
715 lineEnd = i
716 }
717 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
718 lineEnd++
719 }
720
721 // optional (space|tab)* spacer after a newline
722 if lineEnd > 0 {
723 i = lineEnd + 1
724 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
725 i++
726 }
727 }
728
729 // optional title: any non-newline sequence enclosed in '"() alone on its line
730 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
731 i++
732 titleOffset = i
733
734 // look for EOL
735 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
736 i++
737 }
738 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
739 titleEnd = i + 1
740 } else {
741 titleEnd = i
742 }
743
744 // step back
745 i--
746 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
747 i--
748 }
749 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
750 lineEnd = titleEnd
751 titleEnd = i
752 }
753 }
754
755 return
756}
757
758// The first bit of this logic is the same as (*parser).listItem, but the rest
759// is much simpler. This function simply finds the entire block and shifts it
760// over by one tab if it is indeed a block (just returns the line if it's not).
761// blockEnd is the end of the section in the input buffer, and contents is the
762// extracted text that was shifted over one tab. It will need to be rendered at
763// the end of the document.
764func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
765 if i == 0 || len(data) == 0 {
766 return
767 }
768
769 // skip leading whitespace on first line
770 for i < len(data) && data[i] == ' ' {
771 i++
772 }
773
774 blockStart = i
775
776 // find the end of the line
777 blockEnd = i
778 for i < len(data) && data[i-1] != '\n' {
779 i++
780 }
781
782 // get working buffer
783 var raw bytes.Buffer
784
785 // put the first line into the working buffer
786 raw.Write(data[blockEnd:i])
787 blockEnd = i
788
789 // process the following lines
790 containsBlankLine := false
791
792gatherLines:
793 for blockEnd < len(data) {
794 i++
795
796 // find the end of this line
797 for i < len(data) && data[i-1] != '\n' {
798 i++
799 }
800
801 // if it is an empty line, guess that it is part of this item
802 // and move on to the next line
803 if p.isEmpty(data[blockEnd:i]) > 0 {
804 containsBlankLine = true
805 blockEnd = i
806 continue
807 }
808
809 n := 0
810 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
811 // this is the end of the block.
812 // we don't want to include this last line in the index.
813 break gatherLines
814 }
815
816 // if there were blank lines before this one, insert a new one now
817 if containsBlankLine {
818 raw.WriteByte('\n')
819 containsBlankLine = false
820 }
821
822 // get rid of that first tab, write to buffer
823 raw.Write(data[blockEnd+n : i])
824 hasBlock = true
825
826 blockEnd = i
827 }
828
829 if data[blockEnd-1] != '\n' {
830 raw.WriteByte('\n')
831 }
832
833 contents = raw.Bytes()
834
835 return
836}
837
838//
839//
840// Miscellaneous helper functions
841//
842//
843
844// Test if a character is a punctuation symbol.
845// Taken from a private function in regexp in the stdlib.
846func ispunct(c byte) bool {
847 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
848 if c == r {
849 return true
850 }
851 }
852 return false
853}
854
855// Test if a character is a whitespace character.
856func isspace(c byte) bool {
857 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
858}
859
860// Test if a character is letter.
861func isletter(c byte) bool {
862 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
863}
864
865// Test if a character is a letter or a digit.
866// TODO: check when this is looking for ASCII alnum and when it should use unicode
867func isalnum(c byte) bool {
868 return (c >= '0' && c <= '9') || isletter(c)
869}
870
871// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
872// always ends output with a newline
873func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
874 // first, check for common cases: no tabs, or only tabs at beginning of line
875 i, prefix := 0, 0
876 slowcase := false
877 for i = 0; i < len(line); i++ {
878 if line[i] == '\t' {
879 if prefix == i {
880 prefix++
881 } else {
882 slowcase = true
883 break
884 }
885 }
886 }
887
888 // no need to decode runes if all tabs are at the beginning of the line
889 if !slowcase {
890 for i = 0; i < prefix*tabSize; i++ {
891 out.WriteByte(' ')
892 }
893 out.Write(line[prefix:])
894 return
895 }
896
897 // the slow case: we need to count runes to figure out how
898 // many spaces to insert for each tab
899 column := 0
900 i = 0
901 for i < len(line) {
902 start := i
903 for i < len(line) && line[i] != '\t' {
904 _, size := utf8.DecodeRune(line[i:])
905 i += size
906 column++
907 }
908
909 if i > start {
910 out.Write(line[start:i])
911 }
912
913 if i >= len(line) {
914 break
915 }
916
917 for {
918 out.WriteByte(' ')
919 column++
920 if column%tabSize == 0 {
921 break
922 }
923 }
924
925 i++
926 }
927}
928
929// Find if a line counts as indented or not.
930// Returns number of characters the indent is (0 = not indented).
931func isIndented(data []byte, indentSize int) int {
932 if len(data) == 0 {
933 return 0
934 }
935 if data[0] == '\t' {
936 return 1
937 }
938 if len(data) < indentSize {
939 return 0
940 }
941 for i := 0; i < indentSize; i++ {
942 if data[i] != ' ' {
943 return 0
944 }
945 }
946 return indentSize
947}
948
949// Create a url-safe slug for fragments
950func slugify(in []byte) []byte {
951 if len(in) == 0 {
952 return in
953 }
954 out := make([]byte, 0, len(in))
955 sym := false
956
957 for _, ch := range in {
958 if isalnum(ch) {
959 sym = false
960 out = append(out, ch)
961 } else if sym {
962 continue
963 } else {
964 out = append(out, '-')
965 sym = true
966 }
967 }
968 var a, b int
969 var ch byte
970 for a, ch = range out {
971 if ch != '-' {
972 break
973 }
974 }
975 for b = len(out) - 1; b > 0; b-- {
976 if out[b] != '-' {
977 break
978 }
979 }
980 return out[a : b+1]
981}