markdown.go (view raw)
1// Blackfriday Markdown Processor
2// Available at http://github.com/russross/blackfriday
3//
4// Copyright © 2011 Russ Ross <russ@russross.com>.
5// Distributed under the Simplified BSD License.
6// See README.md for details.
7
8package blackfriday
9
10import (
11 "bytes"
12 "fmt"
13 "io"
14 "strings"
15 "unicode/utf8"
16)
17
18//
19// Markdown parsing and processing
20//
21
22// Version string of the package.
23const Version = "2.0"
24
25// Extensions is a bitwise or'ed collection of enabled Blackfriday's
26// extensions.
27type Extensions int
28
29// These are the supported markdown parsing extensions.
30// OR these values together to select multiple extensions.
31const (
32 NoExtensions Extensions = 0
33 NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words
34 Tables // Render tables
35 FencedCode // Render fenced code blocks
36 Autolink // Detect embedded URLs that are not explicitly marked
37 Strikethrough // Strikethrough text using ~~test~~
38 LaxHTMLBlocks // Loosen up HTML block parsing rules
39 SpaceHeaders // Be strict about prefix header rules
40 HardLineBreak // Translate newlines into line breaks
41 TabSizeEight // Expand tabs to eight spaces instead of four
42 Footnotes // Pandoc-style footnotes
43 NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block
44 HeaderIDs // specify header IDs with {#id}
45 Titleblock // Titleblock ala pandoc
46 AutoHeaderIDs // Create the header ID from the text
47 BackslashLineBreak // Translate trailing backslashes into line breaks
48 DefinitionLists // Render definition lists
49
50 CommonHTMLFlags HTMLFlags = UseXHTML | Smartypants |
51 SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes
52
53 CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode |
54 Autolink | Strikethrough | SpaceHeaders | HeaderIDs |
55 BackslashLineBreak | DefinitionLists
56)
57
58// DefaultOptions is a convenience variable with all the options that are
59// enabled by default.
60var DefaultOptions = Options{
61 Extensions: CommonExtensions,
62}
63
64// ListType contains bitwise or'ed flags for list and list item objects.
65type ListType int
66
67// These are the possible flag values for the ListItem renderer.
68// Multiple flag values may be ORed together.
69// These are mostly of interest if you are writing a new output format.
70const (
71 ListTypeOrdered ListType = 1 << iota
72 ListTypeDefinition
73 ListTypeTerm
74
75 ListItemContainsBlock
76 ListItemBeginningOfList // TODO: figure out if this is of any use now
77 ListItemEndOfList
78)
79
80// CellAlignFlags holds a type of alignment in a table cell.
81type CellAlignFlags int
82
83// These are the possible flag values for the table cell renderer.
84// Only a single one of these values will be used; they are not ORed together.
85// These are mostly of interest if you are writing a new output format.
86const (
87 TableAlignmentLeft CellAlignFlags = 1 << iota
88 TableAlignmentRight
89 TableAlignmentCenter = (TableAlignmentLeft | TableAlignmentRight)
90)
91
92// The size of a tab stop.
93const (
94 TabSizeDefault = 4
95 TabSizeDouble = 8
96)
97
98// blockTags is a set of tags that are recognized as HTML block tags.
99// Any of these can be included in markdown text without special escaping.
100var blockTags = map[string]struct{}{
101 "blockquote": struct{}{},
102 "del": struct{}{},
103 "div": struct{}{},
104 "dl": struct{}{},
105 "fieldset": struct{}{},
106 "form": struct{}{},
107 "h1": struct{}{},
108 "h2": struct{}{},
109 "h3": struct{}{},
110 "h4": struct{}{},
111 "h5": struct{}{},
112 "h6": struct{}{},
113 "iframe": struct{}{},
114 "ins": struct{}{},
115 "math": struct{}{},
116 "noscript": struct{}{},
117 "ol": struct{}{},
118 "pre": struct{}{},
119 "p": struct{}{},
120 "script": struct{}{},
121 "style": struct{}{},
122 "table": struct{}{},
123 "ul": struct{}{},
124
125 // HTML5
126 "address": struct{}{},
127 "article": struct{}{},
128 "aside": struct{}{},
129 "canvas": struct{}{},
130 "figcaption": struct{}{},
131 "figure": struct{}{},
132 "footer": struct{}{},
133 "header": struct{}{},
134 "hgroup": struct{}{},
135 "main": struct{}{},
136 "nav": struct{}{},
137 "output": struct{}{},
138 "progress": struct{}{},
139 "section": struct{}{},
140 "video": struct{}{},
141}
142
143// Renderer is the rendering interface.
144// This is mostly of interest if you are implementing a new rendering format.
145//
146// When a byte slice is provided, it contains the (rendered) contents of the
147// element.
148//
149// When a callback is provided instead, it will write the contents of the
150// respective element directly to the output buffer and return true on success.
151// If the callback returns false, the rendering function should reset the
152// output buffer as though it had never been called.
153//
154// Only an HTML implementation is provided in this repository,
155// see the README for external implementations.
156type Renderer interface {
157 Render(ast *Node) []byte
158 RenderNode(w io.Writer, node *Node, entering bool) WalkStatus
159}
160
161// Callback functions for inline parsing. One such function is defined
162// for each character that triggers a response when parsing inline data.
163type inlineParser func(p *parser, data []byte, offset int) (int, *Node)
164
165// Parser holds runtime state used by the parser.
166// This is constructed by the Markdown function.
167type parser struct {
168 refOverride ReferenceOverrideFunc
169 refs map[string]*reference
170 inlineCallback [256]inlineParser
171 flags Extensions
172 nesting int
173 maxNesting int
174 insideLink bool
175
176 // Footnotes need to be ordered as well as available to quickly check for
177 // presence. If a ref is also a footnote, it's stored both in refs and here
178 // in notes. Slice is nil if footnotes not enabled.
179 notes []*reference
180
181 doc *Node
182 tip *Node // = doc
183 oldTip *Node
184 lastMatchedContainer *Node // = doc
185 allClosed bool
186}
187
188func (p *parser) getRef(refid string) (ref *reference, found bool) {
189 if p.refOverride != nil {
190 r, overridden := p.refOverride(refid)
191 if overridden {
192 if r == nil {
193 return nil, false
194 }
195 return &reference{
196 link: []byte(r.Link),
197 title: []byte(r.Title),
198 noteID: 0,
199 hasBlock: false,
200 text: []byte(r.Text)}, true
201 }
202 }
203 // refs are case insensitive
204 ref, found = p.refs[strings.ToLower(refid)]
205 return ref, found
206}
207
208func (p *parser) finalize(block *Node) {
209 above := block.Parent
210 block.open = false
211 p.tip = above
212}
213
214func (p *parser) addChild(node NodeType, offset uint32) *Node {
215 return p.addExistingChild(NewNode(node), offset)
216}
217
218func (p *parser) addExistingChild(node *Node, offset uint32) *Node {
219 for !p.tip.canContain(node.Type) {
220 p.finalize(p.tip)
221 }
222 p.tip.AppendChild(node)
223 p.tip = node
224 return node
225}
226
227func (p *parser) closeUnmatchedBlocks() {
228 if !p.allClosed {
229 for p.oldTip != p.lastMatchedContainer {
230 parent := p.oldTip.Parent
231 p.finalize(p.oldTip)
232 p.oldTip = parent
233 }
234 p.allClosed = true
235 }
236}
237
238//
239//
240// Public interface
241//
242//
243
244// Reference represents the details of a link.
245// See the documentation in Options for more details on use-case.
246type Reference struct {
247 // Link is usually the URL the reference points to.
248 Link string
249 // Title is the alternate text describing the link in more detail.
250 Title string
251 // Text is the optional text to override the ref with if the syntax used was
252 // [refid][]
253 Text string
254}
255
256// ReferenceOverrideFunc is expected to be called with a reference string and
257// return either a valid Reference type that the reference string maps to or
258// nil. If overridden is false, the default reference logic will be executed.
259// See the documentation in Options for more details on use-case.
260type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
261
262// Options represents configurable overrides and callbacks (in addition to the
263// extension flag set) for configuring a Markdown parse.
264type Options struct {
265 // Extensions is a flag set of bit-wise ORed extension bits. See the
266 // Extensions flags defined in this package.
267 Extensions Extensions
268
269 // ReferenceOverride is an optional function callback that is called every
270 // time a reference is resolved.
271 //
272 // In Markdown, the link reference syntax can be made to resolve a link to
273 // a reference instead of an inline URL, in one of the following ways:
274 //
275 // * [link text][refid]
276 // * [refid][]
277 //
278 // Usually, the refid is defined at the bottom of the Markdown document. If
279 // this override function is provided, the refid is passed to the override
280 // function first, before consulting the defined refids at the bottom. If
281 // the override function indicates an override did not occur, the refids at
282 // the bottom will be used to fill in the link details.
283 ReferenceOverride ReferenceOverrideFunc
284}
285
286// MarkdownBasic is a convenience function for simple rendering.
287// It processes markdown input with no extensions enabled.
288func MarkdownBasic(input []byte) []byte {
289 // set up the HTML renderer
290 renderer := NewHTMLRenderer(HTMLRendererParameters{
291 Flags: UseXHTML,
292 })
293
294 // set up the parser
295 return Markdown(input, renderer, Options{})
296}
297
298// MarkdownCommon is a convenience function for simple rendering. It calls
299// Markdown with most useful extensions enabled, including:
300//
301// * Smartypants processing with smart fractions and LaTeX dashes
302//
303// * Intra-word emphasis suppression
304//
305// * Tables
306//
307// * Fenced code blocks
308//
309// * Autolinking
310//
311// * Strikethrough support
312//
313// * Strict header parsing
314//
315// * Custom Header IDs
316func MarkdownCommon(input []byte) []byte {
317 // set up the HTML renderer
318 renderer := NewHTMLRenderer(HTMLRendererParameters{
319 Flags: CommonHTMLFlags,
320 })
321 return Markdown(input, renderer, DefaultOptions)
322}
323
324// Markdown is the main rendering function.
325// It parses and renders a block of markdown-encoded text.
326// The supplied Renderer is used to format the output, and extensions dictates
327// which non-standard extensions are enabled.
328//
329// To use the supplied HTML renderer, see NewHTMLRenderer.
330func Markdown(input []byte, renderer Renderer, options Options) []byte {
331 if renderer == nil {
332 return nil
333 }
334 return renderer.Render(Parse(input, options))
335}
336
337// Parse is an entry point to the parsing part of Blackfriday. It takes an
338// input markdown document and produces a syntax tree for its contents. This
339// tree can then be rendered with a default or custom renderer, or
340// analyzed/transformed by the caller to whatever non-standard needs they have.
341func Parse(input []byte, opts Options) *Node {
342 extensions := opts.Extensions
343
344 // fill in the render structure
345 p := new(parser)
346 p.flags = extensions
347 p.refOverride = opts.ReferenceOverride
348 p.refs = make(map[string]*reference)
349 p.maxNesting = 16
350 p.insideLink = false
351
352 docNode := NewNode(Document)
353 p.doc = docNode
354 p.tip = docNode
355 p.oldTip = docNode
356 p.lastMatchedContainer = docNode
357 p.allClosed = true
358
359 // register inline parsers
360 p.inlineCallback[' '] = maybeLineBreak
361 p.inlineCallback['*'] = emphasis
362 p.inlineCallback['_'] = emphasis
363 if extensions&Strikethrough != 0 {
364 p.inlineCallback['~'] = emphasis
365 }
366 p.inlineCallback['`'] = codeSpan
367 p.inlineCallback['\n'] = lineBreak
368 p.inlineCallback['['] = link
369 p.inlineCallback['<'] = leftAngle
370 p.inlineCallback['\\'] = escape
371 p.inlineCallback['&'] = entity
372 p.inlineCallback['!'] = maybeImage
373 p.inlineCallback['^'] = maybeInlineFootnote
374
375 if extensions&Autolink != 0 {
376 p.inlineCallback['h'] = maybeAutoLink
377 p.inlineCallback['m'] = maybeAutoLink
378 p.inlineCallback['f'] = maybeAutoLink
379 p.inlineCallback['H'] = maybeAutoLink
380 p.inlineCallback['M'] = maybeAutoLink
381 p.inlineCallback['F'] = maybeAutoLink
382 }
383
384 if extensions&Footnotes != 0 {
385 p.notes = make([]*reference, 0)
386 }
387
388 p.block(preprocess(p, input))
389 // Walk the tree and finish up some of unfinished blocks
390 for p.tip != nil {
391 p.finalize(p.tip)
392 }
393 // Walk the tree again and process inline markdown in each block
394 p.doc.Walk(func(node *Node, entering bool) WalkStatus {
395 if node.Type == Paragraph || node.Type == Header || node.Type == TableCell {
396 p.inline(node, node.content)
397 node.content = nil
398 }
399 return GoToNext
400 })
401 p.parseRefsToAST()
402 return p.doc
403}
404
405func (p *parser) parseRefsToAST() {
406 if p.flags&Footnotes == 0 || len(p.notes) == 0 {
407 return
408 }
409 p.tip = p.doc
410 block := p.addBlock(List, nil)
411 block.IsFootnotesList = true
412 block.ListFlags = ListTypeOrdered
413 flags := ListItemBeginningOfList
414 // Note: this loop is intentionally explicit, not range-form. This is
415 // because the body of the loop will append nested footnotes to p.notes and
416 // we need to process those late additions. Range form would only walk over
417 // the fixed initial set.
418 for i := 0; i < len(p.notes); i++ {
419 ref := p.notes[i]
420 p.addExistingChild(ref.footnote, 0)
421 block := ref.footnote
422 block.ListFlags = flags | ListTypeOrdered
423 block.RefLink = ref.link
424 if ref.hasBlock {
425 flags |= ListItemContainsBlock
426 p.block(ref.title)
427 } else {
428 p.inline(block, ref.title)
429 }
430 flags &^= ListItemBeginningOfList | ListItemContainsBlock
431 }
432 above := block.Parent
433 finalizeList(block)
434 p.tip = above
435 block.Walk(func(node *Node, entering bool) WalkStatus {
436 if node.Type == Paragraph || node.Type == Header {
437 p.inline(node, node.content)
438 node.content = nil
439 }
440 return GoToNext
441 })
442}
443
444// preprocess does a preparatory first pass over the input:
445// - normalize newlines
446// - expand tabs (outside of fenced code blocks)
447// - copy everything else
448func preprocess(p *parser, input []byte) []byte {
449 var out bytes.Buffer
450 tabSize := TabSizeDefault
451 if p.flags&TabSizeEight != 0 {
452 tabSize = TabSizeDouble
453 }
454 beg := 0
455 lastFencedCodeBlockEnd := 0
456 for beg < len(input) {
457 // Find end of this line, then process the line.
458 end := beg
459 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
460 end++
461 }
462
463 if p.flags&FencedCode != 0 {
464 // track fenced code block boundaries to suppress tab expansion
465 // and reference extraction inside them:
466 if beg >= lastFencedCodeBlockEnd {
467 if i := p.fencedCodeBlock(input[beg:], false); i > 0 {
468 lastFencedCodeBlockEnd = beg + i
469 }
470 }
471 }
472
473 // add the line body if present
474 if end > beg {
475 if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
476 out.Write(input[beg:end])
477 } else {
478 expandTabs(&out, input[beg:end], tabSize)
479 }
480 }
481
482 if end < len(input) && input[end] == '\r' {
483 end++
484 }
485 if end < len(input) && input[end] == '\n' {
486 end++
487 }
488 out.WriteByte('\n')
489
490 beg = end
491 }
492
493 // empty input?
494 if out.Len() == 0 {
495 out.WriteByte('\n')
496 }
497
498 return out.Bytes()
499}
500
501//
502// Link references
503//
504// This section implements support for references that (usually) appear
505// as footnotes in a document, and can be referenced anywhere in the document.
506// The basic format is:
507//
508// [1]: http://www.google.com/ "Google"
509// [2]: http://www.github.com/ "Github"
510//
511// Anywhere in the document, the reference can be linked by referring to its
512// label, i.e., 1 and 2 in this example, as in:
513//
514// This library is hosted on [Github][2], a git hosting site.
515//
516// Actual footnotes as specified in Pandoc and supported by some other Markdown
517// libraries such as php-markdown are also taken care of. They look like this:
518//
519// This sentence needs a bit of further explanation.[^note]
520//
521// [^note]: This is the explanation.
522//
523// Footnotes should be placed at the end of the document in an ordered list.
524// Inline footnotes such as:
525//
526// Inline footnotes^[Not supported.] also exist.
527//
528// are not yet supported.
529
530// reference holds all information necessary for a reference-style links or
531// footnotes.
532//
533// Consider this markdown with reference-style links:
534//
535// [link][ref]
536//
537// [ref]: /url/ "tooltip title"
538//
539// It will be ultimately converted to this HTML:
540//
541// <p><a href=\"/url/\" title=\"title\">link</a></p>
542//
543// And a reference structure will be populated as follows:
544//
545// p.refs["ref"] = &reference{
546// link: "/url/",
547// title: "tooltip title",
548// }
549//
550// Alternatively, reference can contain information about a footnote. Consider
551// this markdown:
552//
553// Text needing a footnote.[^a]
554//
555// [^a]: This is the note
556//
557// A reference structure will be populated as follows:
558//
559// p.refs["a"] = &reference{
560// link: "a",
561// title: "This is the note",
562// noteID: <some positive int>,
563// }
564//
565// TODO: As you can see, it begs for splitting into two dedicated structures
566// for refs and for footnotes.
567type reference struct {
568 link []byte
569 title []byte
570 noteID int // 0 if not a footnote ref
571 hasBlock bool
572 footnote *Node // a link to the Item node within a list of footnotes
573
574 text []byte // only gets populated by refOverride feature with Reference.Text
575}
576
577func (r *reference) String() string {
578 return fmt.Sprintf("{link: %q, title: %q, text: %q, noteID: %d, hasBlock: %v}",
579 r.link, r.title, r.text, r.noteID, r.hasBlock)
580}
581
582// Check whether or not data starts with a reference link.
583// If so, it is parsed and stored in the list of references
584// (in the render struct).
585// Returns the number of bytes to skip to move past it,
586// or zero if the first line is not a reference.
587func isReference(p *parser, data []byte, tabSize int) int {
588 // up to 3 optional leading spaces
589 if len(data) < 4 {
590 return 0
591 }
592 i := 0
593 for i < 3 && data[i] == ' ' {
594 i++
595 }
596
597 noteID := 0
598
599 // id part: anything but a newline between brackets
600 if data[i] != '[' {
601 return 0
602 }
603 i++
604 if p.flags&Footnotes != 0 {
605 if i < len(data) && data[i] == '^' {
606 // we can set it to anything here because the proper noteIds will
607 // be assigned later during the second pass. It just has to be != 0
608 noteID = 1
609 i++
610 }
611 }
612 idOffset := i
613 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
614 i++
615 }
616 if i >= len(data) || data[i] != ']' {
617 return 0
618 }
619 idEnd := i
620 // footnotes can have empty ID, like this: [^], but a reference can not be
621 // empty like this: []. Break early if it's not a footnote and there's no ID
622 if noteID == 0 && idOffset == idEnd {
623 return 0
624 }
625 // spacer: colon (space | tab)* newline? (space | tab)*
626 i++
627 if i >= len(data) || data[i] != ':' {
628 return 0
629 }
630 i++
631 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
632 i++
633 }
634 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
635 i++
636 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
637 i++
638 }
639 }
640 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
641 i++
642 }
643 if i >= len(data) {
644 return 0
645 }
646
647 var (
648 linkOffset, linkEnd int
649 titleOffset, titleEnd int
650 lineEnd int
651 raw []byte
652 hasBlock bool
653 )
654
655 if p.flags&Footnotes != 0 && noteID != 0 {
656 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
657 lineEnd = linkEnd
658 } else {
659 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
660 }
661 if lineEnd == 0 {
662 return 0
663 }
664
665 // a valid ref has been found
666
667 ref := &reference{
668 noteID: noteID,
669 hasBlock: hasBlock,
670 }
671
672 if noteID > 0 {
673 // reusing the link field for the id since footnotes don't have links
674 ref.link = data[idOffset:idEnd]
675 // if footnote, it's not really a title, it's the contained text
676 ref.title = raw
677 } else {
678 ref.link = data[linkOffset:linkEnd]
679 ref.title = data[titleOffset:titleEnd]
680 }
681
682 // id matches are case-insensitive
683 id := string(bytes.ToLower(data[idOffset:idEnd]))
684
685 p.refs[id] = ref
686
687 return lineEnd
688}
689
690func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
691 // link: whitespace-free sequence, optionally between angle brackets
692 if data[i] == '<' {
693 i++
694 }
695 linkOffset = i
696 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
697 i++
698 }
699 if i == len(data) {
700 return
701 }
702 linkEnd = i
703 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
704 linkOffset++
705 linkEnd--
706 }
707
708 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
709 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
710 i++
711 }
712 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
713 return
714 }
715
716 // compute end-of-line
717 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
718 lineEnd = i
719 }
720 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
721 lineEnd++
722 }
723
724 // optional (space|tab)* spacer after a newline
725 if lineEnd > 0 {
726 i = lineEnd + 1
727 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
728 i++
729 }
730 }
731
732 // optional title: any non-newline sequence enclosed in '"() alone on its line
733 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
734 i++
735 titleOffset = i
736
737 // look for EOL
738 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
739 i++
740 }
741 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
742 titleEnd = i + 1
743 } else {
744 titleEnd = i
745 }
746
747 // step back
748 i--
749 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
750 i--
751 }
752 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
753 lineEnd = titleEnd
754 titleEnd = i
755 }
756 }
757
758 return
759}
760
761// The first bit of this logic is the same as (*parser).listItem, but the rest
762// is much simpler. This function simply finds the entire block and shifts it
763// over by one tab if it is indeed a block (just returns the line if it's not).
764// blockEnd is the end of the section in the input buffer, and contents is the
765// extracted text that was shifted over one tab. It will need to be rendered at
766// the end of the document.
767func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
768 if i == 0 || len(data) == 0 {
769 return
770 }
771
772 // skip leading whitespace on first line
773 for i < len(data) && data[i] == ' ' {
774 i++
775 }
776
777 blockStart = i
778
779 // find the end of the line
780 blockEnd = i
781 for i < len(data) && data[i-1] != '\n' {
782 i++
783 }
784
785 // get working buffer
786 var raw bytes.Buffer
787
788 // put the first line into the working buffer
789 raw.Write(data[blockEnd:i])
790 blockEnd = i
791
792 // process the following lines
793 containsBlankLine := false
794
795gatherLines:
796 for blockEnd < len(data) {
797 i++
798
799 // find the end of this line
800 for i < len(data) && data[i-1] != '\n' {
801 i++
802 }
803
804 // if it is an empty line, guess that it is part of this item
805 // and move on to the next line
806 if p.isEmpty(data[blockEnd:i]) > 0 {
807 containsBlankLine = true
808 blockEnd = i
809 continue
810 }
811
812 n := 0
813 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
814 // this is the end of the block.
815 // we don't want to include this last line in the index.
816 break gatherLines
817 }
818
819 // if there were blank lines before this one, insert a new one now
820 if containsBlankLine {
821 raw.WriteByte('\n')
822 containsBlankLine = false
823 }
824
825 // get rid of that first tab, write to buffer
826 raw.Write(data[blockEnd+n : i])
827 hasBlock = true
828
829 blockEnd = i
830 }
831
832 if data[blockEnd-1] != '\n' {
833 raw.WriteByte('\n')
834 }
835
836 contents = raw.Bytes()
837
838 return
839}
840
841//
842//
843// Miscellaneous helper functions
844//
845//
846
847// Test if a character is a punctuation symbol.
848// Taken from a private function in regexp in the stdlib.
849func ispunct(c byte) bool {
850 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
851 if c == r {
852 return true
853 }
854 }
855 return false
856}
857
858// Test if a character is a whitespace character.
859func isspace(c byte) bool {
860 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
861}
862
863// Test if a character is letter.
864func isletter(c byte) bool {
865 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
866}
867
868// Test if a character is a letter or a digit.
869// TODO: check when this is looking for ASCII alnum and when it should use unicode
870func isalnum(c byte) bool {
871 return (c >= '0' && c <= '9') || isletter(c)
872}
873
874// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
875// always ends output with a newline
876func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
877 // first, check for common cases: no tabs, or only tabs at beginning of line
878 i, prefix := 0, 0
879 slowcase := false
880 for i = 0; i < len(line); i++ {
881 if line[i] == '\t' {
882 if prefix == i {
883 prefix++
884 } else {
885 slowcase = true
886 break
887 }
888 }
889 }
890
891 // no need to decode runes if all tabs are at the beginning of the line
892 if !slowcase {
893 for i = 0; i < prefix*tabSize; i++ {
894 out.WriteByte(' ')
895 }
896 out.Write(line[prefix:])
897 return
898 }
899
900 // the slow case: we need to count runes to figure out how
901 // many spaces to insert for each tab
902 column := 0
903 i = 0
904 for i < len(line) {
905 start := i
906 for i < len(line) && line[i] != '\t' {
907 _, size := utf8.DecodeRune(line[i:])
908 i += size
909 column++
910 }
911
912 if i > start {
913 out.Write(line[start:i])
914 }
915
916 if i >= len(line) {
917 break
918 }
919
920 for {
921 out.WriteByte(' ')
922 column++
923 if column%tabSize == 0 {
924 break
925 }
926 }
927
928 i++
929 }
930}
931
932// Find if a line counts as indented or not.
933// Returns number of characters the indent is (0 = not indented).
934func isIndented(data []byte, indentSize int) int {
935 if len(data) == 0 {
936 return 0
937 }
938 if data[0] == '\t' {
939 return 1
940 }
941 if len(data) < indentSize {
942 return 0
943 }
944 for i := 0; i < indentSize; i++ {
945 if data[i] != ' ' {
946 return 0
947 }
948 }
949 return indentSize
950}
951
952// Create a url-safe slug for fragments
953func slugify(in []byte) []byte {
954 if len(in) == 0 {
955 return in
956 }
957 out := make([]byte, 0, len(in))
958 sym := false
959
960 for _, ch := range in {
961 if isalnum(ch) {
962 sym = false
963 out = append(out, ch)
964 } else if sym {
965 continue
966 } else {
967 out = append(out, '-')
968 sym = true
969 }
970 }
971 var a, b int
972 var ch byte
973 for a, ch = range out {
974 if ch != '-' {
975 break
976 }
977 }
978 for b = len(out) - 1; b > 0; b-- {
979 if out[b] != '-' {
980 break
981 }
982 }
983 return out[a : b+1]
984}