markdown.go (view raw)
1//
2// Blackfriday Markdown Processor
3// Available at http://github.com/russross/blackfriday
4//
5// Copyright © 2011 Russ Ross <russ@russross.com>.
6// Distributed under the Simplified BSD License.
7// See README.md for details.
8//
9
10//
11//
12// Markdown parsing and processing
13//
14//
15
16// Blackfriday markdown processor.
17//
18// Translates plain text with simple formatting rules into HTML or LaTeX.
19package blackfriday
20
21import (
22 "bytes"
23 "strings"
24 "unicode/utf8"
25)
26
27const VERSION = "1.1"
28
29// These are the supported markdown parsing extensions.
30// OR these values together to select multiple extensions.
31const (
32 EXTENSION_NO_INTRA_EMPHASIS = 1 << iota // ignore emphasis markers inside words
33 EXTENSION_TABLES // render tables
34 EXTENSION_FENCED_CODE // render fenced code blocks
35 EXTENSION_AUTOLINK // detect embedded URLs that are not explicitly marked
36 EXTENSION_STRIKETHROUGH // strikethrough text using ~~test~~
37 EXTENSION_LAX_HTML_BLOCKS // loosen up HTML block parsing rules
38 EXTENSION_SPACE_HEADERS // be strict about prefix header rules
39 EXTENSION_HARD_LINE_BREAK // translate newlines into line breaks
40 EXTENSION_TAB_SIZE_EIGHT // expand tabs to eight spaces instead of four
41 EXTENSION_FOOTNOTES // Pandoc-style footnotes
42 EXTENSION_NO_EMPTY_LINE_BEFORE_BLOCK // No need to insert an empty line to start a (code, quote, order list, unorder list)block
43 EXTENSION_HEADER_IDS // specify header IDs with {#id}
44 EXTENSION_TITLEBLOCK // Titleblock ala pandoc
45 EXTENSION_AUTO_HEADER_IDS // Create the header ID from the text
46 EXTENSION_BACKSLASH_LINE_BREAK // translate trailing backslashes into line breaks
47
48 commonHtmlFlags = 0 |
49 HTML_USE_XHTML |
50 HTML_USE_SMARTYPANTS |
51 HTML_SMARTYPANTS_FRACTIONS |
52 HTML_SMARTYPANTS_LATEX_DASHES
53
54 commonExtensions = 0 |
55 EXTENSION_NO_INTRA_EMPHASIS |
56 EXTENSION_TABLES |
57 EXTENSION_FENCED_CODE |
58 EXTENSION_AUTOLINK |
59 EXTENSION_STRIKETHROUGH |
60 EXTENSION_SPACE_HEADERS |
61 EXTENSION_HEADER_IDS |
62 EXTENSION_BACKSLASH_LINE_BREAK
63)
64
65// These are the possible flag values for the link renderer.
66// Only a single one of these values will be used; they are not ORed together.
67// These are mostly of interest if you are writing a new output format.
68const (
69 LINK_TYPE_NOT_AUTOLINK = iota
70 LINK_TYPE_NORMAL
71 LINK_TYPE_EMAIL
72)
73
74// These are the possible flag values for the ListItem renderer.
75// Multiple flag values may be ORed together.
76// These are mostly of interest if you are writing a new output format.
77const (
78 LIST_TYPE_ORDERED = 1 << iota
79 LIST_ITEM_CONTAINS_BLOCK
80 LIST_ITEM_BEGINNING_OF_LIST
81 LIST_ITEM_END_OF_LIST
82)
83
84// These are the possible flag values for the table cell renderer.
85// Only a single one of these values will be used; they are not ORed together.
86// These are mostly of interest if you are writing a new output format.
87const (
88 TABLE_ALIGNMENT_LEFT = 1 << iota
89 TABLE_ALIGNMENT_RIGHT
90 TABLE_ALIGNMENT_CENTER = (TABLE_ALIGNMENT_LEFT | TABLE_ALIGNMENT_RIGHT)
91)
92
93// The size of a tab stop.
94const (
95 TAB_SIZE_DEFAULT = 4
96 TAB_SIZE_EIGHT = 8
97)
98
99// These are the tags that are recognized as HTML block tags.
100// Any of these can be included in markdown text without special escaping.
101var blockTags = map[string]bool{
102 "p": true,
103 "dl": true,
104 "h1": true,
105 "h2": true,
106 "h3": true,
107 "h4": true,
108 "h5": true,
109 "h6": true,
110 "ol": true,
111 "ul": true,
112 "del": true,
113 "div": true,
114 "ins": true,
115 "pre": true,
116 "form": true,
117 "math": true,
118 "table": true,
119 "iframe": true,
120 "script": true,
121 "fieldset": true,
122 "noscript": true,
123 "blockquote": true,
124
125 // HTML5
126 "video": true,
127 "aside": true,
128 "canvas": true,
129 "figure": true,
130 "footer": true,
131 "header": true,
132 "hgroup": true,
133 "output": true,
134 "article": true,
135 "section": true,
136 "progress": true,
137 "figcaption": true,
138}
139
140// Renderer is the rendering interface.
141// This is mostly of interest if you are implementing a new rendering format.
142//
143// When a byte slice is provided, it contains the (rendered) contents of the
144// element.
145//
146// When a callback is provided instead, it will write the contents of the
147// respective element directly to the output buffer and return true on success.
148// If the callback returns false, the rendering function should reset the
149// output buffer as though it had never been called.
150//
151// Currently Html and Latex implementations are provided
152type Renderer interface {
153 // block-level callbacks
154 BlockCode(out *bytes.Buffer, text []byte, lang string)
155 BlockQuote(out *bytes.Buffer, text []byte)
156 BlockHtml(out *bytes.Buffer, text []byte)
157 Header(out *bytes.Buffer, text func() bool, level int, id string)
158 HRule(out *bytes.Buffer)
159 List(out *bytes.Buffer, text func() bool, flags int)
160 ListItem(out *bytes.Buffer, text []byte, flags int)
161 Paragraph(out *bytes.Buffer, text func() bool)
162 Table(out *bytes.Buffer, header []byte, body []byte, columnData []int)
163 TableRow(out *bytes.Buffer, text []byte)
164 TableHeaderCell(out *bytes.Buffer, text []byte, flags int)
165 TableCell(out *bytes.Buffer, text []byte, flags int)
166 Footnotes(out *bytes.Buffer, text func() bool)
167 FootnoteItem(out *bytes.Buffer, name, text []byte, flags int)
168 TitleBlock(out *bytes.Buffer, text []byte)
169
170 // Span-level callbacks
171 AutoLink(out *bytes.Buffer, link []byte, kind int)
172 CodeSpan(out *bytes.Buffer, text []byte)
173 DoubleEmphasis(out *bytes.Buffer, text []byte)
174 Emphasis(out *bytes.Buffer, text []byte)
175 Image(out *bytes.Buffer, link []byte, title []byte, alt []byte)
176 LineBreak(out *bytes.Buffer)
177 Link(out *bytes.Buffer, link []byte, title []byte, content []byte)
178 RawHtmlTag(out *bytes.Buffer, tag []byte)
179 TripleEmphasis(out *bytes.Buffer, text []byte)
180 StrikeThrough(out *bytes.Buffer, text []byte)
181 FootnoteRef(out *bytes.Buffer, ref []byte, id int)
182
183 // Low-level callbacks
184 Entity(out *bytes.Buffer, entity []byte)
185 NormalText(out *bytes.Buffer, text []byte)
186
187 // Header and footer
188 DocumentHeader(out *bytes.Buffer)
189 DocumentFooter(out *bytes.Buffer)
190
191 GetFlags() int
192}
193
194// Callback functions for inline parsing. One such function is defined
195// for each character that triggers a response when parsing inline data.
196type inlineParser func(p *parser, out *bytes.Buffer, data []byte, offset int) int
197
198// Parser holds runtime state used by the parser.
199// This is constructed by the Markdown function.
200type parser struct {
201 r Renderer
202 refOverride ReferenceOverrideFunc
203 refs map[string]*reference
204 inlineCallback [256]inlineParser
205 flags int
206 nesting int
207 maxNesting int
208 insideLink bool
209
210 // Footnotes need to be ordered as well as available to quickly check for
211 // presence. If a ref is also a footnote, it's stored both in refs and here
212 // in notes. Slice is nil if footnotes not enabled.
213 notes []*reference
214}
215
216func (p *parser) getRef(refid string) (ref *reference, found bool) {
217 if p.refOverride != nil {
218 r, overridden := p.refOverride(refid)
219 if overridden {
220 if r == nil {
221 return nil, false
222 }
223 return &reference{
224 link: []byte(r.Link),
225 title: []byte(r.Title),
226 noteId: 0,
227 hasBlock: false,
228 text: []byte(r.Text)}, true
229 }
230 }
231 // refs are case insensitive
232 ref, found = p.refs[strings.ToLower(refid)]
233 return ref, found
234}
235
236//
237//
238// Public interface
239//
240//
241
242// Reference represents the details of a link.
243// See the documentation in Options for more details on use-case.
244type Reference struct {
245 // Link is usually the URL the reference points to.
246 Link string
247 // Title is the alternate text describing the link in more detail.
248 Title string
249 // Text is the optional text to override the ref with if the syntax used was
250 // [refid][]
251 Text string
252}
253
254// ReferenceOverrideFunc is expected to be called with a reference string and
255// return either a valid Reference type that the reference string maps to or
256// nil. If overridden is false, the default reference logic will be executed.
257// See the documentation in Options for more details on use-case.
258type ReferenceOverrideFunc func(reference string) (ref *Reference, overridden bool)
259
260// Options represents configurable overrides and callbacks (in addition to the
261// extension flag set) for configuring a Markdown parse.
262type Options struct {
263 // Extensions is a flag set of bit-wise ORed extension bits. See the
264 // EXTENSION_* flags defined in this package.
265 Extensions int
266
267 // ReferenceOverride is an optional function callback that is called every
268 // time a reference is resolved.
269 //
270 // In Markdown, the link reference syntax can be made to resolve a link to
271 // a reference instead of an inline URL, in one of the following ways:
272 //
273 // * [link text][refid]
274 // * [refid][]
275 //
276 // Usually, the refid is defined at the bottom of the Markdown document. If
277 // this override function is provided, the refid is passed to the override
278 // function first, before consulting the defined refids at the bottom. If
279 // the override function indicates an override did not occur, the refids at
280 // the bottom will be used to fill in the link details.
281 ReferenceOverride ReferenceOverrideFunc
282}
283
284// MarkdownBasic is a convenience function for simple rendering.
285// It processes markdown input with no extensions enabled.
286func MarkdownBasic(input []byte) []byte {
287 // set up the HTML renderer
288 htmlFlags := HTML_USE_XHTML
289 renderer := HtmlRenderer(htmlFlags, "", "")
290
291 // set up the parser
292 return MarkdownOptions(input, renderer, Options{Extensions: 0})
293}
294
295// Call Markdown with most useful extensions enabled
296// MarkdownCommon is a convenience function for simple rendering.
297// It processes markdown input with common extensions enabled, including:
298//
299// * Smartypants processing with smart fractions and LaTeX dashes
300//
301// * Intra-word emphasis suppression
302//
303// * Tables
304//
305// * Fenced code blocks
306//
307// * Autolinking
308//
309// * Strikethrough support
310//
311// * Strict header parsing
312//
313// * Custom Header IDs
314func MarkdownCommon(input []byte) []byte {
315 // set up the HTML renderer
316 renderer := HtmlRenderer(commonHtmlFlags, "", "")
317 return MarkdownOptions(input, renderer, Options{
318 Extensions: commonExtensions})
319}
320
321// Markdown is the main rendering function.
322// It parses and renders a block of markdown-encoded text.
323// The supplied Renderer is used to format the output, and extensions dictates
324// which non-standard extensions are enabled.
325//
326// To use the supplied Html or LaTeX renderers, see HtmlRenderer and
327// LatexRenderer, respectively.
328func Markdown(input []byte, renderer Renderer, extensions int) []byte {
329 return MarkdownOptions(input, renderer, Options{
330 Extensions: extensions})
331}
332
333// MarkdownOptions is just like Markdown but takes additional options through
334// the Options struct.
335func MarkdownOptions(input []byte, renderer Renderer, opts Options) []byte {
336 // no point in parsing if we can't render
337 if renderer == nil {
338 return nil
339 }
340
341 extensions := opts.Extensions
342
343 // fill in the render structure
344 p := new(parser)
345 p.r = renderer
346 p.flags = extensions
347 p.refOverride = opts.ReferenceOverride
348 p.refs = make(map[string]*reference)
349 p.maxNesting = 16
350 p.insideLink = false
351
352 // register inline parsers
353 p.inlineCallback['*'] = emphasis
354 p.inlineCallback['_'] = emphasis
355 if extensions&EXTENSION_STRIKETHROUGH != 0 {
356 p.inlineCallback['~'] = emphasis
357 }
358 p.inlineCallback['`'] = codeSpan
359 p.inlineCallback['\n'] = lineBreak
360 p.inlineCallback['['] = link
361 p.inlineCallback['<'] = leftAngle
362 p.inlineCallback['\\'] = escape
363 p.inlineCallback['&'] = entity
364
365 if extensions&EXTENSION_AUTOLINK != 0 {
366 p.inlineCallback[':'] = autoLink
367 }
368
369 if extensions&EXTENSION_FOOTNOTES != 0 {
370 p.notes = make([]*reference, 0)
371 }
372
373 first := firstPass(p, input)
374 second := secondPass(p, first)
375 return second
376}
377
378// first pass:
379// - extract references
380// - expand tabs
381// - normalize newlines
382// - copy everything else
383// - add missing newlines before fenced code blocks
384func firstPass(p *parser, input []byte) []byte {
385 var out bytes.Buffer
386 tabSize := TAB_SIZE_DEFAULT
387 if p.flags&EXTENSION_TAB_SIZE_EIGHT != 0 {
388 tabSize = TAB_SIZE_EIGHT
389 }
390 beg, end := 0, 0
391 lastLineWasBlank := false
392 lastFencedCodeBlockEnd := 0
393 for beg < len(input) { // iterate over lines
394 if end = isReference(p, input[beg:], tabSize); end > 0 {
395 beg += end
396 } else { // skip to the next line
397 end = beg
398 for end < len(input) && input[end] != '\n' && input[end] != '\r' {
399 end++
400 }
401
402 if p.flags&EXTENSION_FENCED_CODE != 0 {
403 // when last line was none blank and a fenced code block comes after
404 if beg >= lastFencedCodeBlockEnd {
405 if i := p.fencedCode(&out, input[beg:], false); i > 0 {
406 if !lastLineWasBlank {
407 out.WriteByte('\n') // need to inject additional linebreak
408 }
409 lastFencedCodeBlockEnd = beg + i
410 }
411 }
412 lastLineWasBlank = end == beg
413 }
414
415 // add the line body if present
416 if end > beg {
417 if end < lastFencedCodeBlockEnd { // Do not expand tabs while inside fenced code blocks.
418 out.Write(input[beg:end])
419 } else {
420 expandTabs(&out, input[beg:end], tabSize)
421 }
422 }
423 out.WriteByte('\n')
424
425 if end < len(input) && input[end] == '\r' {
426 end++
427 }
428 if end < len(input) && input[end] == '\n' {
429 end++
430 }
431
432 beg = end
433 }
434 }
435
436 // empty input?
437 if out.Len() == 0 {
438 out.WriteByte('\n')
439 }
440
441 return out.Bytes()
442}
443
444// second pass: actual rendering
445func secondPass(p *parser, input []byte) []byte {
446 var output bytes.Buffer
447
448 p.r.DocumentHeader(&output)
449 p.block(&output, input)
450
451 if p.flags&EXTENSION_FOOTNOTES != 0 && len(p.notes) > 0 {
452 p.r.Footnotes(&output, func() bool {
453 flags := LIST_ITEM_BEGINNING_OF_LIST
454 for _, ref := range p.notes {
455 var buf bytes.Buffer
456 if ref.hasBlock {
457 flags |= LIST_ITEM_CONTAINS_BLOCK
458 p.block(&buf, ref.title)
459 } else {
460 p.inline(&buf, ref.title)
461 }
462 p.r.FootnoteItem(&output, ref.link, buf.Bytes(), flags)
463 flags &^= LIST_ITEM_BEGINNING_OF_LIST | LIST_ITEM_CONTAINS_BLOCK
464 }
465
466 return true
467 })
468 }
469
470 p.r.DocumentFooter(&output)
471
472 if p.nesting != 0 {
473 panic("Nesting level did not end at zero")
474 }
475
476 return output.Bytes()
477}
478
479//
480// Link references
481//
482// This section implements support for references that (usually) appear
483// as footnotes in a document, and can be referenced anywhere in the document.
484// The basic format is:
485//
486// [1]: http://www.google.com/ "Google"
487// [2]: http://www.github.com/ "Github"
488//
489// Anywhere in the document, the reference can be linked by referring to its
490// label, i.e., 1 and 2 in this example, as in:
491//
492// This library is hosted on [Github][2], a git hosting site.
493//
494// Actual footnotes as specified in Pandoc and supported by some other Markdown
495// libraries such as php-markdown are also taken care of. They look like this:
496//
497// This sentence needs a bit of further explanation.[^note]
498//
499// [^note]: This is the explanation.
500//
501// Footnotes should be placed at the end of the document in an ordered list.
502// Inline footnotes such as:
503//
504// Inline footnotes^[Not supported.] also exist.
505//
506// are not yet supported.
507
508// References are parsed and stored in this struct.
509type reference struct {
510 link []byte
511 title []byte
512 noteId int // 0 if not a footnote ref
513 hasBlock bool
514 text []byte
515}
516
517// Check whether or not data starts with a reference link.
518// If so, it is parsed and stored in the list of references
519// (in the render struct).
520// Returns the number of bytes to skip to move past it,
521// or zero if the first line is not a reference.
522func isReference(p *parser, data []byte, tabSize int) int {
523 // up to 3 optional leading spaces
524 if len(data) < 4 {
525 return 0
526 }
527 i := 0
528 for i < 3 && data[i] == ' ' {
529 i++
530 }
531
532 noteId := 0
533
534 // id part: anything but a newline between brackets
535 if data[i] != '[' {
536 return 0
537 }
538 i++
539 if p.flags&EXTENSION_FOOTNOTES != 0 {
540 if data[i] == '^' {
541 // we can set it to anything here because the proper noteIds will
542 // be assigned later during the second pass. It just has to be != 0
543 noteId = 1
544 i++
545 }
546 }
547 idOffset := i
548 for i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != ']' {
549 i++
550 }
551 if i >= len(data) || data[i] != ']' {
552 return 0
553 }
554 idEnd := i
555
556 // spacer: colon (space | tab)* newline? (space | tab)*
557 i++
558 if i >= len(data) || data[i] != ':' {
559 return 0
560 }
561 i++
562 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
563 i++
564 }
565 if i < len(data) && (data[i] == '\n' || data[i] == '\r') {
566 i++
567 if i < len(data) && data[i] == '\n' && data[i-1] == '\r' {
568 i++
569 }
570 }
571 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
572 i++
573 }
574 if i >= len(data) {
575 return 0
576 }
577
578 var (
579 linkOffset, linkEnd int
580 titleOffset, titleEnd int
581 lineEnd int
582 raw []byte
583 hasBlock bool
584 )
585
586 if p.flags&EXTENSION_FOOTNOTES != 0 && noteId != 0 {
587 linkOffset, linkEnd, raw, hasBlock = scanFootnote(p, data, i, tabSize)
588 lineEnd = linkEnd
589 } else {
590 linkOffset, linkEnd, titleOffset, titleEnd, lineEnd = scanLinkRef(p, data, i)
591 }
592 if lineEnd == 0 {
593 return 0
594 }
595
596 // a valid ref has been found
597
598 ref := &reference{
599 noteId: noteId,
600 hasBlock: hasBlock,
601 }
602
603 if noteId > 0 {
604 // reusing the link field for the id since footnotes don't have links
605 ref.link = data[idOffset:idEnd]
606 // if footnote, it's not really a title, it's the contained text
607 ref.title = raw
608 } else {
609 ref.link = data[linkOffset:linkEnd]
610 ref.title = data[titleOffset:titleEnd]
611 }
612
613 // id matches are case-insensitive
614 id := string(bytes.ToLower(data[idOffset:idEnd]))
615
616 p.refs[id] = ref
617
618 return lineEnd
619}
620
621func scanLinkRef(p *parser, data []byte, i int) (linkOffset, linkEnd, titleOffset, titleEnd, lineEnd int) {
622 // link: whitespace-free sequence, optionally between angle brackets
623 if data[i] == '<' {
624 i++
625 }
626 linkOffset = i
627 for i < len(data) && data[i] != ' ' && data[i] != '\t' && data[i] != '\n' && data[i] != '\r' {
628 i++
629 }
630 linkEnd = i
631 if data[linkOffset] == '<' && data[linkEnd-1] == '>' {
632 linkOffset++
633 linkEnd--
634 }
635
636 // optional spacer: (space | tab)* (newline | '\'' | '"' | '(' )
637 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
638 i++
639 }
640 if i < len(data) && data[i] != '\n' && data[i] != '\r' && data[i] != '\'' && data[i] != '"' && data[i] != '(' {
641 return
642 }
643
644 // compute end-of-line
645 if i >= len(data) || data[i] == '\r' || data[i] == '\n' {
646 lineEnd = i
647 }
648 if i+1 < len(data) && data[i] == '\r' && data[i+1] == '\n' {
649 lineEnd++
650 }
651
652 // optional (space|tab)* spacer after a newline
653 if lineEnd > 0 {
654 i = lineEnd + 1
655 for i < len(data) && (data[i] == ' ' || data[i] == '\t') {
656 i++
657 }
658 }
659
660 // optional title: any non-newline sequence enclosed in '"() alone on its line
661 if i+1 < len(data) && (data[i] == '\'' || data[i] == '"' || data[i] == '(') {
662 i++
663 titleOffset = i
664
665 // look for EOL
666 for i < len(data) && data[i] != '\n' && data[i] != '\r' {
667 i++
668 }
669 if i+1 < len(data) && data[i] == '\n' && data[i+1] == '\r' {
670 titleEnd = i + 1
671 } else {
672 titleEnd = i
673 }
674
675 // step back
676 i--
677 for i > titleOffset && (data[i] == ' ' || data[i] == '\t') {
678 i--
679 }
680 if i > titleOffset && (data[i] == '\'' || data[i] == '"' || data[i] == ')') {
681 lineEnd = titleEnd
682 titleEnd = i
683 }
684 }
685
686 return
687}
688
689// The first bit of this logic is the same as (*parser).listItem, but the rest
690// is much simpler. This function simply finds the entire block and shifts it
691// over by one tab if it is indeed a block (just returns the line if it's not).
692// blockEnd is the end of the section in the input buffer, and contents is the
693// extracted text that was shifted over one tab. It will need to be rendered at
694// the end of the document.
695func scanFootnote(p *parser, data []byte, i, indentSize int) (blockStart, blockEnd int, contents []byte, hasBlock bool) {
696 if i == 0 || len(data) == 0 {
697 return
698 }
699
700 // skip leading whitespace on first line
701 for i < len(data) && data[i] == ' ' {
702 i++
703 }
704
705 blockStart = i
706
707 // find the end of the line
708 blockEnd = i
709 for i < len(data) && data[i-1] != '\n' {
710 i++
711 }
712
713 // get working buffer
714 var raw bytes.Buffer
715
716 // put the first line into the working buffer
717 raw.Write(data[blockEnd:i])
718 blockEnd = i
719
720 // process the following lines
721 containsBlankLine := false
722
723gatherLines:
724 for blockEnd < len(data) {
725 i++
726
727 // find the end of this line
728 for i < len(data) && data[i-1] != '\n' {
729 i++
730 }
731
732 // if it is an empty line, guess that it is part of this item
733 // and move on to the next line
734 if p.isEmpty(data[blockEnd:i]) > 0 {
735 containsBlankLine = true
736 blockEnd = i
737 continue
738 }
739
740 n := 0
741 if n = isIndented(data[blockEnd:i], indentSize); n == 0 {
742 // this is the end of the block.
743 // we don't want to include this last line in the index.
744 break gatherLines
745 }
746
747 // if there were blank lines before this one, insert a new one now
748 if containsBlankLine {
749 raw.WriteByte('\n')
750 containsBlankLine = false
751 }
752
753 // get rid of that first tab, write to buffer
754 raw.Write(data[blockEnd+n : i])
755 hasBlock = true
756
757 blockEnd = i
758 }
759
760 if data[blockEnd-1] != '\n' {
761 raw.WriteByte('\n')
762 }
763
764 contents = raw.Bytes()
765
766 return
767}
768
769//
770//
771// Miscellaneous helper functions
772//
773//
774
775// Test if a character is a punctuation symbol.
776// Taken from a private function in regexp in the stdlib.
777func ispunct(c byte) bool {
778 for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
779 if c == r {
780 return true
781 }
782 }
783 return false
784}
785
786// Test if a character is a whitespace character.
787func isspace(c byte) bool {
788 return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'
789}
790
791// Test if a character is letter.
792func isletter(c byte) bool {
793 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
794}
795
796// Test if a character is a letter or a digit.
797// TODO: check when this is looking for ASCII alnum and when it should use unicode
798func isalnum(c byte) bool {
799 return (c >= '0' && c <= '9') || isletter(c)
800}
801
802// Replace tab characters with spaces, aligning to the next TAB_SIZE column.
803// always ends output with a newline
804func expandTabs(out *bytes.Buffer, line []byte, tabSize int) {
805 // first, check for common cases: no tabs, or only tabs at beginning of line
806 i, prefix := 0, 0
807 slowcase := false
808 for i = 0; i < len(line); i++ {
809 if line[i] == '\t' {
810 if prefix == i {
811 prefix++
812 } else {
813 slowcase = true
814 break
815 }
816 }
817 }
818
819 // no need to decode runes if all tabs are at the beginning of the line
820 if !slowcase {
821 for i = 0; i < prefix*tabSize; i++ {
822 out.WriteByte(' ')
823 }
824 out.Write(line[prefix:])
825 return
826 }
827
828 // the slow case: we need to count runes to figure out how
829 // many spaces to insert for each tab
830 column := 0
831 i = 0
832 for i < len(line) {
833 start := i
834 for i < len(line) && line[i] != '\t' {
835 _, size := utf8.DecodeRune(line[i:])
836 i += size
837 column++
838 }
839
840 if i > start {
841 out.Write(line[start:i])
842 }
843
844 if i >= len(line) {
845 break
846 }
847
848 for {
849 out.WriteByte(' ')
850 column++
851 if column%tabSize == 0 {
852 break
853 }
854 }
855
856 i++
857 }
858}
859
860// Find if a line counts as indented or not.
861// Returns number of characters the indent is (0 = not indented).
862func isIndented(data []byte, indentSize int) int {
863 if len(data) == 0 {
864 return 0
865 }
866 if data[0] == '\t' {
867 return 1
868 }
869 if len(data) < indentSize {
870 return 0
871 }
872 for i := 0; i < indentSize; i++ {
873 if data[i] != ' ' {
874 return 0
875 }
876 }
877 return indentSize
878}
879
880// Create a url-safe slug for fragments
881func slugify(in []byte) []byte {
882 if len(in) == 0 {
883 return in
884 }
885 out := make([]byte, 0, len(in))
886 sym := false
887
888 for _, ch := range in {
889 if isalnum(ch) {
890 sym = false
891 out = append(out, ch)
892 } else if sym {
893 continue
894 } else {
895 out = append(out, '-')
896 sym = true
897 }
898 }
899 var a, b int
900 var ch byte
901 for a, ch = range out {
902 if ch != '-' {
903 break
904 }
905 }
906 for b = len(out) - 1; b > 0; b-- {
907 if out[b] != '-' {
908 break
909 }
910 }
911 return out[a : b+1]
912}