Run Smartypants as a separate pass over the AST Separate Smartypants somewhat from the HTML renderer. Move its flags from HtmlFlags to Extensions (probably should be moved to its own set of flags, but not now). With that done, do a separate walk of the tree and either run Smartypants processor if it's enabled, or simply escape text nodes.
Vytautas Šaltenis vytas@rtfb.lt
Fri, 01 Apr 2016 10:44:22 +0300
6 files changed,
104 insertions(+),
96 deletions(-)
M
block_test.go
→
block_test.go
@@ -23,13 +23,13 @@ return string(Markdown([]byte(input), renderer, extensions))
} func runMarkdownBlock(input string, extensions Extensions) string { - renderer := HtmlRenderer(UseXHTML, "", "") + renderer := HtmlRenderer(UseXHTML, extensions, "", "") return runMarkdownBlockWithRenderer(input, extensions, renderer) } func runnerWithRendererParameters(parameters HtmlRendererParameters) func(string, Extensions) string { return func(input string, extensions Extensions) string { - renderer := HtmlRendererWithParameters(UseXHTML, "", "", parameters) + renderer := HtmlRendererWithParameters(UseXHTML, extensions, "", "", parameters) return runMarkdownBlockWithRenderer(input, extensions, renderer) } }
M
html.go
→
html.go
@@ -29,25 +29,20 @@ type HtmlFlags int
// HTML renderer configuration options. const ( - HtmlFlagsNone HtmlFlags = 0 - SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks - SkipStyle // Skip embedded <style> elements - SkipImages // Skip embedded images - SkipLinks // Skip all links - Safelink // Only link to trusted protocols - NofollowLinks // Only link with rel="nofollow" - NoreferrerLinks // Only link with rel="noreferrer" - HrefTargetBlank // Add a blank target - Toc // Generate a table of contents - OmitContents // Skip the main contents (for a standalone table of contents) - CompletePage // Generate a complete HTML page - UseXHTML // Generate XHTML output instead of HTML - UseSmartypants // Enable smart punctuation substitutions - SmartypantsFractions // Enable smart fractions (with UseSmartypants) - SmartypantsDashes // Enable smart dashes (with UseSmartypants) - SmartypantsLatexDashes // Enable LaTeX-style dashes (with UseSmartypants) - SmartypantsAngledQuotes // Enable angled double quotes (with UseSmartypants) for double quotes rendering - FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source + HtmlFlagsNone HtmlFlags = 0 + SkipHTML HtmlFlags = 1 << iota // Skip preformatted HTML blocks + SkipStyle // Skip embedded <style> elements + SkipImages // Skip embedded images + SkipLinks // Skip all links + Safelink // Only link to trusted protocols + NofollowLinks // Only link with rel="nofollow" + NoreferrerLinks // Only link with rel="noreferrer" + HrefTargetBlank // Add a blank target + Toc // Generate a table of contents + OmitContents // Skip the main contents (for a standalone table of contents) + CompletePage // Generate a complete HTML page + UseXHTML // Generate XHTML output instead of HTML + FootnoteReturnLinks // Generate a link at the end of a footnote to return to the source TagName = "[A-Za-z][A-Za-z0-9-]*" AttributeName = "[a-zA-Z_:][a-zA-Z0-9:._-]*"@@ -109,10 +104,11 @@
// Track header IDs to prevent ID collision in a single generation. headerIDs map[string]int - smartypants *SPRenderer w HtmlWriter lastOutputLen int disableTags int + + extensions Extensions // This gives Smartypants renderer access to flags } const (@@ -127,8 +123,8 @@ // flags is a set of HtmlFlags ORed together.
// title is the title of the document, and css is a URL for the document's // stylesheet. // title and css are only used when HTML_COMPLETE_PAGE is selected. -func HtmlRenderer(flags HtmlFlags, title string, css string) Renderer { - return HtmlRendererWithParameters(flags, title, css, HtmlRendererParameters{}) +func HtmlRenderer(flags HtmlFlags, extensions Extensions, title string, css string) Renderer { + return HtmlRendererWithParameters(flags, extensions, title, css, HtmlRendererParameters{}) } type HtmlWriter struct {@@ -157,7 +153,7 @@ func (r *HTML) Write(b []byte) (int, error) {
return r.w.Write(b) } -func HtmlRendererWithParameters(flags HtmlFlags, title string, +func HtmlRendererWithParameters(flags HtmlFlags, extensions Extensions, title string, css string, renderParameters HtmlRendererParameters) Renderer { // configure the rendering engine closeTag := htmlClose@@ -172,6 +168,7 @@
var writer HtmlWriter return &HTML{ flags: flags, + extensions: extensions, closeTag: closeTag, title: title, css: css,@@ -183,8 +180,7 @@ toc: new(bytes.Buffer),
headerIDs: make(map[string]int), - smartypants: NewSmartypantsRenderer(flags), - w: writer, + w: writer, } }@@ -688,41 +684,15 @@ r.w.Write(entity)
} func (r *HTML) NormalText(text []byte) { - if r.flags&UseSmartypants != 0 { + if r.extensions&Smartypants != 0 { r.Smartypants(text) } else { r.attrEscape(text) } } -func (r *HTML) Smartypants2(text []byte) []byte { - var buff bytes.Buffer - // first do normal entity escaping - text = attrEscape2(text) - mark := 0 - for i := 0; i < len(text); i++ { - if action := r.smartypants.callbacks[text[i]]; action != nil { - if i > mark { - buff.Write(text[mark:i]) - } - previousChar := byte(0) - if i > 0 { - previousChar = text[i-1] - } - var tmp bytes.Buffer - i += action(&tmp, previousChar, text[i:]) - buff.Write(tmp.Bytes()) - mark = i + 1 - } - } - if mark < len(text) { - buff.Write(text[mark:]) - } - return buff.Bytes() -} - func (r *HTML) Smartypants(text []byte) { - r.w.Write(r.Smartypants2(text)) + r.w.Write(NewSmartypantsRenderer(r.extensions).Process(text)) } func (r *HTML) DocumentHeader() {@@ -1133,12 +1103,7 @@ func (r *HTML) RenderNode(w io.Writer, node *Node, entering bool) {
attrs := []string{} switch node.Type { case Text: - if r.flags&UseSmartypants != 0 && isSmartypantable(node) { - // TODO: don't do that in renderer, do that at parse time! - r.out(w, r.Smartypants2(node.Literal)) - } else { - r.out(w, esc(node.Literal, false)) - } + r.out(w, node.Literal) break case Softbreak: r.out(w, []byte("\n"))@@ -1431,6 +1396,17 @@
func (r *HTML) Render(ast *Node) []byte { //println("render_Blackfriday") //dump(ast) + // Run Smartypants if it's enabled or simply escape text if not + sr := NewSmartypantsRenderer(r.extensions) + ForEachNode(ast, func(node *Node, entering bool) { + if node.Type == Text { + if r.extensions&Smartypants != 0 { + node.Literal = sr.Process(node.Literal) + } else { + node.Literal = esc(node.Literal, false) + } + } + }) var buff bytes.Buffer ForEachNode(ast, func(node *Node, entering bool) { r.RenderNode(&buff, node, entering)
M
inline_test.go
→
inline_test.go
@@ -26,7 +26,7 @@ opts.Extensions |= Strikethrough
htmlFlags |= UseXHTML - renderer := HtmlRendererWithParameters(htmlFlags, "", "", params) + renderer := HtmlRendererWithParameters(htmlFlags, opts.Extensions, "", "", params) return string(MarkdownOptions([]byte(input), renderer, opts)) }@@ -1084,7 +1084,7 @@
"blahblah\n<!--- foo -->\nrhubarb\n", "<p>blahblah\n<!--- foo -->\nrhubarb</p>\n", } - doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{}) + doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{}) } func TestSmartDoubleQuotes(t *testing.T) {@@ -1096,7 +1096,7 @@ "<p>this “ single double</p>\n",
"two pair of \"some\" quoted \"text\".\n", "<p>two pair of “some” quoted “text”.</p>\n"} - doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{}) + doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{}) } func TestSmartAngledDoubleQuotes(t *testing.T) {@@ -1108,7 +1108,7 @@ "<p>this « single double</p>\n",
"two pair of \"some\" quoted \"text\".\n", "<p>two pair of «some» quoted «text».</p>\n"} - doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsAngledQuotes, HtmlRendererParameters{}) + doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsAngledQuotes}, 0, HtmlRendererParameters{}) } func TestSmartFractions(t *testing.T) {@@ -1118,7 +1118,7 @@ "<p>½, ¼ and ¾; ¼th and ¾ths</p>\n",
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n", "<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"} - doTestsInlineParam(t, tests, Options{}, UseSmartypants, HtmlRendererParameters{}) + doTestsInlineParam(t, tests, Options{Extensions: Smartypants}, 0, HtmlRendererParameters{}) tests = []string{ "1/2, 2/3, 81/100 and 1000000/1048576.\n",@@ -1126,7 +1126,7 @@ "<p><sup>1</sup>⁄<sub>2</sub>, <sup>2</sup>⁄<sub>3</sub>, <sup>81</sup>⁄<sub>100</sub> and <sup>1000000</sup>⁄<sub>1048576</sub>.</p>\n",
"1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.\n", "<p>1/2/2015, 1/4/2015, 3/4/2015; 2015/1/2, 2015/1/4, 2015/3/4.</p>\n"} - doTestsInlineParam(t, tests, Options{}, UseSmartypants|SmartypantsFractions, HtmlRendererParameters{}) + doTestsInlineParam(t, tests, Options{Extensions: Smartypants | SmartypantsFractions}, 0, HtmlRendererParameters{}) } func TestDisableSmartDashes(t *testing.T) {@@ -1145,7 +1145,7 @@ "foo -- bar\n",
"<p>foo — bar</p>\n", "foo --- bar\n", "<p>foo —– bar</p>\n", - }, Options{}, UseSmartypants|SmartypantsDashes, HtmlRendererParameters{}) + }, Options{Extensions: Smartypants | SmartypantsDashes}, 0, HtmlRendererParameters{}) doTestsInlineParam(t, []string{ "foo - bar\n", "<p>foo - bar</p>\n",@@ -1153,7 +1153,7 @@ "foo -- bar\n",
"<p>foo – bar</p>\n", "foo --- bar\n", "<p>foo — bar</p>\n", - }, Options{}, UseSmartypants|SmartypantsLatexDashes|SmartypantsDashes, + }, Options{Extensions: Smartypants | SmartypantsLatexDashes | SmartypantsDashes}, 0, HtmlRendererParameters{}) doTestsInlineParam(t, []string{ "foo - bar\n",@@ -1162,7 +1162,7 @@ "foo -- bar\n",
"<p>foo -- bar</p>\n", "foo --- bar\n", "<p>foo --- bar</p>\n", - }, Options{}, - UseSmartypants|SmartypantsLatexDashes, + }, Options{Extensions: Smartypants | SmartypantsLatexDashes}, + 0, HtmlRendererParameters{}) }
M
markdown.go
→
markdown.go
@@ -32,30 +32,35 @@
// These are the supported markdown parsing extensions. // OR these values together to select multiple extensions. const ( - NoExtensions Extensions = 0 - NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words - Tables // Render tables - FencedCode // Render fenced code blocks - Autolink // Detect embedded URLs that are not explicitly marked - Strikethrough // Strikethrough text using ~~test~~ - LaxHTMLBlocks // Loosen up HTML block parsing rules - SpaceHeaders // Be strict about prefix header rules - HardLineBreak // Translate newlines into line breaks - TabSizeEight // Expand tabs to eight spaces instead of four - Footnotes // Pandoc-style footnotes - NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block - HeaderIDs // specify header IDs with {#id} - Titleblock // Titleblock ala pandoc - AutoHeaderIDs // Create the header ID from the text - BackslashLineBreak // Translate trailing backslashes into line breaks - DefinitionLists // Render definition lists + NoExtensions Extensions = 0 + NoIntraEmphasis Extensions = 1 << iota // Ignore emphasis markers inside words + Tables // Render tables + FencedCode // Render fenced code blocks + Autolink // Detect embedded URLs that are not explicitly marked + Strikethrough // Strikethrough text using ~~test~~ + LaxHTMLBlocks // Loosen up HTML block parsing rules + SpaceHeaders // Be strict about prefix header rules + HardLineBreak // Translate newlines into line breaks + TabSizeEight // Expand tabs to eight spaces instead of four + Footnotes // Pandoc-style footnotes + NoEmptyLineBeforeBlock // No need to insert an empty line to start a (code, quote, ordered list, unordered list) block + HeaderIDs // specify header IDs with {#id} + Titleblock // Titleblock ala pandoc + AutoHeaderIDs // Create the header ID from the text + BackslashLineBreak // Translate trailing backslashes into line breaks + DefinitionLists // Render definition lists + Smartypants // Enable smart punctuation substitutions + SmartypantsFractions // Enable smart fractions (with Smartypants) + SmartypantsDashes // Enable smart dashes (with Smartypants) + SmartypantsLatexDashes // Enable LaTeX-style dashes (with Smartypants) + SmartypantsAngledQuotes // Enable angled double quotes (with Smartypants) for double quotes rendering - CommonHtmlFlags HtmlFlags = UseXHTML | UseSmartypants | - SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes + CommonHtmlFlags HtmlFlags = UseXHTML CommonExtensions Extensions = NoIntraEmphasis | Tables | FencedCode | Autolink | Strikethrough | SpaceHeaders | HeaderIDs | - BackslashLineBreak | DefinitionLists + BackslashLineBreak | DefinitionLists | Smartypants | + SmartypantsFractions | SmartypantsDashes | SmartypantsLatexDashes ) var DefaultOptions = Options{@@ -340,7 +345,7 @@ // It processes markdown input with no extensions enabled.
func MarkdownBasic(input []byte) []byte { // set up the HTML renderer htmlFlags := UseXHTML - renderer := HtmlRenderer(htmlFlags, "", "") + renderer := HtmlRenderer(htmlFlags, CommonExtensions, "", "") // set up the parser return MarkdownOptions(input, renderer, Options{Extensions: 0})@@ -367,7 +372,7 @@ //
// * Custom Header IDs func MarkdownCommon(input []byte) []byte { // set up the HTML renderer - renderer := HtmlRenderer(CommonHtmlFlags, "", "") + renderer := HtmlRenderer(CommonHtmlFlags, CommonExtensions, "", "") return MarkdownOptions(input, renderer, DefaultOptions) }@@ -441,10 +446,11 @@ }
first := firstPass(p, input) secondPass(p, first) - // walk the tree and finish up some of unfinished blocks: + // Walk the tree and finish up some of unfinished blocks for p.tip != nil { p.finalize(p.tip) } + // Walk the tree again and process inline markdown in each block ForEachNode(p.doc, func(node *Node, entering bool) { if node.Type == Paragraph || node.Type == Header || node.Type == TableCell { p.currBlock = node
M
ref_test.go
→
ref_test.go
@@ -20,7 +20,7 @@ "testing"
) func runMarkdownReference(input string, flag Extensions) string { - renderer := HtmlRenderer(0, "", "") + renderer := HtmlRenderer(0, flag, "", "") return string(Markdown([]byte(input), renderer, flag)) }
M
smartypants.go
→
smartypants.go
@@ -366,7 +366,7 @@ }
type smartCallback func(out *bytes.Buffer, previousChar byte, text []byte) int -func NewSmartypantsRenderer(flags HtmlFlags) *SPRenderer { +func NewSmartypantsRenderer(flags Extensions) *SPRenderer { var r SPRenderer if flags&SmartypantsAngledQuotes == 0 { r.callbacks['"'] = r.smartDoubleQuote@@ -397,3 +397,29 @@ r.callbacks['<'] = r.smartLeftAngle
r.callbacks['`'] = r.smartBacktick return &r } + +func (sr *SPRenderer) Process(text []byte) []byte { + var buff bytes.Buffer + // first do normal entity escaping + text = attrEscape2(text) + mark := 0 + for i := 0; i < len(text); i++ { + if action := sr.callbacks[text[i]]; action != nil { + if i > mark { + buff.Write(text[mark:i]) + } + previousChar := byte(0) + if i > 0 { + previousChar = text[i-1] + } + var tmp bytes.Buffer + i += action(&tmp, previousChar, text[i:]) + buff.Write(tmp.Bytes()) + mark = i + 1 + } + } + if mark < len(text) { + buff.Write(text[mark:]) + } + return buff.Bytes() +}