1//
  2// Copyright (c) 2019 Ted Unangst <tedu@tedunangst.com>
  3//
  4// Permission to use, copy, modify, and distribute this software for any
  5// purpose with or without fee is hereby granted, provided that the above
  6// copyright notice and this permission notice appear in all copies.
  7//
  8// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  9// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 10// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 11// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 12// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 13// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 14// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 15
 16package main
 17
 18import (
 19	"fmt"
 20	"io"
 21	"net/http"
 22	"os"
 23	"regexp"
 24	"strings"
 25
 26	"github.com/andybalholm/cascadia"
 27	"golang.org/x/net/html"
 28	"humungus.tedunangst.com/r/webs/htfilter"
 29	"humungus.tedunangst.com/r/webs/templates"
 30)
 31
 32var tweetsel = cascadia.MustCompile("div[data-testid=tweetText]")
 33var linksel = cascadia.MustCompile("a time")
 34var replyingto = cascadia.MustCompile(".ReplyingToContextBelowAuthor")
 35var imgsel = cascadia.MustCompile("div[data-testid=tweetPhoto] img")
 36var authorregex = regexp.MustCompile("twitter.com/([^/]+)")
 37
 38var re_hoots = regexp.MustCompile(`hoot: ?https://\S+`)
 39var re_removepics = regexp.MustCompile(`pic\.twitter\.com/[[:alnum:]]+`)
 40
 41func hootextractor(r io.Reader, url string, seen map[string]bool) string {
 42	root, err := html.Parse(r)
 43	if err != nil {
 44		elog.Printf("error parsing hoot: %s", err)
 45		return url
 46	}
 47
 48	url = strings.Replace(url, "mobile.twitter.com", "twitter.com", -1)
 49	wantmatch := authorregex.FindStringSubmatch(url)
 50	var wanted string
 51	if len(wantmatch) == 2 {
 52		wanted = wantmatch[1]
 53	}
 54
 55	var htf htfilter.Filter
 56	htf.Imager = func(node *html.Node) string {
 57		alt := htfilter.GetAttr(node, "alt")
 58		src := htfilter.GetAttr(node, "src")
 59		if htfilter.HasClass(node, "Emoji") || strings.HasSuffix(src, ".svg") {
 60			return alt
 61		}
 62		return string(templates.Sprintf(" <img src='%s' alt='%s'>", src, alt))
 63	}
 64
 65	var buf strings.Builder
 66	fmt.Fprintf(&buf, "%s\n", url)
 67
 68	divs := tweetsel.MatchAll(root)
 69	for i, div := range divs {
 70		{
 71			twp := div.Parent.Parent.Parent.Parent.Parent
 72			link := url
 73			alink := linksel.MatchFirst(twp)
 74			if alink == nil {
 75				if i != 0 {
 76					dlog.Printf("missing link")
 77					continue
 78				}
 79			} else {
 80				alink = alink.Parent
 81				link = "https://twitter.com" + htfilter.GetAttr(alink, "href")
 82			}
 83			authormatch := authorregex.FindStringSubmatch(link)
 84			if len(authormatch) < 2 {
 85				dlog.Printf("no author?: %s", link)
 86				continue
 87			}
 88			author := authormatch[1]
 89			if wanted == "" {
 90				wanted = author
 91			}
 92			if author != wanted {
 93				continue
 94			}
 95			for _, img := range imgsel.MatchAll(twp) {
 96				img.Parent.RemoveChild(img)
 97				div.AppendChild(img)
 98			}
 99			text := htf.NodeText(div)
100			text = strings.Replace(text, "\n", " ", -1)
101			fmt.Fprintf(&buf, "> @%s: %s\n", author, text)
102			continue
103		}
104
105		twp := div.Parent.Parent.Parent.Parent.Parent
106		link := url
107		alink := linksel.MatchFirst(twp)
108		if alink == nil {
109			if i != 0 {
110				dlog.Printf("missing link")
111				continue
112			}
113		} else {
114			link = "https://twitter.com" + htfilter.GetAttr(alink, "href")
115		}
116		replto := replyingto.MatchFirst(twp)
117		if replto != nil {
118			continue
119		}
120		authormatch := authorregex.FindStringSubmatch(link)
121		if len(authormatch) < 2 {
122			dlog.Printf("no author?: %s", link)
123			continue
124		}
125		author := authormatch[1]
126		if wanted == "" {
127			wanted = author
128		}
129		if author != wanted {
130			continue
131		}
132		for _, img := range imgsel.MatchAll(twp) {
133			img.Parent.RemoveChild(img)
134			div.AppendChild(img)
135		}
136		text := htf.NodeText(div)
137		text = strings.Replace(text, "\n", " ", -1)
138		text = re_removepics.ReplaceAllString(text, "")
139
140		if seen[text] {
141			continue
142		}
143
144		fmt.Fprintf(&buf, "> @%s: %s\n", author, text)
145		seen[text] = true
146	}
147	return buf.String()
148}
149
150func hooterize(noise string) string {
151	seen := make(map[string]bool)
152
153	hootfetcher := func(hoot string) string {
154		url := hoot[5:]
155		if url[0] == ' ' {
156			url = url[1:]
157		}
158		url = strings.Replace(url, "mobile.twitter.com", "twitter.com", -1)
159		dlog.Printf("hooterizing %s", url)
160		req, err := http.NewRequest("GET", url, nil)
161		if err != nil {
162			ilog.Printf("error: %s", err)
163			return hoot
164		}
165		req.Header.Set("User-Agent", "Bot")
166		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
167		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
168		resp, err := http.DefaultClient.Do(req)
169		if err != nil {
170			ilog.Printf("error: %s", err)
171			return hoot
172		}
173		defer resp.Body.Close()
174		if resp.StatusCode != 200 {
175			ilog.Printf("error getting %s: %d", url, resp.StatusCode)
176			return hoot
177		}
178		ld, _ := os.Create("lasthoot.html")
179		r := io.TeeReader(resp.Body, ld)
180		return hootextractor(r, url, seen)
181	}
182
183	return re_hoots.ReplaceAllStringFunc(noise, hootfetcher)
184}