Merge commit 'b67691a419' into masto
Anirudh Oppiliappan x@icyphox.sh
Sat, 02 Mar 2024 13:00:46 +0200
43 files changed,
1646 insertions(+),
880 deletions(-)
jump to
M
README
→
README
@@ -20,7 +20,7 @@
## build It should be sufficient to type make after unpacking a release. -You'll need a go compiler version 1.16 or later. And libsqlite3. +You'll need a go compiler version 1.18 or later. And libsqlite3. Even on a fast machine, building from source can take several seconds.
A
TODO
@@ -0,0 +1,11 @@
+A few things that could be hacked on. + +Remove httpsig key fetching from the input path. + If the key is known, check signature and reply 200 or 401. + If the key is unknown, immediately reply 202 and process later. + +The gob encoding for backend rpc uses more memory than needed. + A custom encoding could reduce allocations. + Maybe the backend could fetch the data itself. + +
M
activity.go
→
activity.go
@@ -30,8 +30,8 @@ "regexp"
"strings" "time" - "humungus.tedunangst.com/r/webs/cache" "humungus.tedunangst.com/r/webs/gate" + "humungus.tedunangst.com/r/webs/gencache" "humungus.tedunangst.com/r/webs/httpsig" "humungus.tedunangst.com/r/webs/junk" "humungus.tedunangst.com/r/webs/templates"@@ -43,8 +43,12 @@ var falsenames = []string{
`application/ld+json`, `application/activity+json`, } -var itiswhatitis = "https://www.w3.org/ns/activitystreams" -var thewholeworld = "https://www.w3.org/ns/activitystreams#Public" + +const itiswhatitis = "https://www.w3.org/ns/activitystreams" +const papersplease = "https://w3id.org/security/v1" +const thewholeworld = "https://www.w3.org/ns/activitystreams#Public" +const tinyworld = "as:Public" +const chatKeyProp = "chatKeyV0" var fastTimeout time.Duration = 5 var slowTimeout time.Duration = 30@@ -59,13 +63,18 @@ }
return false } -var honkClient = http.Client{} +var honkTransport = http.Transport{ + MaxIdleConns: 120, + MaxConnsPerHost: 4, +} + +var honkClient = http.Client{ + Transport: &honkTransport, +} func gogglesDoNothing() { - honkClient.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, + honkTransport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, } }@@ -88,36 +97,40 @@ resp, err := honkClient.Do(req)
if err != nil { return err } - resp.Body.Close() + defer resp.Body.Close() switch resp.StatusCode { case 200: case 201: case 202: default: + var buf [4096]byte + n, _ := resp.Body.Read(buf[:]) + dlog.Printf("post failure message: %s", buf[:n]) return fmt.Errorf("http post status: %d", resp.StatusCode) } ilog.Printf("successful post: %s %d", url, resp.StatusCode) return nil } -func GetJunk(userid int64, url string) (junk.Junk, error) { - return GetJunkTimeout(userid, url, slowTimeout*time.Second) +func GetJunk(userid UserID, url string) (junk.Junk, error) { + return GetJunkTimeout(userid, url, slowTimeout*time.Second, nil) } -func GetJunkFast(userid int64, url string) (junk.Junk, error) { - return GetJunkTimeout(userid, url, fastTimeout*time.Second) +func GetJunkFast(userid UserID, url string) (junk.Junk, error) { + return GetJunkTimeout(userid, url, fastTimeout*time.Second, nil) } -func GetJunkHardMode(userid int64, url string) (junk.Junk, error) { +func GetJunkHardMode(userid UserID, url string) (junk.Junk, error) { j, err := GetJunk(userid, url) if err != nil { emsg := err.Error() - if emsg == "http get status: 502" || strings.Contains(emsg, "timeout") { - ilog.Printf("trying again after error: %s", emsg) + if emsg == "http get status: 429" || emsg == "http get status: 502" || + strings.Contains(emsg, "timeout") { + ilog.Printf("trying %s again after error: %s", url, emsg) time.Sleep(time.Duration(60+notrand.Int63n(60)) * time.Second) j, err = GetJunk(userid, url) if err != nil { - ilog.Printf("still couldn't get it") + ilog.Printf("still couldn't get %s", url) } else { ilog.Printf("retry success!") }@@ -128,14 +141,16 @@ }
var flightdeck = gate.NewSerializer() -func GetJunkTimeout(userid int64, url string, timeout time.Duration) (junk.Junk, error) { +func GetJunkTimeout(userid UserID, url string, timeout time.Duration, final *string) (junk.Junk, error) { if rejectorigin(userid, url, false) { return nil, fmt.Errorf("rejected origin: %s", url) } + if final != nil { + *final = url + } sign := func(req *http.Request) error { - var ki *KeyInfo - ok := ziggies.Get(userid, &ki) - if ok { + ki := ziggy(userid) + if ki != nil { httpsig.SignRequest(ki.keyname, ki.seckey, req, nil) } return nil@@ -148,6 +163,9 @@ client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 { return fmt.Errorf("stopped after 5 redirects") } + if final != nil { + *final = req.URL.String() + } if sign != nil { sign(req) }@@ -158,12 +176,13 @@ at := theonetruename
if strings.Contains(url, ".well-known/webfinger?resource") { at = "application/jrd+json" } - j, err := junk.Get(url, junk.GetArgs{ + j, err := getsomejunk(url, junk.GetArgs{ Accept: at, Agent: "honksnonk/5.0; " + serverName, Timeout: timeout, Client: &client, Fixup: sign, + Limit: 1 * 1024 * 1024, }) return j, err }@@ -176,6 +195,56 @@ j := ji.(junk.Junk)
return j, nil } +func getsomejunk(url string, args junk.GetArgs) (junk.Junk, error) { + client := http.DefaultClient + if args.Client != nil { + client = args.Client + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + if args.Accept != "" { + req.Header.Set("Accept", args.Accept) + } + if args.Agent != "" { + req.Header.Set("User-Agent", args.Agent) + } + if args.Fixup != nil { + err = args.Fixup(req) + if err != nil { + return nil, err + } + } + if args.Timeout != 0 { + ctx, cancel := context.WithTimeout(context.Background(), args.Timeout) + defer cancel() + req = req.WithContext(ctx) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + switch resp.StatusCode { + case 200: + case 201: + case 202: + default: + return nil, fmt.Errorf("http get status: %d", resp.StatusCode) + } + ct := resp.Header.Get("Content-Type") + if args.Accept != "application/jrd+json" && !friendorfoe(ct) { + return nil, fmt.Errorf("incompatible content type %s", ct) + } + var r io.Reader = resp.Body + if args.Limit > 0 { + r = io.LimitReader(r, args.Limit) + } + return junk.Read(r) +} + func fetchsome(url string) ([]byte, error) { req, err := http.NewRequest("GET", url, nil) if err != nil {@@ -200,10 +269,12 @@ default:
return nil, fmt.Errorf("http get not 200: %d %s", resp.StatusCode, url) } var buf bytes.Buffer - limiter := io.LimitReader(resp.Body, 10*1024*1024) + limiter := io.LimitReader(resp.Body, maxFetchSize) io.Copy(&buf, limiter) return buf.Bytes(), nil } + +const maxFetchSize = 14 * 1024 * 1024 func savedonk(url string, name, desc, media string, localize bool) *Donk { if url == "" {@@ -226,7 +297,7 @@ goto saveit
} data = ii.([]byte) - if len(data) == 10*1024*1024 { + if len(data) == maxFetchSize { ilog.Printf("truncation likely") } if strings.HasPrefix(media, "image") {@@ -262,7 +333,7 @@ donk.FileID = fileid
return donk } -func iszonked(userid int64, xid string) bool { +func iszonked(userid UserID, xid string) bool { var id int64 row := stmtFindZonk.QueryRow(userid, xid) err := row.Scan(&id)@@ -314,7 +385,7 @@ }
return true } -func eradicatexonk(userid int64, xid string) { +func eradicatexonk(userid UserID, xid string) { xonk := getxonk(userid, xid) if xonk != nil { deletehonk(xonk.ID)@@ -338,7 +409,7 @@ Out string
Shared string } -var boxofboxes = cache.New(cache.Options{Filler: func(ident string) (*Box, bool) { +var boxofboxes = gencache.New(gencache.Options[string, *Box]{Fill: func(ident string) (*Box, bool) { var info string row := stmtGetXonker.QueryRow(ident, "boxes") err := row.Scan(&info)@@ -362,6 +433,15 @@ }
return nil, false }}) +var gettergate = gate.NewLimiter(1) + +func getsomemore(user *WhatAbout, page string) { + time.Sleep(5 * time.Second) + gettergate.Start() + defer gettergate.Finish() + gimmexonks(user, page) +} + func gimmexonks(user *WhatAbout, outbox string) { dlog.Printf("getting outbox: %s", outbox) j, err := GetJunk(user.ID, outbox)@@ -371,7 +451,7 @@ return
} t, _ := j.GetString("type") origin := originate(outbox) - if t == "OrderedCollection" { + if t == "OrderedCollection" || t == "CollectionPage" { items, _ := j.GetArray("orderedItems") if items == nil { items, _ = j.GetArray("items")@@ -480,8 +560,22 @@ }
return "" } +func grabhonk(user *WhatAbout, xid string) { + if x := getxonk(user.ID, xid); x != nil { + dlog.Printf("already have it: %s", xid) + return + } + var final string + j, err := GetJunkTimeout(user.ID, xid, fastTimeout*time.Second, &final) + if err != nil { + dlog.Printf("unable to fetch xid: %s", err) + return + } + xonksaver(user, j, originate(final)) +} + var re_mast0link = regexp.MustCompile(`https://[[:alnum:].]+/users/[[:alnum:]]+/statuses/[[:digit:]]+`) -var re_masto1ink = regexp.MustCompile(`https://([[:alnum:].]+)/@([[:alnum:]]+)/([[:digit:]]+)`) +var re_masto1ink = regexp.MustCompile(`https://([[:alnum:].]+)/@([[:alnum:]]+)(@[[:alnum:].]+)?/([[:digit:]]+)`) var re_misslink = regexp.MustCompile(`https://[[:alnum:].]+/notes/[[:alnum:]]+`) var re_honklink = regexp.MustCompile(`https://[[:alnum:].]+/u/[[:alnum:]]+/h/[[:alnum:]]+`) var re_r0malink = regexp.MustCompile(`https://[[:alnum:].]+/objects/[[:alnum:]-]+`)@@ -495,7 +589,7 @@ currenttid := ""
goingup := 0 var xonkxonkfn func(junk.Junk, string, bool, string) *Honk - qutify := func(user *WhatAbout, content string) string { + qutify := func(user *WhatAbout, qurl, content string) string { if depth >= maxdepth { ilog.Printf("in too deep") return content@@ -504,6 +598,10 @@ // well this is gross
malcontent := strings.ReplaceAll(content, `</span><span class="ellipsis">`, "") malcontent = strings.ReplaceAll(malcontent, `</span><span class="invisible">`, "") mlinks := re_qtlinks.FindAllString(malcontent, -1) + if qurl != "" { + mlinks = append(mlinks, ">"+qurl+"<") + } + mlinks = oneofakind(mlinks) for _, m := range mlinks { tryit := false m = m[1 : len(m)-1]@@ -512,20 +610,33 @@ re_honklink.MatchString(m) || re_r0malink.MatchString(m) ||
re_roma1ink.MatchString(m) { tryit = true } else if re_masto1ink.MatchString(m) { - m = re_masto1ink.ReplaceAllString(m, "https://$1/users/$2/statuses/$3") tryit = true } if tryit { + dlog.Printf("trying to get a quote from %s", m) + var prefix string + if m == qurl { + prefix += fmt.Sprintf("<p><a href=\"%s\">%s</a>", m, m) + } + var final string if x := getxonk(user.ID, m); x != nil { - content = fmt.Sprintf("%s<blockquote>%s</blockquote>", content, x.Noise) - } else if j, err := GetJunk(user.ID, m); err == nil { + dlog.Printf("already had it") + content = fmt.Sprintf("%s%s<blockquote>%s</blockquote>", content, prefix, x.Noise) + } else { + j, err := GetJunkTimeout(user.ID, m, fastTimeout*time.Second, &final) + if err != nil { + dlog.Printf("unable to fetch quote: %s", err) + continue + } q, ok := j.GetString("content") if ok { - content = fmt.Sprintf("%s<blockquote>%s</blockquote>", content, q) + content = fmt.Sprintf("%s%s<blockquote>%s</blockquote>", content, prefix, q) + } else { + dlog.Printf("apparently no content") } prevdepth := depth depth = maxdepth - xonkxonkfn(j, originate(m), false, "") + xonkxonkfn(j, originate(final), false, "") depth = prevdepth } }@@ -606,8 +717,8 @@ case "Announce":
obj, ok = item.GetMap("object") if ok { // peek ahead some - what, ok := obj.GetString("type") - if ok && (what == "Create" || what == "Update") { + what := firstofmany(obj, "type") + if what == "Create" || what == "Update" { if what == "Update" { isUpdate = true }@@ -628,6 +739,12 @@ if !isUpdate && !needbonkid(user, xid) {
return nil } bonker, _ = item.GetString("actor") + if originate(bonker) != origin { + ilog.Printf("out of bounds actor in bonk: %s not from %s", bonker, origin) + item.Write(ilog.Writer()) + return nil + } + bonker, _ = item.GetString("actor") origin = originate(xid) if ok && originate(id) == origin { dlog.Printf("using object in announce for %s", xid)@@ -649,7 +766,7 @@ if !ok {
xid, _ = item.GetString("object") dlog.Printf("getting created honk: %s", xid) if originate(xid) != origin { - ilog.Printf("out of bounds %s not from %s", xid, origin) + ilog.Printf("out of bounds object in create: %s not from %s", xid, origin) return nil } obj, err = GetJunkHardMode(user.ID, xid)@@ -754,7 +871,12 @@ xonk.XID = xid
xonk.UserID = user.ID xonk.Honker, _ = item.GetString("actor") if xonk.Honker == "" { - xonk.Honker, _ = item.GetString("attributedTo") + xonk.Honker = extractattrto(item) + } + if originate(xonk.Honker) != origin { + ilog.Printf("out of bounds honker %s from %s", xonk.Honker, origin) + item.Write(ilog.Writer()) + return nil } if obj != nil { if xonk.Honker == "" {@@ -770,11 +892,16 @@ xonk.Audience = newphone(nil, obj)
} xonk.Audience = append(xonk.Audience, xonk.Honker) xonk.Audience = oneofakind(xonk.Audience) + for i, a := range xonk.Audience { + if a == tinyworld { + xonk.Audience[i] = thewholeworld + } + } xonk.Public = loudandproud(xonk.Audience) var mentions []Mention if obj != nil { - ot, _ := obj.GetString("type") + ot := firstofmany(obj, "type") url, _ = obj.GetString("url") if dt2, ok := obj.GetString("published"); ok { dt = dt2@@ -798,7 +925,8 @@ content += fmt.Sprintf(`<p><a href="%s">%s</a>`, url, url)
url = xid } if user.Options.InlineQuotes { - content = qutify(user, content) + qurl, _ := obj.GetString("quoteUrl") + content = qutify(user, qurl, content) } rid, ok = obj.GetString("inReplyTo") if !ok {@@ -1056,7 +1184,9 @@ // init xonk
xonk.What = what xonk.RID = rid xonk.Date, _ = time.Parse(time.RFC3339, dt) - xonk.URL = url + if originate(url) == originate(xonk.XID) { + xonk.URL = url + } xonk.Format = "html" xonk.Convoy = convoy xonk.Mentions = mentions@@ -1068,10 +1198,26 @@ }
imaginate(&xonk) if what == "chonk" { - target, _ := obj.GetString("to") + // undo damage above + xonk.Noise = strings.TrimPrefix(xonk.Noise, "<p>") + target := firstofmany(obj, "to") if target == user.URL { target = xonk.Honker } + enc, _ := obj.GetString(chatKeyProp) + if enc != "" { + var dec string + if pubkey, ok := getchatkey(xonk.Honker); ok { + dec, err = decryptString(xonk.Noise, user.ChatSecKey, pubkey) + if err != nil { + ilog.Printf("failed to decrypt chonk") + } + } + if err == nil { + dlog.Printf("successful decrypt from %s", xonk.Honker) + xonk.Noise = dec + } + } ch := Chonk{ UserID: xonk.UserID, XID: xid,@@ -1115,7 +1261,7 @@ if convoy == "" {
convoy = currenttid } if convoy == "" { - convoy = "data:,missing-" + xfiltrate() + convoy = xonk.XID currenttid = convoy } xonk.Convoy = convoy@@ -1221,6 +1367,9 @@ j := junk.New()
j["id"] = user.URL + "/" + h.What + "/" + shortxid(h.XID) j["actor"] = user.URL j["published"] = dt + if h.Public { + h.Audience = append(h.Audience, user.URL+"/followers") + } j["to"] = h.Audience[0] if len(h.Audience) > 1 { j["cc"] = h.Audience[1:]@@ -1291,7 +1440,7 @@ for _, o := range h.Onts {
t := junk.New() t["type"] = "Hashtag" o = strings.ToLower(o) - t["href"] = fmt.Sprintf("https://%s/o/%s", serverName, o[1:]) + t["href"] = serverURL("/o/%s", o[1:]) t["name"] = o tags = append(tags, t) }@@ -1346,10 +1495,22 @@ jo["duration"] = "PT" + strings.ToUpper(t.Duration.String())
} } atts := activatedonks(h.Donks) + if h.Link != "" { + jo["type"] = "Page" + jl := junk.New() + jl["type"] = "Link" + jl["href"] = h.Link + atts = append(atts, jl) + } if len(atts) > 0 { jo["attachment"] = atts } - jo["summary"] = h.Precis + if h.LegalName != "" { + jo["name"] = h.LegalName + } + if h.Precis != "" { + jo["summary"] = h.Precis + } jo["content"] = h.Noise j["object"] = jo case "bonk":@@ -1401,7 +1562,7 @@
return j, jo } -var oldjonks = cache.New(cache.Options{Filler: func(xid string) ([]byte, bool) { +var oldjonks = gencache.New(gencache.Options[string, []byte]{Fill: func(xid string) ([]byte, bool) { row := stmtAnyXonk.QueryRow(xid) honk := scanhonk(row) if honk == nil || !honk.Public {@@ -1426,8 +1587,7 @@ return j.ToBytes(), true
}, Limit: 128}) func gimmejonk(xid string) ([]byte, bool) { - var j []byte - ok := oldjonks.Get(xid, &j) + j, ok := oldjonks.Get(xid) return j, ok }@@ -1441,8 +1601,7 @@ if a[0] == '%' {
rcpts[a] = true continue } - var box *Box - ok := boxofboxes.Get(a, &box) + box, ok := boxofboxes.Get(a) if ok && useshared && box.Shared != "" { rcpts["%"+box.Shared] = true } else {@@ -1452,7 +1611,7 @@ }
return rcpts } -func chonkifymsg(user *WhatAbout, ch *Chonk) []byte { +func chonkifymsg(user *WhatAbout, rcpt string, ch *Chonk) []byte { dt := ch.Date.Format(time.RFC3339) aud := []string{ch.Target}@@ -1462,7 +1621,19 @@ jo["type"] = "ChatMessage"
jo["published"] = dt jo["attributedTo"] = user.URL jo["to"] = aud - jo["content"] = ch.HTML + content := string(ch.HTML) + if user.ChatSecKey.key != nil { + if pubkey, ok := getchatkey(rcpt); ok { + var err error + content, err = encryptString(content, user.ChatSecKey, pubkey) + if err != nil { + ilog.Printf("failure encrypting chonk: %s", err) + } + jo[chatKeyProp] = user.Options.ChatPubKey + } + } + jo["content"] = content + atts := activatedonks(ch.Donks) if len(atts) > 0 { jo["attachment"] = atts@@ -1497,11 +1668,10 @@ return j.ToBytes()
} func sendchonk(user *WhatAbout, ch *Chonk) { - msg := chonkifymsg(user, ch) - rcpts := make(map[string]bool) rcpts[ch.Target] = true for a := range rcpts { + msg := chonkifymsg(user, a, ch) go deliverate(user.ID, a, msg) } }@@ -1518,8 +1688,7 @@ for _, h := range getdubs(user.ID) {
if h.XID == user.URL { continue } - var box *Box - ok := boxofboxes.Get(h.XID, &box) + box, ok := boxofboxes.Get(h.XID) if ok && box.Shared != "" { rcpts["%"+box.Shared] = true } else {@@ -1530,8 +1699,7 @@ for _, f := range getbacktracks(honk.XID) {
if f[0] == '%' { rcpts[f] = true } else { - var box *Box - ok := boxofboxes.Get(f, &box) + box, ok := boxofboxes.Get(f) if ok && box.Shared != "" { rcpts["%"+box.Shared] = true } else {@@ -1561,11 +1729,10 @@ j["type"] = "Add"
j["id"] = user.URL + "/add/" + shortxid(ont+honk.XID) j["actor"] = user.URL j["object"] = honk.XID - j["target"] = fmt.Sprintf("https://%s/o/%s", serverName, ont[1:]) + j["target"] = serverURL("/o/%s", ont[1:]) rcpts := make(map[string]bool) for _, dub := range dubs { - var box *Box - ok := boxofboxes.Get(dub.XID, &box) + box, ok := boxofboxes.Get(dub.XID) if ok && box.Shared != "" { rcpts["%"+box.Shared] = true } else {@@ -1581,7 +1748,7 @@ }
func junkuser(user *WhatAbout) junk.Junk { j := junk.New() - j["@context"] = itiswhatitis + j["@context"] = []string{itiswhatitis, papersplease} j["id"] = user.URL j["inbox"] = user.URL + "/inbox" j["outbox"] = user.URL + "/outbox"@@ -1597,7 +1764,7 @@ for _, o := range user.Onts {
t := junk.New() t["type"] = "Hashtag" o = strings.ToLower(o) - t["href"] = fmt.Sprintf("https://%s/o/%s", serverName, o[1:]) + t["href"] = serverURL("/o/%s", o[1:]) t["name"] = o tags = append(tags, t) }@@ -1630,11 +1797,12 @@ k["id"] = user.URL + "#key"
k["owner"] = user.URL k["publicKeyPem"] = user.Key j["publicKey"] = k + j[chatKeyProp] = user.Options.ChatPubKey return j } -var oldjonkers = cache.New(cache.Options{Filler: func(name string) ([]byte, bool) { +var oldjonkers = gencache.New(gencache.Options[string, []byte]{Fill: func(name string) ([]byte, bool) { user, err := butwhatabout(name) if err != nil { return nil, false@@ -1644,12 +1812,11 @@ return j.ToBytes(), true
}, Duration: 1 * time.Minute}) func asjonker(name string) ([]byte, bool) { - var j []byte - ok := oldjonkers.Get(name, &j) + j, ok := oldjonkers.Get(name) return j, ok } -var handfull = cache.New(cache.Options{Filler: func(name string) (string, bool) { +var handfull = gencache.New(gencache.Options[string, string]{Fill: func(name string) (string, bool) { m := strings.Split(name, "@") if len(m) != 2 { dlog.Printf("bad fish name: %s", name)@@ -1692,8 +1859,7 @@ func gofish(name string) string {
if name[0] == '@' { name = name[1:] } - var href string - handfull.Get(name, &href) + href, _ := handfull.Get(name) return href }@@ -1753,12 +1919,27 @@ return info, nil
} func allinjest(origin string, obj junk.Junk) { + ident, _ := obj.GetString("id") + if ident == "" { + return + } + if originate(ident) != origin { + return + } keyobj, ok := obj.GetMap("publicKey") if ok { ingestpubkey(origin, keyobj) } ingestboxes(origin, obj) ingesthandle(origin, obj) + chatkey, ok := obj.GetString(chatKeyProp) + if ok { + when := time.Now().UTC().Format(dbtimeformat) + _, err := stmtSaveXonker.Exec(ident, chatkey, chatKeyProp, when) + if err != nil { + elog.Printf("error saving chatkey: %s", err) + } + } } func ingestpubkey(origin string, obj junk.Junk) {@@ -1857,8 +2038,7 @@ }
} func updateMe(username string) { - var user *WhatAbout - somenamedusers.Get(username, &user) + user, _ := somenamedusers.Get(username) dt := time.Now().UTC().Format(time.RFC3339) j := junk.New() j["@context"] = itiswhatitis@@ -1876,9 +2056,8 @@ for _, f := range getdubs(user.ID) {
if f.XID == user.URL { continue } - var box *Box - boxofboxes.Get(f.XID, &box) - if box != nil && box.Shared != "" { + box, ok := boxofboxes.Get(f.XID) + if ok && box.Shared != "" { rcpts["%"+box.Shared] = true } else { rcpts[f.XID] = true
M
avatar.go
→
avatar.go
@@ -108,7 +108,7 @@ func avatarURL(user *WhatAbout) string {
if ava := user.Options.Avatar; ava != "" { return ava } - return fmt.Sprintf("https://%s/a?a=%s", serverName, url.QueryEscape(user.URL)) + return serverURL("/a?a=%s", url.QueryEscape(user.URL)) } func showflag(writer http.ResponseWriter, req *http.Request) {@@ -176,7 +176,7 @@ h.Noise = re_flags.ReplaceAllStringFunc(h.Noise, func(m string) string {
count++ var e Emu e.Name = fmt.Sprintf(":flag%d:", count) - e.ID = fmt.Sprintf("https://%s/flag/%s", serverName, m[5:]) + e.ID = serverURL("/flag/%s", m[5:]) emus = append(emus, e) return e.Name })@@ -186,7 +186,7 @@
func renderflags(h *Honk) { h.Noise = re_flags.ReplaceAllStringFunc(h.Noise, func(m string) string { code := m[5:] - src := fmt.Sprintf("https://%s/flag/%s", serverName, code) + src := serverURL("/flag/%s", code) return fmt.Sprintf(`<img class="emu" title="%s" src="%s">`, "flag", src) }) }
M
backend.go
→
backend.go
@@ -23,7 +23,10 @@ "net/http"
"net/rpc" "os" "os/exec" + "os/signal" "strings" + "sync" + "syscall" "humungus.tedunangst.com/r/webs/gate" "humungus.tedunangst.com/r/webs/image"@@ -58,18 +61,33 @@ func backendSockname() string {
return dataDir + "/backend.sock" } +var bomFuck = []byte{0xef, 0xbb, 0xbf} + func isSVG(data []byte) bool { + if bytes.HasPrefix(data, bomFuck) { + data = data[3:] + } ct := http.DetectContentType(data) - if strings.HasPrefix(ct, "text/xml") { - return strings.Index(string(data), "<!DOCTYPE svg PUBLIC") != -1 - } - if strings.HasPrefix(ct, "text/plain") { - return bytes.HasPrefix(data, []byte("<svg ")) + if strings.HasPrefix(ct, "text/xml") || strings.HasPrefix(ct, "text/plain") { + // this seems suboptimal + prefixes := []string{ + `<svg `, + `<!DOCTYPE svg PUBLIC`, + `<?xml version="1.0" encoding="UTF-8"?> <svg `, + } + for _, pre := range prefixes { + if bytes.HasPrefix(data, []byte(pre)) { + return true + } + } } return ct == "image/svg+xml" } func imageFromSVG(data []byte) (*image.Image, error) { + if bytes.HasPrefix(data, bomFuck) { + data = data[3:] + } if len(data) > 100000 { return nil, errors.New("my svg is too big") }@@ -80,7 +98,7 @@ }
return svg, nil } -func bigshrink(data []byte) (*image.Image, error) { +func callshrink(data []byte, params image.Params) (*image.Image, error) { if isSVG(data) { return imageFromSVG(data) }@@ -91,13 +109,8 @@ }
defer cl.Close() var res ShrinkerResult err = cl.Call("Shrinker.Shrink", &ShrinkerArgs{ - Buf: data, - Params: image.Params{ - LimitSize: 14200 * 4200, - MaxWidth: 2600, - MaxHeight: 2048, - MaxSize: 768 * 1024, - }, + Buf: data, + Params: params, }, &res) if err != nil { return nil, err@@ -105,28 +118,34 @@ }
return res.Image, nil } -func shrinkit(data []byte) (*image.Image, error) { - if isSVG(data) { - return imageFromSVG(data) +func lilshrink(data []byte) (*image.Image, error) { + params := image.Params{ + LimitSize: 14200 * 4200, + MaxWidth: 256, + MaxHeight: 256, + MaxSize: 16 * 1024, } - cl, err := rpc.Dial("unix", backendSockname()) - if err != nil { - return nil, err + return callshrink(data, params) +} +func bigshrink(data []byte) (*image.Image, error) { + params := image.Params{ + LimitSize: 14200 * 4200, + MaxWidth: 2600, + MaxHeight: 2048, + MaxSize: 768 * 1024, } - defer cl.Close() - var res ShrinkerResult - err = cl.Call("Shrinker.Shrink", &ShrinkerArgs{ - Buf: data, - Params: image.Params{LimitSize: 4200 * 4200, MaxWidth: 2048, MaxHeight: 2048}, - }, &res) - if err != nil { - return nil, err + return callshrink(data, params) +} + +func shrinkit(data []byte) (*image.Image, error) { + params := image.Params{ + LimitSize: 4200 * 4200, + MaxWidth: 2048, + MaxHeight: 2048, } - return res.Image, nil + return callshrink(data, params) } -var backendhooks []func() - func orphancheck() { var b [1]byte os.Stdin.Read(b[:])@@ -136,7 +155,9 @@ }
func backendServer() { dlog.Printf("backend server running") + closedatabases() go orphancheck() + signal.Ignore(syscall.SIGINT) shrinker := new(Shrinker) srv := rpc.NewServer() err := srv.Register(shrinker)@@ -158,9 +179,7 @@ err = setLimits()
if err != nil { elog.Printf("error setting backend limits: %s", err) } - for _, h := range backendhooks { - h() - } + securitizebackend() srv.Accept(lis) }@@ -177,9 +196,23 @@ err = proc.Start()
if err != nil { elog.Panicf("can't exec backend: %s", err) } + workinprogress++ + var mtx sync.Mutex go func() { - proc.Wait() - elog.Printf("lost the backend: %s", err) + <-endoftheworld + mtx.Lock() + defer mtx.Unlock() w.Close() + w = nil + readyalready <- true + }() + go func() { + err := proc.Wait() + mtx.Lock() + defer mtx.Unlock() + if w != nil { + elog.Printf("lost the backend: %s", err) + w.Close() + } }() }
M
database.go
→
database.go
@@ -30,7 +30,7 @@ "sync"
"time" "github.com/jmoiron/sqlx" - "humungus.tedunangst.com/r/webs/cache" + "humungus.tedunangst.com/r/webs/gencache" "humungus.tedunangst.com/r/webs/htfilter" "humungus.tedunangst.com/r/webs/httpsig" "humungus.tedunangst.com/r/webs/login"@@ -53,13 +53,15 @@ if err != nil {
return nil, err } if user.ID > 0 { - user.URL = fmt.Sprintf("https://%s/%s/%s", serverName, userSep, user.Name) + user.URL = serverURL("/%s/%s", userSep, user.Name) err = unjsonify(options, &user.Options) if err != nil { elog.Printf("error processing user options: %s", err) } + user.ChatPubKey.key, _ = b64tokey(user.Options.ChatPubKey) + user.ChatSecKey.key, _ = b64tokey(user.Options.ChatSecKey) } else { - user.URL = fmt.Sprintf("https://%s/%s", serverName, user.Name) + user.URL = serverURL("/%s", user.Name) } if user.Options.Reaction == "" { user.Options.Reaction = "none"@@ -68,7 +70,7 @@
return user, nil } -var somenamedusers = cache.New(cache.Options{Filler: func(name string) (*WhatAbout, bool) { +var somenamedusers = gencache.New(gencache.Options[string, *WhatAbout]{Fill: func(name string) (*WhatAbout, bool) { row := stmtUserByName.QueryRow(name) user, err := userfromrow(row) if err != nil {@@ -82,7 +84,7 @@ user.Onts = marker.HashTags
return user, true }}) -var somenumberedusers = cache.New(cache.Options{Filler: func(userid int64) (*WhatAbout, bool) { +var somenumberedusers = gencache.New(gencache.Options[UserID, *WhatAbout]{Fill: func(userid UserID) (*WhatAbout, bool) { row := stmtUserByNumber.QueryRow(userid) user, err := userfromrow(row) if err != nil {@@ -94,8 +96,7 @@ return user, true
}}) func getserveruser() *WhatAbout { - var user *WhatAbout - ok := somenumberedusers.Get(serverUID, &user) + user, ok := somenumberedusers.Get(serverUID) if !ok { elog.Panicf("lost server user") }@@ -103,17 +104,16 @@ return user
} func butwhatabout(name string) (*WhatAbout, error) { - var user *WhatAbout - ok := somenamedusers.Get(name, &user) + user, ok := somenamedusers.Get(name) if !ok { return nil, fmt.Errorf("no user: %s", name) } return user, nil } -var honkerinvalidator cache.Invalidator +var honkerinvalidator gencache.Invalidator[UserID] -func gethonkers(userid int64) []*Honker { +func gethonkers(userid UserID) []*Honker { rows, err := stmtHonkers.Query(userid) if err != nil { elog.Printf("error querying honkers: %s", err)@@ -138,12 +138,12 @@ }
return honkers } -func getdubs(userid int64) []*Honker { +func getdubs(userid UserID) []*Honker { rows, err := stmtDubbers.Query(userid) return dubsfromrows(rows, err) } -func getnameddubs(userid int64, name string) []*Honker { +func getnameddubs(userid UserID, name string) []*Honker { rows, err := stmtNamedDubbers.Query(userid, name) return dubsfromrows(rows, err) }@@ -179,12 +179,15 @@ }
return users } -func getxonk(userid int64, xid string) *Honk { - row := stmtOneXonk.QueryRow(userid, xid) +func getxonk(userid UserID, xid string) *Honk { + if xid == "" { + return nil + } + row := stmtOneXonk.QueryRow(userid, xid, xid) return scanhonk(row) } -func getbonk(userid int64, xid string) *Honk { +func getbonk(userid UserID, xid string) *Honk { row := stmtOneBonk.QueryRow(userid, xid) return scanhonk(row) }@@ -194,7 +197,7 @@ dt := time.Now().Add(-honkwindow).UTC().Format(dbtimeformat)
rows, err := stmtPublicHonks.Query(dt, 100) return getsomehonks(rows, err) } -func geteventhonks(userid int64) []*Honk { +func geteventhonks(userid UserID) []*Honk { rows, err := stmtEventHonks.Query(userid, 25) honks := getsomehonks(rows, err) sort.Slice(honks, func(i, j int) bool {@@ -235,58 +238,65 @@ }
rows, err := stmtUserHonks.Query(wanted, whofore, name, dt, limit) return getsomehonks(rows, err) } -func gethonksforuser(userid int64, wanted int64) []*Honk { +func gethonksforuser(userid UserID, wanted int64) []*Honk { dt := time.Now().Add(-honkwindow).UTC().Format(dbtimeformat) rows, err := stmtHonksForUser.Query(wanted, userid, dt, userid, userid) return getsomehonks(rows, err) } -func gethonksforuserfirstclass(userid int64, wanted int64) []*Honk { +func gethonksforuserfirstclass(userid UserID, wanted int64) []*Honk { dt := time.Now().Add(-honkwindow).UTC().Format(dbtimeformat) rows, err := stmtHonksForUserFirstClass.Query(wanted, userid, dt, userid, userid) return getsomehonks(rows, err) } -func gethonksforme(userid int64, wanted int64) []*Honk { +func gethonksforme(userid UserID, wanted int64) []*Honk { dt := time.Now().Add(-honkwindow).UTC().Format(dbtimeformat) rows, err := stmtHonksForMe.Query(wanted, userid, dt, userid, 250) return getsomehonks(rows, err) } -func gethonksfromlongago(userid int64, wanted int64) []*Honk { +func gethonksfromlongago(userid UserID, wanted int64) []*Honk { + var params []interface{} + var wheres []string + params = append(params, wanted) + params = append(params, userid) now := time.Now() - var honks []*Honk - for i := 1; i <= 4; i++ { + for i := 1; i <= 5; i++ { dt := time.Date(now.Year()-i, now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), 0, now.Location()) dt1 := dt.Add(-36 * time.Hour).UTC().Format(dbtimeformat) dt2 := dt.Add(12 * time.Hour).UTC().Format(dbtimeformat) - rows, err := stmtHonksFromLongAgo.Query(wanted, userid, dt1, dt2, userid) - honks = append(honks, getsomehonks(rows, err)...) + wheres = append(wheres, "(dt > ? and dt < ?)") + params = append(params, dt1, dt2) } - return honks + params = append(params, userid) + sql := strings.ReplaceAll(sqlHonksFromLongAgo, "WHERECLAUSE", strings.Join(wheres, " or ")) + db := opendatabase() + rows, err := db.Query(sql, params...) + return getsomehonks(rows, err) } -func getsavedhonks(userid int64, wanted int64) []*Honk { +func getsavedhonks(userid UserID, wanted int64) []*Honk { rows, err := stmtHonksISaved.Query(wanted, userid) return getsomehonks(rows, err) } -func gethonksbyhonker(userid int64, honker string, wanted int64) []*Honk { +func gethonksbyhonker(userid UserID, honker string, wanted int64) []*Honk { rows, err := stmtHonksByHonker.Query(wanted, userid, honker, userid) return getsomehonks(rows, err) } -func gethonksbyxonker(userid int64, xonker string, wanted int64) []*Honk { +func gethonksbyxonker(userid UserID, xonker string, wanted int64) []*Honk { rows, err := stmtHonksByXonker.Query(wanted, userid, xonker, xonker, userid) return getsomehonks(rows, err) } -func gethonksbycombo(userid int64, combo string, wanted int64) []*Honk { +func gethonksbycombo(userid UserID, combo string, wanted int64) []*Honk { combo = "% " + combo + " %" rows, err := stmtHonksByCombo.Query(wanted, userid, userid, combo, userid, wanted, userid, combo, userid) return getsomehonks(rows, err) } -func gethonksbyconvoy(userid int64, convoy string, wanted int64) []*Honk { +func gethonksbyconvoy(userid UserID, convoy string, wanted int64) []*Honk { rows, err := stmtHonksByConvoy.Query(wanted, userid, userid, convoy) honks := getsomehonks(rows, err) return honks } -func gethonksbysearch(userid int64, q string, wanted int64) []*Honk { +func gethonksbysearch(userid UserID, q string, wanted int64) []*Honk { var queries []string var params []interface{} queries = append(queries, "honks.honkid > ?")@@ -359,7 +369,7 @@ rows, err := opendatabase().Query(selecthonks+where+butnotthose+limit, params...)
honks := getsomehonks(rows, err) return honks } -func gethonksbyontology(userid int64, name string, wanted int64) []*Honk { +func gethonksbyontology(userid UserID, name string, wanted int64) []*Honk { rows, err := stmtHonksByOntology.Query(wanted, name, userid, userid) honks := getsomehonks(rows, err) return honks@@ -508,6 +518,14 @@ if err != nil {
elog.Printf("error parsing badonks: %s", err) continue } + case "seealso": + h.SeeAlso = j + case "onties": + h.Onties = j + case "link": + h.Link = j + case "legalname": + h.LegalName = j case "oldrev": default: elog.Printf("unknown meta genus: %s", genus)@@ -586,7 +604,7 @@ elog.Printf("error checking file hash: %s", err)
return 0, "", err } if url == "" { - url = fmt.Sprintf("https://%s/d/%s", serverName, xid) + url = serverURL("/d/%s", xid) } }@@ -652,9 +670,8 @@ }
return err } -func chatplusone(tx *sql.Tx, userid int64) { - var user *WhatAbout - ok := somenumberedusers.Get(userid, &user) +func chatplusone(tx *sql.Tx, userid UserID) { + user, ok := somenumberedusers.Get(userid) if !ok { return }@@ -671,9 +688,8 @@ somenamedusers.Clear(user.Name)
somenumberedusers.Clear(user.ID) } -func chatnewnone(userid int64) { - var user *WhatAbout - ok := somenumberedusers.Get(userid, &user) +func chatnewnone(userid UserID) { + user, ok := somenumberedusers.Get(userid) if !ok || user.Options.ChatCount == 0 { return }@@ -691,9 +707,8 @@ somenamedusers.Clear(user.Name)
somenumberedusers.Clear(user.ID) } -func meplusone(tx *sql.Tx, userid int64) { - var user *WhatAbout - ok := somenumberedusers.Get(userid, &user) +func meplusone(tx *sql.Tx, userid UserID) { + user, ok := somenumberedusers.Get(userid) if !ok { return }@@ -710,9 +725,8 @@ somenamedusers.Clear(user.Name)
somenumberedusers.Clear(user.ID) } -func menewnone(userid int64) { - var user *WhatAbout - ok := somenumberedusers.Get(userid, &user) +func menewnone(userid UserID) { + user, ok := somenumberedusers.Get(userid) if !ok || user.Options.MeCount == 0 { return }@@ -730,7 +744,7 @@ somenamedusers.Clear(user.Name)
somenumberedusers.Clear(user.ID) } -func loadchatter(userid int64) []*Chatter { +func loadchatter(userid UserID) []*Chatter { duedt := time.Now().Add(-3 * 24 * time.Hour).UTC().Format(dbtimeformat) rows, err := stmtLoadChonks.Query(userid, duedt) if err != nil {@@ -847,6 +861,7 @@ err = saveextras(tx, h)
} if err == nil { if h.Whofore == 1 { + dlog.Printf("another one for me: %s", h.XID) meplusone(tx, h.UserID) } err = tx.Commit()@@ -969,6 +984,34 @@ elog.Printf("error saving mentions: %s", err)
return err } } + if onties := h.Onties; onties != "" { + _, err := tx.Stmt(stmtSaveMeta).Exec(h.ID, "onties", onties) + if err != nil { + elog.Printf("error saving onties: %s", err) + return err + } + } + if legalname := h.LegalName; legalname != "" { + _, err := tx.Stmt(stmtSaveMeta).Exec(h.ID, "legalname", legalname) + if err != nil { + elog.Printf("error saving legalname: %s", err) + return err + } + } + if seealso := h.SeeAlso; seealso != "" { + _, err := tx.Stmt(stmtSaveMeta).Exec(h.ID, "seealso", seealso) + if err != nil { + elog.Printf("error saving seealso: %s", err) + return err + } + } + if link := h.Link; link != "" { + _, err := tx.Stmt(stmtSaveMeta).Exec(h.ID, "link", link) + if err != nil { + elog.Printf("error saving link: %s", err) + return err + } + } return nil }@@ -1036,7 +1079,7 @@ func savexonker(what, value, flav, when string) {
stmtSaveXonker.Exec(what, value, flav, when) } -func savehonker(user *WhatAbout, url, name, flavor, combos, mj string) (int64, error) { +func savehonker(user *WhatAbout, url, name, flavor, combos, mj string) (int64, string, error) { var owner string if url[0] == '#' { flavor = "peep"@@ -1044,11 +1087,18 @@ if name == "" {
name = url[1:] } owner = url + } else if strings.HasSuffix(url, ".rss") { + flavor = "peep" + if name == "" { + name = url[strings.LastIndexByte(url, '/')+1:] + } + owner = url + } else { info, err := investigate(url) if err != nil { ilog.Printf("failed to investigate honker: %s", err) - return 0, err + return 0, "", err } url = info.XID if name == "" {@@ -1067,16 +1117,16 @@ elog.Printf("honker scan err: %s", err)
} else { err = fmt.Errorf("it seems you are already subscribed to them") } - return 0, err + return 0, "", err } res, err := stmtSaveHonker.Exec(user.ID, name, url, flavor, combos, owner, mj) if err != nil { elog.Print(err) - return 0, err + return 0, "", err } honkerid, _ := res.LastInsertId() - return honkerid, nil + return honkerid, flavor, nil } func cleanupdb(arg string) {@@ -1106,8 +1156,8 @@ doordie(db, "delete from zonkers where userid = ? and wherefore = 'zonvoy' and zonkerid < (select zonkerid from zonkers where userid = ? and wherefore = 'zonvoy' order by zonkerid desc limit 1 offset 200)", u.UserID, u.UserID)
} filexids := make(map[string]bool) - blobdb := openblobdb() - rows, err := blobdb.Query("select xid from filedata") + g_blobdb = openblobdb() + rows, err := g_blobdb.Query("select xid from filedata") if err != nil { elog.Fatal(err) }@@ -1130,7 +1180,7 @@ }
delete(filexids, xid) } rows.Close() - tx, err := blobdb.Begin() + tx, err := g_blobdb.Begin() if err != nil { elog.Fatal(err) }@@ -1144,6 +1194,7 @@ err = tx.Commit()
if err != nil { elog.Fatal(err) } + closedatabases() } func getusercount() int {@@ -1209,7 +1260,7 @@ var stmtHonkers, stmtDubbers, stmtNamedDubbers, stmtSaveHonker, stmtUpdateFlavor, stmtUpdateHonker *sql.Stmt
var stmtDeleteHonker *sql.Stmt var stmtAnyXonk, stmtOneXonk, stmtPublicHonks, stmtUserHonks, stmtHonksByCombo, stmtHonksByConvoy *sql.Stmt var stmtHonksByOntology, stmtHonksForUser, stmtHonksForMe, stmtSaveDub, stmtHonksByXonker *sql.Stmt -var stmtHonksFromLongAgo *sql.Stmt +var sqlHonksFromLongAgo string var stmtHonksByHonker, stmtSaveHonk, stmtUserByName, stmtUserByNumber *sql.Stmt var stmtEventHonks, stmtOneBonk, stmtFindZonk, stmtFindXonk, stmtSaveDonk *sql.Stmt var stmtFindFile, stmtFindFileId, stmtGetFileData, stmtSaveFileData, stmtSaveFile *sql.Stmt@@ -1251,6 +1302,21 @@ }
return stmt } +var g_blobdb *sql.DB + +func closedatabases() { + err := alreadyopendb.Close() + if err != nil { + elog.Printf("error closing database: %s", err) + } + if g_blobdb != nil { + err = g_blobdb.Close() + if err != nil { + elog.Printf("error closing database: %s", err) + } + } +} + func prepareStatements(db *sql.DB) { stmtHonkers = preparetodie(db, "select honkerid, userid, name, xid, flavor, combos, meta from honkers where userid = ? and (flavor = 'presub' or flavor = 'sub' or flavor = 'peep' or flavor = 'unsub') order by name") stmtSaveHonker = preparetodie(db, "insert into honkers (userid, name, xid, flavor, combos, owner, meta, folxid) values (?, ?, ?, ?, ?, ?, ?, '')")@@ -1265,7 +1331,7 @@ selecthonks := "select honks.honkid, honks.userid, username, what, honker, oonker, honks.xid, rid, dt, url, audience, noise, precis, format, convoy, whofore, flags from honks join users on honks.userid = users.userid "
limit := " order by honks.honkid desc limit 250" smalllimit := " order by honks.honkid desc limit ?" butnotthose := " and convoy not in (select name from zonkers where userid = ? and wherefore = 'zonvoy' order by zonkerid desc limit 100)" - stmtOneXonk = preparetodie(db, selecthonks+"where honks.userid = ? and xid = ?") + stmtOneXonk = preparetodie(db, selecthonks+"where honks.userid = ? and (xid = ? or url = ?)") stmtAnyXonk = preparetodie(db, selecthonks+"where xid = ? and what <> 'bonk' order by honks.honkid asc") stmtOneBonk = preparetodie(db, selecthonks+"where honks.userid = ? and xid = ? and what = 'bonk' and whofore = 2") stmtPublicHonks = preparetodie(db, selecthonks+"where whofore = 2 and dt > ?"+smalllimit)@@ -1275,7 +1341,7 @@ myhonkers := " and honker in (select xid from honkers where userid = ? and (flavor = 'sub' or flavor = 'peep' or flavor = 'presub') and combos not like '% - %')"
stmtHonksForUser = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ?"+myhonkers+butnotthose+limit) stmtHonksForUserFirstClass = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ? and (rid = '' or what = 'bonk')"+myhonkers+butnotthose+limit) stmtHonksForMe = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ? and whofore = 1"+butnotthose+smalllimit) - stmtHonksFromLongAgo = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and dt > ? and dt < ? and whofore = 2"+butnotthose+limit) + sqlHonksFromLongAgo = selecthonks + "where honks.honkid > ? and honks.userid = ? and (WHERECLAUSE) and (whofore = 2 or flags & 4)" + butnotthose + limit stmtHonksISaved = preparetodie(db, selecthonks+"where honks.honkid > ? and honks.userid = ? and flags & 4 order by honks.honkid desc") stmtHonksByHonker = preparetodie(db, selecthonks+"join honkers on (honkers.xid = honks.honker or honkers.xid = honks.oonker) where honks.honkid > ? and honks.userid = ? and honkers.name = ?"+butnotthose+limit) stmtHonksByXonker = preparetodie(db, selecthonks+" where honks.honkid > ? and honks.userid = ? and (honker = ? or oonker = ?)"+butnotthose+limit)@@ -1295,10 +1361,10 @@ stmtDeleteOnts = preparetodie(db, "delete from onts where honkid = ?")
stmtSaveDonk = preparetodie(db, "insert into donks (honkid, chonkid, fileid) values (?, ?, ?)") stmtDeleteDonks = preparetodie(db, "delete from donks where honkid = ?") stmtSaveFile = preparetodie(db, "insert into filemeta (xid, name, description, url, media, local) values (?, ?, ?, ?, ?, ?)") - blobdb := openblobdb() - stmtSaveFileData = preparetodie(blobdb, "insert into filedata (xid, media, hash, content) values (?, ?, ?, ?)") - stmtCheckFileData = preparetodie(blobdb, "select xid from filedata where hash = ?") - stmtGetFileData = preparetodie(blobdb, "select media, content from filedata where xid = ?") + g_blobdb = openblobdb() + stmtSaveFileData = preparetodie(g_blobdb, "insert into filedata (xid, media, hash, content) values (?, ?, ?, ?)") + stmtCheckFileData = preparetodie(g_blobdb, "select xid from filedata where hash = ?") + stmtGetFileData = preparetodie(g_blobdb, "select media, content from filedata where xid = ?") stmtFindXonk = preparetodie(db, "select honkid from honks where userid = ? and xid = ?") stmtFindFile = preparetodie(db, "select fileid, xid from filemeta where url = ? and local = 1") stmtFindFileId = preparetodie(db, "select xid, local, description from filemeta where fileid = ? and url = ? and local = 1")
M
deliverator.go
→
deliverator.go
@@ -29,7 +29,7 @@
type Doover struct { ID int64 When time.Time - Userid int64 + Userid UserID Tries int64 Rcpt string Msgs [][]byte@@ -61,10 +61,12 @@ default:
} } +const nearlyDead = 8 + func lethaldose(err error) int64 { str := err.Error() if strings.Contains(str, "no such host") { - return 8 + return nearlyDead } return 0 }@@ -74,12 +76,15 @@ str := err.Error()
if strings.Contains(str, "http post status: 400") { return true } + if strings.Contains(str, "http post status: 422") { + return true + } return false } var dqmtx sync.Mutex -func delinquent(userid int64, rcpt string, msg []byte) bool { +func delinquent(userid UserID, rcpt string, msg []byte) bool { dqmtx.Lock() defer dqmtx.Unlock() row := stmtDeliquentCheck.QueryRow(userid, rcpt)@@ -103,7 +108,7 @@ }
return true } -func deliverate(userid int64, rcpt string, msg []byte) { +func deliverate(userid UserID, rcpt string, msg []byte) { if delinquent(userid, rcpt, msg) { return }@@ -118,13 +123,14 @@
var garage = gate.NewLimiter(40) func deliveration(doover Doover) { + requestWG.Add(1) + defer requestWG.Done() rcpt := doover.Rcpt garage.StartKey(rcpt) defer garage.FinishKey(rcpt) - var ki *KeyInfo - ok := ziggies.Get(doover.Userid, &ki) - if !ok { + ki := ziggy(doover.Userid) + if ki == nil { elog.Printf("lost key for delivery") return }@@ -133,10 +139,12 @@ // already did the box indirection
if rcpt[0] == '%' { inbox = rcpt[1:] } else { - var box *Box - ok := boxofboxes.Get(rcpt, &box) + box, ok := boxofboxes.Get(rcpt) if !ok { ilog.Printf("failed getting inbox for %s", rcpt) + if doover.Tries < nearlyDead { + doover.Tries = nearlyDead + } sayitagain(doover) return }@@ -206,6 +214,7 @@ return nil
} func redeliverator() { + workinprogress++ sleeper := time.NewTimer(5 * time.Second) for { select {@@ -215,6 +224,9 @@ <-sleeper.C
} time.Sleep(5 * time.Second) case <-sleeper.C: + case <-endoftheworld: + readyalready <- true + return } doovers := getdoovers()
M
docs/changelog.txt
→
docs/changelog.txt
@@ -1,6 +1,64 @@
changelog -### next +### 1.3.0 Big Bonsai + ++ Some performance tuning. + ++ Follow .rss feeds for hashtags. + ++ Easier to inline attached images. + ++ Optional .json urls for activities. + ++ cc, link, name, and tags fields for advanced metadata. + ++ <big> tag supported in html. + +### 1.2.3 Regarded Reflection + ++ Don't serve attachments to clients expecting activities. + +### 1.2.2 Nameless Neophyte + ++ Ensure fetched activities are compatible content types. + ++ Some federation interop improvements. + ++ More complete API support. + ++ More compact image arrangement. + +### 1.2.1 Solipsist Satisfaction + ++ Federation reliability and compat improvements. + ++ Fix 32 bit support. + ++ Close databases to give the wal file a chance to checkpoint. + ++ Dim images in darkmode. + +- Remove the hoot: feature. The bird is dead. + +### 1.2.0 Forgotten Followup + ++ Filter option to match unknown actors. + ++ Update some dependencies. + ++ Watch local.css for changes. + ++ MacOS support. lol. + ++ Wait for requests to drain on shutdown. + ++ Handle quoteUrl property. + ++ Reroute memes to donks in emergencies. + ++ Fix handling of svg with bom fucks. + ++ FastCGI listening. + Finally fix slow public queries.
A
docs/encrypted-messages.txt
@@ -0,0 +1,45 @@
+Informal spec for an encrypted chat message prototype + +This is an extension to the ChatMessage activity. +It could apply to other types, but has limited utility for public activities. + +The encryption used is the "nacl box" combination of Curve25519, Salsa20, and Poly1305. +It's widely available in user proof crypto libraries. + +A 32 byte user public key is added to the actor object in base64 format in the "chatKeyV0" property. + +If a ChatMessage object has an "chatKeyV0" property it should be decrypted. +The "content" property is now a base64 encoded message consisting of nonce[24] || cipher[...]. + +To send an encrypted ChatMessage: +1. Add "chatKeyV0" with public key (base64) to one's own actor object. +2. Look up chatKeyV0 for remote actor. +3. Generate a random nonce and call crypto_box. +4. Base64 encode nonce and cipher text, storing as "content". +5. Add "chatKeyV0" property with key. + +Receiving and decrypting: +1. Check for "chatKeyV0" property. +2. Look up chatKeyV0 for remote actor. +3. Base64 decode "content" property. +4. Split into nonce and cipher text, call crypto_box_open. +5. Replace message content. + +The public key is duplicated in the actor and the message. + +Notes + +This doesn't support shared group keys. Messages need to be encrypted per recipient. + +This doesn't use any advanced ratcheting techniques. +Not sure how well they fit in with ActivityPub. +Limited library availability. + +Random nonces are fine and should be used. +ActivityPub IDs should be unique, but it's better to avoid the possiblity of duplicates. + +Keys should be verified at some point. + +It's only secure if the secret keys are kept somewhere secret. + +It's V0.
M
docs/hfcs.1
→
docs/hfcs.1
@@ -44,10 +44,8 @@ .Fa Ar actor
property. .It Ar include audience Previous match is applied against -.Fa to -and -.Fa cc -fields as well. +.It Ar only unknowns +Previous (domain) match is applied only to unknown actors. .It Ar text Regular expression match against the post .Fa content .@@ -97,6 +95,9 @@ that can be matched against and will appear if the post is collapsed.
.Pp An optional expiration may be specified as a duration. XdYhZm for X days, Y hours, and Z minutes. +.Sh EXAMPLES +A rudimentary spam filter may be constructed with where set to mastodon.social, +checking the only unknowns option, with an action of reject. .Sh SEE ALSO .Xr honk 1 .Sh CAVEATS
M
docs/honk.1
→
docs/honk.1
@@ -71,6 +71,7 @@ In addition to honkers, it is possible to subscribe to a hashtag collection.
(Where supported.) Enter the collection URL for .Ar url . +Alternatively, RSS feeds may be followed if the URL ends in .rss. .Pp Separately, hashtags may be added to a combo by creating a honker with a .Ar url
M
docs/honk.5
→
docs/honk.5
@@ -60,6 +60,8 @@ Inline images with img tags.
.Bd -literal <img alt="Lifecycle of a honk" src="https://example.com/diagram.png"> .Ed +.Pp +A src of a single number (1, 2, etc.) may refer to an attachment. .It links URLs beginning with .Dq http@@ -98,11 +100,6 @@ .Ql @https://example.com
works as well. When honking back, the author of the parent post is automatically mentioned. .Ss Extras -Threads from the tiny bird site may be included as quotes in a post via the -.Ar hoot -operator followed by the URL. -.Dl hoot: https://twitter.com/tedunangst/status/850379741492367360 -.Pp Custom emoji may be included by colon wrapping the image name. .Pq :hellsyeah: A meme (sticker, reaction gif) may be included with the@@ -128,6 +125,8 @@ Images are automatically rescaled and reduced in size for federation.
A description, or caption, is encouraged. Text files and PDFs are also supported as attachments. Other formats are not supported. +Multiple files may be attached to the some post, but be wary of depending +on a particular presentation order. .Pp One may also live dangerously by posting assassination coordinates. The available fields, all optional, are@@ -153,6 +152,16 @@ .Pp
When everything is at last ready to go, press the .Dq it's gonna be honked button. +.Ss Advanced +Some additional fields exist which modify the post's metadata. +.Pp +cc to add a recipient without inlining their address. +.Pp +link to add an http Link to the post. +.Pp +name to give a title to the link. +.Pp +tags to add additional hashtags without cluttering the text. .Sh EXAMPLES (Slightly dated screenshots.) .Pp
M
docs/honk.8
→
docs/honk.8
@@ -38,10 +38,12 @@ Make sure to pass the Host header, if necessary (as for nginx).
.Bd -literal -offset indent proxy_set_header Host $http_host; .Ed +.Pp +FastCGI can be used by prefixing the listen address with "fcgi:". .Ss Build Building .Nm -requires a go compiler 1.16 and libsqlite. +requires a go compiler 1.18 and libsqlite. On .Ox this is the go and sqlite3 packages.@@ -212,7 +214,7 @@ .Ss Export
User data may be exported to a zip archive using the .Ic export command. -This will export the user's outbox and inbox in ActvityPub json format, +This will export the user's outbox and inbox in ActivityPub json format, along with associated media. .Dl ./honk export username zipname .Ss Advanced Options
M
docs/vim.3
→
docs/vim.3
@@ -47,8 +47,6 @@ .It hfcs.go
Filtering framework. .It honk.go Just a few data types. -.It hoot.go -Twitter scraper. .It import.go Importers from other service data dumps. .It markitzero.go
A
encrypt.go
@@ -0,0 +1,115 @@
+package main + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "fmt" + "io" + "strings" + "time" + + "golang.org/x/crypto/nacl/box" + "humungus.tedunangst.com/r/webs/gencache" +) + +type boxSecKey struct { + key *[32]byte +} +type boxPubKey struct { + key *[32]byte +} + +func encryptString(plain string, seckey boxSecKey, pubkey boxPubKey) (string, error) { + var nonce [24]byte + rand.Read(nonce[:]) + out := box.Seal(nil, []byte(plain), &nonce, pubkey.key, seckey.key) + + var sb strings.Builder + b64 := base64.NewEncoder(base64.StdEncoding, &sb) + b64.Write(nonce[:]) + b64.Write(out) + b64.Close() + return sb.String(), nil +} + +func decryptString(encmsg string, seckey boxSecKey, pubkey boxPubKey) (string, error) { + var buf bytes.Buffer + b64 := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encmsg)) + io.Copy(&buf, b64) + data := buf.Bytes() + if len(data) < 24 { + return "", fmt.Errorf("not enough data") + } + var nonce [24]byte + copy(nonce[:], data) + data = data[24:] + out, ok := box.Open(nil, data, &nonce, pubkey.key, seckey.key) + if !ok { + return "", fmt.Errorf("error decrypting chonk") + } + return string(out), nil +} + +func b64tokey(s string) (*[32]byte, error) { + var buf bytes.Buffer + b64 := base64.NewDecoder(base64.StdEncoding, strings.NewReader(s)) + n, _ := io.Copy(&buf, b64) + if n != 32 { + return nil, fmt.Errorf("bad key size") + } + var key [32]byte + copy(key[:], buf.Bytes()) + return &key, nil +} + +func tob64(data []byte) string { + var sb strings.Builder + b64 := base64.NewEncoder(base64.StdEncoding, &sb) + b64.Write(data) + b64.Close() + return sb.String() +} + +func newChatKeys() (boxPubKey, boxSecKey) { + pub, sec, _ := box.GenerateKey(rand.Reader) + return boxPubKey{pub}, boxSecKey{sec} +} + +var chatkeys = gencache.New(gencache.Options[string, boxPubKey]{Fill: func(xonker string) (boxPubKey, bool) { + data := getxonker(xonker, chatKeyProp) + if data == "" { + dlog.Printf("hitting the webs for missing chatkey: %s", xonker) + j, err := GetJunk(readyLuserOne, xonker) + if err != nil { + ilog.Printf("error getting %s: %s", xonker, err) + when := time.Now().UTC().Format(dbtimeformat) + stmtSaveXonker.Exec(xonker, "failed", chatKeyProp, when) + return boxPubKey{}, true + } + allinjest(originate(xonker), j) + data = getxonker(xonker, chatKeyProp) + if data == "" { + ilog.Printf("key not found after ingesting") + when := time.Now().UTC().Format(dbtimeformat) + stmtSaveXonker.Exec(xonker, "failed", chatKeyProp, when) + return boxPubKey{}, true + } + } + if data == "failed" { + ilog.Printf("lookup previously failed chatkey %s", xonker) + return boxPubKey{}, true + } + var pubkey boxPubKey + var err error + pubkey.key, err = b64tokey(data) + if err != nil { + ilog.Printf("error decoding %s pubkey: %s", xonker, err) + } + return pubkey, true +}, Limit: 512}) + +func getchatkey(xonker string) (boxPubKey, bool) { + pubkey, _ := chatkeys.Get(xonker) + return pubkey, pubkey.key != nil +}
M
fun.go
→
fun.go
@@ -24,13 +24,15 @@ "io"
"net/http" "net/url" "os" + "path" "regexp" + "strconv" "strings" "time" "github.com/dustin/go-humanize" "golang.org/x/net/html" - "humungus.tedunangst.com/r/webs/cache" + "humungus.tedunangst.com/r/webs/gencache" "humungus.tedunangst.com/r/webs/htfilter" "humungus.tedunangst.com/r/webs/httpsig" "humungus.tedunangst.com/r/webs/mz"@@ -85,10 +87,8 @@ }
return humanize.CustomRelTime(d, time.Now(), "", "from now", customMags) } - -func reverbolate(userid int64, honks []*Honk) { - var user *WhatAbout - somenumberedusers.Get(userid, &user) +func reverbolate(userid UserID, honks []*Honk) { + user, _ := somenumberedusers.Get(userid) for _, h := range honks { // idk where else to put this h.DatePretty = prettifydate(h.Date)@@ -177,7 +177,7 @@
zap := make(map[string]bool) { var htf htfilter.Filter - htf.Imager = replaceimgsand(zap, false) + htf.Imager = replaceimgsand(zap, false, h) htf.SpanClasses = allowedclasses htf.BaseURL, _ = url.Parse(h.XID) emuxifier := func(e string) string {@@ -190,9 +190,8 @@ }
} } if local && h.What != "bonked" { - var emu Emu - emucache.Get(e, &emu) - if emu.ID != "" { + emu, _ := emucache.Get(e) + if emu != nil { return fmt.Sprintf(`<img class="emu" title="%s" src="%s">`, emu.Name, emu.ID) } }@@ -242,7 +241,7 @@ }
} } -func replaceimgsand(zap map[string]bool, absolute bool) func(node *html.Node) string { +func replaceimgsand(zap map[string]bool, absolute bool, honk *Honk) func(node *html.Node) string { return func(node *html.Node) string { src := htfilter.GetAttr(node, "src") alt := htfilter.GetAttr(node, "alt")@@ -250,12 +249,19 @@ //title := GetAttr(node, "title")
if htfilter.HasClass(node, "Emoji") && alt != "" { return alt } - d := finddonk(src) + base := path.Base(src) + didx, _ := strconv.Atoi(base) + var d *Donk + if strings.HasPrefix(src, serverPrefix) && didx > 0 && didx <= len(honk.Donks) { + d = honk.Donks[didx-1] + } else { + d = finddonk(src) + } if d != nil { zap[d.XID] = true base := "" if absolute { - base = "https://" + serverName + base = serverURL("") } return string(templates.Sprintf(`<img alt="%s" title="%s" src="%s/d/%s">`, alt, alt, base, d.XID)) }@@ -293,9 +299,8 @@ }
} } if local { - var emu Emu - emucache.Get(e, &emu) - if emu.ID != "" { + emu, _ := emucache.Get(e) + if emu != nil { return fmt.Sprintf(`<img class="emu" title="%s" src="%s">`, emu.Name, emu.ID) } }@@ -327,6 +332,12 @@ func inlineimgsfor(honk *Honk) func(node *html.Node) string {
return func(node *html.Node) string { src := htfilter.GetAttr(node, "src") alt := htfilter.GetAttr(node, "alt") + base := path.Base(src) + didx, _ := strconv.Atoi(base) + if strings.HasPrefix(src, serverPrefix) && didx > 0 && didx <= len(honk.Donks) { + dlog.Printf("skipping inline image %s", src) + return "" + } d := savedonk(src, "image", alt, "image", true) if d != nil { honk.Donks = append(honk.Donks, d)@@ -357,6 +368,7 @@ honk.Precis = noise[:idx]
noise = noise[idx+1:] } var marker mz.Marker + marker.Short = true honk.Precis = marker.Mark(strings.TrimSpace(honk.Precis)) honk.Noise = noise }@@ -375,16 +387,25 @@ marker.AllowImages = true
noise = strings.TrimSpace(noise) noise = marker.Mark(noise) honk.Noise = noise - honk.Onts = oneofakind(append(honk.Onts, marker.HashTags...)) + honk.Onts = append(honk.Onts, marker.HashTags...) honk.Mentions = bunchofgrapes(marker.Mentions) + for _, t := range oneofakind(strings.Split(honk.Onties, " ")) { + if t[0] != '#' { + t = "#" + t + } + honk.Onts = append(honk.Onts, t) + } + honk.Onts = oneofakind(honk.Onts) + honk.Mentions = append(honk.Mentions, bunchofgrapes(oneofakind(strings.Split(honk.SeeAlso, " ")))...) } func redoimages(honk *Honk) { zap := make(map[string]bool) { var htf htfilter.Filter - htf.Imager = replaceimgsand(zap, true) + htf.Imager = replaceimgsand(zap, true, honk) htf.SpanClasses = allowedclasses + htf.BaseURL, _ = url.Parse(honk.XID) p, _ := htf.String(honk.Precis) n, _ := htf.String(honk.Noise) honk.Precis = string(p)@@ -435,9 +456,14 @@
func bunchofgrapes(m []string) []Mention { var mentions []Mention for i := range m { - where := gofish(m[i]) + who := m[i] + if strings.HasPrefix(who, "@https://") { + mentions = append(mentions, Mention{Who: who, Where: who[1:]}) + continue + } + where := gofish(who) if where != "" { - mentions = append(mentions, Mention{Who: m[i], Where: where}) + mentions = append(mentions, Mention{Who: who, Where: where}) } } return mentions@@ -451,7 +477,7 @@ }
var re_emus = regexp.MustCompile(`:[[:alnum:]_-]+:`) -var emucache = cache.New(cache.Options{Filler: func(ename string) (Emu, bool) { +var emucache = gencache.New(gencache.Options[string, *Emu]{Fill: func(ename string) (*Emu, bool) { fname := ename[1 : len(ename)-1] exts := []string{".png", ".gif"} for _, ext := range exts {@@ -459,23 +485,22 @@ _, err := os.Stat(dataDir + "/emus/" + fname + ext)
if err != nil { continue } - url := fmt.Sprintf("https://%s/emu/%s%s", serverName, fname, ext) + url := serverURL("/emu/%s%s", fname, ext) if develMode { url = fmt.Sprintf("/emu/%s%s", fname, ext) } - return Emu{ID: url, Name: ename, Type: "image/" + ext[1:]}, true + return &Emu{ID: url, Name: ename, Type: "image/" + ext[1:]}, true } - return Emu{Name: ename, ID: "", Type: "image/png"}, true + return nil, true }, Duration: 10 * time.Second}) -func herdofemus(noise string) []Emu { +func herdofemus(noise string) []*Emu { m := re_emus.FindAllString(noise, -1) m = oneofakind(m) - var emus []Emu + var emus []*Emu for _, e := range m { - var emu Emu - emucache.Get(e, &emu) - if emu.ID == "" { + emu, _ := emucache.Get(e) + if emu == nil { continue } emus = append(emus, emu)@@ -506,7 +531,7 @@ n, _ := fd.Read(peek[:])
ct := http.DetectContentType(peek[:n]) fd.Close() - url := fmt.Sprintf("https://%s/meme/%s", serverName, name) + url := serverURL("/meme/%s", name) fileid, err := savefile(name, name, url, ct, false, nil) if err != nil { elog.Printf("error saving meme: %s", err)@@ -545,7 +570,7 @@ }
var re_quickmention = regexp.MustCompile("(^|[ \n])@[[:alnum:]_]+([ \n:;.,']|$)") -func quickrename(s string, userid int64) string { +func quickrename(s string, userid UserID) string { nonstop := true for nonstop { nonstop = false@@ -580,7 +605,7 @@ }
return s } -var shortnames = cache.New(cache.Options{Filler: func(userid int64) (map[string]string, bool) { +var shortnames = gencache.New(gencache.Options[UserID, map[string]string]{Fill: func(userid UserID) (map[string]string, bool) { honkers := gethonkers(userid) m := make(map[string]string) for _, h := range honkers {@@ -589,16 +614,15 @@ }
return m, true }, Invalidator: &honkerinvalidator}) -func shortname(userid int64, xid string) string { - var m map[string]string - ok := shortnames.Get(userid, &m) +func shortname(userid UserID, xid string) string { + m, ok := shortnames.Get(userid) if ok { return m[xid] } return "" } -var fullnames = cache.New(cache.Options{Filler: func(userid int64) (map[string]string, bool) { +var fullnames = gencache.New(gencache.Options[UserID, map[string]string]{Fill: func(userid UserID) (map[string]string, bool) { honkers := gethonkers(userid) m := make(map[string]string) for _, h := range honkers {@@ -607,9 +631,8 @@ }
return m, true }, Invalidator: &honkerinvalidator}) -func fullname(name string, userid int64) string { - var m map[string]string - ok := fullnames.Get(userid, &m) +func fullname(name string, userid UserID) string { + m, ok := fullnames.Get(userid) if ok { return m[name] }@@ -618,6 +641,9 @@ }
func attoreplacer(m string) string { fill := `<span class="h-card"><a class="u-url mention" href="%s">%s</a></span>` + if strings.HasPrefix(m, "@https://") { + return fmt.Sprintf(fill, html.EscapeString(m[1:]), html.EscapeString(m)) + } where := gofish(m) if where == "" { return m@@ -627,8 +653,8 @@ return fmt.Sprintf(fill, html.EscapeString(where), html.EscapeString(who))
} func ontoreplacer(h string) string { - return fmt.Sprintf(`<a class="mention hashtag" href="https://%s/o/%s">%s</a>`, serverName, - strings.ToLower(h[1:]), h) + return fmt.Sprintf(`<a class="mention hashtag" href="%s">%s</a>`, + serverURL("/o/%s", strings.ToLower(h[1:])), h) } var re_unurl = regexp.MustCompile("https://([^/]+).*/([^/]+)")@@ -642,7 +668,7 @@ }
return "" } -var allhandles = cache.New(cache.Options{Filler: func(xid string) (string, bool) { +var allhandles = gencache.New(gencache.Options[string, string]{Fill: func(xid string) (string, bool) { handle := getxonker(xid, "handle") if handle == "" { dlog.Printf("need to get a handle: %s", xid)@@ -666,8 +692,7 @@ func handles(xid string) (string, string) {
if xid == "" || xid == thewholeworld || strings.HasSuffix(xid, "/followers") { return "", "" } - var handle string - allhandles.Get(xid, &handle) + handle, _ := allhandles.Get(xid) if handle == xid { return xid, xid }@@ -709,9 +734,8 @@ }
return a[:j] } -var ziggies = cache.New(cache.Options{Filler: func(userid int64) (*KeyInfo, bool) { - var user *WhatAbout - ok := somenumberedusers.Get(userid, &user) +var ziggies = gencache.New(gencache.Options[UserID, *KeyInfo]{Fill: func(userid UserID) (*KeyInfo, bool) { + user, ok := somenumberedusers.Get(userid) if !ok { return nil, false }@@ -721,13 +745,12 @@ ki.seckey = user.SecKey
return ki, true }}) -func ziggy(userid int64) *KeyInfo { - var ki *KeyInfo - ziggies.Get(userid, &ki) +func ziggy(userid UserID) *KeyInfo { + ki, _ := ziggies.Get(userid) return ki } -var zaggies = cache.New(cache.Options{Filler: func(keyname string) (httpsig.PublicKey, bool) { +var zaggies = gencache.New(gencache.Options[string, httpsig.PublicKey]{Fill: func(keyname string) (httpsig.PublicKey, bool) { data := getxonker(keyname, "pubkey") if data == "" { dlog.Printf("hitting the webs for missing pubkey: %s", keyname)@@ -760,8 +783,7 @@ return key, true
}, Limit: 512}) func zaggy(keyname string) (httpsig.PublicKey, error) { - var key httpsig.PublicKey - zaggies.Get(keyname, &key) + key, _ := zaggies.Get(keyname) return key, nil }
M
go.mod
→
go.mod
@@ -3,20 +3,29 @@
go 1.18 require ( - github.com/andybalholm/cascadia v1.3.2 github.com/dustin/go-humanize v1.0.1 github.com/gorilla/handlers v1.5.2 - github.com/gorilla/mux v1.8.0 + github.com/gorilla/mux v1.8.1 github.com/jmoiron/sqlx v1.3.5 github.com/mattn/go-runewidth v0.0.15 - golang.org/x/crypto v0.12.0 - golang.org/x/net v0.14.0 + github.com/mmcdole/gofeed v1.2.1 + golang.org/x/crypto v0.19.0 + golang.org/x/net v0.21.0 humungus.tedunangst.com/r/go-sqlite3 v1.1.3 - humungus.tedunangst.com/r/webs v0.7.7 + humungus.tedunangst.com/r/gonix v0.1.4 + humungus.tedunangst.com/r/webs v0.7.10 ) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - golang.org/x/image v0.11.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/image v0.15.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect )
M
go.sum
→
go.sum
@@ -1,68 +1,68 @@
-github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= -github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= +github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= +github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= +github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= -golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= +golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= humungus.tedunangst.com/r/go-sqlite3 v1.1.3 h1:G2N4wzDS0NbuvrZtQJhh4F+3X+s7BF8b9ga8k38geUI= humungus.tedunangst.com/r/go-sqlite3 v1.1.3/go.mod h1:FtEEmQM7U2Ey1TuEEOyY1BmphTZnmiEjPsNLEAkpf/M= -humungus.tedunangst.com/r/webs v0.7.7 h1:1IITkwf5T3Zv2mG0rzmORPVcXuP/YGoTPeh6uBpUkCo= -humungus.tedunangst.com/r/webs v0.7.7/go.mod h1:ylhqHSPI0Oi7b4nsnx5mSO7AjLXN7wFpEHayLfN/ugk= +humungus.tedunangst.com/r/gonix v0.1.4 h1:FuvWYQlFIzmfHxfvIfq5SYpSiHhFcpJqq3pi+w45s78= +humungus.tedunangst.com/r/gonix v0.1.4/go.mod h1:VFBc2bPDXr1ayHOmHUutxYu8fSM+pkwK8o36h4rkORg= +humungus.tedunangst.com/r/webs v0.7.10 h1:DPEsA7DU1P1uOBWYrhJWjqDtll6SGJkWQtJ/2N6P8DI= +humungus.tedunangst.com/r/webs v0.7.10/go.mod h1:ylhqHSPI0Oi7b4nsnx5mSO7AjLXN7wFpEHayLfN/ugk=
M
hfcs.go
→
hfcs.go
@@ -19,10 +19,14 @@ import (
"net/http" "regexp" "sort" + "strconv" + "strings" "time" "unicode" "humungus.tedunangst.com/r/webs/cache" + "humungus.tedunangst.com/r/webs/gencache" + "humungus.tedunangst.com/r/webs/login" ) type Filter struct {@@ -32,6 +36,7 @@ Name string
Date time.Time Actor string `json:",omitempty"` IncludeAudience bool `json:",omitempty"` + OnlyUnknowns bool `json:",omitempty"` Text string `json:",omitempty"` re_text *regexp.Regexp IsReply bool `json:",omitempty"`@@ -68,15 +73,18 @@ }
type afiltermap map[filtType][]*Filter -var filtInvalidator cache.Invalidator -var filtcache *cache.Cache +var filtInvalidator gencache.Invalidator[UserID] +var filtcache *gencache.Cache[UserID, afiltermap] func init() { // resolve init loop - filtcache = cache.New(cache.Options{Filler: filtcachefiller, Invalidator: &filtInvalidator}) + filtcache = gencache.New(gencache.Options[UserID, afiltermap]{ + Fill: filtcachefiller, + Invalidator: &filtInvalidator, + }) } -func filtcachefiller(userid int64) (afiltermap, bool) { +func filtcachefiller(userid UserID) (afiltermap, bool) { rows, err := stmtGetFilters.Query(userid) if err != nil { elog.Printf("error querying filters: %s", err)@@ -175,14 +183,13 @@ }
return filtmap, true } -func filtcacheclear(userid int64, dur time.Duration) { +func filtcacheclear(userid UserID, dur time.Duration) { time.Sleep(dur + time.Second) filtInvalidator.Clear(userid) } -func getfilters(userid int64, scope filtType) []*Filter { - var filtmap afiltermap - ok := filtcache.Get(userid, &filtmap) +func getfilters(userid UserID, scope filtType) []*Filter { + filtmap, ok := filtcache.Get(userid) if ok { return filtmap[scope] }@@ -193,7 +200,7 @@ type arejectmap map[string][]*Filter
var rejectAnyKey = "..." -var rejectcache = cache.New(cache.Options{Filler: func(userid int64) (arejectmap, bool) { +var rejectcache = gencache.New(gencache.Options[UserID, arejectmap]{Fill: func(userid UserID) (arejectmap, bool) { m := make(arejectmap) filts := getfilters(userid, filtReject) for _, f := range filts {@@ -214,18 +221,20 @@ }
return m, true }, Invalidator: &filtInvalidator}) -func rejectfilters(userid int64, name string) []*Filter { - var m arejectmap - rejectcache.Get(userid, &m) +func rejectfilters(userid UserID, name string) []*Filter { + m, _ := rejectcache.Get(userid) return m[name] } -func rejectorigin(userid int64, origin string, isannounce bool) bool { +func rejectorigin(userid UserID, origin string, isannounce bool) bool { if o := originate(origin); o != "" { origin = o } filts := rejectfilters(userid, origin) for _, f := range filts { + if f.OnlyUnknowns { + continue + } if isannounce && f.IsAnnounce { if f.AnnounceOf == origin { return true@@ -238,7 +247,7 @@ }
return false } -func rejectactor(userid int64, actor string) bool { +func rejectactor(userid UserID, actor string) bool { filts := rejectfilters(userid, actor) for _, f := range filts { if f.IsAnnounce {@@ -259,6 +268,13 @@ if f.IsAnnounce {
continue } if f.Actor == origin { + if f.OnlyUnknowns { + if unknownActor(userid, actor) { + ilog.Printf("rejecting unknown actor: %s", actor) + return true + } + continue + } ilog.Printf("rejecting actor: %s", actor) return true }@@ -266,9 +282,22 @@ }
return false } -func stealthmode(userid int64, r *http.Request) bool { - agent := r.UserAgent() - agent = originate(agent) +var knownknowns = gencache.New(gencache.Options[UserID, map[string]bool]{Fill: func(userid UserID) (map[string]bool, bool) { + m := make(map[string]bool) + honkers := gethonkers(userid) + for _, h := range honkers { + m[h.XID] = true + } + return m, true +}, Invalidator: &honkerinvalidator}) + +func unknownActor(userid UserID, actor string) bool { + knowns, _ := knownknowns.Get(userid) + return !knowns[actor] +} + +func stealthmode(userid UserID, r *http.Request) bool { + agent := requestActor(r) if agent != "" { fake := rejectorigin(userid, agent, false) if fake {@@ -292,7 +321,7 @@ if f.Actor == h.Honker || f.Actor == h.Oonker {
match = true rv = f.Actor } - if !match && (f.Actor == originate(h.Honker) || + if !match && !f.OnlyUnknowns && (f.Actor == originate(h.Honker) || f.Actor == originate(h.Oonker) || f.Actor == originate(h.XID)) { match = true@@ -317,10 +346,11 @@ }
} if match && f.IsAnnounce { match = false - if (f.AnnounceOf == "" && h.Oonker != "") || f.AnnounceOf == h.Oonker || - f.AnnounceOf == originate(h.Oonker) { - match = true - rv += " announce" + if h.Oonker != "" { + if f.AnnounceOf == "" || f.AnnounceOf == h.Oonker || f.AnnounceOf == originate(h.Oonker) { + match = true + rv += " announce" + } } } if match && f.Text != "" && f.Text != "." {@@ -357,8 +387,7 @@ return ""
} func rejectxonk(xonk *Honk) bool { - var m arejectmap - rejectcache.Get(xonk.UserID, &m) + m, _ := rejectcache.Get(xonk.UserID) filts := m[rejectAnyKey] filts = append(filts, m[xonk.Honker]...) filts = append(filts, m[originate(xonk.Honker)]...)@@ -387,7 +416,7 @@ }
return false } -func unsee(honks []*Honk, userid int64) { +func unsee(honks []*Honk, userid UserID) { if userid != -1 { colfilts := getfilters(userid, filtCollapse) rwfilts := getfilters(userid, filtRewrite)@@ -425,7 +454,7 @@ }
} } -var untagged = cache.New(cache.Options{Filler: func(userid int64) (map[string]bool, bool) { +var untagged = cache.New(cache.Options{Filler: func(userid UserID) (map[string]bool, bool) { rows, err := stmtUntagged.Query(userid) if err != nil { elog.Printf("error query untagged: %s", err)@@ -451,7 +480,7 @@ }
return bad, true }}) -func osmosis(honks []*Honk, userid int64, withfilt bool) []*Honk { +func osmosis(honks []*Honk, userid UserID, withfilt bool) []*Honk { var badparents map[string]bool untagged.GetAndLock(userid, &badparents) j := 0@@ -485,3 +514,56 @@ }
honks = honks[0:j] return honks } + +func savehfcs(w http.ResponseWriter, r *http.Request) { + userid := UserID(login.GetUserInfo(r).UserID) + itsok := r.FormValue("itsok") + if itsok == "iforgiveyou" { + hfcsid, _ := strconv.ParseInt(r.FormValue("hfcsid"), 10, 0) + _, err := stmtDeleteFilter.Exec(userid, hfcsid) + if err != nil { + elog.Printf("error deleting filter: %s", err) + } + filtInvalidator.Clear(userid) + http.Redirect(w, r, "/hfcs", http.StatusSeeOther) + return + } + + filt := new(Filter) + filt.Name = strings.TrimSpace(r.FormValue("name")) + filt.Date = time.Now().UTC() + filt.Actor = strings.TrimSpace(r.FormValue("actor")) + filt.IncludeAudience = r.FormValue("incaud") == "yes" + filt.OnlyUnknowns = r.FormValue("unknowns") == "yes" + filt.Text = strings.TrimSpace(r.FormValue("filttext")) + filt.IsReply = r.FormValue("isreply") == "yes" + filt.IsAnnounce = r.FormValue("isannounce") == "yes" + filt.AnnounceOf = strings.TrimSpace(r.FormValue("announceof")) + filt.Reject = r.FormValue("doreject") == "yes" + filt.SkipMedia = r.FormValue("doskipmedia") == "yes" + filt.Hide = r.FormValue("dohide") == "yes" + filt.Collapse = r.FormValue("docollapse") == "yes" + filt.Rewrite = strings.TrimSpace(r.FormValue("filtrewrite")) + filt.Replace = strings.TrimSpace(r.FormValue("filtreplace")) + if dur := parseDuration(r.FormValue("filtduration")); dur > 0 { + filt.Expiration = time.Now().UTC().Add(dur) + } + filt.Notes = strings.TrimSpace(r.FormValue("filtnotes")) + + if filt.Actor == "" && filt.Text == "" && !filt.IsAnnounce { + ilog.Printf("blank filter") + http.Error(w, "can't save a blank filter", http.StatusInternalServerError) + return + } + + j, err := jsonify(filt) + if err == nil { + _, err = stmtSaveFilter.Exec(userid, j) + } + if err != nil { + elog.Printf("error saving filter: %s", err) + } + + filtInvalidator.Clear(userid) + http.Redirect(w, r, "/hfcs", http.StatusSeeOther) +}
M
honk.go
→
honk.go
@@ -25,16 +25,18 @@ "humungus.tedunangst.com/r/webs/httpsig"
) type WhatAbout struct { - ID int64 - Name string - Display string - About string - HTAbout template.HTML - Onts []string - Key string - URL string - Options UserOptions - SecKey httpsig.PrivateKey + ID UserID + Name string + Display string + About string + HTAbout template.HTML + Onts []string + Key string + URL string + Options UserOptions + SecKey httpsig.PrivateKey + ChatPubKey boxPubKey + ChatSecKey boxSecKey } type UserOptions struct {@@ -50,6 +52,8 @@ MapLink string `json:",omitempty"`
Reaction string `json:",omitempty"` MeCount int64 ChatCount int64 + ChatPubKey string + ChatSecKey string } type KeyInfo struct {@@ -57,12 +61,14 @@ keyname string
seckey httpsig.PrivateKey } -const serverUID int64 = -2 -const readyLuserOne int64 = 1 +type UserID int64 + +const serverUID UserID = -2 +const readyLuserOne UserID = 1 type Honk struct { ID int64 - UserID int64 + UserID UserID Username string DisplayName string What string@@ -93,8 +99,12 @@ Donks []*Donk
Onts []string Place *Place Time *Time + Link string Mentions []Mention Badonks []Badonk + SeeAlso string + Onties string + LegalName string } type Badonk struct {@@ -104,7 +114,7 @@ }
type Chonk struct { ID int64 - UserID int64 + UserID UserID XID string Who string Target string@@ -243,7 +253,7 @@ }
type Honker struct { ID int64 - UserID int64 + UserID UserID Name string XID string Handle string
D
hoot.go
@@ -1,139 +0,0 @@
-// -// Copyright (c) 2019 Ted Unangst <tedu@tedunangst.com> -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -package main - -import ( - "fmt" - "io" - "net/http" - "os" - "regexp" - "strings" - - "github.com/andybalholm/cascadia" - "golang.org/x/net/html" - "humungus.tedunangst.com/r/webs/htfilter" - "humungus.tedunangst.com/r/webs/templates" -) - -var tweetsel = cascadia.MustCompile("div[data-testid=tweetText]") -var linksel = cascadia.MustCompile("a time") -var replyingto = cascadia.MustCompile(".ReplyingToContextBelowAuthor") -var imgsel = cascadia.MustCompile("div[data-testid=tweetPhoto] img") -var authorregex = regexp.MustCompile("twitter.com/([^/]+)") - -var re_hoots = regexp.MustCompile(`hoot: ?https://\S+`) -var re_removepics = regexp.MustCompile(`pic\.twitter\.com/[[:alnum:]]+`) - -func hootextractor(r io.Reader, url string, seen map[string]bool) string { - root, err := html.Parse(r) - if err != nil { - elog.Printf("error parsing hoot: %s", err) - return url - } - - url = strings.Replace(url, "mobile.twitter.com", "twitter.com", -1) - wantmatch := authorregex.FindStringSubmatch(url) - var wanted string - if len(wantmatch) == 2 { - wanted = wantmatch[1] - } - - var htf htfilter.Filter - htf.Imager = func(node *html.Node) string { - alt := htfilter.GetAttr(node, "alt") - src := htfilter.GetAttr(node, "src") - if htfilter.HasClass(node, "Emoji") || strings.HasSuffix(src, ".svg") { - return alt - } - return string(templates.Sprintf(" <img src='%s' alt='%s'>", src, alt)) - } - - var buf strings.Builder - fmt.Fprintf(&buf, "%s\n", url) - - divs := tweetsel.MatchAll(root) - for i, div := range divs { - twp := div.Parent.Parent.Parent.Parent.Parent - link := url - alink := linksel.MatchFirst(twp) - if alink == nil { - if i != 0 { - dlog.Printf("missing link") - continue - } - } else { - alink = alink.Parent - link = "https://twitter.com" + htfilter.GetAttr(alink, "href") - } - authormatch := authorregex.FindStringSubmatch(link) - if len(authormatch) < 2 { - dlog.Printf("no author?: %s", link) - continue - } - author := authormatch[1] - if wanted == "" { - wanted = author - } - if author != wanted { - continue - } - for _, img := range imgsel.MatchAll(twp) { - img.Parent.RemoveChild(img) - div.AppendChild(img) - } - text := htf.NodeText(div) - text = strings.Replace(text, "\n", " ", -1) - fmt.Fprintf(&buf, "> @%s: %s\n", author, text) - } - return buf.String() -} - -func hooterize(noise string) string { - seen := make(map[string]bool) - - hootfetcher := func(hoot string) string { - url := hoot[5:] - if url[0] == ' ' { - url = url[1:] - } - url = strings.Replace(url, "mobile.twitter.com", "twitter.com", -1) - dlog.Printf("hooterizing %s", url) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - ilog.Printf("error: %s", err) - return hoot - } - req.Header.Set("User-Agent", "Bot") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - req.Header.Set("Accept-Language", "en-US,en;q=0.9") - resp, err := http.DefaultClient.Do(req) - if err != nil { - ilog.Printf("error: %s", err) - return hoot - } - defer resp.Body.Close() - if resp.StatusCode != 200 { - ilog.Printf("error getting %s: %d", url, resp.StatusCode) - return hoot - } - ld, _ := os.Create("lasthoot.html") - r := io.TeeReader(resp.Body, ld) - return hootextractor(r, url, seen) - } - - return re_hoots.ReplaceAllStringFunc(noise, hootfetcher) -}
D
hoot_test.go
@@ -1,19 +0,0 @@
-package main - -import ( - "fmt" - "log" - "os" - "testing" -) - -func TestHooterize(t *testing.T) { - dlog = log.Default() - fd, err := os.Open("lasthoot.html") - if err != nil { - return - } - seen := make(map[string]bool) - hoots := hootextractor(fd, "lasthoot.html", seen) - fmt.Printf("hoots: %s\n", hoots) -}
M
main.go
→
main.go
@@ -23,6 +23,7 @@ golog "log"
"log/syslog" notrand "math/rand" "os" + "runtime/pprof" "strconv" "time"@@ -45,12 +46,21 @@ var serverMsg template.HTML
var aboutMsg template.HTML var loginMsg template.HTML +func serverURL(u string, args ...interface{}) string { + return fmt.Sprintf("https://"+serverName+u, args...) +} + func ElaborateUnitTests() { + user, _ := butwhatabout("test") + syndicate(user, "https://mastodon.social/tags/mastoadmin.rss") } func unplugserver(hostname string) { db := opendatabase() - xid := fmt.Sprintf("%%https://%s/%%", hostname) + xid := fmt.Sprintf("https://%s", hostname) + db.Exec("delete from honkers where xid = ? and flavor = 'dub'", xid) + db.Exec("delete from doovers where rcpt = ?", xid) + xid += "/%" db.Exec("delete from honkers where xid like ? and flavor = 'dub'", xid) db.Exec("delete from doovers where rcpt like ?", xid) }@@ -69,10 +79,28 @@ fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1) } +var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") +var memprofile = flag.String("memprofile", "", "write memory profile to this file") +var memprofilefd *os.File + func main() { flag.StringVar(&dataDir, "datadir", dataDir, "data directory") flag.StringVar(&viewDir, "viewdir", viewDir, "view directory") flag.Parse() + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + errx("can't open cpu profile: %s", err) + } + pprof.StartCPUProfile(f) + } + if *memprofile != "" { + f, err := os.Create(*memprofile) + if err != nil { + errx("can't open mem profile: %s", err) + } + memprofilefd = f + } log.Init(log.Options{Progname: "honk", Facility: syslog.LOG_UUCP}) elog = log.E@@ -111,7 +139,7 @@ getconfig("masqname", &masqName)
if masqName == "" { masqName = serverName } - serverPrefix = fmt.Sprintf("https://%s/", serverName) + serverPrefix = serverURL("/") getconfig("usersep", &userSep) getconfig("honksep", &honkSep) getconfig("devel", &develMode)@@ -188,11 +216,13 @@ errx("user %s not found", args[1])
} var meta HonkerMeta mj, _ := jsonify(&meta) - honkerid, err := savehonker(user, args[2], "", "presub", "", mj) + honkerid, flavor, err := savehonker(user, args[2], "", "presub", "", mj) if err != nil { errx("had some trouble with that: %s", err) } - followyou(user, honkerid, true) + if flavor == "presub" { + followyou(user, honkerid, true) + } case "unfollow": if len(args) < 3 { errx("usage: honk unfollow username url")
M
masto.go
→
masto.go
@@ -248,8 +248,7 @@ }
// https://docs.joinmastodon.org/methods/accounts/#verify_credentials func verifycreds(rw http.ResponseWriter, r *http.Request) { - var user *WhatAbout - ok := somenumberedusers.Get(int64(1), &user) + user, ok := somenumberedusers.Get(UserID(1)) if !ok { elog.Fatalf("masto: no user number 1???") }
M
preflight.sh
→
preflight.sh
@@ -1,19 +1,19 @@
set -e -go version > /dev/null 2>&1 || (echo go 1.16+ is required && false) +go version > /dev/null 2>&1 || (echo go 1.18+ is required && false) v=`go version | egrep -o "go1\.[^.]+"` || echo failed to identify go version -if [ "$v" \< "go1.16" ] ; then +if [ "$v" \< "go1.18" ] ; then echo go version is too old: $v - echo go 1.16+ is required + echo go 1.18+ is required false fi -# if [ \! \( -e /usr/include/sqlite3.h -o -e /usr/local/include/sqlite3.h \) ] ; then -# echo unable to find sqlite3.h header -# echo please install libsqlite3 dev package -# false -# fi +if [ \! \( -e /usr/include/sqlite3.h -o -e /usr/local/include/sqlite3.h -o `uname` = "Darwin" \) ] ; then + echo unable to find sqlite3.h header + echo please install libsqlite3 dev package + false +fi touch .preflightcheck
M
schema.sql
→
schema.sql
@@ -14,9 +14,11 @@ create table masto (clientname text, redirecturis text, scopes text, clientid text, clientsecret text, vapidkey text, authtoken text);
create table mastokens (clientid text, accesstoken text) create index idx_honksxid on honks(xid); +create index idx_honksurl on honks(url); create index idx_honksconvoy on honks(convoy); create index idx_honkshonker on honks(honker); create index idx_honksoonker on honks(oonker); +create index idx_honksforme on honks(whofore) where whofore = 1; create index idx_honkswhotwo on honks(whofore) where whofore = 2; create index idx_donkshonk on donks(honkid); create index idx_donkschonk on donks(chonkid);
M
skulduggery.go
→
skulduggery.go
@@ -32,7 +32,7 @@ for _, c := range s {
if c == '\n' { continue } - if runewidth.RuneWidth(c) == 0 { + if c > 127 && runewidth.RuneWidth(c) == 0 { zw = true break }@@ -41,7 +41,7 @@ if zw {
x := make([]byte, 0, len(s)) zw = false for _, c := range s { - if runewidth.RuneWidth(c) == 0 { + if c > 127 && runewidth.RuneWidth(c) == 0 { if zw { continue }
A
syndicate.go
@@ -0,0 +1,53 @@
+package main + +import ( + "bytes" + notrand "math/rand" + "strings" + "time" + + "github.com/mmcdole/gofeed" +) + +func syndicate(user *WhatAbout, url string) { + data, err := fetchsome(url) + if err != nil { + dlog.Printf("error fetching feed: %s", err) + return + } + parser := gofeed.NewParser() + rss, err := parser.Parse(bytes.NewReader(data)) + if err != nil { + dlog.Printf("error parsing feed: %s", err) + return + } + reverseItems(rss.Items) + for _, item := range rss.Items { + dlog.Printf("link: %s", item.Link) + grabhonk(user, item.Link) + } +} + +func syndicator() { + for { + dur := 8 * time.Hour + dur += time.Duration(notrand.Int63n(int64(dur / 4))) + time.Sleep(dur) + users := allusers() + for _, ui := range users { + user, _ := butwhatabout(ui.Username) + honkers := gethonkers(user.ID) + for _, h := range honkers { + if strings.HasSuffix(h.XID, ".rss") { + syndicate(user, h.XID) + } + } + } + } +} + +func reverseItems(items []*gofeed.Item) { + for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { + items[i], items[j] = items[j], items[i] + } +}
M
unveil.go
→
unveil.go
@@ -1,6 +1,3 @@
-//go:build openbsd -// +build openbsd - // // Copyright (c) 2019 Ted Unangst <tedu@tedunangst.com> //@@ -18,50 +15,38 @@ // OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
package main -/* -#include <stdlib.h> -#include <unistd.h> -*/ -import "C" - import ( - "unsafe" + "humungus.tedunangst.com/r/gonix" ) -func Unveil(path string, perms string) { - cpath := C.CString(path) - defer C.free(unsafe.Pointer(cpath)) - cperms := C.CString(perms) - defer C.free(unsafe.Pointer(cperms)) - - rv, err := C.unveil(cpath, cperms) - if rv != 0 { - elog.Fatalf("unveil(%s, %s) failure (%d)", path, perms, err) +func securitizeweb() { + err := gonix.Unveil("/etc/ssl", "r") + if err != nil { + elog.Fatalf("unveil(%s, %s) failure (%d)", "/etc/ssl", "r", err) + } + if viewDir != dataDir { + err = gonix.Unveil(viewDir, "r") + if err != nil { + elog.Fatalf("unveil(%s, %s) failure (%d)", viewDir, "r", err) + } + } + err = gonix.Unveil(dataDir, "rwc") + if err != nil { + elog.Fatalf("unveil(%s, %s) failure (%d)", dataDir, "rwc", err) + } + gonix.UnveilEnd() + promises := "stdio rpath wpath cpath flock dns inet unix" + err = gonix.Pledge(promises) + if err != nil { + elog.Fatalf("pledge(%s) failure (%d)", promises, err) } } -func Pledge(promises string) { - cpromises := C.CString(promises) - defer C.free(unsafe.Pointer(cpromises)) - - rv, err := C.pledge(cpromises, nil) - if rv != 0 { +func securitizebackend() { + gonix.UnveilEnd() + promises := "stdio unix" + err := gonix.Pledge(promises) + if err != nil { elog.Fatalf("pledge(%s) failure (%d)", promises, err) } } - -func init() { - preservehooks = append(preservehooks, func() { - Unveil("/etc/ssl", "r") - if viewDir != dataDir { - Unveil(viewDir, "r") - } - Unveil(dataDir, "rwc") - C.unveil(nil, nil) - Pledge("stdio rpath wpath cpath flock dns inet unix") - }) - backendhooks = append(backendhooks, func() { - C.unveil(nil, nil) - Pledge("stdio unix") - }) -}
M
upgradedb.go
→
upgradedb.go
@@ -23,7 +23,7 @@
"humungus.tedunangst.com/r/webs/htfilter" ) -var myVersion = 46 // idx whotwo +var myVersion = 49 // index honks url type dbexecer interface { Exec(query string, args ...interface{}) (sql.Result, error)@@ -47,27 +47,31 @@ elog.Fatal("database is too old to upgrade")
} var err error var tx *sql.Tx - try := func(s string, args ...interface{}) { - if tx != nil { - _, err = tx.Exec(s, args...) + try := func(s string, args ...interface{}) *sql.Rows { + var rows *sql.Rows + if strings.HasPrefix(s, "select") { + if tx != nil { + rows, err = tx.Query(s, args...) + } else { + rows, err = db.Query(s, args...) + } } else { - _, err = db.Exec(s, args...) + if tx != nil { + _, err = tx.Exec(s, args...) + } else { + _, err = db.Exec(s, args...) + } } if err != nil { elog.Fatalf("can't run %s: %s", s, err) } + return rows } setV := func(ver int64) { try("update config set value = ? where key = 'dbversion'", ver) } switch dbversion { - case 40: - doordie(db, "PRAGMA journal_mode=WAL") - blobdb := openblobdb() - doordie(blobdb, "PRAGMA journal_mode=WAL") - doordie(db, "update config set value = 41 where key = 'dbversion'") - fallthrough case 41: tx, err := db.Begin() if err != nil {@@ -176,7 +180,42 @@ try("create index idx_honkswhotwo on honks(whofore) where whofore = 2")
setV(46) fallthrough case 46: + try("create index idx_honksforme on honks(whofore) where whofore = 1") + setV(47) + fallthrough + case 47: + rows := try("select userid, options from users where userid > 0") + var users []*WhatAbout + for rows.Next() { + var user WhatAbout + var jopt string + err = rows.Scan(&user.ID, &jopt) + if err != nil { + elog.Fatal(err) + } + err = unjsonify(jopt, &user.Options) + if err != nil { + elog.Fatal(err) + } + users = append(users, &user) + } + rows.Close() + for _, user := range users { + chatpubkey, chatseckey := newChatKeys() + user.Options.ChatPubKey = tob64(chatpubkey.key[:]) + user.Options.ChatSecKey = tob64(chatseckey.key[:]) + jopt, _ := jsonify(user.Options) + try("update users set options = ? where userid = ?", jopt, user.ID) + } + setV(48) + fallthrough + case 48: + try("create index idx_honksurl on honks(url)") + setV(49) + fallthrough + case 49: try("analyze") + closedatabases() default: elog.Fatalf("can't upgrade unknown version %d", dbversion)
M
util.go
→
util.go
@@ -36,10 +36,8 @@ import (
"bufio" "crypto/rand" "crypto/rsa" - "crypto/sha512" "database/sql" "fmt" - "io/ioutil" "net" "os" "os/signal"@@ -53,24 +51,8 @@ "humungus.tedunangst.com/r/webs/httpsig"
"humungus.tedunangst.com/r/webs/login" ) -var savedassetparams = make(map[string]string) - var re_plainname = regexp.MustCompile("^[[:alnum:]_-]+$") -func getassetparam(file string) string { - if p, ok := savedassetparams[file]; ok { - return p - } - data, err := ioutil.ReadFile(file) - if err != nil { - return "" - } - hasher := sha512.New() - hasher.Write(data) - - return fmt.Sprintf("?v=%.8x", hasher.Sum(nil)) -} - var dbtimeformat = "2006-01-02 15:04:05" var alreadyopendb *sql.DB@@ -78,6 +60,7 @@ var alreadyopendbx *sqlx.DB
var stmtConfig *sql.Stmt func initdb() { + blobdbname := dataDir + "/blob.db" dbname := dataDir + "/honk.db" _, err := os.Stat(dbname) if err == nil {@@ -95,6 +78,7 @@ alreadyopendbx = dbx
defer func() { os.Remove(dbname) + os.Remove(blobdbname) os.Exit(1) }() c := make(chan os.Signal, 1)@@ -104,6 +88,7 @@ <-c
C.termecho(1) fmt.Printf("\n") os.Remove(dbname) + os.Remove(blobdbname) os.Exit(1) }()@@ -121,7 +106,7 @@ }
} r := bufio.NewReader(os.Stdin) - initblobdb() + initblobdb(blobdbname) prepareStatements(db) prepareStatementsx(dbx)@@ -178,8 +163,7 @@ fmt.Printf("done.\n")
os.Exit(0) } -func initblobdb() { - blobdbname := dataDir + "/blob.db" +func initblobdb(blobdbname string) { _, err := os.Stat(blobdbname) if err == nil { elog.Fatalf("%s already exists", blobdbname)@@ -290,7 +274,7 @@ if err != nil {
elog.Print(err) return } - err = login.SetPassword(user.ID, pass) + err = login.SetPassword(int64(user.ID), pass) if err != nil { elog.Print(err) return@@ -351,8 +335,13 @@ seckey, err := httpsig.EncodeKey(k)
if err != nil { return err } + chatpubkey, chatseckey := newChatKeys() + var opts UserOptions + opts.ChatPubKey = tob64(chatpubkey.key[:]) + opts.ChatSecKey = tob64(chatseckey.key[:]) + jopt, _ := jsonify(opts) about := "what about me?" - _, err = db.Exec("insert into users (username, displayname, about, hash, pubkey, seckey, options) values (?, ?, ?, ?, ?, ?, ?)", name, name, about, hash, pubkey, seckey, "{}") + _, err = db.Exec("insert into users (username, displayname, about, hash, pubkey, seckey, options) values (?, ?, ?, ?, ?, ?, ?)", name, name, about, hash, pubkey, seckey, jopt) if err != nil { return err }@@ -476,6 +465,10 @@ err := getconfig("listenaddr", &listenAddr)
if err != nil { return nil, err } + if strings.HasPrefix(listenAddr, "fcgi:") { + listenAddr = listenAddr[5:] + usefcgi = true + } if listenAddr == "" { return nil, fmt.Errorf("must have listenaddr") }@@ -494,5 +487,6 @@ }
if proto == "unix" { os.Chmod(listenAddr, 0777) } + listenSocket = listener return listener, nil }
M
views/about.html
→
views/about.html
@@ -5,10 +5,10 @@ {{ .AboutMsg }}
<p> <table class="font08em"> <tbody> -<tr><td>version:<td style="text-align:right">{{ .HonkVersion }} -<tr><td>memory:<td style="text-align:right">{{ printf "%.02f" .Sensors.Memory }}MB -<tr><td>uptime:<td style="text-align:right">{{ .Sensors.Uptime }} -<tr><td>cputime:<td style="text-align:right">{{ printf "%.02f" .Sensors.CPU }}s +<tr><td>version:<td class="text-right">{{ .HonkVersion }} +<tr><td>memory:<td class="text-right">{{ printf "%.02f" .Sensors.Memory }}MB +<tr><td>uptime:<td class="text-right">{{ printf "%.02f" .Sensors.Uptime }}s +<tr><td>cputime:<td class="text-right">{{ printf "%.02f" .Sensors.CPU }}s </table> <p> </div>
M
views/hfcs.html
→
views/hfcs.html
@@ -18,6 +18,8 @@ <p><label for="actor">who or where:</label><br>
<input tabindex=1 type="text" name="actor" value="" autocomplete=off> <p><span><label class=button for="incaud">include audience: <input tabindex=1 type="checkbox" id="incaud" name="incaud" value="yes"><span></span></label></span> +<span><label class=button for="unknowns">only unknowns: +<input tabindex=1 type="checkbox" id="unknowns" name="unknowns" value="yes"><span></span></label></span> <p><label for="filttext">text matches:</label><br> <input tabindex=1 type="text" name="filttext" value="" autocomplete=off> <p><span><label class=button for="isreply">is reply:@@ -55,7 +57,7 @@ <section class="honk">
<p>Name: {{ .Name }} {{ with .Notes }}<p>Notes: {{ . }}{{ end }} <p>Date: {{ .Date.Format "2006-01-02" }} -{{ with .Actor }}<p>Who: {{ . }}{{ end }} {{ with .IncludeAudience }} (inclusive) {{ end }} +{{ with .Actor }}<p>Who: {{ . }}{{ end }}{{ if .IncludeAudience }} (inclusive){{ end }}{{ if .OnlyUnknowns }} (unknowns){{ end }} {{ if .IsReply }}<p>Reply: y{{ end }} {{ if .IsAnnounce }}<p>Announce: {{ .AnnounceOf }}{{ end }} {{ with .Text }}<p>Text: {{ . }}{{ end }}
M
views/honk.html
→
views/honk.html
@@ -49,39 +49,41 @@ </ul>
{{ end }} </header> <p> -{{ if .HTPrecis }} -<details class="noise"> +<details class="noise" {{ with .Open }}{{.}}{{end}}> <summary class="noise">{{ .HTPrecis }}<p></summary> -{{ end }} +<p>{{ .HTPrecis }} <p class="content">{{ .HTML }} +{{ if .Link }} +<p><a href="{{ .Link }}">{{ or .LegalName .Link }}</a> +{{ end }} {{ with .Time }} -<p>Time: {{ .StartTime.Local.Format "03:04PM EDT Mon Jan 02"}} -{{ if .Duration }}<br>Duration: {{ .Duration }}{{ end }} +<p>Time: {{ .StartTime.Local.Format "03:04PM MST Mon Jan 02"}} +{{ if .Duration }}<br>Duration: {{ .Duration }}{{ end }}</p> {{ end }} {{ with .Place }} -<p>Location: {{ with .Url }}<a href="{{ . }}" rel=noreferrer>{{ end }}{{ .Name }}{{ if .Url }}</a>{{ end }}{{ if or .Latitude .Longitude }} <a href="{{ if eq $maplink "apple" }}https://maps.apple.com/?q={{ or .Name "here" }}&z=16&ll={{ .Latitude }},{{ .Longitude }}{{ else }}https://www.openstreetmap.org/?mlat={{ .Latitude }}&mlon={{ .Longitude}}#map=16/{{ .Latitude }}/{{ .Longitude }}{{ end }}" rel=noreferrer>{{ .Latitude }} {{ .Longitude }}</a>{{ end }} +<p>Location: {{ with .Url }}<a href="{{ . }}" rel=noreferrer>{{ end }}{{ .Name }}{{ if .Url }}</a>{{ end }}{{ if or .Latitude .Longitude }} <a href="{{ if eq $maplink "apple" }}https://maps.apple.com/?q={{ or .Name "here" }}&z=16&ll={{ .Latitude }},{{ .Longitude }}{{ else }}https://www.openstreetmap.org/?mlat={{ .Latitude }}&mlon={{ .Longitude}}#map=16/{{ .Latitude }}/{{ .Longitude }}{{ end }}" rel=noreferrer>{{ .Latitude }} {{ .Longitude }}</a></p>{{ end }} {{ end }} {{ range .Donks }} {{ if .Local }} {{ if eq .Media "text/plain" }} -<p><a href="/d/{{ .XID }}">Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }} +<p><a href="/d/{{ .XID }}">Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}</p> {{ else if eq .Media "application/pdf" }} -<p><a href="/d/{{ .XID }}">Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }} +<p><a href="/d/{{ .XID }}">Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}</p> {{ else }} {{ if $omitimages }} -<p><a href="/d/{{ .XID }}">Image: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }} +<p><a href="/d/{{ .XID }}">Image: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}</p> {{ else }} -<p><img src="/d/{{ .XID }}" loading=lazy title="{{ .Desc }}" alt="{{ .Desc }}"> +<img class="donk donklink" src="/d/{{ .XID }}" loading=lazy title="{{ .Desc }}" alt="{{ .Desc }}"> {{ end }} {{ end }} {{ else }} {{ if .External }} -<p><a href="{{ .URL }}" rel=noreferrer>External Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }} +<p><a href="{{ .URL }}" rel=noreferrer>External Attachment: {{ .Name }}</a>{{ if not (eq .Desc .Name) }} {{ .Desc }}{{ end }}</p> {{ else }} {{ if eq .Media "video/mp4" }} -<p><video controls src="{{ .URL }}">{{ .Name }}</video> +<p><video controls src="{{ .URL }}">{{ .Name }}</video></p> {{ else }} -<p><img src="{{ .URL }}" title="{{ .Desc }}" alt="{{ .Desc }}"> +<p><img src="{{ .URL }}" title="{{ .Desc }}" alt="{{ .Desc }}"></p> {{ end }} {{ end }} {{ end }}@@ -125,7 +127,11 @@ <button disabled>untagged</button>
{{ else }} <button class="flogit-untag">untag me</button> {{ end }} +{{ if eq .Honk.Honker .UserURL }} <button><a href="/edit?xid={{ .Honk.XID }}">edit</a></button> +{{ else }} +<button disabled>nope</button> +{{ end }} {{ if not (eq .Badonk "none") }} {{ if .Honk.IsReacted }} <button disabled>badonked</button>
M
views/honkform.html
→
views/honkform.html
@@ -9,8 +9,7 @@ <p>
<details> <summary>more options</summary> <p> -<label class=button id="donker">attach: <input type="file" multiple name="donk"><span>{{ .SavedFile }}</span></label> -<input type="hidden" id="saveddonkxid" name="donkxid" value="{{ .SavedFile }}"> +<label class=button id="donker">attach: <input type="file" multiple name="donk"><span>{{ .SavedFile }}</span></label><input type="hidden" id="saveddonkxid" name="donkxid" value="{{ .SavedFile }}"> <p id="donkdescriptor"><label for=donkdesc>description:</label><br> <input type="text" name="donkdesc" value="{{ .DonkDesc }}" autocomplete=off> {{ with .SavedPlace }}@@ -45,6 +44,17 @@ <line x1="15" y1="10" x2="15.01" y2="10"></line>
</svg> <div id="emupicker"> </div> +</details> +<details> +<summary>advanced</summary> +<p><label for=seealso>cc:</label><br> +<input type="text" name="seealso" value="{{ .SeeAlso }}"> +<p><label for=legalname>name:</label><br> +<input type="text" name="legalname" value="{{ .LegalName }}"> +<p><label for=link>link:</label><br> +<input type="text" name="link" value="{{ .Link }}"> +<p><label for=onties>tags:</label><br> +<input type="text" name="onties" value="{{ .Onties }}"> </details> <p> <textarea name="noise" id="honknoise">{{ .Noise }}</textarea>
M
views/honkfrags.html
→
views/honkfrags.html
@@ -2,6 +2,7 @@ {{ $BonkCSRF := .HonkCSRF }}
{{ $MapLink := .MapLink }} {{ $Badonk := .User.Options.Reaction }} {{ $OmitImages := .User.Options.OmitImages }} +{{ $UserURL := .User.URL }} {{ range .Honks }} -{{ template "honk.html" map "Honk" . "MapLink" $MapLink "BonkCSRF" $BonkCSRF "Badonk" $Badonk "OmitImages" $OmitImages }} +{{ template "honk.html" map "Honk" . "MapLink" $MapLink "BonkCSRF" $BonkCSRF "Badonk" $Badonk "OmitImages" $OmitImages "UserURL" $UserURL }} {{ end }}
M
views/honkpage.html
→
views/honkpage.html
@@ -2,6 +2,7 @@ {{ template "header.html" . }}
<main> <div class="info" id="infobox"> <div id="srvmsg"> +<div> {{ if .Name }} <p>{{ .Name }} <span style="margin-left:1em;"><a href="/u/{{ .Name }}/rss">rss</a></span> <p>{{ .WhatAbout }}@@ -34,8 +35,9 @@ {{ $IsPreview := .IsPreview }}
{{ $MapLink := .MapLink }} {{ $Badonk := .User.Options.Reaction }} {{ $OmitImages := .User.Options.OmitImages }} +{{ $UserURL := .User.URL }} {{ range .Honks }} -{{ template "honk.html" map "Honk" . "MapLink" $MapLink "BonkCSRF" $BonkCSRF "IsPreview" $IsPreview "Badonk" $Badonk "OmitImages" $OmitImages }} +{{ template "honk.html" map "Honk" . "MapLink" $MapLink "BonkCSRF" $BonkCSRF "IsPreview" $IsPreview "Badonk" $Badonk "OmitImages" $OmitImages "UserURL" $UserURL }} {{ end }} </div> </div>
M
views/honkpage.js
→
views/honkpage.js
@@ -19,14 +19,15 @@ x.timeout = 30 * 1000
x.setRequestHeader("Content-Type", "application/x-www-form-urlencoded") x.send(data) } -function get(url, whendone, whentimedout) { +function get(url, whendone, errfunction) { var x = new XMLHttpRequest() x.open("GET", url) x.timeout = 15 * 1000 x.responseType = "json" x.onload = function() { whendone(x) } - if (whentimedout) { - x.ontimeout = function(e) { whentimedout(x, e) } + if (errfunction) { + x.ontimeout = function(e) { errfunction(" timed out") } + x.onerror = function(e) { errfunction(" error") } } x.send() }@@ -82,7 +83,7 @@
function oldestnewest(btn) { var els = document.getElementsByClassName("glow") if (els.length) { - els[els.length-1].scrollIntoView() + els[els.length-1].scrollIntoView({ behavior: "smooth" }) } } function removeglow() {@@ -132,7 +133,7 @@ var honksonpage = document.getElementById("honksonpage")
var holder = honksonpage.children[0] var lenhonks = honks.length for (var i = honks.length; i > 0; i--) { - var h = honks[i-1] + var h = honks[frontload ? i-1 : 0] if (glowit) h.classList.add("glow") if (frontload) {@@ -181,10 +182,10 @@ refreshupdate(" " + lenhonks + " new")
} else { refreshupdate(" status: " + xhr.status) } - }, function(xhr, e) { + }, function(err) { btn.innerHTML = "refresh" btn.disabled = false - refreshupdate(" timed out") + refreshupdate(err) }) } function statechanger(evt) {@@ -230,8 +231,8 @@ fillinhonks(xhr, false)
} else { refreshupdate(" status: " + xhr.status) } - }, function(xhr, e) { - refreshupdate(" timed out") + }, function(err) { + refreshupdate(err) }) } refreshupdate("")@@ -276,6 +277,16 @@ var xid = el.getAttribute("data-xid")
el.onclick = pageswitcher("honker", xid) el.classList.remove("honkerlink") } + els = document.getElementsByClassName("donklink") + while (els.length) { + let el = els[0] + el.onclick = function() { + el.classList.remove("donk") + el.onclick = null + return false + } + el.classList.remove("donklink") + } els = document.querySelectorAll("#honksonpage article button") els.forEach(function(el) {@@ -345,6 +356,7 @@ }
function showhonkform(elem, rid, hname) { var form = lehonkform form.style = "display: block" + form.reset() if (elem) { form.remove() elem.parentElement.parentElement.parentElement.insertAdjacentElement('beforebegin', form)@@ -353,6 +365,8 @@ hideelement(lehonkbutton)
elem = document.getElementById("honkformhost") elem.insertAdjacentElement('afterend', form) } + var donker = document.getElementById("donker") + donker.children[1].textContent = "" var ridinput = document.getElementById("ridinput") var honknoise = document.getElementById("honknoise") if (rid) {@@ -368,6 +382,8 @@ honknoise.value = ""
} var updateinput = document.getElementById("updatexidinput") updateinput.value = "" + var savedfile = document.getElementById("saveddonkxid") + savedfile.value = "" honknoise.focus() return false }@@ -448,10 +464,15 @@ }
} function hotkey(e) { + if (e.ctrlKey || e.altKey) + return + if (e.code == "Escape") { + var menu = document.getElementById("topmenu") + menu.open = false + return + } if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) return - if (e.ctrlKey || e.altKey) - return switch (e.code) { case "KeyR":@@ -468,12 +489,12 @@ scrollprevioushonk();
break; case "KeyM": var menu = document.getElementById("topmenu") - menu.open = true - menu.querySelector("a").focus() - break - case "Escape": - var menu = document.getElementById("topmenu") - menu.open = false + if (!menu.open) { + menu.open = true + menu.querySelector("a").focus() + } else { + menu.open = false + } break case "Slash": document.getElementById("topmenu").open = true
M
views/onts.html
→
views/onts.html
@@ -3,8 +3,12 @@ <main>
<div class="info"> <p>ontologies of interest {{ $firstrune := .FirstRune }} -{{ $letter := 0 }} <ul> +<li><p> +{{ range .Pops }} +<span class="wsnowrap"><a href="/o/{{ .Name }}">#{{ .Name }}</a> ({{ .Count }})</span> +{{ end }} +{{ $letter := 0 }} {{ range .Onts }} {{ if not (eq $letter (call $firstrune .Name)) }} {{ $letter = (call $firstrune .Name) }}
M
views/style.css
→
views/style.css
@@ -34,6 +34,15 @@
* { font-size: 1rem !important; } +@media (prefers-color-scheme: dark) { +.noise img:not(.emu) { + opacity: .5; + transition: opacity .5s ease-in-out; +} +.noise img:not(.emu):hover { + opacity: 1; +} +} body { padding: 0px;@@ -456,6 +465,11 @@ max-height: 600px;
} .noise img:not(.emu) { display: block; +} +.noise img.donk { + max-height: 400px; + max-width: 48%; + display: inline; } img.emu { height: 2em;
M
web.go
→
web.go
@@ -18,29 +18,35 @@
import ( "bytes" "context" + "crypto/sha512" "database/sql" + "errors" "fmt" "html/template" "io" notrand "math/rand" "mime/multipart" + "net" "net/http" + "net/http/fcgi" "net/url" "os" "os/signal" "path" "path/filepath" "regexp" + "runtime/pprof" "sort" "strconv" "strings" + "sync" "syscall" "time" "unicode/utf8" "github.com/gorilla/handlers" "github.com/gorilla/mux" - "humungus.tedunangst.com/r/webs/cache" + "humungus.tedunangst.com/r/gonix" "humungus.tedunangst.com/r/webs/gencache" "humungus.tedunangst.com/r/webs/httpsig" "humungus.tedunangst.com/r/webs/junk"@@ -98,8 +104,7 @@ }
if u := login.GetUserInfo(r); u != nil { templinfo["UserInfo"], _ = butwhatabout(u.Username) templinfo["UserStyle"] = getuserstyle(u) - var combos []string - combocache.Get(u.UserID, &combos) + combos, _ := combocache.Get(UserID(u.UserID)) templinfo["Combos"] = combos } return templinfo@@ -109,7 +114,7 @@ var oldnews = gencache.New(gencache.Options[string, []byte]{
Fill: func(url string) ([]byte, bool) { templinfo := getInfo(nil) var honks []*Honk - var userid int64 = -1 + var userid UserID = -1 templinfo["ServerMessage"] = serverMsg switch url {@@ -150,7 +155,7 @@ return
} templinfo := getInfo(r) var honks []*Honk - var userid int64 = -1 + var userid UserID = -1 templinfo["ServerMessage"] = serverMsg if u == nil || r.URL.Path == "/front" {@@ -163,7 +168,7 @@ templinfo["ShowRSS"] = true
honks = getpublichonks() } } else { - userid = u.UserID + userid = UserID(u.UserID) switch r.URL.Path { case "/atme": templinfo["ServerMessage"] = "at me!"@@ -252,7 +257,7 @@ honks = getpublichonks()
} reverbolate(-1, honks) - home := fmt.Sprintf("https://%s/", serverName) + home := serverURL("/") base := home if name != "" { home += "u/" + name@@ -275,7 +280,7 @@ continue
} desc := string(honk.HTML) if t := honk.Time; t != nil { - desc += fmt.Sprintf(`<p>Time: %s`, t.StartTime.Local().Format("03:04PM EDT Mon Jan 02")) + desc += fmt.Sprintf(`<p>Time: %s`, t.StartTime.Local().Format("03:04PM MST Mon Jan 02")) if t.Duration != 0 { desc += fmt.Sprintf(`<br>Duration: %s`, t.Duration) }@@ -329,15 +334,14 @@ func ping(user *WhatAbout, who string) {
if targ := fullname(who, user.ID); targ != "" { who = targ } - if !strings.HasPrefix(who, "https:") { + if !strings.HasPrefix(who, "https://") { who = gofish(who) } if who == "" { ilog.Printf("nobody to ping!") return } - var box *Box - ok := boxofboxes.Get(who, &box) + box, ok := boxofboxes.Get(who) if !ok { ilog.Printf("no inbox to ping %s", who) return@@ -362,8 +366,7 @@ ilog.Printf("sent ping to %s: %s", who, j["id"])
} func pong(user *WhatAbout, who string, obj string) { - var box *Box - ok := boxofboxes.Get(who, &box) + box, ok := boxofboxes.Get(who) if !ok { ilog.Printf("no inbox to pong %s", who) return@@ -412,10 +415,19 @@
if crappola(j) { return } - what, _ := j.GetString("type") + what := firstofmany(j, "type") obj, _ := j.GetString("object") - if what == "Like" || what == "Dislike" || (what == "EmojiReact" && originate(obj) != serverName) { + switch what { + case "Like": + return + case "Dislike": + return + case "Listen": return + case "EmojiReact": + if originate(obj) != serverName { + return + } } who, _ := j.GetString("actor") if rejectactor(user.ID, who) {@@ -432,12 +444,25 @@ ilog.Printf("inbox message failed signature for %s from %s: %s", keyname, r.Header.Get("X-Forwarded-For"), err)
if keyname != "" { ilog.Printf("bad signature from %s", keyname) } - http.Error(w, "what did you call me?", http.StatusTeapot) + http.Error(w, "what did you call me?", http.StatusUnauthorized) return } origin := keymatch(keyname, who) if origin == "" { ilog.Printf("keyname actor mismatch: %s <> %s", keyname, who) + if what == "Create" { + var xid string + obj, ok := j.GetMap("object") + if ok { + xid, _ = obj.GetString("id") + } else { + xid, _ = j.GetString("object") + } + if xid != "" { + dlog.Printf("getting forwarded create from %s: %s", keyname, xid) + go grabhonk(user, xid) + } + } return }@@ -461,7 +486,7 @@ nofollowyou2(user, j)
case "Update": obj, ok := j.GetMap("object") if ok { - what, _ := obj.GetString("type") + what := firstofmany(obj, "type") switch what { case "Service": fallthrough@@ -481,7 +506,7 @@ unfollowme(user, "", "", j)
} return } - what, _ := obj.GetString("type") + what := firstofmany(obj, "type") switch what { case "Follow": unfollowme(user, who, who, j)@@ -545,7 +570,7 @@ ilog.Printf("inbox message failed signature for %s from %s: %s", keyname, r.Header.Get("X-Forwarded-For"), err)
if keyname != "" { ilog.Printf("bad signature from %s", keyname) } - http.Error(w, "what did you call me?", http.StatusTeapot) + http.Error(w, "what did you call me?", http.StatusUnauthorized) return } who, _ := j.GetString("actor")@@ -557,8 +582,8 @@ }
if rejectactor(user.ID, who) { return } - re_ont := regexp.MustCompile("https://" + serverName + "/o/([\\pL[:digit:]]+)") - what, _ := j.GetString("type") + re_ont := regexp.MustCompile(serverURL("/o") + "/([\\pL[:digit:]]+)") + what := firstofmany(j, "type") dlog.Printf("server got a %s", what) switch what { case "Follow":@@ -581,7 +606,7 @@ if !ok {
ilog.Printf("unknown undo no object") return } - what, _ := obj.GetString("type") + what := firstofmany(obj, "type") if what != "Follow" { ilog.Printf("unknown undo: %s", what) return@@ -613,13 +638,13 @@
func ximport(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) xid := strings.TrimSpace(r.FormValue("q")) - xonk := getxonk(u.UserID, xid) + xonk := getxonk(UserID(u.UserID), xid) if xonk == nil { p, _ := investigate(xid) if p != nil { xid = p.XID } - j, err := GetJunk(u.UserID, xid) + j, err := GetJunk(UserID(u.UserID), xid) if err != nil { http.Error(w, "error getting external object", http.StatusInternalServerError) ilog.Printf("error getting external object: %s", err)@@ -677,7 +702,7 @@ elog.Print(err)
} } -var oldoutbox = cache.New(cache.Options{Filler: func(name string) ([]byte, bool) { +var oldoutbox = gencache.New(gencache.Options[string, []byte]{Fill: func(name string) ([]byte, bool) { user, err := butwhatabout(name) if err != nil { return nil, false@@ -715,8 +740,7 @@ if stealthmode(user.ID, r) {
http.NotFound(w, r) return } - var j []byte - ok := oldoutbox.Get(name, &j) + j, ok := oldoutbox.Get(name) if ok { w.Header().Set("Content-Type", theonetruename) w.Write(j)@@ -725,12 +749,12 @@ http.NotFound(w, r)
} } -var oldempties = cache.New(cache.Options{Filler: func(url string) ([]byte, bool) { +var oldempties = gencache.New(gencache.Options[string, []byte]{Fill: func(url string) ([]byte, bool) { colname := "/followers" if strings.HasSuffix(url, "/following") { colname = "/following" } - user := fmt.Sprintf("https://%s%s", serverName, url[:len(url)-10]) + user := serverURL("%s", url[:len(url)-10]) j := junk.New() j["@context"] = itiswhatitis j["id"] = user + colname@@ -753,8 +777,7 @@ if stealthmode(user.ID, r) {
http.NotFound(w, r) return } - var j []byte - ok := oldempties.Get(r.URL.Path, &j) + j, ok := oldempties.Get(r.URL.Path) if ok { w.Header().Set("Content-Type", theonetruename) w.Write(j)@@ -775,7 +798,11 @@ if stealthmode(user.ID, r) {
http.NotFound(w, r) return } - if friendorfoe(r.Header.Get("Accept")) { + wantjson := false + if strings.HasSuffix(r.URL.Path, ".json") { + wantjson = true + } + if friendorfoe(r.Header.Get("Accept")) || wantjson { j, ok := asjonker(name) if ok { w.Header().Set("Content-Type", theonetruename)@@ -794,6 +821,7 @@ templinfo := getInfo(r)
templinfo["PageName"] = "user" templinfo["PageArg"] = name templinfo["Name"] = user.Name + templinfo["Honkology"] = oguser(user) templinfo["WhatAbout"] = user.HTAbout templinfo["ServerMessage"] = "" templinfo["APAltLink"] = templates.Sprintf("<link href='%s' rel='alternate' type='application/activity+json'>", user.URL)@@ -809,9 +837,9 @@ name := mux.Vars(r)["name"]
var honks []*Honk if name == "" { name = r.FormValue("xid") - honks = gethonksbyxonker(u.UserID, name, 0) + honks = gethonksbyxonker(UserID(u.UserID), name, 0) } else { - honks = gethonksbyhonker(u.UserID, name, 0) + honks = gethonksbyhonker(UserID(u.UserID), name, 0) } miniform := templates.Sprintf(`<form action="/submithonker" method="POST"> <input type="hidden" name="CSRF" value="%s">@@ -830,8 +858,8 @@
func showcombo(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] u := login.GetUserInfo(r) - honks := gethonksbycombo(u.UserID, name, 0) - honks = osmosis(honks, u.UserID, true) + honks := gethonksbycombo(UserID(u.UserID), name, 0) + honks = osmosis(honks, UserID(u.UserID), true) templinfo := getInfo(r) templinfo["PageName"] = "combo" templinfo["PageArg"] = name@@ -842,12 +870,12 @@ }
func showconvoy(w http.ResponseWriter, r *http.Request) { c := r.FormValue("c") u := login.GetUserInfo(r) - honks := gethonksbyconvoy(u.UserID, c, 0) + honks := gethonksbyconvoy(UserID(u.UserID), c, 0) templinfo := getInfo(r) if len(honks) > 0 { templinfo["TopHID"] = honks[0].ID } - honks = osmosis(honks, u.UserID, false) + honks = osmosis(honks, UserID(u.UserID), false) //reversehonks(honks) honks = threadsort(honks) templinfo["PageName"] = "convoy"@@ -863,7 +891,7 @@ ximport(w, r)
return } u := login.GetUserInfo(r) - honks := gethonksbysearch(u.UserID, q, 0) + honks := gethonksbysearch(UserID(u.UserID), q, 0) templinfo := getInfo(r) templinfo["PageName"] = "search" templinfo["PageArg"] = q@@ -874,9 +902,9 @@ }
func showontology(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] u := login.GetUserInfo(r) - var userid int64 = -1 + var userid UserID = -1 if u != nil { - userid = u.UserID + userid = UserID(u.UserID) } honks := gethonksbyontology(userid, "#"+name, 0) if friendorfoe(r.Header.Get("Accept")) {@@ -893,7 +921,7 @@ user := getserveruser()
j := junk.New() j["@context"] = itiswhatitis - j["id"] = fmt.Sprintf("https://%s/o/%s", serverName, name) + j["id"] = serverURL("/o/%s", name) j["name"] = "#" + name j["attributedTo"] = user.URL j["type"] = "OrderedCollection"@@ -917,9 +945,9 @@ }
func thelistingoftheontologies(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) - var userid int64 = -1 + var userid UserID = -1 if u != nil { - userid = u.UserID + userid = UserID(u.UserID) } rows, err := stmtAllOnts.Query(userid) if err != nil {@@ -927,7 +955,7 @@ elog.Printf("selection error: %s", err)
return } defer rows.Close() - var onts []Ont + var onts, pops []Ont for rows.Next() { var o Ont err := rows.Scan(&o.Name, &o.Count)@@ -940,14 +968,24 @@ continue
} o.Name = o.Name[1:] onts = append(onts, o) + if o.Count > 1 { + pops = append(pops, o) + } } sort.Slice(onts, func(i, j int) bool { return onts[i].Name < onts[j].Name }) + sort.Slice(pops, func(i, j int) bool { + return pops[i].Count > pops[j].Count + }) + if len(pops) > 40 { + pops = pops[:40] + } if u == nil && !develMode { w.Header().Set("Cache-Control", "max-age=300") } templinfo := getInfo(r) + templinfo["Pops"] = pops templinfo["Onts"] = onts templinfo["FirstRune"] = func(s string) rune { r, _ := utf8.DecodeRuneInString(s); return r } err = readviews.Execute(w, "onts.html", templinfo)@@ -1072,16 +1110,17 @@ }
var re_keyholder = regexp.MustCompile(`keyId="([^"]+)"`) -func trackback(xid string, r *http.Request) { - agent := r.UserAgent() - who := originate(agent) - sig := r.Header.Get("Signature") - if sig != "" { - m := re_keyholder.FindStringSubmatch(sig) - if len(m) == 2 { - who = m[1] +func requestActor(r *http.Request) string { + if sig := r.Header.Get("Signature"); sig != "" { + if m := re_keyholder.FindStringSubmatch(sig); len(m) == 2 { + return m[1] } } + return "" +} + +func trackback(xid string, r *http.Request) { + who := requestActor(r) if who != "" { trackchan <- Track{xid: xid, who: who} }@@ -1098,6 +1137,18 @@ }
return n1 == n2 } +func threadposes(honks []*Honk, wanted int64) ([]*Honk, []int) { + var poses []int + var newhonks []*Honk + for i, honk := range honks { + if honk.ID > wanted { + newhonks = append(newhonks, honk) + poses = append(poses, i) + } + } + return newhonks, poses +} + func threadsort(honks []*Honk) []*Honk { sort.Slice(honks, func(i, j int) bool { return honks[i].Date.Before(honks[j].Date)@@ -1166,9 +1217,24 @@ }
return thread } +func oguser(user *WhatAbout) template.HTML { + short := user.About + if len(short) > 160 { + short = short[0:160] + "..." + } + title := user.Display + imgurl := avatarURL(user) + return templates.Sprintf( + `<meta property="og:title" content="%s" /> +<meta property="og:type" content="website" /> +<meta property="og:url" content="%s" /> +<meta property="og:image" content="%s" /> +<meta property="og:description" content="%s" />`, + title, user.URL, imgurl, short) +} + func honkology(honk *Honk) template.HTML { - var user *WhatAbout - ok := somenumberedusers.Get(honk.UserID, &user) + user, ok := somenumberedusers.Get(honk.UserID) if !ok { return "" }@@ -1205,9 +1271,15 @@ if stealthmode(user.ID, r) {
http.NotFound(w, r) return } - xid := fmt.Sprintf("https://%s%s", serverName, r.URL.Path) + wantjson := false + path := r.URL.Path + if strings.HasSuffix(path, ".json") { + path = path[:len(path)-5] + wantjson = true + } + xid := serverURL("%s", path) - if friendorfoe(r.Header.Get("Accept")) { + if friendorfoe(r.Header.Get("Accept")) || wantjson { j, ok := gimmejonk(xid) if ok { trackback(xid, r)@@ -1224,7 +1296,7 @@ http.NotFound(w, r)
return } u := login.GetUserInfo(r) - if u != nil && u.UserID != user.ID { + if u != nil && UserID(u.UserID) != user.ID { u = nil } if !honk.Public {@@ -1268,21 +1340,9 @@ honkpage(w, u, honks, templinfo)
} func honkpage(w http.ResponseWriter, u *login.UserInfo, honks []*Honk, templinfo map[string]interface{}) { - var emunames []string - dir, err := os.Open(dataDir + "/emus") - if err == nil { - emunames, _ = dir.Readdirnames(0) - dir.Close() - } - for i, e := range emunames { - if len(e) > 4 { - emunames[i] = e[:len(e)-4] - } - } - templinfo["Emus"] = emunames - var userid int64 = -1 + var userid UserID = -1 if u != nil { - userid = u.UserID + userid = UserID(u.UserID) templinfo["User"], _ = butwhatabout(u.Username) } reverbolate(userid, honks)@@ -1298,7 +1358,7 @@ }
if u == nil && !develMode { w.Header().Set("Cache-Control", "max-age=60") } - err = readviews.Execute(w, "honkpage.html", templinfo) + err := readviews.Execute(w, "honkpage.html", templinfo) if err != nil { elog.Print(err) }@@ -1312,31 +1372,12 @@ user, _ := butwhatabout(u.Username)
db := opendatabase() options := user.Options - if r.FormValue("skinny") == "skinny" { - options.SkinnyCSS = true - } else { - options.SkinnyCSS = false - } - if r.FormValue("omitimages") == "omitimages" { - options.OmitImages = true - } else { - options.OmitImages = false - } - if r.FormValue("mentionall") == "mentionall" { - options.MentionAll = true - } else { - options.MentionAll = false - } - if r.FormValue("inlineqts") == "inlineqts" { - options.InlineQuotes = true - } else { - options.InlineQuotes = false - } - if r.FormValue("maps") == "apple" { - options.MapLink = "apple" - } else { - options.MapLink = "" - } + options.SkinnyCSS = r.FormValue("skinny") == "skinny" + options.OmitImages = r.FormValue("omitimages") == "omitimages" + options.MentionAll = r.FormValue("mentionall") == "mentionall" + options.InlineQuotes = r.FormValue("inlineqts") == "inlineqts" + options.MapLink = r.FormValue("maps") + options.Reaction = r.FormValue("reaction") sendupdate := false if r.FormValue("displayname") != "" {@@ -1359,7 +1400,7 @@ ava = ava[7:]
if ava[0] == ' ' { ava = ava[1:] } - ava = fmt.Sprintf("https://%s/meme/%s", serverName, ava) + ava = serverURL("/meme/%s", ava) } if ava != options.Avatar { options.Avatar = ava@@ -1372,7 +1413,7 @@ ban = ban[7:]
if ban[0] == ' ' { ban = ban[1:] } - ban = fmt.Sprintf("https://%s/meme/%s", serverName, ban) + ban = serverURL("/meme/%s", ban) } if ban != options.Banner { options.Banner = ban@@ -1390,7 +1431,7 @@ if err != nil {
elog.Printf("error bouting what: %s", err) } somenamedusers.Clear(u.Username) - somenumberedusers.Clear(u.UserID) + somenumberedusers.Clear(user.ID) oldjonkers.Clear(u.Username) if sendupdate {@@ -1491,11 +1532,11 @@
func zonkit(w http.ResponseWriter, r *http.Request) { wherefore := r.FormValue("wherefore") what := r.FormValue("what") - userinfo := login.GetUserInfo(r) - user, _ := butwhatabout(userinfo.Username) + u := login.GetUserInfo(r) + user, _ := butwhatabout(u.Username) if wherefore == "save" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil { _, err := stmtUpdateFlags.Exec(flagIsSaved, xonk.ID) if err != nil {@@ -1506,7 +1547,7 @@ return
} if wherefore == "unsave" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil { _, err := stmtClearFlags.Exec(flagIsSaved, xonk.ID) if err != nil {@@ -1524,7 +1565,7 @@ }
if reaction == "none" { return } - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil { _, err := stmtUpdateFlags.Exec(flagIsReacted, xonk.ID) if err != nil {@@ -1539,7 +1580,7 @@ // my hammer is too big, oh well
defer oldjonks.Flush() if wherefore == "ack" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil && !xonk.IsAcked() { _, err := stmtUpdateFlags.Exec(flagIsAcked, xonk.ID) if err != nil {@@ -1551,7 +1592,7 @@ return
} if wherefore == "deack" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil && xonk.IsAcked() { _, err := stmtClearFlags.Exec(flagIsAcked, xonk.ID) if err != nil {@@ -1563,16 +1604,15 @@ return
} if wherefore == "bonk" { - user, _ := butwhatabout(userinfo.Username) bonkit(what, user) return } if wherefore == "unbonk" { - xonk := getbonk(userinfo.UserID, what) + xonk := getbonk(user.ID, what) if xonk != nil { deletehonk(xonk.ID) - xonk = getxonk(userinfo.UserID, what) + xonk = getxonk(user.ID, what) _, err := stmtClearFlags.Exec(flagIsBonked, xonk.ID) if err != nil { elog.Printf("error unbonking: %s", err)@@ -1583,7 +1623,7 @@ return
} if wherefore == "untag" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil { _, err := stmtUpdateFlags.Exec(flagIsUntagged, xonk.ID) if err != nil {@@ -1591,7 +1631,7 @@ elog.Printf("error untagging: %s", err)
} } var badparents map[string]bool - untagged.GetAndLock(userinfo.UserID, &badparents) + untagged.GetAndLock(user.ID, &badparents) badparents[what] = true untagged.Unlock() return@@ -1599,7 +1639,7 @@ }
ilog.Printf("zonking %s %s", wherefore, what) if wherefore == "zonk" { - xonk := getxonk(userinfo.UserID, what) + xonk := getxonk(user.ID, what) if xonk != nil { deletehonk(xonk.ID) if xonk.Whofore == 2 || xonk.Whofore == 3 {@@ -1607,7 +1647,7 @@ sendzonkofsorts(xonk, user, "zonk", "")
} } } - _, err := stmtSaveZonker.Exec(userinfo.UserID, what, wherefore) + _, err := stmtSaveZonker.Exec(user.ID, what, wherefore) if err != nil { elog.Printf("error saving zonker: %s", err) return@@ -1618,17 +1658,21 @@ func edithonkpage(w http.ResponseWriter, r *http.Request) {
u := login.GetUserInfo(r) user, _ := butwhatabout(u.Username) xid := r.FormValue("xid") - honk := getxonk(u.UserID, xid) + honk := getxonk(user.ID, xid) if !canedithonk(user, honk) { http.Error(w, "no editing that please", http.StatusInternalServerError) return } - noise := honk.Noise + var savedfiles []string honks := []*Honk{honk} donksforhonks(honks) - reverbolate(u.UserID, honks) + for _, d := range honk.Donks { + savedfiles = append(savedfiles, fmt.Sprintf("%s:%d", d.XID, d.FileID)) + } + reverbolate(user.ID, honks) + templinfo := getInfo(r) templinfo["HonkCSRF"] = login.GetCSRF("honkhonk", r) templinfo["Honks"] = honks@@ -1642,14 +1686,14 @@ if tm.Duration != 0 {
templinfo["Duration"] = tm.Duration } } + templinfo["Onties"] = honk.Onties + templinfo["SeeAlso"] = honk.SeeAlso + templinfo["Link"] = honk.Link + templinfo["LegalName"] = honk.LegalName templinfo["ServerMessage"] = "honk edit" templinfo["IsPreview"] = true templinfo["UpdateXID"] = honk.XID - if len(honk.Donks) > 0 { - var savedfiles []string - for _, d := range honk.Donks { - savedfiles = append(savedfiles, fmt.Sprintf("%s:%d", d.XID, d.FileID)) - } + if len(savedfiles) > 0 { templinfo["SavedFile"] = strings.Join(savedfiles, ",") } err := readviews.Execute(w, "honkpage.html", templinfo)@@ -1663,7 +1707,7 @@ u := login.GetUserInfo(r)
rid := r.FormValue("rid") noise := "" - xonk := getxonk(u.UserID, rid) + xonk := getxonk(UserID(u.UserID), rid) if xonk != nil { _, replto := handles(xonk.Honker) if replto != "" {@@ -1795,7 +1839,8 @@ h := submithonk(w, r)
if h == nil { return } - http.Redirect(w, r, h.XID[len(serverName)+8:], http.StatusSeeOther) + redir := h.XID[len(serverURL("")):] + http.Redirect(w, r, redir, http.StatusSeeOther) } // what a hot mess this function is@@ -1811,14 +1856,14 @@ http.Error(w, "unknown format", 500)
return nil } - userinfo := login.GetUserInfo(r) - user, _ := butwhatabout(userinfo.Username) + u := login.GetUserInfo(r) + user, _ := butwhatabout(u.Username) dt := time.Now().UTC() updatexid := r.FormValue("updatexid") var honk *Honk if updatexid != "" { - honk = getxonk(userinfo.UserID, updatexid) + honk = getxonk(user.ID, updatexid) if !canedithonk(user, honk) { http.Error(w, "no editing that please", http.StatusInternalServerError) return nil@@ -1830,8 +1875,8 @@ } else {
xid := fmt.Sprintf("%s/%s/%s", user.URL, honkSep, xfiltrate()) what := "honk" honk = &Honk{ - UserID: userinfo.UserID, - Username: userinfo.Username, + UserID: user.ID, + Username: user.Name, What: what, Honker: user.URL, XID: xid,@@ -1839,6 +1884,10 @@ Date: dt,
Format: format, } } + honk.SeeAlso = strings.TrimSpace(r.FormValue("seealso")) + honk.Onties = strings.TrimSpace(r.FormValue("onties")) + honk.Link = strings.TrimSpace(r.FormValue("link")) + honk.LegalName = strings.TrimSpace(r.FormValue("legalname")) var convoy string noise = strings.Replace(noise, "\r", "", -1)@@ -1852,8 +1901,7 @@ }
return "" }) } - noise = quickrename(noise, userinfo.UserID) - noise = hooterize(noise) + noise = quickrename(noise, user.ID) honk.Noise = noise precipitate(honk) noise = honk.Noise@@ -1861,7 +1909,7 @@ recategorize(honk)
translate(honk) if rid != "" { - xonk := getxonk(userinfo.UserID, rid) + xonk := getxonk(user.ID, rid) if xonk == nil { http.Error(w, "replyto disappeared", http.StatusNotFound) return nil@@ -1886,11 +1934,12 @@ }
} else if updatexid == "" { honk.Audience = []string{thewholeworld} } - if honk.Noise != "" && honk.Noise[0] == '@' { + if noise != "" && noise[0] == '@' { honk.Audience = append(grapevine(honk.Mentions), honk.Audience...) } else { honk.Audience = append(honk.Audience, grapevine(honk.Mentions)...) } + honk.Convoy = convoy if convoy == "" { convoy = fmt.Sprintf("data:,%s-", masqName) + xfiltrate()@@ -1903,7 +1952,7 @@ http.Error(w, "honk to nowhere...", http.StatusNotFound)
return nil } honk.Public = loudandproud(honk.Audience) - honk.Convoy = convoy + donkxid := strings.Join(r.Form["donkxid"], ",") if donkxid == "" { donks, err := submitdonk(w, r)@@ -1926,7 +1975,7 @@ break
} p := strings.Split(xid, ":") xid = p[0] - url := fmt.Sprintf("https://%s/d/%s", serverName, xid) + url := serverURL("/d/%s", xid) var donk *Donk if len(p) > 1 { fileid, _ := strconv.ParseInt(p[1], 10, 0)@@ -1992,11 +2041,11 @@ honk.Noise = noise
if r.FormValue("preview") == "preview" { honks := []*Honk{honk} - reverbolate(userinfo.UserID, honks) + reverbolate(user.ID, honks) templinfo := getInfo(r) templinfo["HonkCSRF"] = login.GetCSRF("honkhonk", r) templinfo["Honks"] = honks - templinfo["MapLink"] = getmaplink(userinfo) + templinfo["MapLink"] = getmaplink(u) templinfo["InReplyTo"] = r.FormValue("rid") templinfo["Noise"] = r.FormValue("noise") templinfo["SavedFile"] = donkxid@@ -2038,9 +2087,9 @@ return honk
} func showhonkers(w http.ResponseWriter, r *http.Request) { - userinfo := login.GetUserInfo(r) + userid := UserID(login.GetUserInfo(r).UserID) templinfo := getInfo(r) - templinfo["Honkers"] = gethonkers(userinfo.UserID) + templinfo["Honkers"] = gethonkers(userid) templinfo["HonkerCSRF"] = login.GetCSRF("submithonker", r) err := readviews.Execute(w, "honkers.html", templinfo) if err != nil {@@ -2050,8 +2099,8 @@ }
func showchatter(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) - chatnewnone(u.UserID) - chatter := loadchatter(u.UserID) + chatnewnone(UserID(u.UserID)) + chatter := loadchatter(UserID(u.UserID)) for _, chat := range chatter { for _, ch := range chat.Chonks { filterchonk(ch)@@ -2077,14 +2126,14 @@ dt := time.Now().UTC()
xid := fmt.Sprintf("%s/%s/%s", user.URL, "chonk", xfiltrate()) if !strings.HasPrefix(target, "https://") { - target = fullname(target, u.UserID) + target = fullname(target, user.ID) } if target == "" { http.Error(w, "who is that?", http.StatusInternalServerError) return } ch := Chonk{ - UserID: u.UserID, + UserID: user.ID, XID: xid, Who: user.URL, Target: target,@@ -2110,7 +2159,7 @@
http.Redirect(w, r, "/chatter", http.StatusSeeOther) } -var combocache = cache.New(cache.Options{Filler: func(userid int64) ([]string, bool) { +var combocache = gencache.New(gencache.Options[UserID, []string]{Fill: func(userid UserID) ([]string, bool) { honkers := gethonkers(userid) var combos []string for _, h := range honkers {@@ -2127,9 +2176,6 @@ return combos, true
}, Invalidator: &honkerinvalidator}) func showcombos(w http.ResponseWriter, r *http.Request) { - userinfo := login.GetUserInfo(r) - var combos []string - combocache.Get(userinfo.UserID, &combos) templinfo := getInfo(r) err := readviews.Execute(w, "combos.html", templinfo) if err != nil {@@ -2165,7 +2211,7 @@ var meta HonkerMeta
meta.Notes = strings.TrimSpace(r.FormValue("notes")) mj, _ := jsonify(&meta) - defer honkerinvalidator.Clear(u.UserID) + defer honkerinvalidator.Clear(user.ID) // mostly dummy, fill in later... h := &Honker{@@ -2184,7 +2230,7 @@ }
if r.FormValue("sub") == "sub" { followyou(user, honkerid, false) } - _, err := stmtUpdateHonker.Exec(name, combos, mj, honkerid, u.UserID) + _, err := stmtUpdateHonker.Exec(name, combos, mj, honkerid, user.ID) if err != nil { elog.Printf("update honker err: %s", err) return nil@@ -2203,7 +2249,7 @@ flavor = "peep"
} var err error - honkerid, err = savehonker(user, url, name, flavor, combos, mj) + honkerid, flavor, err = savehonker(user, url, name, flavor, combos, mj) if err != nil { http.Error(w, "had some trouble with that: "+err.Error(), http.StatusInternalServerError) return nil@@ -2216,9 +2262,9 @@ return h
} func hfcspage(w http.ResponseWriter, r *http.Request) { - userinfo := login.GetUserInfo(r) + u := login.GetUserInfo(r) - filters := getfilters(userinfo.UserID, filtAny) + filters := getfilters(UserID(u.UserID), filtAny) templinfo := getInfo(r) templinfo["Filters"] = filters@@ -2229,58 +2275,6 @@ elog.Print(err)
} } -func savehfcs(w http.ResponseWriter, r *http.Request) { - userinfo := login.GetUserInfo(r) - itsok := r.FormValue("itsok") - if itsok == "iforgiveyou" { - hfcsid, _ := strconv.ParseInt(r.FormValue("hfcsid"), 10, 0) - _, err := stmtDeleteFilter.Exec(userinfo.UserID, hfcsid) - if err != nil { - elog.Printf("error deleting filter: %s", err) - } - filtInvalidator.Clear(userinfo.UserID) - http.Redirect(w, r, "/hfcs", http.StatusSeeOther) - return - } - - filt := new(Filter) - filt.Name = strings.TrimSpace(r.FormValue("name")) - filt.Date = time.Now().UTC() - filt.Actor = strings.TrimSpace(r.FormValue("actor")) - filt.IncludeAudience = r.FormValue("incaud") == "yes" - filt.Text = strings.TrimSpace(r.FormValue("filttext")) - filt.IsReply = r.FormValue("isreply") == "yes" - filt.IsAnnounce = r.FormValue("isannounce") == "yes" - filt.AnnounceOf = strings.TrimSpace(r.FormValue("announceof")) - filt.Reject = r.FormValue("doreject") == "yes" - filt.SkipMedia = r.FormValue("doskipmedia") == "yes" - filt.Hide = r.FormValue("dohide") == "yes" - filt.Collapse = r.FormValue("docollapse") == "yes" - filt.Rewrite = strings.TrimSpace(r.FormValue("filtrewrite")) - filt.Replace = strings.TrimSpace(r.FormValue("filtreplace")) - if dur := parseDuration(r.FormValue("filtduration")); dur > 0 { - filt.Expiration = time.Now().UTC().Add(dur) - } - filt.Notes = strings.TrimSpace(r.FormValue("filtnotes")) - - if filt.Actor == "" && filt.Text == "" && !filt.IsAnnounce { - ilog.Printf("blank filter") - http.Error(w, "can't save a blank filter", http.StatusInternalServerError) - return - } - - j, err := jsonify(filt) - if err == nil { - _, err = stmtSaveFilter.Exec(userinfo.UserID, j) - } - if err != nil { - elog.Printf("error saving filter: %s", err) - } - - filtInvalidator.Clear(userinfo.UserID) - http.Redirect(w, r, "/hfcs", http.StatusSeeOther) -} - func accountpage(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) user, _ := butwhatabout(u.Username)@@ -2310,7 +2304,7 @@ }
http.Redirect(w, r, "/account", http.StatusSeeOther) } -var oldfingers = cache.New(cache.Options{Filler: func(orig string) ([]byte, bool) { +var oldfingers = gencache.New(gencache.Options[string, []byte]{Fill: func(orig string) ([]byte, bool) { if strings.HasPrefix(orig, "acct:") { orig = orig[5:] } else {@@ -2378,8 +2372,7 @@ orig := r.FormValue("resource")
dlog.Printf("finger lick: %s", orig) - var j []byte - ok := oldfingers.Get(orig, &j) + j, ok := oldfingers.Get(orig) if ok { w.Header().Set("Content-Type", "application/jrd+json") w.Write(j)@@ -2456,7 +2449,7 @@
if isurl(n) { uinfo := login.GetUserInfo(r) if uinfo != nil { - j, err := GetJunkFast(uinfo.UserID, n) + j, err := GetJunkFast(UserID(uinfo.UserID), n) if err != nil { dlog.Println("avatating: getting junk:", err) a = avatateautogen(r)@@ -2540,11 +2533,23 @@ func servememe(w http.ResponseWriter, r *http.Request) {
meme := mux.Vars(r)["meme"] w.Header().Set("Cache-Control", "max-age="+somedays()) - http.ServeFile(w, r, dataDir+"/memes/"+meme) + _, err := os.Stat(dataDir + "/memes/" + meme) + if err == nil { + http.ServeFile(w, r, dataDir+"/memes/"+meme) + } else { + mux.Vars(r)["xid"] = meme + servefile(w, r) + } } func servefile(w http.ResponseWriter, r *http.Request) { + if friendorfoe(r.Header.Get("Accept")) { + dlog.Printf("incompatible accept for donk") + http.Error(w, "there are no activities here", http.StatusNotAcceptable) + return + } xid := mux.Vars(r)["xid"] + preview := r.FormValue("preview") == "1" var media string var data []byte row := stmtGetFileData.QueryRow(xid)@@ -2554,6 +2559,12 @@ elog.Printf("error loading file: %s", err)
http.NotFound(w, r) return } + if preview && strings.HasPrefix(media, "image") { + img, err := lilshrink(data) + if err == nil { + data = img.Data + } + } w.Header().Set("Content-Type", media) w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Cache-Control", "max-age="+somedays())@@ -2579,11 +2590,12 @@ Srvmsg template.HTML
Honks string MeCount int64 ChatCount int64 + Poses []int } func webhydra(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) - userid := u.UserID + userid := UserID(u.UserID) templinfo := getInfo(r) templinfo["HonkCSRF"] = login.GetCSRF("honkhonk", r) page := r.FormValue("page")@@ -2622,10 +2634,10 @@ honks = osmosis(honks, userid, false)
hydra.Srvmsg = templates.Sprintf("honks by combo: %s", c) case "convoy": c := r.FormValue("c") - honks = gethonksbyconvoy(userid, c, wanted) + honks = gethonksbyconvoy(userid, c, 0) honks = osmosis(honks, userid, false) honks = threadsort(honks) - reversehonks(honks) + honks, hydra.Poses = threadposes(honks, wanted) hydra.Srvmsg = templates.Sprintf("honks in convoy: %s", c) case "honker": xid := r.FormValue("xid")@@ -2646,7 +2658,11 @@ http.NotFound(w, r)
} if len(honks) > 0 { - hydra.Tophid = honks[0].ID + if page == "convoy" { + hydra.Tophid = honks[len(honks)-1].ID + } else { + hydra.Tophid = honks[0].ID + } } else { hydra.Tophid = wanted }@@ -2685,7 +2701,7 @@ }
func apihandler(w http.ResponseWriter, r *http.Request) { u := login.GetUserInfo(r) - userid := u.UserID + userid := UserID(u.UserID) action := r.FormValue("action") wait, _ := strconv.ParseInt(r.FormValue("wait"), 10, 0) dlog.Printf("api request '%s' on behalf of %s", action, u.Username)@@ -2732,6 +2748,24 @@ honks = osmosis(honks, userid, true)
case "myhonks": honks = gethonksbyuser(u.Username, true, wanted) honks = osmosis(honks, userid, true) + case "saved": + honks = getsavedhonks(userid, wanted) + case "combo": + c := r.FormValue("c") + honks = gethonksbycombo(userid, c, wanted) + honks = osmosis(honks, userid, false) + case "convoy": + c := r.FormValue("c") + honks = gethonksbyconvoy(userid, c, 0) + honks = osmosis(honks, userid, false) + honks = threadsort(honks) + honks, _ = threadposes(honks, wanted) + case "honker": + xid := r.FormValue("xid") + honks = gethonksbyxonker(userid, xid, wanted) + case "search": + q := r.FormValue("q") + honks = gethonksbysearch(userid, q, wanted) default: http.Error(w, "unknown page", http.StatusNotFound) return@@ -2747,12 +2781,15 @@ case <-waitchan:
} } reverbolate(userid, honks) + user, _ := butwhatabout(u.Username) j := junk.New() j["honks"] = honks + j["mecount"] = user.Options.MeCount + j["chatcount"] = user.Options.ChatCount j.Write(w) case "sendactivity": - user, _ := butwhatabout(u.Username) public := r.FormValue("public") == "1" + user, _ := butwhatabout(u.Username) rcpts := boxuprcpts(user, r.Form["rcpt"], public) msg := []byte(r.FormValue("msg")) for rcpt := range rcpts {@@ -2760,7 +2797,7 @@ go deliverate(userid, rcpt, msg)
} case "gethonkers": j := junk.New() - j["honkers"] = gethonkers(u.UserID) + j["honkers"] = gethonkers(userid) j.Write(w) case "savehonker": h := submithonker(w, r)@@ -2791,25 +2828,40 @@
var endoftheworld = make(chan bool) var readyalready = make(chan bool) var workinprogress = 0 +var requestWG sync.WaitGroup +var listenSocket net.Listener func enditall() { sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) <-sig ilog.Printf("stopping...") + listenSocket.Close() for i := 0; i < workinprogress; i++ { endoftheworld <- true } + if *cpuprofile != "" { + pprof.StopCPUProfile() + } + if *memprofile != "" { + pprof.WriteHeapProfile(memprofilefd) + memprofilefd.Close() + } ilog.Printf("waiting...") + go func() { + time.Sleep(10 * time.Second) + elog.Printf("timed out waiting for requests to finish") + os.Exit(0) + }() for i := 0; i < workinprogress; i++ { <-readyalready } + requestWG.Wait() ilog.Printf("apocalypse") + closedatabases() os.Exit(0) } -var preservehooks []func() - func bgmonitor() { for { when := time.Now().Add(-3 * 24 * time.Hour).UTC().Format(dbtimeformat)@@ -2820,6 +2872,19 @@ }
zaggies.Flush() time.Sleep(50 * time.Minute) } +} + +func addcspheaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestWG.Add(1) + defer requestWG.Done() + policy := "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self'; media-src 'self'" + if develMode { + policy += "; report-uri /csp-violation" + } + w.Header().Set("Content-Security-Policy", policy) + next.ServeHTTP(w, r) + }) } func emuinit() {@@ -2857,6 +2922,48 @@ }
http.Redirect(w, r, path.Join(aturl, last), http.StatusMovedPermanently) } +var savedassetparams = make(map[string]string) + +func getassetparam(file string) string { + if p, ok := savedassetparams[file]; ok { + return p + } + data, err := os.ReadFile(file) + if err != nil { + return "" + } + hasher := sha512.New() + hasher.Write(data) + + return fmt.Sprintf("?v=%.8x", hasher.Sum(nil)) +} + +func startWatcher() { + watcher, err := gonix.NewWatcher() + if err != nil { + return + } + go func() { + s := dataDir + "/views/local.css" + for { + err := watcher.WatchFile(s) + if err != nil { + break + } + err = watcher.WaitForChange() + if err != nil { + dlog.Printf("can't wait: %s", err) + break + } + dlog.Printf("local.css changed") + delete(savedassetparams, s) + savedassetparams[s] = getassetparam(s) + } + }() +} + +var usefcgi bool + func serve() { db := opendatabase() login.Init(login.InitArgs{Db: db, Logger: ilog, Insecure: develMode, SameSiteStrict: !develMode})@@ -2869,32 +2976,22 @@ runBackendServer()
go enditall() go redeliverator() go tracker() + go syndicator() go bgmonitor() go qotd() loadLingo() emuinit() - readviews = templates.Load(develMode, - viewDir+"/views/honkpage.html", - viewDir+"/views/honkfrags.html", - viewDir+"/views/honkers.html", - viewDir+"/views/chatter.html", - viewDir+"/views/hfcs.html", - viewDir+"/views/combos.html", - viewDir+"/views/honkform.html", - viewDir+"/views/honk.html", - viewDir+"/views/account.html", - viewDir+"/views/about.html", - viewDir+"/views/funzone.html", - viewDir+"/views/login.html", - viewDir+"/views/xzone.html", - viewDir+"/views/msg.html", - viewDir+"/views/header.html", - viewDir+"/views/onts.html", - viewDir+"/views/emus.html", - viewDir+"/views/oauthlogin.html", - viewDir+"/views/honkpage.js", - ) + var toload []string + dents, _ := os.ReadDir(viewDir + "/views") + for _, dent := range dents { + name := dent.Name() + if strings.HasSuffix(name, ".html") { + toload = append(toload, viewDir+"/views/"+name) + } + } + + readviews = templates.Load(develMode, toload...) if !develMode { assets := []string{ viewDir + "/views/style.css",@@ -2910,10 +3007,9 @@ savedassetparams[s] = getassetparam(s)
} loadAvatarColors() } + startWatcher() - for _, h := range preservehooks { - h() - } + securitizeweb() mux := mux.NewRouter() mux.Use(login.Checker)@@ -2930,8 +3026,11 @@ getters.HandleFunc("/events", homepage)
getters.HandleFunc("/robots.txt", nomoroboto) getters.HandleFunc("/rss", showrss) getters.HandleFunc("/@{name:[\\pL[:digit:]]+}", showuser) + getters.HandleFunc("/@{name:[\\pL[:digit:]]+}.json", showuser) + getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}.json", showuser) getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}", redirectPretty) getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}/"+honkSep+"/{xid:[\\pL[:digit:]]+}", showonehonk) + getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}/"+honkSep+"/{xid:[\\pL[:digit:]]+}.json", showonehonk) getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}/rss", showrss) posters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}/inbox", inbox) getters.HandleFunc("/"+userSep+"/{name:[\\pL[:digit:]]+}/outbox", outbox)@@ -3019,6 +3118,16 @@ err = http.Serve(listener, loggedmux)
if err != nil { elog.Fatal(err) } + if usefcgi { + err = fcgi.Serve(listener, mux) + } else { + err = http.Serve(listener, mux) + } + if err != nil && !errors.Is(err, net.ErrClosed) { + elog.Printf("serve error: %s", err) + } + time.Sleep(15 * time.Second) + elog.Printf("fell off the bottom") } // Verifies that accesstoken is valid and injects the associated