hfcs.go (view raw)
1//
2// Copyright (c) 2019 Ted Unangst <tedu@tedunangst.com>
3//
4// Permission to use, copy, modify, and distribute this software for any
5// purpose with or without fee is hereby granted, provided that the above
6// copyright notice and this permission notice appear in all copies.
7//
8// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15
16package main
17
18import (
19 "net/http"
20 "regexp"
21 "sort"
22 "strconv"
23 "strings"
24 "time"
25 "unicode"
26
27 "humungus.tedunangst.com/r/webs/cache"
28 "humungus.tedunangst.com/r/webs/login"
29)
30
31type Filter struct {
32 ID int64 `json:"-"`
33 Actions []filtType `json:"-"`
34 Name string
35 Date time.Time
36 Actor string `json:",omitempty"`
37 IncludeAudience bool `json:",omitempty"`
38 OnlyUnknowns bool `json:",omitempty"`
39 Text string `json:",omitempty"`
40 re_text *regexp.Regexp
41 IsReply bool `json:",omitempty"`
42 IsAnnounce bool `json:",omitempty"`
43 AnnounceOf string `json:",omitempty"`
44 Reject bool `json:",omitempty"`
45 SkipMedia bool `json:",omitempty"`
46 Hide bool `json:",omitempty"`
47 Collapse bool `json:",omitempty"`
48 Rewrite string `json:",omitempty"`
49 re_rewrite *regexp.Regexp
50 Replace string `json:",omitempty"`
51 Expiration time.Time
52 Notes string
53}
54
55type filtType uint
56
57const (
58 filtNone filtType = iota
59 filtAny
60 filtReject
61 filtSkipMedia
62 filtHide
63 filtCollapse
64 filtRewrite
65)
66
67var filtNames = []string{"None", "Any", "Reject", "SkipMedia", "Hide", "Collapse", "Rewrite"}
68
69func (ft filtType) String() string {
70 return filtNames[ft]
71}
72
73type afiltermap map[filtType][]*Filter
74
75var filtInvalidator cache.Invalidator
76var filtcache *cache.Cache
77
78func init() {
79 // resolve init loop
80 filtcache = cache.New(cache.Options{Filler: filtcachefiller, Invalidator: &filtInvalidator})
81}
82
83func filtcachefiller(userid int64) (afiltermap, bool) {
84 rows, err := stmtGetFilters.Query(userid)
85 if err != nil {
86 elog.Printf("error querying filters: %s", err)
87 return nil, false
88 }
89 defer rows.Close()
90
91 now := time.Now()
92
93 var expflush time.Time
94
95 filtmap := make(afiltermap)
96 for rows.Next() {
97 filt := new(Filter)
98 var j string
99 var filterid int64
100 err = rows.Scan(&filterid, &j)
101 if err == nil {
102 err = unjsonify(j, filt)
103 }
104 if err != nil {
105 elog.Printf("error scanning filter: %s", err)
106 continue
107 }
108 if !filt.Expiration.IsZero() {
109 if filt.Expiration.Before(now) {
110 continue
111 }
112 if expflush.IsZero() || filt.Expiration.Before(expflush) {
113 expflush = filt.Expiration
114 }
115 }
116 if t := filt.Text; t != "" && t != "." {
117 wordfront := unicode.IsLetter(rune(t[0]))
118 wordtail := unicode.IsLetter(rune(t[len(t)-1]))
119 t = "(?i:" + t + ")"
120 if wordfront {
121 t = "\\b" + t
122 }
123 if wordtail {
124 t = t + "\\b"
125 }
126 filt.re_text, err = regexp.Compile(t)
127 if err != nil {
128 elog.Printf("error compiling filter text: %s", err)
129 continue
130 }
131 }
132 if t := filt.Rewrite; t != "" {
133 wordfront := unicode.IsLetter(rune(t[0]))
134 wordtail := unicode.IsLetter(rune(t[len(t)-1]))
135 t = "(?i:" + t + ")"
136 if wordfront {
137 t = "\\b" + t
138 }
139 if wordtail {
140 t = t + "\\b"
141 }
142 filt.re_rewrite, err = regexp.Compile(t)
143 if err != nil {
144 elog.Printf("error compiling filter rewrite: %s", err)
145 continue
146 }
147 }
148 filt.ID = filterid
149 if filt.Reject {
150 filt.Actions = append(filt.Actions, filtReject)
151 filtmap[filtReject] = append(filtmap[filtReject], filt)
152 }
153 if filt.SkipMedia {
154 filt.Actions = append(filt.Actions, filtSkipMedia)
155 filtmap[filtSkipMedia] = append(filtmap[filtSkipMedia], filt)
156 }
157 if filt.Hide {
158 filt.Actions = append(filt.Actions, filtHide)
159 filtmap[filtHide] = append(filtmap[filtHide], filt)
160 }
161 if filt.Collapse {
162 filt.Actions = append(filt.Actions, filtCollapse)
163 filtmap[filtCollapse] = append(filtmap[filtCollapse], filt)
164 }
165 if filt.Rewrite != "" {
166 filt.Actions = append(filt.Actions, filtRewrite)
167 filtmap[filtRewrite] = append(filtmap[filtRewrite], filt)
168 }
169 filtmap[filtAny] = append(filtmap[filtAny], filt)
170 }
171 sorting := filtmap[filtAny]
172 sort.Slice(filtmap[filtAny], func(i, j int) bool {
173 return sorting[i].Name < sorting[j].Name
174 })
175 if !expflush.IsZero() {
176 dur := expflush.Sub(now)
177 go filtcacheclear(userid, dur)
178 }
179 return filtmap, true
180}
181
182func filtcacheclear(userid int64, dur time.Duration) {
183 time.Sleep(dur + time.Second)
184 filtInvalidator.Clear(userid)
185}
186
187func getfilters(userid int64, scope filtType) []*Filter {
188 var filtmap afiltermap
189 ok := filtcache.Get(userid, &filtmap)
190 if ok {
191 return filtmap[scope]
192 }
193 return nil
194}
195
196type arejectmap map[string][]*Filter
197
198var rejectAnyKey = "..."
199
200var rejectcache = cache.New(cache.Options{Filler: func(userid int64) (arejectmap, bool) {
201 m := make(arejectmap)
202 filts := getfilters(userid, filtReject)
203 for _, f := range filts {
204 if f.Text != "" {
205 key := rejectAnyKey
206 m[key] = append(m[key], f)
207 continue
208 }
209 if f.IsAnnounce && f.AnnounceOf != "" {
210 key := f.AnnounceOf
211 m[key] = append(m[key], f)
212 }
213 if f.Actor != "" {
214 key := f.Actor
215 m[key] = append(m[key], f)
216 }
217 }
218 return m, true
219}, Invalidator: &filtInvalidator})
220
221func rejectfilters(userid int64, name string) []*Filter {
222 var m arejectmap
223 rejectcache.Get(userid, &m)
224 return m[name]
225}
226
227func rejectorigin(userid int64, origin string, isannounce bool) bool {
228 if o := originate(origin); o != "" {
229 origin = o
230 }
231 filts := rejectfilters(userid, origin)
232 for _, f := range filts {
233 if f.OnlyUnknowns {
234 continue
235 }
236 if isannounce && f.IsAnnounce {
237 if f.AnnounceOf == origin {
238 return true
239 }
240 }
241 if f.Actor == origin {
242 return true
243 }
244 }
245 return false
246}
247
248func rejectactor(userid int64, actor string) bool {
249 filts := rejectfilters(userid, actor)
250 for _, f := range filts {
251 if f.IsAnnounce {
252 continue
253 }
254 if f.Actor == actor {
255 ilog.Printf("rejecting actor: %s", actor)
256 return true
257 }
258 }
259 origin := originate(actor)
260 if origin == "" {
261 return false
262 }
263 filts = rejectfilters(userid, origin)
264 for _, f := range filts {
265 if f.IsAnnounce {
266 continue
267 }
268 if f.Actor == origin {
269 if f.OnlyUnknowns {
270 if unknownActor(userid, actor) {
271 ilog.Printf("rejecting unknown actor: %s", actor)
272 return true
273 }
274 continue
275 }
276 ilog.Printf("rejecting actor: %s", actor)
277 return true
278 }
279 }
280 return false
281}
282
283var knownknowns = cache.New(cache.Options{Filler: func(userid int64) (map[string]bool, bool) {
284 m := make(map[string]bool)
285 honkers := gethonkers(userid)
286 for _, h := range honkers {
287 m[h.XID] = true
288 }
289 return m, true
290}, Invalidator: &honkerinvalidator})
291
292func unknownActor(userid int64, actor string) bool {
293 var knowns map[string]bool
294 knownknowns.Get(userid, &knowns)
295 return !knowns[actor]
296}
297
298func stealthmode(userid int64, r *http.Request) bool {
299 agent := r.UserAgent()
300 agent = originate(agent)
301 if agent != "" {
302 fake := rejectorigin(userid, agent, false)
303 if fake {
304 ilog.Printf("faking 404 for %s", agent)
305 return true
306 }
307 }
308 return false
309}
310
311func matchfilter(h *Honk, f *Filter) bool {
312 return matchfilterX(h, f) != ""
313}
314
315func matchfilterX(h *Honk, f *Filter) string {
316 rv := ""
317 match := true
318 if match && f.Actor != "" {
319 match = false
320 if f.Actor == h.Honker || f.Actor == h.Oonker {
321 match = true
322 rv = f.Actor
323 }
324 if !match && !f.OnlyUnknowns && (f.Actor == originate(h.Honker) ||
325 f.Actor == originate(h.Oonker) ||
326 f.Actor == originate(h.XID)) {
327 match = true
328 rv = f.Actor
329 }
330 if !match && f.IncludeAudience {
331 for _, a := range h.Audience {
332 if f.Actor == a || f.Actor == originate(a) {
333 match = true
334 rv = f.Actor
335 break
336 }
337 }
338 }
339 }
340 if match && f.IsReply {
341 match = false
342 if h.RID != "" {
343 match = true
344 rv += " reply"
345 }
346 }
347 if match && f.IsAnnounce {
348 match = false
349 if h.Oonker != "" {
350 if f.AnnounceOf == "" || f.AnnounceOf == h.Oonker || f.AnnounceOf == originate(h.Oonker) {
351 match = true
352 rv += " announce"
353 }
354 }
355 }
356 if match && f.Text != "" && f.Text != "." {
357 match = false
358 re := f.re_text
359 m := re.FindString(h.Precis)
360 if m == "" {
361 m = re.FindString(h.Noise)
362 }
363 if m == "" {
364 for _, d := range h.Donks {
365 m = re.FindString(d.Desc)
366 if m != "" {
367 break
368 }
369 }
370 }
371 if m != "" {
372 match = true
373 rv = m
374 }
375 }
376 if match && f.Text == "." {
377 match = false
378 if h.Precis != "" {
379 match = true
380 rv = h.Precis
381 }
382 }
383 if match {
384 return rv
385 }
386 return ""
387}
388
389func rejectxonk(xonk *Honk) bool {
390 var m arejectmap
391 rejectcache.Get(xonk.UserID, &m)
392 filts := m[rejectAnyKey]
393 filts = append(filts, m[xonk.Honker]...)
394 filts = append(filts, m[originate(xonk.Honker)]...)
395 filts = append(filts, m[xonk.Oonker]...)
396 filts = append(filts, m[originate(xonk.Oonker)]...)
397 for _, a := range xonk.Audience {
398 filts = append(filts, m[a]...)
399 filts = append(filts, m[originate(a)]...)
400 }
401 for _, f := range filts {
402 if cause := matchfilterX(xonk, f); cause != "" {
403 ilog.Printf("rejecting %s because %s", xonk.XID, cause)
404 return true
405 }
406 }
407 return false
408}
409
410func skipMedia(xonk *Honk) bool {
411 filts := getfilters(xonk.UserID, filtSkipMedia)
412 for _, f := range filts {
413 if matchfilter(xonk, f) {
414 return true
415 }
416 }
417 return false
418}
419
420func unsee(honks []*Honk, userid int64) {
421 if userid != -1 {
422 colfilts := getfilters(userid, filtCollapse)
423 rwfilts := getfilters(userid, filtRewrite)
424 for _, h := range honks {
425 for _, f := range colfilts {
426 if bad := matchfilterX(h, f); bad != "" {
427 if h.Precis == "" {
428 h.Precis = bad
429 }
430 h.Open = ""
431 break
432 }
433 }
434 if h.Open == "open" && h.Precis == "unspecified horror" {
435 h.Precis = ""
436 }
437 for _, f := range rwfilts {
438 if matchfilter(h, f) {
439 h.Noise = f.re_rewrite.ReplaceAllString(h.Noise, f.Replace)
440 }
441 }
442 if len(h.Noise) > 6000 && h.Open == "open" {
443 if h.Precis == "" {
444 h.Precis = "really freaking long"
445 }
446 h.Open = ""
447 }
448 }
449 } else {
450 for _, h := range honks {
451 if h.Precis != "" {
452 h.Open = ""
453 }
454 }
455 }
456}
457
458var untagged = cache.New(cache.Options{Filler: func(userid int64) (map[string]bool, bool) {
459 rows, err := stmtUntagged.Query(userid)
460 if err != nil {
461 elog.Printf("error query untagged: %s", err)
462 return nil, false
463 }
464 defer rows.Close()
465 bad := make(map[string]bool)
466 for rows.Next() {
467 var xid, rid string
468 var flags int64
469 err = rows.Scan(&xid, &rid, &flags)
470 if err != nil {
471 elog.Printf("error scanning untag: %s", err)
472 continue
473 }
474 if flags&flagIsUntagged != 0 {
475 bad[xid] = true
476 }
477 if bad[rid] {
478 bad[xid] = true
479 }
480 }
481 return bad, true
482}})
483
484func osmosis(honks []*Honk, userid int64, withfilt bool) []*Honk {
485 var badparents map[string]bool
486 untagged.GetAndLock(userid, &badparents)
487 j := 0
488 reversehonks(honks)
489 for _, h := range honks {
490 if badparents[h.RID] {
491 badparents[h.XID] = true
492 continue
493 }
494 honks[j] = h
495 j++
496 }
497 untagged.Unlock()
498 honks = honks[0:j]
499 reversehonks(honks)
500 if !withfilt {
501 return honks
502 }
503 filts := getfilters(userid, filtHide)
504 j = 0
505outer:
506 for _, h := range honks {
507 for _, f := range filts {
508 if matchfilter(h, f) {
509 continue outer
510 }
511 }
512 honks[j] = h
513 j++
514 }
515 honks = honks[0:j]
516 return honks
517}
518
519func savehfcs(w http.ResponseWriter, r *http.Request) {
520 userinfo := login.GetUserInfo(r)
521 itsok := r.FormValue("itsok")
522 if itsok == "iforgiveyou" {
523 hfcsid, _ := strconv.ParseInt(r.FormValue("hfcsid"), 10, 0)
524 _, err := stmtDeleteFilter.Exec(userinfo.UserID, hfcsid)
525 if err != nil {
526 elog.Printf("error deleting filter: %s", err)
527 }
528 filtInvalidator.Clear(userinfo.UserID)
529 http.Redirect(w, r, "/hfcs", http.StatusSeeOther)
530 return
531 }
532
533 filt := new(Filter)
534 filt.Name = strings.TrimSpace(r.FormValue("name"))
535 filt.Date = time.Now().UTC()
536 filt.Actor = strings.TrimSpace(r.FormValue("actor"))
537 filt.IncludeAudience = r.FormValue("incaud") == "yes"
538 filt.OnlyUnknowns = r.FormValue("unknowns") == "yes"
539 filt.Text = strings.TrimSpace(r.FormValue("filttext"))
540 filt.IsReply = r.FormValue("isreply") == "yes"
541 filt.IsAnnounce = r.FormValue("isannounce") == "yes"
542 filt.AnnounceOf = strings.TrimSpace(r.FormValue("announceof"))
543 filt.Reject = r.FormValue("doreject") == "yes"
544 filt.SkipMedia = r.FormValue("doskipmedia") == "yes"
545 filt.Hide = r.FormValue("dohide") == "yes"
546 filt.Collapse = r.FormValue("docollapse") == "yes"
547 filt.Rewrite = strings.TrimSpace(r.FormValue("filtrewrite"))
548 filt.Replace = strings.TrimSpace(r.FormValue("filtreplace"))
549 if dur := parseDuration(r.FormValue("filtduration")); dur > 0 {
550 filt.Expiration = time.Now().UTC().Add(dur)
551 }
552 filt.Notes = strings.TrimSpace(r.FormValue("filtnotes"))
553
554 if filt.Actor == "" && filt.Text == "" && !filt.IsAnnounce {
555 ilog.Printf("blank filter")
556 http.Error(w, "can't save a blank filter", http.StatusInternalServerError)
557 return
558 }
559
560 j, err := jsonify(filt)
561 if err == nil {
562 _, err = stmtSaveFilter.Exec(userinfo.UserID, j)
563 }
564 if err != nil {
565 elog.Printf("error saving filter: %s", err)
566 }
567
568 filtInvalidator.Clear(userinfo.UserID)
569 http.Redirect(w, r, "/hfcs", http.StatusSeeOther)
570}