behold the syndicate
Ted Unangst tedu@tedunangst.com
Tue, 27 Feb 2024 16:12:00 -0500
7 files changed,
117 insertions(+),
9 deletions(-)
M
database.go
→
database.go
@@ -1068,7 +1068,7 @@ func savexonker(what, value, flav, when string) {
stmtSaveXonker.Exec(what, value, flav, when) } -func savehonker(user *WhatAbout, url, name, flavor, combos, mj string) (int64, error) { +func savehonker(user *WhatAbout, url, name, flavor, combos, mj string) (int64, string, error) { var owner string if url[0] == '#' { flavor = "peep"@@ -1076,11 +1076,18 @@ if name == "" {
name = url[1:] } owner = url + } else if strings.HasSuffix(url, ".rss") { + flavor = "peep" + if name == "" { + name = url[strings.LastIndexByte(url, '/')+1:] + } + owner = url + } else { info, err := investigate(url) if err != nil { ilog.Printf("failed to investigate honker: %s", err) - return 0, err + return 0, "", err } url = info.XID if name == "" {@@ -1099,16 +1106,16 @@ elog.Printf("honker scan err: %s", err)
} else { err = fmt.Errorf("it seems you are already subscribed to them") } - return 0, err + return 0, "", err } res, err := stmtSaveHonker.Exec(user.ID, name, url, flavor, combos, owner, mj) if err != nil { elog.Print(err) - return 0, err + return 0, "", err } honkerid, _ := res.LastInsertId() - return honkerid, nil + return honkerid, flavor, nil } func cleanupdb(arg string) {
M
go.mod
→
go.mod
@@ -5,6 +5,7 @@
require ( github.com/gorilla/mux v1.8.1 github.com/mattn/go-runewidth v0.0.15 + github.com/mmcdole/gofeed v1.2.1 golang.org/x/crypto v0.19.0 golang.org/x/net v0.21.0 humungus.tedunangst.com/r/go-sqlite3 v1.1.3@@ -13,7 +14,14 @@ humungus.tedunangst.com/r/webs v0.7.10
) require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/sys v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect )
M
go.sum
→
go.sum
@@ -1,12 +1,33 @@
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= +github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= +github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= +github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=@@ -15,14 +36,22 @@ golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= humungus.tedunangst.com/r/go-sqlite3 v1.1.3 h1:G2N4wzDS0NbuvrZtQJhh4F+3X+s7BF8b9ga8k38geUI= humungus.tedunangst.com/r/go-sqlite3 v1.1.3/go.mod h1:FtEEmQM7U2Ey1TuEEOyY1BmphTZnmiEjPsNLEAkpf/M= humungus.tedunangst.com/r/gonix v0.1.4 h1:FuvWYQlFIzmfHxfvIfq5SYpSiHhFcpJqq3pi+w45s78=
M
main.go
→
main.go
@@ -51,6 +51,8 @@ return fmt.Sprintf("https://"+serverName+u, args...)
} func ElaborateUnitTests() { + user, _ := butwhatabout("test") + syndicate(user, "https://mastodon.social/tags/mastoadmin.rss") } func unplugserver(hostname string) {@@ -208,11 +210,13 @@ errx("user %s not found", args[1])
} var meta HonkerMeta mj, _ := jsonify(&meta) - honkerid, err := savehonker(user, args[2], "", "presub", "", mj) + honkerid, flavor, err := savehonker(user, args[2], "", "presub", "", mj) if err != nil { errx("had some trouble with that: %s", err) } - followyou(user, honkerid, true) + if flavor == "presub" { + followyou(user, honkerid, true) + } case "unfollow": if len(args) < 3 { errx("usage: honk unfollow username url")
A
syndicate.go
@@ -0,0 +1,59 @@
+package main + +import ( + "bytes" + notrand "math/rand" + "strings" + "time" + + "github.com/mmcdole/gofeed" +) + +func syndicate(user *WhatAbout, url string) { + data, err := fetchsome(url) + if err != nil { + dlog.Printf("error fetching feed: %s", err) + return + } + parser := gofeed.NewParser() + rss, err := parser.Parse(bytes.NewReader(data)) + if err != nil { + dlog.Printf("error parsing feed: %s", err) + return + } + reverseItems(rss.Items) + for _, item := range rss.Items { + var final string + dlog.Printf("link: %s", item.Link) + j, err := GetJunkTimeout(user.ID, item.Link, fastTimeout*time.Second, &final) + if err != nil { + dlog.Printf("unable to fetch link: %s", err) + continue + } + xonksaver(user, j, originate(final)) + } +} + +func syndicator() { + for { + dur := 12 * time.Hour + dur += time.Duration(notrand.Int63n(int64(dur / 4))) + time.Sleep(dur) + users := allusers() + for _, ui := range users { + user, _ := butwhatabout(ui.Username) + honkers := gethonkers(user.ID) + for _, h := range honkers { + if strings.HasSuffix(h.XID, ".rss") { + syndicate(user, h.XID) + } + } + } + } +} + +func reverseItems(items []*gofeed.Item) { + for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { + items[i], items[j] = items[j], items[i] + } +}
M
web.go
→
web.go
@@ -2209,7 +2209,7 @@ flavor = "peep"
} var err error - honkerid, err = savehonker(user, url, name, flavor, combos, mj) + honkerid, flavor, err = savehonker(user, url, name, flavor, combos, mj) if err != nil { http.Error(w, "had some trouble with that: "+err.Error(), http.StatusInternalServerError) return nil@@ -2832,6 +2832,7 @@ runBackendServer()
go enditall() go redeliverator() go tracker() + go syndicator() go bgmonitor() go qotd() loadLingo()