matching import for the honk export. roughly roundtrips now.
Ted Unangst tedu@tedunangst.com
Fri, 18 Aug 2023 20:53:42 -0400
3 files changed,
71 insertions(+),
21 deletions(-)
M
docs/honk.8
→
docs/honk.8
@@ -194,9 +194,12 @@ .Ss Import
Data may be imported and converted from other services using the .Ic import command. -Currently supports Mastodon, Twitter, and Instagram exported data. +Currently supports Honk, Mastodon, Twitter, and Instagram exported data. Posts are imported and backdated to appear as old honks. The Mastodon following list is imported, but must be refollowed. +.Pp +To prepare a Honk data archive, extract the export.zip file. +.Dl ./honk import username honk source-directory .Pp To prepare a Mastodon data archive, extract the archive-longhash.tar.gz file. .Dl ./honk import username mastodon source-directory@@ -208,6 +211,13 @@ .Dl ./honk import username twitter source-directory
.Pp To prepare an Instagram data archive, extract the igusername.zip file. .Dl ./honk import username instagram source-directory +.Ss Export +User data may be exported to a zip archive using the +.Ic export +command. +This will export the user's outbox and inbox in ActvityPub json format, +along with associated media. +.Dl ./honk export username zipname .Ss Advanced Options Advanced configuration values may be set by running the .Ic setconfig Ar key value
M
import.go
→
import.go
@@ -36,6 +36,8 @@ func importMain(username, flavor, source string) {
switch flavor { case "mastodon": importMastodon(username, source) + case "honk": + importHonk(username, source) case "twitter": importTwitter(username, source) case "instagram":@@ -45,11 +47,13 @@ elog.Fatal("unknown source flavor")
} } -type TootObject struct { +type ActivityObject struct { + AttributedTo string Summary string Content string InReplyTo string Conversation string + Context string Published time.Time Tag []struct { Type string@@ -63,10 +67,10 @@ Name string
} } -type PlainTootObject TootObject +type PlainActivityObject ActivityObject -func (obj *TootObject) UnmarshalJSON(b []byte) error { - p := (*PlainTootObject)(obj) +func (obj *ActivityObject) UnmarshalJSON(b []byte) error { + p := (*PlainActivityObject)(obj) json.Unmarshal(b, p) return nil }@@ -77,8 +81,9 @@ if err != nil {
elog.Fatal(err) } - if _, err := os.Stat(source + "/outbox.json"); err == nil { - importMastotoots(user, source) + outbox := source + "/outbox.json" + if _, err := os.Stat(outbox); err == nil { + importActivities(user, outbox, source) } else { ilog.Printf("skipping outbox.json!") }@@ -89,19 +94,33 @@ ilog.Printf("skipping following_accounts.csv!")
} } -func importMastotoots(user *WhatAbout, source string) { - type Toot struct { +func importHonk(username, source string) { + user, err := butwhatabout(username) + if err != nil { + elog.Fatal(err) + } + + outbox := source + "/outbox.json" + if _, err := os.Stat(outbox); err == nil { + importActivities(user, outbox, source) + } else { + ilog.Printf("skipping outbox.json!") + } +} + +func importActivities(user *WhatAbout, filename, source string) { + type Activity struct { Id string Type string - To []string + To interface{} Cc []string - Object TootObject + Object ActivityObject } var outbox struct { - OrderedItems []Toot + OrderedItems []Activity } ilog.Println("Importing honks...") - fd, err := os.Open(source + "/outbox.json") + fd, err := os.Open(filename) if err != nil { elog.Fatal(err) }@@ -123,7 +142,11 @@ return false
} re_tootid := regexp.MustCompile("[^/]+$") - for _, item := range outbox.OrderedItems { + items := outbox.OrderedItems + for i, j := 0, len(items)-1; i < j; i, j = i+1, j-1 { + items[i], items[j] = items[j], items[i] + } + for _, item := range items { toot := item if toot.Type != "Create" { continue@@ -136,6 +159,21 @@ xid := fmt.Sprintf("%s/%s/%s", user.URL, honkSep, tootid)
if havetoot(xid) { continue } + + convoy := toot.Object.Context + if convoy == "" { + convoy = toot.Object.Conversation + } + var audience []string + to, ok := toot.To.(string) + if ok { + audience = append(audience, to) + } else { + for _, t := range toot.To.([]interface{}) { + audience = append(audience, t.(string)) + } + } + audience = append(audience, toot.Cc...) honk := Honk{ UserID: user.ID, What: "honk",@@ -144,9 +182,9 @@ XID: xid,
RID: toot.Object.InReplyTo, Date: toot.Object.Published, URL: xid, - Audience: append(toot.To, toot.Cc...), + Audience: audience, Noise: toot.Object.Content, - Convoy: toot.Object.Conversation, + Convoy: convoy, Whofore: 2, Format: "html", Precis: toot.Object.Summary,@@ -537,13 +575,14 @@ var jonks []junk.Junk
rows, err := stmtUserHonks.Query(0, 3, user.Name, "0", 1234567) honks := getsomehonks(rows, err) for _, honk := range honks { + for _, donk := range honk.Donks { + donk.URL = "media/" + donk.XID + donks[donk.XID] = true + } noise := honk.Noise j, jo := jonkjonk(user, honk) if honk.Format == "markdown" { jo["source"] = noise - } - for _, donk := range honk.Donks { - donks[donk.XID] = true } jonks = append(jonks, j) }@@ -565,10 +604,11 @@ var jonks []junk.Junk
rows, err := stmtHonksForMe.Query(0, user.ID, "0", user.ID, 1234567) honks := getsomehonks(rows, err) for _, honk := range honks { - j, _ := jonkjonk(user, honk) for _, donk := range honk.Donks { + donk.URL = "media/" + donk.XID donks[donk.XID] = true } + j, _ := jonkjonk(user, honk) jonks = append(jonks, j) } j := junk.New()