1//
  2// Copyright (c) 2019 Ted Unangst <tedu@tedunangst.com>
  3//
  4// Permission to use, copy, modify, and distribute this software for any
  5// purpose with or without fee is hereby granted, provided that the above
  6// copyright notice and this permission notice appear in all copies.
  7//
  8// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  9// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 10// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 11// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 12// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 13// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 14// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 15
 16package main
 17
 18import (
 19	"encoding/json"
 20	"fmt"
 21	"html"
 22	"io/ioutil"
 23	"log"
 24	"os"
 25	"sort"
 26	"strings"
 27	"time"
 28)
 29
 30func importMain(username, flavor, source string) {
 31	switch flavor {
 32	case "twitter":
 33		importTwitter(username, source)
 34	default:
 35		log.Fatal("unknown source flavor")
 36	}
 37}
 38
 39func importTwitter(username, source string) {
 40	user, err := butwhatabout(username)
 41	if err != nil {
 42		log.Fatal(err)
 43	}
 44
 45	type Tweet struct {
 46		ID_str                  string
 47		Created_at              string
 48		Full_text               string
 49		In_reply_to_screen_name string
 50		In_reply_to_status_id   string
 51		Entities                struct {
 52			Hashtags []struct {
 53				Text string
 54			}
 55			Media []struct {
 56				Url       string
 57				Media_url string
 58			}
 59			Urls []struct {
 60				Url          string
 61				Expanded_url string
 62			}
 63		}
 64		date   time.Time
 65		convoy string
 66	}
 67
 68	var tweets []*Tweet
 69	fd, err := os.Open(source + "/tweet.js")
 70	if err != nil {
 71		log.Fatal(err)
 72	}
 73	// skip past window.YTD.tweet.part0 =
 74	fd.Seek(25, 0)
 75	dec := json.NewDecoder(fd)
 76	err = dec.Decode(&tweets)
 77	if err != nil {
 78		log.Fatalf("error parsing json: %s", err)
 79	}
 80	fd.Close()
 81	tweetmap := make(map[string]*Tweet)
 82	for _, t := range tweets {
 83		t.date, _ = time.Parse("Mon Jan 02 15:04:05 -0700 2006", t.Created_at)
 84		tweetmap[t.ID_str] = t
 85	}
 86	sort.Slice(tweets, func(i, j int) bool {
 87		return tweets[i].date.Before(tweets[j].date)
 88	})
 89	havetwid := func(xid string) bool {
 90		var id int64
 91		row := stmtFindXonk.QueryRow(user.ID, xid)
 92		err := row.Scan(&id)
 93		if err == nil {
 94			return true
 95		}
 96		return false
 97	}
 98
 99	for _, t := range tweets {
100		xid := fmt.Sprintf("%s/%s/%s", user.URL, honkSep, t.ID_str)
101		if havetwid(xid) {
102			continue
103		}
104		what := "honk"
105		noise := ""
106		if parent := tweetmap[t.In_reply_to_status_id]; parent != nil {
107			t.convoy = parent.convoy
108			what = "tonk"
109		} else {
110			t.convoy = "data:,acoustichonkytonk-" + t.ID_str
111			if t.In_reply_to_screen_name != "" {
112				noise = fmt.Sprintf("re: https://twitter.com/%s/status/%s\n\n",
113					t.In_reply_to_screen_name, t.In_reply_to_status_id)
114				what = "tonk"
115			}
116		}
117		audience := []string{thewholeworld}
118		honk := Honk{
119			UserID:   user.ID,
120			Username: user.Name,
121			What:     what,
122			Honker:   user.URL,
123			XID:      xid,
124			Date:     t.date,
125			Format:   "markdown",
126			Audience: audience,
127			Convoy:   t.convoy,
128			Public:   true,
129			Whofore:  2,
130		}
131		noise += t.Full_text
132		// unbelievable
133		noise = html.UnescapeString(noise)
134		for _, r := range t.Entities.Urls {
135			noise = strings.Replace(noise, r.Url, r.Expanded_url, -1)
136		}
137		for _, m := range t.Entities.Media {
138			u := m.Media_url
139			idx := strings.LastIndexByte(u, '/')
140			u = u[idx+1:]
141			fname := fmt.Sprintf("%s/tweet_media/%s-%s", source, t.ID_str, u)
142			data, err := ioutil.ReadFile(fname)
143			if err != nil {
144				log.Printf("error reading media: %s", fname)
145				continue
146			}
147			newurl := fmt.Sprintf("https://%s/d/%s", serverName, u)
148
149			fileid, err := savefile(u, u, u, newurl, "image/jpg", true, data)
150			if err != nil {
151				log.Printf("error saving media: %s", fname)
152				continue
153			}
154			donk := &Donk{
155				FileID: fileid,
156			}
157			honk.Donks = append(honk.Donks, donk)
158			noise = strings.Replace(noise, m.Url, "", -1)
159		}
160		for _, ht := range t.Entities.Hashtags {
161			honk.Onts = append(honk.Onts, "#"+ht.Text)
162		}
163		honk.Noise = noise
164		savehonk(&honk)
165	}
166}