all repos — navani @ ea8ad151310499d429bb7aa02cdb4acdac813db0

forlater's primary mail processing service

Attach non-text/html pages
Anirudh Oppiliappan x@icyphox.sh
Wed, 29 Sep 2021 16:12:53 +0530
commit

ea8ad151310499d429bb7aa02cdb4acdac813db0

parent

394e3cf44fa0b3ba70b74595dc6bf877c9d0b767

6 files changed, 140 insertions(+), 33 deletions(-)

jump to
M go.modgo.mod

@@ -6,10 +6,9 @@ require (

blitiri.com.ar/go/spf v1.2.0 github.com/go-shiori/go-readability v0.0.0-20210627123243-82cc33435520 github.com/gomodule/redigo v1.8.5 - github.com/joegrasse/mail v1.0.0 - github.com/joegrasse/mime v0.0.0-20151001172835-f543c4783e35 // indirect github.com/joho/godotenv v1.3.0 github.com/microcosm-cc/bluemonday v1.0.15 + github.com/xhit/go-simple-mail/v2 v2.10.0 golang.org/x/net v0.0.0-20210908191846-a5e095526f91 mvdan.cc/xurls/v2 v2.3.0 )
M go.sumgo.sum

@@ -57,10 +57,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=

github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/joegrasse/mail v1.0.0 h1:yi8Mk55sXYRZ5OhOyCzozKlj9l3HGPY6P3KEhZMCUKQ= -github.com/joegrasse/mail v1.0.0/go.mod h1:4bofkOaSoOo/INXkD1lCV5OP2MH0K6BQBQtmPX8Hf54= -github.com/joegrasse/mime v0.0.0-20151001172835-f543c4783e35 h1:onVWojd9sn2EXatgzU6X61N05cZVga4bTLyTuJY/W5M= -github.com/joegrasse/mime v0.0.0-20151001172835-f543c4783e35/go.mod h1:HCNVm2oxH9F8N1jYFa03+MWzfPddTG6n5R5P7qdgAZk= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=

@@ -120,6 +116,8 @@ github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=

github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xhit/go-simple-mail/v2 v2.10.0 h1:nib6RaJ4qVh5HD9UE9QJqnUZyWp3upv+Z6CFxaMj0V8= +github.com/xhit/go-simple-mail/v2 v2.10.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
M mail/send.gomail/send.go

@@ -2,12 +2,14 @@ package mail

import ( "fmt" + "io" "log" + "mime" "os" "git.icyphox.sh/forlater/navani/reader" - "github.com/joegrasse/mail" "github.com/joho/godotenv" + mail "github.com/xhit/go-simple-mail/v2" ) func init() {

@@ -18,14 +20,7 @@ }

} func SendArticle(article *reader.Article, to string, readable bool) error { - var ( - EMAIL_USER_SECRET = os.Getenv("EMAIL_USER_SECRET") - EMAIL_PASSWORD = os.Getenv("EMAIL_PASSWORD") - EMAIL_FROM = os.Getenv("EMAIL_FROM") - SMTP_HOST = os.Getenv("SMTP_HOST") - SMTP_PORT = os.Getenv("SMTP_PORT") - ) - + var EMAIL_FROM = os.Getenv("EMAIL_FROM") htmlContent, err := RenderTemplate("html.tpl", article) if err != nil { return err

@@ -36,25 +31,61 @@ if err != nil {

return fmt.Errorf("making plaintext: %w\n", err) } - email := mail.New() - email.Encryption = mail.EncryptionTLS + email := mail.NewMSG() email.SetFrom(fmt.Sprintf("saved forlater <%s>", EMAIL_FROM)) email.AddTo(to) if readable { email.SetSubject(article.Title) - email.SetBody("text/plain", string(plainContent)) - email.AddAlternative("text/html", string(htmlContent)) + email.SetBodyData(mail.TextPlain, plainContent) + email.AddAlternative(mail.TextHTML, string(htmlContent)) } else { email.SetSubject(article.URL.String()) - email.SetBody("text/plain", fmt.Sprintf( + email.SetBody(mail.TextPlain, fmt.Sprintf( "We were unable to parse your link: %s", article.URL.String(), )) } - email.Username = EMAIL_USER_SECRET - email.Password = EMAIL_PASSWORD + + c, err := mailClient() + if err != nil { + return fmt.Errorf("mail: %w\n", err) + } + err = email.Send(c) + if err != nil { + return err + } + return nil +} - err = email.Send(SMTP_HOST + ":" + SMTP_PORT) +func SendAttachment(r reader.Response, to string, url string) error { + var EMAIL_FROM = os.Getenv("EMAIL_FROM") + email := mail.NewMSG() + email.SetFrom(fmt.Sprintf("saved forlater <%s>", EMAIL_FROM)) + email.AddTo(to) + + b, err := io.ReadAll(r.Body) + if err != nil { + return fmt.Errorf("read attachment: %w\n", err) + } + fmt.Println(len(b)) + + ext, _ := mime.ExtensionsByType(r.MIMEType) + var name string + if ext != nil { + name = "file" + ext[0] + } else { + name = "file" + } + email.SetSubject(url) + email.Attach(&mail.File{MimeType: r.MIMEType, Data: b, Name: name, Inline: true}) + email.SetBody(mail.TextPlain, fmt.Sprintf(`That didn't look like a HTML page; we found %s. +We've attached it to this email.`, r.MIMEType)) + + c, err := mailClient() + if err != nil { + return fmt.Errorf("mail: %w\n", err) + } + err = email.Send(c) if err != nil { return err }
A mail/smtp.go

@@ -0,0 +1,48 @@

+package mail + +import ( + "crypto/tls" + "os" + "strconv" + "time" + + mail "github.com/xhit/go-simple-mail/v2" +) + +func mailClient() (*mail.SMTPClient, error) { + var ( + EMAIL_USER_SECRET = os.Getenv("EMAIL_USER_SECRET") + EMAIL_PASSWORD = os.Getenv("EMAIL_PASSWORD") + SMTP_HOST = os.Getenv("SMTP_HOST") + SMTP_PORT = os.Getenv("SMTP_PORT") + ) + server := mail.NewSMTPClient() + + // SMTP Server + server.Host = SMTP_HOST + server.Port, _ = strconv.Atoi(SMTP_PORT) + server.Username = EMAIL_USER_SECRET + server.Password = EMAIL_PASSWORD + server.Encryption = mail.EncryptionSTARTTLS + + // Variable to keep alive connection + server.KeepAlive = false + + // Timeout for connect to SMTP Server + server.ConnectTimeout = 10 * time.Second + + // Timeout for send the data and wait respond + server.SendTimeout = 10 * time.Second + + // Set TLSConfig to provide custom TLS configuration. For example, + // to skip TLS verification (useful for testing): + server.TLSConfig = &tls.Config{InsecureSkipVerify: true} + + // SMTP client + smtpClient, err := server.Connect() + if err != nil { + return nil, err + } + + return smtpClient, nil +}
M main.gomain.go

@@ -32,12 +32,22 @@ if err != nil {

log.Printf("url parse: %v\n", err) } - f, err := reader.Fetch(parsedURL.String()) + resp, err := reader.Fetch(parsedURL.String()) if err != nil { log.Printf("reader fetch: %v\n", err) } - article, err := reader.Readable(f, parsedURL) + if resp.MIMEType != "text/html" { + err = mail.SendAttachment(resp, m.From, u) + if err != nil { + log.Printf("error sending attachment to: %s: %v\n", m.From, err) + } else { + log.Printf("sent attachment to %s: %s\n", m.From, resp.MIMEType) + } + break + } + + article, err := reader.Readable(resp.Body, parsedURL) if err == nil { err = mail.SendArticle(&article, m.From, true) if err != nil {
M reader/fetch.goreader/fetch.go

@@ -6,6 +6,7 @@ "crypto/sha1"

"encoding/hex" "fmt" "io" + "mime" "net/http" "net/url" "strings"

@@ -26,14 +27,19 @@ b := h.Sum(nil)

return hex.EncodeToString(b) } +type Response struct { + Body io.Reader + MIMEType string +} + // Fetches the web page and stores the hash of the URL against // the response body in cache. Returns an io.Reader. -func Fetch(url string) (io.Reader, error) { +func Fetch(url string) (Response, error) { client := &http.Client{} sum := checksum([]byte(url)) c, err := cache.NewConn() if err != nil { - return nil, fmt.Errorf("cache error: %w\n", err) + return Response{}, fmt.Errorf("cache error: %w\n", err) } body, err := c.Get(sum)

@@ -41,13 +47,27 @@ // Not in cache.

if err != nil { req, err := http.NewRequest("GET", url, nil) if err != nil { - return nil, fmt.Errorf("http error: %w\n", err) + return Response{}, fmt.Errorf("http error: %w\n", err) } req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36") resp, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("http client error: %w\n", err) + return Response{}, fmt.Errorf("http client error: %w\n", err) + } + + mt, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil { + return Response{}, fmt.Errorf("parse mime: %w\n", err) + } + + // If page isn't text/html, just return the body; no caching. + if mt != "text/html" { + if err != nil { + return Response{}, fmt.Errorf("reading non-html body: %w\n", err) + } + + return Response{resp.Body, mt}, nil } buf := bytes.Buffer{}

@@ -56,16 +76,17 @@ // Cache and return!

r := io.TeeReader(resp.Body, &buf) b, err := io.ReadAll(r) if err != nil { - return nil, fmt.Errorf("io error: %w\n", err) + return Response{}, fmt.Errorf("io error: %w\n", err) } _, err = c.Set(sum, b) if err != nil { - return nil, fmt.Errorf("cache error: %w\n", err) + return Response{}, fmt.Errorf("cache error: %w\n", err) } - return &buf, nil + return Response{&buf, mt}, nil } - return strings.NewReader(body), nil + // We can safely assume it's text/html + return Response{strings.NewReader(body), "text/html"}, nil } // Makes a given html body readable. Returns an error if it