toys/fetch.go (view raw)
1package main
2
3import (
4 "bytes"
5 "context"
6 "crypto/tls"
7 "fmt"
8 "io"
9 "log"
10 "net/http"
11 "time"
12)
13
14var debugClient = &http.Client{
15 Transport: &http.Transport{
16 TLSClientConfig: &tls.Config{
17 InsecureSkipVerify: true,
18 },
19 },
20}
21
22func fetchsome(url string) ([]byte, error) {
23 client := http.DefaultClient
24 if debugMode {
25 client = debugClient
26 }
27 req, err := http.NewRequest("GET", url, nil)
28 if err != nil {
29 log.Printf("error fetching %s: %s", url, err)
30 return nil, err
31 }
32 req.Header.Set("User-Agent", "honksnonk/4.0")
33 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
34 defer cancel()
35 req = req.WithContext(ctx)
36 resp, err := client.Do(req)
37 if err != nil {
38 log.Printf("error fetching %s: %s", url, err)
39 return nil, err
40 }
41 defer resp.Body.Close()
42 switch resp.StatusCode {
43 case 200:
44 case 201:
45 case 202:
46 default:
47 return nil, fmt.Errorf("http get not 200: %d %s", resp.StatusCode, url)
48 }
49 var buf bytes.Buffer
50 limiter := io.LimitReader(resp.Body, 10*1024*1024)
51 io.Copy(&buf, limiter)
52 return buf.Bytes(), nil
53}