1package main
 2
 3import (
 4	"bytes"
 5	"context"
 6	"crypto/tls"
 7	"fmt"
 8	"io"
 9	"log"
10	"net/http"
11	"time"
12)
13
14var debugClient = &http.Client{
15	Transport: &http.Transport{
16		TLSClientConfig: &tls.Config{
17			InsecureSkipVerify: true,
18		},
19	},
20}
21
22func fetchsome(url string) ([]byte, error) {
23	client := http.DefaultClient
24	if debugMode {
25		client = debugClient
26	}
27	req, err := http.NewRequest("GET", url, nil)
28	if err != nil {
29		log.Printf("error fetching %s: %s", url, err)
30		return nil, err
31	}
32	req.Header.Set("User-Agent", "honksnonk/4.0")
33	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
34	defer cancel()
35	req = req.WithContext(ctx)
36	resp, err := client.Do(req)
37	if err != nil {
38		log.Printf("error fetching %s: %s", url, err)
39		return nil, err
40	}
41	defer resp.Body.Close()
42	switch resp.StatusCode {
43	case 200:
44	case 201:
45	case 202:
46	default:
47		return nil, fmt.Errorf("http get not 200: %d %s", resp.StatusCode, url)
48	}
49	var buf bytes.Buffer
50	limiter := io.LimitReader(resp.Body, 10*1024*1024)
51	io.Copy(&buf, limiter)
52	return buf.Bytes(), nil
53}