一个简单的go语言爬虫

package main

import (
        "bufio"
        "fmt"
        "golang.org/x/net/html/charset"
        "golang.org/x/text/encoding"
        "golang.org/x/text/transform"
        "io"
        "io/ioutil"
        "net/http"
)

func determineEncoding(r io.Reader) encoding.Encoding {
       //编码判断
        bytes, err := bufio.NewReader(r).Peek(1024)
        if err != nil {
                panic(err)
        }
        e, _, _ := charset.DetermineEncoding(bytes, ",")
        return e

}
func main() {
        resp, err := http.Get("http://www.baidu.com")
        if err != nil {
                panic(err)
        }
        defer resp.Body.Close() //最后关闭，先进后出

        if resp.StatusCode != http.StatusOK {
                fmt.Println("Error:status code", resp.StatusCode)
                return
        }
        e := determineEncoding(resp.Body)
        utf8Reader := transform.NewReader(resp.Body, e.NewDecoder())
        all, err := ioutil.ReadAll(utf8Reader)
        //utf8Reader:=transform.NewReader(resp.Body,simplifiedchinese.GBK.NewDecoder())

        //下面是不加编码判断的
        //all, err := ioutil.ReadAll(resp.Body)
        if err != nil {
                panic(err)
        }
        fmt.Printf("%s\n", all)

}