自行开发的爬虫框架，采用多种语言，python和golang语言-豆瓣网爬取案例

package fetcher import ( "bufio" "fmt" "golang.org/x/net/html/charset" "golang.org/x/text/encoding" "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" "io/ioutil" "log" "net/http" "time" "zdcrawl/config" ) var ( rateLimiter = time.Tick( time.Second / config.Qps) verboseLogging = config.VerboseLogger ) func SetVerboseLogging() { verboseLogging = true } func Fetch(url string) ([]byte, error) { <-rateLimiter if verboseLogging { log.Printf("Fetching url %s", url) } client := &http.Client{} /* 说明：此处如果采用resp ：= http.Get(url), 则会产生 wrong status code: 418 的错误 */ request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } request.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36") resp, _ := client.Do(request) defer resp.Body.Close() log.Printf("Fetching url %s code:%d\n", url, resp.StatusCode) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode) } bodyReader := bufio.NewReader(resp.Body) e := determineEncoding(bodyReader) utf8Reader := transform.NewReader(bodyReader, e.NewDecoder()) return ioutil.ReadAll(utf8Reader) } func determineEncoding( r *bufio.Reader) encoding.Encoding { bytes, err := r.Peek(1024) if err != nil { log.Printf("Fetcher error: %v", err) return unicode.UTF8 } e, _, _ := charset.DetermineEncoding( bytes, "") return e }