Golang网页下载示例
package main
/*
* 中文编码问题
*/
import (
"errors"
"flag"
"fmt"
query "github.com/PuerkitoBio/goquery"
"golang.org/x/text/encoding/simplifiedchinese"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
)
var (
np = runtime.NumCPU()
_ = runtime.GOMAXPROCS(np)
)
var wg sync.WaitGroup
type Folder struct {
Url string
Dir string
}
type File struct {
Url string
Dir string
Name string
}
func checkErr(err error) {
if err != nil {
fmt.Printf("%v\n", err.Error())
os.Exit(1)
}
}
func decodeToGBK(text string) (string, error) {
dst := make([]byte, len(text)*2)
tr := simplifiedchinese.GB18030.NewDecoder()
nDst, _, err := tr.Transform(dst, []byte(text), true)
if err != nil {
return text, err
}
return string(dst[:nDst]), nil
}
func printEach(index int, item *query.Selection) {
fmt.Println("Selection: ", item.Text())
}
func isDir(path string) bool {
return strings.HasSuffix(path, "/")
}
func makeFolder(item *query.Selection, url, dir string) (f *Folder, err error) {
tx := item.Text()
href, ok := item.Attr("href")
name, err := decodeToGBK(tx)
if err != nil {
return
}
if !ok {
err = errors.New("makeFolder : " + tx + " href属性不存在")
return
}
f = &Folder{Url: url + href, Dir: filepath.Join(dir, name)}
return
}
func makeFile(item *query.Selection, url, dir string) (f *File, err error) {
tx := item.Text()
href, ok := item.Attr("href")
if !ok {
err = errors.New("makeFile : " + tx + " href属性不存在")
return
}
name, err := decodeToGBK(tx)
if err != nil {
return
}
f = &File{Url: url + href, Dir: dir, Name: name}
return
}
func crawl(url, localDir string) {
doc, err := query.NewDocument(url)
// checkErr(err)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
items := doc.Find("a")
dir := localDir
if !strings.HasSuffix(url, "/") {
url += "/"
}
crawlEach := func(i int, item *query.Selection) {
tx := item.Text()
if isDir(tx) {
folder, err := makeFolder(item, url, dir)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
wg.Add(1)
go crawlFolder(folder)
} else {
file, err := makeFile(item, url, dir)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
download(file)
}
}
items.Each(crawlEach)
}
func download(file *File) {
dir := file.Dir
url := file.Url
name := file.Name
if err := os.MkdirAll(dir, os.ModePerm); os.IsExist(err) {
fmt.Printf("%x is exist\n", dir)
} else {
os.Chmod(dir, os.ModePerm)
}
resp, err := http.Get(url)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Printf("%v\n", err.Error())
return
}
fp := string([]rune(filepath.Join(dir, name)))
err = ioutil.WriteFile(fp, body, 0777)
if err != nil {
fmt.Printf("%v fp:[%v]\n", err.Error(), fp)
return
}
fmt.Printf("Download: %+v\n", file)
}
func crawlFolder(folder *Folder) {
url := folder.Url
dir := folder.Dir
crawl(url, dir)
wg.Done()
}
func main() {
host := flag.String("host", "http://localhost:8000", "HTTP服务地址Host")
location := flag.String("locate", "E:/Crawler下载文件", "本地文件系统绝对路径")
flag.Parse()
crawl(*host, *location)
wg.Wait()
}