$ go get -u golang.org/x/net
我们需要安装库。
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Colour</title>
</head>
<body>
<p>
A list of colours
</p>
<ul>
<li>red</li>
<li>green</li>
<li>blue</li>
<li>yellow</li>
<li>orange</li>
<li>brown</li>
<li>pink</li>
</ul>
<footer>
A footer
</footer>
</body>
</html>
部分示例使用此HTML文件。
去解析HTML列表
在下一个示例中，我们使用标记器API解析HTML列表。
package main
import (
"fmt"
"golang.org/x/net/html"
"io/ioutil"
"log"
"strings"
)
func readHtmlFromFile(fileName string) (string, error) {
bs, err := ioutil.ReadFile(fileName)
if err != nil {
return "", err
}
return string(bs), nil
}
func parse(text string) (data []string) {
tkn := html.NewTokenizer(strings.NewReader(text))
var vals []string
var isLi bool
for {
tt := tkn.Next()
switch {
case tt == html.ErrorToken:
return vals
case tt == html.StartTagToken:
t := tkn.Token()
isLi = t.Data == "li"
case tt == html.TextToken:
t := tkn.Token()
if isLi {
vals = append(vals, t.Data)
}
isLi = false
}
}
}
func main() {
fileName := "index.html"
text, err := readHtmlFromFile(fileName)
if err != nil {
log.Fatal(err)
}
data := parse(text)
fmt.Println(data)
}
该示例打印列表中颜色的名称。
tkn := html.NewTokenizer(strings.NewReader(text))
html.NewTokenizerfor {
tt := tkn.Next()
...
Nextcase tt == html.ErrorToken:
return vals
我们在解析结束时终止for循环并返回数据。
case tt == html.StartTagToken:
t := tkn.Token()
isLi = t.Data == "li"
TokenliisLicase tt == html.TextToken:
t := tkn.Token()
if isLi {
vals = append(vals, t.Data)
}
isLi = false
valsisLili$ go run parse_list.go [red green blue yellow orange brown pink]
去解析HTML表格
在下一个例子中，我们解析一个HTML列表。
package main
import (
"fmt"
"golang.org/x/net/html"
"io/ioutil"
"log"
"net/http"
"strings"
)
func getHtmlPage(webPage string) (string, error) {
resp, err := http.Get(webPage)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(body), nil
}
func parseAndShow(text string) {
tkn := html.NewTokenizer(strings.NewReader(text))
var isTd bool
var n int
for {
tt := tkn.Next()
switch {
case tt == html.ErrorToken:
return
case tt == html.StartTagToken:
t := tkn.Token()
isTd = t.Data == "td"
case tt == html.TextToken:
t := tkn.Token()
if isTd {
fmt.Printf("%s ", t.Data)
n++
}
if isTd && n % 3 == 0 {
fmt.Println()
}
isTd = false
}
}
}
func main() {
webPage := "http://webcode.me/countries.html"
data, err := getHtmlPage(webPage)
if err != nil {
log.Fatal(err)
}
parseAndShow(data)
}
td$ go run parse_table.go Id Name Population 1 China 1382050000 2 India 1313210000 3 USA 324666000 4 Indonesia 260581000 5 Brazil 207221000 6 Pakistan 196626000 ...
去解析HTML列表II
在下一个示例中，我们使用解析API解析HTML列表。
package main
import (
"fmt"
"golang.org/x/net/html"
"io/ioutil"
"log"
"strings"
)
func main() {
fileName := "index.html"
bs, err := ioutil.ReadFile(fileName)
if err != nil {
log.Fatal(err)
}
text := string(bs)
doc, err := html.Parse(strings.NewReader(text))
if err != nil {
log.Fatal(err)
}
var data []string
doTraverse(doc, &data, "li")
fmt.Println(data)
}
func doTraverse(doc *html.Node, data *[]string, tag string) {
var traverse func(n *html.Node, tag string) *html.Node
traverse = func(n *html.Node, tag string) *html.Node {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.TextNode && c.Parent.Data == tag {
*data = append(*data, c.Data)
}
res := traverse(c, tag)
if res != nil {
return res
}
}
return nil
}
traverse(doc, tag)
}
lidoc, err := html.Parse(strings.NewReader(text))
html.Parsetraverse = func(n *html.Node, tag string) *html.Node {
for c := n.FirstChild; c != nil; c = c.NextSibling {
if c.Type == html.TextNode && c.Parent.Data == tag {
*data = append(*data, c.Data)
}
res := traverse(c, tag)
if res != nil {
return res
}
}
return nil
}
lidata$ go run parsing.go [red green blue yellow orange brown pink]
通过id查找标签
idididAttrpackage main
import (
"bytes"
"fmt"
"golang.org/x/net/html"
"io"
"log"
"strings"
)
func getAttribute(n *html.Node, key string) (string, bool) {
for _, attr := range n.Attr {
if attr.Key == key {
return attr.Val, true
}
}
return "", false
}
func renderNode(n *html.Node) string {
var buf bytes.Buffer
w := io.Writer(&buf)
err := html.Render(w, n)
if err != nil {
return ""
}
return buf.String()
}
func checkId(n *html.Node, id string) bool {
if n.Type == html.ElementNode {
s, ok := getAttribute(n, "id")
if ok && s == id {
return true
}
}
return false
}
func traverse(n *html.Node, id string) *html.Node {
if checkId(n, id) {
return n
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
res := traverse(c, id)
if res != nil {
return res
}
}
return nil
}
func getElementById(n *html.Node, id string) *html.Node {
return traverse(n, id)
}
func main() {
doc, err := html.Parse(strings.NewReader(data))
if err != nil {
log.Fatal(err)
}
tag := getElementById(doc, "yellow")
output := renderNode(tag)
fmt.Println(output)
}
var data = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Colour</title>
</head>
<body>
<p>
A list of colours:
</p>
<ul>
<li>red</li>
<li>green</li>
<li>blue</li>
<li id="yellow">yellow</li>
<li>orange</li>
<li>brown</li>
<li>pink</li>
</ul>
</body>
</html>`
我们找到一个特定的标签并呈现它的HTML。我们从多行字符串加载HTML数据。
func getAttribute(n *html.Node, key string) (string, bool) {
for _, attr := range n.Attr {
if attr.Key == key {
return attr.Val, true
}
}
return "", false
}
Attrfunc renderNode(n *html.Node) string {
var buf bytes.Buffer
w := io.Writer(&buf)
err := html.Render(w, n)
if err != nil {
return ""
}
return buf.String()
}
html.Render$ go run find_by_id.go <li id="yellow">yellow</li>
并发解析标题
在下一个示例中，我们同时解析来自各个网站的HTML标题。该示例使用了tokenizerAPI。
package main
import (
"fmt"
"golang.org/x/net/html"
"net/http"
"sync"
)
var wg sync.WaitGroup
func main() {
urls := []string{
"http://webcode.me",
"https://example.com",
"http://httpbin.org",
"https://www.perl.org",
"https://www.php.net",
"https://www.python.org",
"https://code.visualstudio.com",
"https://clojure.org",
}
showTitles(urls)
}
func showTitles(urls []string) {
c := getTitleTags(urls)
for msg := range c {
fmt.Println(msg)
}
}
func getTitleTags(urls []string) chan string {
c := make(chan string)
for _, url := range urls {
wg.Add(1)
go getTitle(url, c)
}
go func() {
wg.Wait()
close(c)
}()
return c
}
func getTitle(url string, c chan string) {
defer wg.Done()
resp, err := http.Get(url)
if err != nil {
c <- "failed to fetch data"
return
}
defer resp.Body.Close()
tkn := html.NewTokenizer(resp.Body)
var isTitle bool
for {
tt := tkn.Next()
switch {
case tt == html.ErrorToken:
return
case tt == html.StartTagToken:
t := tkn.Token()
isTitle = t.Data == "title"
case tt == html.TextToken:
t := tkn.Token()
if isTitle {
c <- t.Data
isTitle = false
}
}
}
}
sync.WaitGroup$ go run parse_titles.go My html page Welcome to Python.org The Perl Programming Language - www.perl.org Clojure PHP: Hypertext Preprocessor Visual Studio Code - Code Editing. Redefined httpbin.org Example Domain
在本教程中，我们使用Go的net/html库解析了HTML。
列出所有Go教程。