Go语言小爬虫--第一步
2017-01-23 15:58
459 查看
解析新浪手机新闻网页,获取5条新的新闻的url存入到数据库
package linksql
import (
"database/sql"
"log"
_"github.com/go-sql-driver/mysql"
)
func Open_url_table() *sql.DB{
db,err:=sql.Open("mysql","root:root@/mysql")
Checkerror(err)
return db
}
func Insertlink(id int,src string,url string){
db:=Open_url_table()
stmt,err1:=db.Prepare("insert into url_table(id,src,url) values(?,?,?)")
if err1!=nil{
log.Fatalln("prepare error",err1)
}
_,err2:=stmt.Exec(id,src,url)
if err2!=nil{
log.Fatalln("Exec error")
}
}
func Checkerror(err error){ //差错
if err!=nil{
log.Fatal(err)
}
}
package search_sina
import (
"net/http"
"log"
"gopkg.in/xmlpath.v2"
"test/linksql"
)
func Get_url(url string){ //找新URL
resp,err:=http.Get(url)
Checkerror(err)
root,err1:=xmlpath.ParseHTML(resp.Body)
Checkerror(err1)
path:=xmlpath.MustCompile(`//dl[@class="f_card"]/../@href`)
iter:=path.Iter(root)
i:=1
for iter.Next() && i<6{
//fmt.Println(iter.Node())
linksql.Insertlink(i,"sina",iter.Node().String())
i++
}
}
func Checkerror(err error){
if err!=nil{
log.Fatalln(err)
}
}
package linksql
import (
"database/sql"
"log"
_"github.com/go-sql-driver/mysql"
)
func Open_url_table() *sql.DB{
db,err:=sql.Open("mysql","root:root@/mysql")
Checkerror(err)
return db
}
func Insertlink(id int,src string,url string){
db:=Open_url_table()
stmt,err1:=db.Prepare("insert into url_table(id,src,url) values(?,?,?)")
if err1!=nil{
log.Fatalln("prepare error",err1)
}
_,err2:=stmt.Exec(id,src,url)
if err2!=nil{
log.Fatalln("Exec error")
}
}
func Checkerror(err error){ //差错
if err!=nil{
log.Fatal(err)
}
}
package search_sina
import (
"net/http"
"log"
"gopkg.in/xmlpath.v2"
"test/linksql"
)
func Get_url(url string){ //找新URL
resp,err:=http.Get(url)
Checkerror(err)
root,err1:=xmlpath.ParseHTML(resp.Body)
Checkerror(err1)
path:=xmlpath.MustCompile(`//dl[@class="f_card"]/../@href`)
iter:=path.Iter(root)
i:=1
for iter.Next() && i<6{
//fmt.Println(iter.Node())
linksql.Insertlink(i,"sina",iter.Node().String())
i++
}
}
func Checkerror(err error){
if err!=nil{
log.Fatalln(err)
}
}
+----+------+-------------------------------------------------------------------------------+ | id | src | url | +----+------+-------------------------------------------------------------------------------+ | 1 | sina | http://tech.sina.cn/tech_zt/meitu_ipo?vt=4&pos=18 | | 2 | sina | http://tech.sina.cn/i/gj/2016-12-15/detail-ifxytqav9330894.d.html?vt=4&pos=18 | | 3 | sina | http://zhuanlan.sina.cn/article?vt=4&pos=18&id=57660&ch=tech | | 4 | sina | http://tech.sina.cn/i/gj/2016-12-15/detail-ifxytqax6095612.d.html?vt=4&pos=18 | | 5 | sina | http://tech.sina.cn/i/gj/2016-12-15/detail-ifxytqav9326149.d.html?vt=4&pos=18 | +----+------+-------------------------------------------------------------------------------+
相关文章推荐
- Codeforces Round #388 (Div. 2)A Bachgold Problem
- 搭建Go开发及调试环境(LiteIDE + GoClipse) -- Windows篇
- vs2015 编译google v8
- 分别通过mongoTemplate聚合查询(带参数模糊查询)分页、Jpa查询(带参数模糊查询)分页
- 按要求输出
- 试用第三方web推送GoEasy
- 日志分析工具goaccess
- go语言实现IOS OTA安装应用
- Go指南练习之《图片》(Images)
- HDU5546 Ancient Go(DFS)
- google map如何根据地址查询经纬度
- google map初次使用总结
- 花信年华--You're gonna love who you turn out to be
- Codeforces 755D-PolandBall and Polygon
- Golang 学习之“”、nil 和 len(s)的对比
- Kubernetes 大咖秀徐超《使用 client-go 控制原生及拓展的 Kubernetes API》
- golang中打印方法
- go操作cookie
- 【Algothrim】 strcmp strcpy
- Polygon,Ring,Segment