一种通用数据采集的schema定义形式

{
"name": "凤凰金融",
"notice": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
　　"comments": "网站通告"
},
"url": {
"data": "attribute",
"value": "http://www.fengjr.com/financing/list?type=cx"
"comments": "本平台数据的采集URL"
},
"project": {
"data": "url",
"url": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
},
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"detail": {
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"member": {
"data": "sub_item",
"sub_item": {
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"src-save": 0,
"url": {
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
}
},
"detail": {
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"src-save": 1
}

补充：

{
"name": "凤凰金融",
"notice": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"url": {
"data": "attribute",
"value": "http://www.fengjr.com/financing/list?type=cx"
},
"project": {
"data": "url",
"url": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
},
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"detail": {
"name": "网贷列表",
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"member": {
"data": "sub_item",
"sub_item": {
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"src-save": 0,
"url": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
}
},
"detail": {
"name": "会员材料",
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"src-save": 1,

　　"crawler": {

　　　　　　"handler":"httpClient|selenium",
　　　　　　"results":"html|json|text",
　　　　　　"next_page": {
　　　　　　　　"matcher": [
　　　　　　　　　　{
　　　　　　　　　　　　"match": "xpath",
　　　　　　　　　　　　"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
　　　　　　　　　　}
　　　　　　　　　],
　　　　　　　　"template": ""
　　　　　　},
　　　　　　"history": "re-crawl|skip|stop"
　　　　}

}