hbase java api操作导入数据

使用hbase存储名人数据集，数据集由名人文字信息以及名人图片组成。

名人文字信息使用scrapy框架从wiki百科上爬取并保存在csv格式中。

图片信息从百度图片上爬取每人30张保存在以该名人姓名命名的文件夹中

因此本文包含以下几个方面：

- 爬取文本的爬虫

- 爬取图片的爬虫

- 将数据导入hbase

scrapy 爬取wiki百科

首先新建scrapy项目



然后在settings.py文件中加入

FEED_URI = u'file:///F:/pySpace/celebrity/info1.csv'
FEED_FORMAT = 'CSV'

即以csv格式保存爬取数据以及文件保存位置

在main.py文件中加入

import sys
reload(sys)
sys.setdefaultencoding('utf-8')
sys.getdefaultencoding()
from scrapy import cmdline
cmdline.execute("scrapy crawl celebrity".split())

<python>
# -*- coding: utf-8 -*-
from scrapy.spiders import CrawlSpider
from scrapy.selector import Selector
from celebrity.items import CelebrityItem
from scrapy.http import Request
import pandas as pd
#读取待爬取的名人姓名列表
with open(r'F:\pySpace\celebrity\name_lists1.txt','r') as f:
url_list = f.read()
url_list = url_list.split('\n')

class Celebrity(CrawlSpider):
len_url = len(url_list)
num =1
name = "celebrity"
front_url = 'https://zh.wikipedia.org/wiki/'
start_urls = [front_url + url_list[num].encode('utf-8')]

def parse(self, response):
item = CelebrityItem()
selector = Selector(response)
body = selector.xpath('//*[@id="mw-content-text"]')[0]
Title = body.xpath('//span[@class="mw-headline"]/text()').extract()
titles = ['简介']
for i in range(len(Title)):
if Title[i] != '参考文献' and Title[i] != '注释' and Title[i] != '外部链接' and Title[i] != '参考资料':
titles.append(Title[i])
Passage = selector.xpath('//*[@id="mw-content-text"]/p')
all_info = []
for eachPassage in Passage:
info =''.join(eachPassage.xpath('.//text()').extract())
if info!= '':
all_info.append(info.strip())
Ul_list = selector.xpath('//*[@id="mw-content-text"]/ul')
for eachul in Ul_list:
info = ''.join(eachul.xpath('.//text()').extract())
if info != '' and info!= '\n' and info != ' ':
all_info.append(info)

# 爬取带标题的
k = 0
epoch = len(all_info) / len(titles)
i=0
if epoch >0:
for i  in range(len(titles)):

if i == len(titles)-1:
item['name'] = url_list[self.num].encode('utf-8')
item['title'] = titles[i]
item['info'] = ''.join(all_info[k:])
else :
item['name'] = url_list[self.num].encode('utf-8')
item['title'] = titles[i]
item['info'] = ''.join(all_info[k:k+epoch])
k = k+epoch
yield item
else :
for j in range(len(all_info)):
item['name'] = url_list[self.num].encode('utf-8')
item['title'] = titles[j]
item['info'] = all_info[j]
yield item

#爬取不带标题的
# for j in range(len(all_info)):
#     item['name'] = url_l
4000
ist[self.num].encode('utf-8')
#     item['info'] = all_info[j]
#     yield item

print item['name']
self.num = self.num + 1
print self.num
if self.num < self.len_url:
nextUrl =self.front_url + url_list[self.num].encode('utf-8')
yield Request(nextUrl,callback=self.parse)
</python>

爬取图片

import urllib2
import re
import os
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

def img_spider(name_file):

user_agent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
headers = {'User-Agent':user_agent}

with open(name_file) as f:
name_list = [name.rstrip().decode('utf-8') for name in f.readlines()]
f.close()

for name in name_list:
if not os.path.exists('F:/pySpace/celebrity/img_data/' + name):
os.makedirs('F:/pySpace/celebrity/img_data/' + name)
try:
url = "http://image.baidu.com/search/avatarjson?tn=resultjsonavatarnew&ie=utf-8&word=" + name.replace(' ','%20') + "&cg=girl&rn=60&pn=60"
req = urllib2.Request(url, headers=headers)
res = urllib2.urlopen(req)
page = res.read()
#print page
img_srcs = re.findall('"objURL":"(.*?)"', page, re.S)
print name,len(img_srcs)
except:
print name," error:"
continue
j = 1
src_txt = ''

for src in img_srcs:
with open('F:/pySpace/celebrity/img_data/' + name + '/' + str(j)+'.jpg','wb') as p:
try:
print "downloading No.%d"%j
req = urllib2.Request(src, headers=headers)
img = urllib2.urlopen(src,timeout=3)
p.write(img.read())
except:
print "No.%d error:"%j
p.close()
continue
p.close()
src_txt = src_txt + src + '\n'
if j==30:
break
j = j+1
#保存src路径为txt
with open('F:/pySpace/celebrity/img_data/' + name + '/' + name +'.txt','wb') as p2:
p2.write(src_txt)
p2.close()
print "save %s txt done"%name

if __name__ == '__main__':
name_file = "name_lists1.txt"
img_spider(name_file)

通过java api 将数据导入hbase

在hbase中建两个表，分别为celebrity（存储图片信息）和celebrity_info（存储文本信息）名人的姓名为rowkey。

<java>
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import com.csvreader.CsvReader;
import com.google.common.primitives.Chars;
import org.junit.Test;
import java.nio.charset.Charset;
import java.io.*;
import javax.swing.ImageIcon;
/**
* Created by mxy on 2016/10/31.
*/
public class CelebrityDataBase {

/*新建表*/
public void createTable(String tablename)throws Exception{
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","node4,node5,node6");
HBaseAdmin admin = new HBaseAdmin(config);
String table = tablename;

if(admin.isTableAvailable(table)){
admin.disableTable(table);
admin.deleteTable(table);
}else {
HTableDescriptor t = new HTableDescriptor(table.getBytes());
HColumnDescriptor cf1 = new HColumnDescriptor("cf1".getBytes()) ;
cf1.setMaxVersions(10);
t.addFamily(cf1);
admin.createTable(t);
}
admin.close();
}
//插入数据csv格式文字数据
public void putInfo()throws Exception{
CsvReader r = new CsvReader("F://pySpace//celebrity//info.csv",',', Charset.forName("utf-8"));
r.readHeaders();
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","node4,node5,node6");
HTable table = new HTable(config,"celebrity_info");
while(r.readRecord()){

System.out.println(r.get("name"));
//          String rowkey = r.get("name");
Put put = new Put(r.get("name").getBytes());
put.add("cf1".getBytes(),r.get("title").getBytes(),r.get("info").getBytes());
table.put(put);

}
r.close();
table.close();

}

//查找图片数据
public void getImage(String celebrity_name,String img_num)throws Exception{
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","node4,node5,node6");
HTable table = new HTable(config,"celebrity");
Get get = new Get(celebrity_name.getBytes());
Result res = table.get(get);
Cell c1 = res.getColumnLatestCell("cf1".getBytes(),img_num.getBytes());
File file=new File("D://"+celebrity_name+img_num);//将输出的二进制流转化后的图片的路径
FileOutputStream fos=new FileOutputStream(file);
fos.write(c1.getValue());
fos.flush();
System.out.println(file.length());
fos.close();
table.close();
}

//查找文本数据
public void getInfo(String name) throws Exception{
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","node4,node5,node6");
HTable table = new HTable(config,"celebrity_info");

Get get = new Get(name.getBytes());
Result res = table.get(get);
Result result = table.get(get);
for(Cell cell : result.rawCells()){
System.out.println("rowKey:" + new String(CellUtil.cloneRow(cell))
+ " cfName:" + new String(CellUtil.cloneFamily(cell))
+ " qualifierName:" + new String(CellUtil.cloneQualifier(cell))
+ " value:" + new String(CellUtil.cloneValue(cell)));
}
table.close();
}

//插入图片数据
public void putImage(String each_celebrity,String each_img)throws Exception{

String str = null;
Configuration config = HBaseConfiguration.create();
config.set("hbase.zookeeper.quorum","node4,node5,node6");
HTable table = new HTable(config,"celebrity");
str = String.format("F://pySpace//celebrity//img_data//%s//%s",each_celebrity,each_img);
File file = new File(str);
int size = 0;
size = (int)file.length();
System.out.println(size);
byte[] bbb = new byte[size];
try {
InputStream a = new FileInputStream(file);
a.read(bbb);
//            System.out.println(bbb);
//            System.out.println(Integer.toBinaryString(bbb));
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String rowkey = each_celebrity;
Put put = new Put(rowkey.getBytes());
put.add("cf1".getBytes(),each_img.getBytes(),bbb);
table.put(put);
table.close();

}

public static void main(String args[]){
CelebrityDatabase pt = new CelebrityDatabase();
try {
pt.createTable("celebrity);
pt.createTable("celebrity_info);
} catch (Exception e) {
e.printStackTrace();
System.out.println("createTable error");
}
String root_path = "F://pySpace//celebrity//img_data";
File file = new File(root_path);
File[] files = file.listFiles();

for(int i = 0;i < files.length;i++){
String each_path = root_path +"//"+ files[i].getName();
File celebrity_file = new File(each_path);
File[] celebrity_files = celebrity_file.listFiles();
System.out.println(each_path);
for(int j = 0;j<celebrity_files.length - 1;j++){
try {
pt.putImage(files[i].getName(),celebrity_files[j].getName());
} catch (Exception e) {
e.printStackTrace();
System.out.println("putImage error");
}
}

}
//存入文字信息
try {
pt.putInfo();
} catch (Exception e) {
e.printStackTrace();
}

//取出图片
try {
pt.getImage("龔照勝","13.jpg");
} catch (Exception e) {
e.printStackTrace();
System.out.println("getImage error")
971b
;
}
//取出文字
try {
pt.getInfo("成龙");
} catch (Exception e) {
e.printStackTrace();
}
}
}

</java>