您的位置:首页 > 其它

hive over hbase方式将微博用户数据导入hbase

2015-04-30 20:59 323 查看
1,创建hbase表WeiboUserData
>>
create 'WeiboUserData','CF'
2,创建hive->hbase外表logic_WeiboUserData,并对应hbase中的WeiboUser表
>>
CREATE EXTERNAL TABLE logic_WeiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,
VIPLevel string,
birthday string,
identityAuthentication string,
educationBackground string,
workingExperience string,
followersCount int,
friendsCount int,
statusesCount int,
favouritesCount int,
biFollowersCount int,
userTags string,
statusTexts string)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,
CF:screenName,
CF:location,
CF:description,
CF:gender,
CF:weiboLevel,
CF:VIPLevel,
CF:birthday,
CF:identityAuthentication,
CF:educationBackground,
CF:workingExperience,
CF:followersCount,
CF:friendsCount,
CF:statusesCount,
CF:favouritesCount,
CF:biFollowersCount,
CF:userTags,
CF:statusTexts")
TBLPROPERTIES("hbase.table.name" = "WeiboUserData");
//创建目录
hadoop fs -mkdir -p /user/maoxiao/2015/aud/weiboUserData/input
//上传文件
hadoop fs -put weiboUserData.txt /user/maoxiao/2015/aud/weiboUserData/input
3,创建数据外表
CREATE EXTERNAL TABLE data_weiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,

VIPLevel string,

birthday string,

identityAuthentication
string,
educationBackground
string,

workingExperience
string,
followersCount int,
friendsCount int,

statusesCount int,

favouritesCount int,

biFollowersCount int,

userTags string,

statusTexts string
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
LOCATION '/user/maoxiao/2015/aud/weiboUserData/input';
4,通过数据外表导入到hbase外表
SET hive.hbase.bulk=true;
INSERT OVERWRITE TABLE logic_WeiboUserData SELECT
rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts

FROM data_weiboUserData ;

put 'WeiboUserData', 'rowkey1', 'CF:UserId', 'value'

rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: