hive over hbase方式将微博用户数据导入hbase
2015-04-30 20:59
323 查看
1,创建hbase表WeiboUserData
>>
create 'WeiboUserData','CF'
2,创建hive->hbase外表logic_WeiboUserData,并对应hbase中的WeiboUser表
>>
CREATE EXTERNAL TABLE logic_WeiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,
VIPLevel string,
birthday string,
identityAuthentication string,
educationBackground string,
workingExperience string,
followersCount int,
friendsCount int,
statusesCount int,
favouritesCount int,
biFollowersCount int,
userTags string,
statusTexts string)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,
CF:screenName,
CF:location,
CF:description,
CF:gender,
CF:weiboLevel,
CF:VIPLevel,
CF:birthday,
CF:identityAuthentication,
CF:educationBackground,
CF:workingExperience,
CF:followersCount,
CF:friendsCount,
CF:statusesCount,
CF:favouritesCount,
CF:biFollowersCount,
CF:userTags,
CF:statusTexts")
TBLPROPERTIES("hbase.table.name" = "WeiboUserData");
//创建目录
hadoop fs -mkdir -p /user/maoxiao/2015/aud/weiboUserData/input
//上传文件
hadoop fs -put weiboUserData.txt /user/maoxiao/2015/aud/weiboUserData/input
3,创建数据外表
CREATE EXTERNAL TABLE data_weiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,
VIPLevel string,
birthday string,
identityAuthentication
string,
educationBackground
string,
workingExperience
string,
followersCount int,
friendsCount int,
statusesCount int,
favouritesCount int,
biFollowersCount int,
userTags string,
statusTexts string
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
LOCATION '/user/maoxiao/2015/aud/weiboUserData/input';
4,通过数据外表导入到hbase外表
SET hive.hbase.bulk=true;
INSERT OVERWRITE TABLE logic_WeiboUserData SELECT
rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts
FROM data_weiboUserData ;
put 'WeiboUserData', 'rowkey1', 'CF:UserId', 'value'
rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts
>>
create 'WeiboUserData','CF'
2,创建hive->hbase外表logic_WeiboUserData,并对应hbase中的WeiboUser表
>>
CREATE EXTERNAL TABLE logic_WeiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,
VIPLevel string,
birthday string,
identityAuthentication string,
educationBackground string,
workingExperience string,
followersCount int,
friendsCount int,
statusesCount int,
favouritesCount int,
biFollowersCount int,
userTags string,
statusTexts string)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,
CF:screenName,
CF:location,
CF:description,
CF:gender,
CF:weiboLevel,
CF:VIPLevel,
CF:birthday,
CF:identityAuthentication,
CF:educationBackground,
CF:workingExperience,
CF:followersCount,
CF:friendsCount,
CF:statusesCount,
CF:favouritesCount,
CF:biFollowersCount,
CF:userTags,
CF:statusTexts")
TBLPROPERTIES("hbase.table.name" = "WeiboUserData");
//创建目录
hadoop fs -mkdir -p /user/maoxiao/2015/aud/weiboUserData/input
//上传文件
hadoop fs -put weiboUserData.txt /user/maoxiao/2015/aud/weiboUserData/input
3,创建数据外表
CREATE EXTERNAL TABLE data_weiboUserData (
rowkey string,
screenName string,
location string,
description string,
gender string,
weiboLevel string,
VIPLevel string,
birthday string,
identityAuthentication
string,
educationBackground
string,
workingExperience
string,
followersCount int,
friendsCount int,
statusesCount int,
favouritesCount int,
biFollowersCount int,
userTags string,
statusTexts string
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
LOCATION '/user/maoxiao/2015/aud/weiboUserData/input';
4,通过数据外表导入到hbase外表
SET hive.hbase.bulk=true;
INSERT OVERWRITE TABLE logic_WeiboUserData SELECT
rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts
FROM data_weiboUserData ;
put 'WeiboUserData', 'rowkey1', 'CF:UserId', 'value'
rowkey,screenName,location,description,gender,weiboLevel,VIPLevel,birthday,identityAuthentication,educationBackground,workingExperience,followersCount, friendsCount, statusesCount, favouritesCount, biFollowersCount, userTags,statusTexts
相关文章推荐
- hive over hbase方式将文本库数据导入hbase
- hive-hbase-handler方式导入hive表数据到hbase表中
- hive over hbase方式将数据导出
- hive-hbase-handler方式导入hive表数据到hbase表中
- hive 导入数据的四种方式
- Hive几种数据导入方式
- Hive表数据导入到Hbase
- Hive几种数据导入方式
- Hive几种数据导入方式
- HBase的几种导入数据的方式
- HBase导入大数据三大方式之(二)——importtsv +completebulkload 方式
- [置顶] Hive几种数据导入方式
- hive-导入数据的方式
- 简单点,搬砖的方式简单点,hive mysql 导入数据的UDF,分享给小伙伴们
- Hive总结(七)Hive四种数据导入方式
- Hive四种数据导入方式介绍
- Hive的几种常见的数据导入方式
- 把hive中的数据导入到hdfs或者本地文件的方式
- 使用Sqoop将HDFS/Hive/HBase与MySQL/Oracle中的数据相互导入、导出