hive学习练习&Sqoop知识补充

Databases in Hive

create database userdb;

create database IF NOT EXISTS userdb;

hive> show databases like ‘u.*’;   (f代表userdb数据库)

/user/hive/warehouse

/user/hive/warehouse/userdb.db

create database userdb
location ‘/my/preferred/directory’;

create database userdb
comment ‘holds all about user tables’;

describe database userdb;

create database userdb
with dbproperties (‘creator=’leon’,’data’=’2016-08-19’);

use userdb;

drop database if exists userdb;

drop database if exists userdb cascade;

alter database userdb set dbproperties(‘edited-by’=’leon’);

基本操作

create table t_order (order_sn string, user_id string,amount int,create_time timestamp)
row format delimited fields terminated by '\t';

create external table t_order_ex (order_sn string, user_id string,amount int,create_time timestamp)
row format delimited fields terminated by '\t'
location '/external/hive/t_order_ex';

create external table t_order_in_ex (order_sn string, user_id string,amount int,create_time timestamp)
row format delimited fields terminated by '\t';

load data local inpath '/root/hivetb.txt' into table t_order;
load data local inpath '/root/hivetb.txt' into table t_order_ex;
load data local inpath '/root/hivetb.txt' into table t_order_in_ex;

select * from t_order;
select * from t_order_ex;
select * from t_order_in_ex;

select count(*) from t_order_wk;

hdfs dfs -put /root/hiveatb.txt /external/hive/t_order_ex

select * from t_order_ex;

hdfs dfs -put /root/hiveatb.txt /user/hive/warehouse/userdb.db/t_order

load data inpath '/hiveatb.txt' into table t_order_in_ex;

drop table t_order;
drop table t_order_ex;
drop table t_order_in_ex;

show tables;

create table t_order_order_sn
as
select order_sn,user_id from t_order;

show tables;

desc t_order_order_sn;

create table t_order_like like t_order;

show tables;

insert overwrite table t_order_like
select * from t_order;

create table t_order_part (order_sn string, user_id string,amount int,create_time timestamp)
partitioned by (month string)
row format delimited fields terminated by '\t';

partitioned by (month string)

partitioned by (pname1 ptype1,pname2 ptype2 ...)

desc t_order_part;

file:///user/hive/project/data1.

load data local inpath '/root/hivetb.txt' into table t_order_part partition (month='20160820');

load data local inpath '/root/hivetb.txt' overwrite into table t_order_part partition (month='20160821');

select * from t_order_part;

select * from t_order_part  where month='20160820';
select count(*) from t_order_part where month='20160820';

ALTER TABLE t_order_part ADD PARTITION (month = '20160822') location '/external/20160822';

show partitions t_order_part;

/external/20160822

hdfs dfs -put /root/hiveatb.txt /external/20160822

select * from t_order_part;

hdfs dfs -mkdir /external/datapart

hdfs dfs -put /root/hiveatb.txt /external/datapart/

/external/datapart

alter table t_order_part add partition (month='20160824') location '/external/datapart';

hive命令的3种调用方式

vi hive_script3.sql

[root@hadoopcswfb iwisdom]# /iwisdom/hive-1.4.4/bin/hive -S -f /hive-script.sql

start_hql.sh

#!/bin/bash
# -S 打印输出mapreduce日志
hive \
-hivevar col1=2016080910 \
-hivevar col2=10086 \
-S -f test.sql

test.sql

use userdb;
select * from t_order where id='${hivevar:col1}' and col2='${hivevar:col2}';

chmod +x start_hql.sh
./start_hql.sh

$PATH=$HIVE_HOME\bin

[root@hadoopcswfb iwisdom]# hive -e 'use userdb';
[root@hadoopcswfb iwisdom]# hive -e 'select * from t_order';

[root@hadoopcswfb iwisdom]# hive -S -e 'select * from t_order';

[root@hadoopcswfb iwisdom]# hive -e 'select * from t_order'  > test.txt

Sqoop拾遗&利用sqoop实现MySQL与Hive互导数据

import

export

sqoop import --connect jdbc:mysql://192.168.0.100/userdb --username root --password 123 --table t_order --hive-import -- --default-character-set=utf-8;

sqoop import \
--connect jdbc:mysql://192.168.0.100:3306/userdb \
--username root \
--password 123 \
--query 'select f_order.*,userinfo.username from f_order join userinfo on (f_order.user_id=userinfo.user_id) where $CONDITIONS' \
--hive-import \
--hive-database userdb \
--create-hive-table \
--hive-table f_order_userinfo \
--target-dir /myinfo/f_order_userinfo \
--split-by f_order.user_id

--query

--table

$CONDITIONS

--target-dir

sqoop job \
--create myjob \
-- import \
--connect jdbc:mysql://192.168.0.100:3306/userdb \
--table t_order \
--username root \
--password 123 \
--hive-import \
--hive-database userdb \
--hive-table f_order_userinfo \
--incremental lastmodified \
--check-column create_time \
--last-value '2016-08-18 11:58:36'  --append

-- import

[root@hadoopcswfb iwisdom]# sqoop job --show myjob;

Warning: /usr/lib/hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/lib/hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Enter password:
Job: myjob
Tool: import
Options:
----------------------------
verbose = false
**incremental.last.value = 2016-08-18 11:58:36**
db.connect.string = jdbc:mysql://192.168.0.100:3306/userdb
codegen.output.delimiters.escape = 0
codegen.output.delimiters.enclose.required = false
codegen.input.delimiters.field = 0
hbase.create.table = false
db.require.password = true
hdfs.append.dir = false
db.table = t_order
import.fetch.size = null
codegen.input.delimiters.escape = 0
codegen.input.delimiters.enclose.required = false
db.username = root
codegen.output.delimiters.record = 10
import.max.inline.lob.size = 16777216
hcatalog.create.table = false
db.clear.staging.table = false
incremental.col = create_time
codegen.input.delimiters.record = 0
enable.compression = false
hive.overwrite.table = false
hive.import = true
codegen.input.delimiters.enclose = 0
hive.table.name = t_order
hive.database.name = userdb
hive.drop.delims = false
codegen.output.delimiters.enclose = 0
hdfs.delete-target.dir = false
codegen.output.dir = .
codegen.auto.compile.dir = true
mapreduce.num.mappers = 4
import.direct.split.size = 0
codegen.output.delimiters.field = 1
export.new.update = UpdateOnly
incremental.mode = DateLastModified
hdfs.file.format = TextFile
codegen.compile.dir = /tmp/sqoop-root/compile/cac84cfd83e2a71d659a57e00fe08a06
direct.import = false
hive.fail.table.exists = false
db.batch = false

[root@hadoopcswfb iwisdom]# sqoop job --exec myjob

[root@hadoopcswfb iwisdom]# sqoop job --show myjob;

Warning: /usr/lib/hbase does not exist! HBase imports will fail.
Please set $HBASE_HOME to the root of your HBase installation.
Warning: /usr/lib/hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Enter password:
Job: myjob
Tool: import
Options:
----------------------------
verbose = false
**incremental.last.value = 2016-08-20 01:48:34.0**
db.connect.string = jdbc:mysql://192.168.0.100:3306/userdb
codegen.output.delimiters.escape = 0
codegen.output.delimiters.enclose.required = false
codegen.input.delimiters.field = 0
hbase.create.table = false
db.require.password = true
hdfs.append.dir = false
db.table = t_order
import.fetch.size = null
codegen.input.delimiters.escape = 0
codegen.input.delimiters.enclose.required = false
db.username = root
codegen.output.delimiters.record = 10
import.max.inline.lob.size = 16777216
hcatalog.create.table = false
db.clear.staging.table = false
incremental.col = create_time
codegen.input.delimiters.record = 0
enable.compression = false
hive.overwrite.table = false
hive.import = true
codegen.input.delimiters.enclose = 0
hive.table.name = t_order
hive.database.name = userdb
hive.drop.delims = false
codegen.output.delimiters.enclose = 0
hdfs.delete-target.dir = false
codegen.output.dir = .
codegen.auto.compile.dir = true
mapreduce.num.mappers = 4
import.direct.split.size = 0
codegen.output.delimiters.field = 1
export.new.update = UpdateOnly
incremental.mode = DateLastModified
hdfs.file.format = TextFile
codegen.compile.dir = /tmp/sqoop-root/compile/64827b1118b076a00a8d0a47a94a3d05
direct.import = false
hive.fail.table.exists = false
db.batch = false

incremental.last.value = 2016-08-20 01:48:34.0

[root@hadoopcswfb iwisdom]# sqoop job --exec myjob

incremental.last.value = 2016-08-20 02:22:57.0

sqoop import \
--connect jdbc:mysql://192.168.0.100:3306/userdb \
--username root \
--password 123 \
--table t_order \
--columns 'order_sn,user_id' \
--hive-import \
--hive-database userdb \
--create-hive-table \
-target-dir /user/hive/warehouse/userdb.db/t_order_sn_uid \
-m 1 \
--hive-table t_order_sn_uid

--create-hive-table

--hive-import

SHOW TABLES

conf/hive-site.xml

<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=hive_metastore_db;create=true</value>
</property>

数据导出工具export

其他1 空值的处理

sqoop import --connect jdbc:oracle:thin:@//myoracle:1521/pdbORCL --username DM --password-file sqoop.pwd --table PUB_DATE_D --where "QUARTER_NAME<'Q2'" --hive-import --hive-database default --null-string '\\N' --null-non-string '\\N'

其他2 合并 hdfs 文件

sqoop merge –new-data /test/p1/person –onto /test/p2/person –target-dir /test/merged –jar-file /opt/data/sqoop/person/Person.jar –class-name Person –merge-key id