java 链接mysql 产生500W数据模拟生成环境

java 插入数据到mysql 通过sqoop 导入到hive 中，kylin模拟见cube 时间和 数据膨胀率 kylin 数据插入到 HBase

Kylin

HBase 1.1.3

Hive 1.2.1

Hadoop 2.5.1

create table infoagetime(
prod_name char(10),
prod_id SMALLINT,
ods_date DATE
)

数据格式

oPmgBZxldW    75    2016-09-04

WSSAnnZrNy    57    2016-09-09

本人Java水平有限，大家自行改善，尤其是批量插入mysql语句

另工程中需要导入 mysql jar ：   mysql-connector-java-5.1.6.jar

V2

package com.wubaiwan.instmysql;

import java.io.File;
import java.io.FileOutputStream;
import java.sql.DriverManager;
import java.util.Random;

public class YiBaiWan {

public static String getRandomString(int length) { // length表示生成字符串的长度
String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
Random random = new Random();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < length; i++) {
int number = random.nextInt(base.length());
sb.append(base.charAt(number));
}
return sb.toString();
}

public static final String url = "jdbc:mysql://192.168.184.168/hive2";
public static final String name = "com.mysql.jdbc.Driver";
public static final String user = "root";
public static final String password = "123";

public static java.sql.Connection conn = null;
public static java.sql.PreparedStatement pst = null;
static FileOutputStream out = null;

public static void main(String args[]) {

try {
Class.forName(name);
conn = DriverManager.getConnection(url, user, password);// 获取连接
conn.setAutoCommit(false);
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} // 指定连接类型

Random random = new Random();
// int k = random.nextInt();
// System.out.println(k);
int x = 0;
try {
out = new FileOutputStream(new File("D:/mysql.txt"));
String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values(?,?,?)";
// System.out.println(sql);
pst = conn.prepareStatement(sql);

while (x < 5000000) {
// System.out.println((int)(Math.random()*100));
// System.out.println(getRandomString(10) + ',' +
// (int)(Math.random()*100));
// String k = getRandomString(10) + ',' +
// (int)(Math.random()*100 )+ ','+ "2016-09-0" +
// (int)(random.nextInt(9)%9 + 1)+"\r\n" ;
// System.out.println(k);
// out.write(k.getBytes());

pst.setString(1, getRandomString(10));
pst.setInt(2, (int) (Math.random() * 100));
pst.setString(3, "2016-09-0" + (int) (random.nextInt(9) % 9 + 1));

pst.addBatch();

if (x % 1000 == 0) {
pst.executeBatch();
//pst.executeUpdate(arg0)
}

x++;
}
System.out.println("Commit");
pst.executeBatch();
conn.commit();
out.close();
conn.close();
pst.close();

} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{

}

}

}

======================================

V1  这个版本会有java memory 异常

package com.wubaiwan.instmysql;

import java.io.File;
import java.io.FileOutputStream;
import java.sql.DriverManager;
import java.util.Random;

public class YiBaiWan {

public static String getRandomString(int length) { //length表示生成字符串的长度
String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
Random random = new Random();
StringBuffer sb = new StringBuffer();
for (int i = 0; i < length; i++) {
int number = random.nextInt(base.length());
sb.append(base.charAt(number));
}
return sb.toString();
}

public static final String url = "jdbc:mysql://192.168.184.168/hive2";
public static final String name = "com.mysql.jdbc.Driver";
public static final String user = "root";
public static final String password = "123";

public static java.sql.Connection conn = null;
public static java.sql.PreparedStatement pst = null;

public static void main(String args[]){

try {
Class.forName(name);
conn = DriverManager.getConnection(url, user, password);//获取连接
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}//指定连接类型

FileOutputStream out = null;

Random random = new Random();
//int k = random.nextInt();
//System.out.println(k);
int x = 0;
try {
out = new FileOutputStream(new File("D:/mysql.txt"));

while(x <5000000){
//System.out.println((int)(Math.random()*100));
//System.out.println(getRandomString(10) + ',' + (int)(Math.random()*100));
//String k = getRandomString(10) + ',' + (int)(Math.random()*100 )+ ','+ "2016-09-0" + (int)(random.nextInt(9)%9 + 1)+"\r\n" ;
//System.out.println(k);
//out.write(k.getBytes());
String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values('"+getRandomString(10)+ "',"+(int)(Math.random()*100 )+","+ "'2016-09-0" + (int)(random.nextInt(9)%9 + 1) + "')"   ;
//System.out.println(sql);
pst = conn.prepareStatement(sql);
pst.executeUpdate();
if(x%10000 == 0 ) {
                System.out.println("Commit");
                conn.commit();
            }
x++;
}
out.close();
conn.close();
pst.close();
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

}