您的位置:首页 > 数据库 > Oracle

查询超大的Excel的数据更新Oracle数据库中的表

2017-08-15 15:42 316 查看
有此一文,是因为本来一个很简单的事情,但是在数据量比较大的时候就变得复杂。当时干这个事情的时候,深受其害,故分享经验,希望大家少走弯路。

场景是查询几百万的Excel中的数据,并更新到oracle数据库中。

多的不说了,上代码:

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.io.FileUtils;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.ap
4000
ache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.util.CellReference;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MultipartFile;

import com.bonc.security.client.SecurityClient;
import com.bonc.security.client.SecurityRestClient;
import com.bonc.springbootdemo.common.datasource.DynamicDataSourceContextHolder;
import com.bonc.springbootdemo.mapper.download.DownloadMapper;

public class ExcleUtil {
public void ReadExcleBIG(String fileName) throws ServletException, IOException {
Map<String, String> b = new HashMap<String, String>();
String savePath = "/home/text/downloads/";
String ReadName = savePath + fileName + ".xlsx";
String endTmie = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(new Date());
String oldds = DynamicDataSourceContextHolder.getDataSourceType();
DynamicDataSourceContextHolder.setDataSourceType("shandong1");
try {
File excel_file = new File(ReadName);// 读取的文件路径
FileInputStream input = new FileInputStream(excel_file); // 读取的文件路径
XSSFWorkbook wb = new XSSFWorkbook(new BufferedInputStream(input));
int sheet_numbers = wb.getNumberOfSheets();// 获取表的总数
System.out.println("\n\n**********共有工作表总数**********:" + sheet_numbers);// 7
String[] sheetnames = new String[sheet_numbers];
int excleToOrcle = 1;

Map<String, Object> param = new HashMap<String, Object>();
param.put("tName", "table_" + endTmie);
downloadMapper.createTable(param);
Map<String, Object> ss = new HashMap<String, Object>();
List<Map<String,Object>> list=new ArrayList<Map<String,Object>>();
ss.put("tName","table_"+ endTmie);
int listNumber=1;
for (int i = 0; i < sheet_numbers; i++) {// 遍历所有表
Sheet sheet = wb.getSheetAt(i); // 获取 某个表
sheetnames[i] = sheet.getSheetName();// 获取表名,存入数组
System.out.println("\n\n---正在读取和匹配工作表\t《" + sheetnames[i] + "》\t的数据---\n");// 7

int rows_num = sheet.getLastRowNum();// 获取行数
System.out.println("\n\n---表\t《" + sheetnames[i] + "》\t 共有数据---:\t" + rows_num + "\t行");// 7
for (int rows = 1; rows < rows_num; rows++) {
Row row = sheet.getRow(rows);// 取得某一行 对象
if (row != null && !(row.equals(""))) {
int columns_num = row.getLastCellNum();// 获取列数

Map<String, Object> s = new HashMap<String, Object>();// 初始化数组长度
for (int columns = 0; columns < columns_num; columns++) {
Cell cell = row.getCell(columns);
if (cell != null) {
s.put("cell"+columns, cell.getStringCellValue());
}else{
s.put("cell"+columns," ");
}

}
list.add(s);
if(listNumber==100){
ss.put("list",list);
downloadMapper.ExcleToOracleAll(ss);//将数据插入新表
listNumber=0;
list=new ArrayList<Map<String,Object>>();;
}
listNumber++;

}
}

}
input.close();

Map<String, Object> createNew = new HashMap<String, Object>();
createNew.put("table2","table_"+endTmie);
String newTime = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss").format(new Date());
createNew.put("table3","table_"+newTime);
downloadMapper.createNew(createNew);//新旧表生成最终的表
downloadMapper.deleteOld(createNew);//删除旧表

downloadMapper.updateNewName(createNew);//将最终表更新为旧表名

downloadMapper.deleteTable(param);//删除新表
} catch (IOException ex) {
ex.printStackTrace();
} finally {
DynamicDataSourceContextHolder.setDataSourceType(oldds);
}
}
}
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签:  java poi excle oracle
相关文章推荐