利用hadoop中的hdfs作为海量文件服务器的实践

 当WEB集群的数据达到海量的时候，一台文件服务器负担会很大，所以可能会换分布式来存储，下面介绍的方法，就是用来调用hadoop的hdfs的存和取。

这个是存入hdfs的方法，仅供参考：

import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;

import java.net.HttpURLConnection;

import java.net.URL;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class UploadImage

{

private static final Log logger = LogFactory.getLog(ShowImage.class);
public void uploadFile(InputStream stream, String fileName, String path)

throws IOException

{

HttpURLConnection con;

try

{

con = getConnection("http://192.168.1.96:50070/webhdfs/v1/",
path + 

fileName, "CREATE", null, "PUT");
byte[] bytes = new byte[1024];

int rc = 0;

while ((rc = stream.read(bytes, 0, bytes.length)) > 0)

con.getOutputStream().write(bytes, 0, rc);

con.getInputStream();

con.disconnect();

} catch (IOException e) {

logger.info(e.getMessage());

e.printStackTrace();

}

stream.close();

}
public HttpURLConnection getConnection(String strurl, String path, String op, String parameters, String request)

{

URL url = null;

HttpURLConnection con = null;

StringBuffer sb = new StringBuffer();

try {

sb.append(strurl);

sb.append(path);

sb.append("?op=");

sb.append(op);

if (parameters != null)

sb.append(parameters);
url = new URL(sb.toString());

con = (HttpURLConnection)url.openConnection();

con.setRequestMethod(request);

con.setRequestProperty("accept", "*/*");

con.setRequestProperty("connection", "Keep-Alive");

String s = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)";

con.setRequestProperty("User-Agent", s);

con.setRequestProperty("Accept-Encoding", "gzip");

con.setDoInput(true);

con.setDoOutput(true);

con.setUseCaches(false);

} catch (IOException e) {

e.printStackTrace();

}

return con;

}

}
下面是读取hdfs的方法：

package image;

import java.io.IOException;

import java.io.InputStream;

import java.net.HttpURLConnection;

import java.net.URL;

import javax.servlet.ServletException;

import javax.servlet.ServletOutputStream;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

public class ShowImage extends HttpServlet {

private static final long serialVersionUID = 1L;

private static final Log log = LogFactory.getLog(ShowImage.class);

public ShowImage() {

super();

}

protected void service(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

doPost(request, response);

}

protected void doGet(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

doPost(request, response);

}

protected void doPost(HttpServletRequest request,

HttpServletResponse response) throws ServletException, IOException {

request.setCharacterEncoding("utf-8");

HttpURLConnection con = null;

ServletOutputStream out = null;

byte[] bytes = (byte[]) null;

String path = request.getRequestURI();

String[] arr = path.split("/");

String name = arr[arr.length - 1];

try {

out = response.getOutputStream();

con = getConnection("http://192.168.1.96:50070/webhdfs/v1/",
"tmp/"

+ name, "OPEN", null, "GET");

InputStream is = con.getInputStream();

bytes = new byte[1024];

while (is.read(bytes) != -1) {

out.write(bytes, 0, bytes.length);

}

out.flush();

out.close();

log.info("****Success****");

} catch (IOException e) {

e.printStackTrace();

}

}

public HttpURLConnection getConnection(String strurl, String path,

String op, String parameters, String request) {

URL url = null;

HttpURLConnection con = null;

StringBuffer sb = new StringBuffer();

try {

sb.append(strurl);

sb.append(path);

sb.append("?op=");

sb.append(op);

if (parameters != null)

sb.append(parameters);

url = new URL(sb.toString());

con = (HttpURLConnection) url.openConnection();

con.setRequestMethod(request);

con.setRequestProperty("accept", "*/*");

con.setRequestProperty("connection", "Keep-Alive");

String s = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)";

con.setRequestProperty("User-Agent", s);

con.setRequestProperty("Accept-Encoding", "gzip");

con.setDoInput(true);

con.setDoOutput(true);

con.setUseCaches(false);

} catch (IOException e) {

e.printStackTrace();

}

return con;

}

}

延伸阅读： http://blog.itpub.net/29806344/viewspace-1364778/
   关于WEB集群中文件服务器的讨论

原文链接：http://blog.itpub.net/29806344/viewspace-1368266/