【hadoop】 2003-hdfs的java接口

一、下载Eclipse版本
http://www.eclipse.org/downloads/?osType=linux

二、Linux下安装Eclipse
1、上传eclipse
通过xshell工具上传eclipse-jee-luna-SR1a-linux-gtk.tar.gz 文件到指定目录

2、解压 eclipse-jee-luna-SR1a-linux-gtk.tar.gz

tar -zxvfeclipse-jee-luna-SR1a-linux-gtk.tar.gz -C app/

三、基于命令行模式开发HDFS程序

1.0查看帮助

hadoop fs -help <cmd>

1.1上传

hadoop fs -put <linux上文件> <hdfs上的路径>

1.2查看文件内容

hadoop fs -cat <hdfs上的路径>

1.3查看文件列表

hadoop fs -ls /

1.4下载文件

hadoop fs -get <hdfs上的路径> <linux上文件>

四、基于Eclipse界面开发HDFS程序

1、新建HDFSDemo

2、在HDFSDemo工程下新建lib





3、拷贝hadoop相关jar到lib，采用命令行模式
[hadoop@cloud01 hadoop]$ pwd

/home/hadoop/app/hadoop-2.4.1/share/hadoop

[hadoop@cloud01 hadoop]$ ls

common hdfs httpfs mapreduce tools yarn

以common目录为例
[hadoop@cloud01 common]$ cd /home/hadoop/app/hadoop-2.4.1/share/hadoop/common
[hadoop@cloud01 common]$ cp hadoop-* /home/hadoop/workspace/HDFSdemo/lib/

[hadoop@cloud01 common]$ cp lib/* /home/hadoop/workspace/HDFSdemo/lib/

其他的目录采用同样的方式拷贝即可

4、选择lib目录，然后添加jar包到PATH下







5、通过Eclipse开发工具演示文件的上传、下载、查看HDFS文件列表

package hdfs;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
/**
* @function:
Java 操作HDFS
* @author shenfl
* @date 2015/2/21
*/
public class HDFSDemo
{

private static final String HDFS_PATH = "hdfs://192.168.2.31:9000";
FileSystem fileSystem= null;
@Before
public void init(){
/**
* Returns the FileSystem for this URI's scheme and authority. The
* scheme of the URI determines a configuration property name,
* <tt>fs.<i>scheme </i>.class</tt> whose
value names the FileSystem
* class. The entire URI is passed to the FileSystem instance's
* initialize method.
*/
try {
fileSystem =
FileSystem. get(new URI( HDFS_PATH), new Configuration());
} catch (Exception e)
{
e.printStackTrace();
}
}
/**
* @eg:
[hadoop@cloud01 ~]$ hadoop fs - put -put:
Not enough arguments:
* expected 1 but got 0 Usage: hadoop fs [generic
options] - put [-f]
* [ -p] <localsrc> ... <dst>
*
*
* hadoop fs - put -f
readme.txt hdfs://cloud01:9000/
* @description:
upload file from localsrc to hdfs
* @author shenfl
* @Date 2015- 2-20
*/
@Test
public void testPut()
{
InputStream in = null;
OutputStream out = null;
int buffSize =
1024 * 4;

try {
in = new FileInputStream( new File("/home/hadoop/workspace/HDFSdemo/src/hdfs/readme2.txt" ));
/**
* Create an FSDataOutputStream at the indicated Path.
*
* @param f
* the file to create
* @param overwrite
* if a file with this name already exists, then if true,
* the file will be overwritten, and if false an
* exception will be thrown.
*/
boolean overwrite = true;
out = fileSystem.create( new Path( HDFS_PATH+"/readme2.txt" ), overwrite);
boolean close = true;
/**
* Copies from in stream to out stream.
*
* @param in
* InputStrem to read from
* @param out
* OutputStream to write to
* @param buffSize
* the size of the buffer
*/
IOUtils. copyBytes(in, out, buffSize, close);
} catch (Exception e)
{
e.printStackTrace();
}
}
/**
* [ hadoop@cloud01 ~]$ hadoop fs -get -get:
Not enough arguments: expected 1
* but got 0 Usage: hadoop fs [generic
options] - get [-p]
[ -ignoreCrc]
* [ -crc] <src> ... <localdst> [hadoop@cloud01
~]$ hadoop fs - get
* hdfs://cloud01:9000/readme.txt readme1.txt
*
* @description:
download file from hdfs
*/
@Test
public void testGet()
{
InputStream in = null;
OutputStream out = null;
int buffSize =
1024 * 4;
boolean close = true;
try {
/**
* Opens an FSDataInputStream at the indicated Path.
*
* @param f
* the file to open
*/
in = fileSystem.open( new Path( "hdfs://cloud01:9000/readme2.txt" ));
// get OutputStream
out =
System. out;
IOUtils. copyBytes(in, out, buffSize, close);
} catch (Exception e)
{
e.printStackTrace();
}
}
/**
*
@eg: hadoop cmd
[ hadoop@cloud01 ~]$ hadoop fs -ls /
Found 2 items
-rw- r--r --
1 hadoop supergroup 41 2015 -02- 20
07:05 /readme.txt
-rw- r--r --
3 hadoop supergroup 16 2015 -02- 21
04:18 /readme2.txt

@return
rw- r--r -- File 1 hadoop supergroup 2015-02 -20
07:05:32 hdfs://192.168.2.31:9000/readme.txt
rw- r--r -- File 3 hadoop supergroup 2015-02 -21
04:18:14 hdfs://192.168.2.31:9000/readme2.txt
*/
@Test
public void testViewHdfsFiles ()
{
FileStatus[] listStatus = null;
try {
listStatus = fileSystem.listStatus( new Path( "/"));
StringBuffer sb = new StringBuffer();
for (FileStatus fileStatus : listStatus)
{
short replication = fileStatus.getReplication();
String type = fileStatus.isDirectory()
? "Directory" : "File" ;
FsPermission permission = fileStatus.getPermission();
String path = fileStatus.getPath().toString();
String owner = fileStatus.getOwner();
String group = fileStatus.getGroup();
String modifiyDate =
secondTranToDate(fileStatus.getModificationTime());
sb.append( permission).append( "\t").append(type ).append("\t" ).append(replication ).append("\t" )
.append( owner).append( "\t").append(group ).append("\t" ).append(modifiyDate ).append("\t" )
.append( path).append( "\n");
}
System. out.println(sb .toString());
} catch (Exception e)
{
e.printStackTrace();
}
}
/**
*
* @param second
* @return
*/
private String
secondTranToDate(long second)
{
SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd
hh:mm:ss");
Date date = new Date( second);
String modifiyDate = sdf.format( date);
return modifiyDate ;
}
}