Java操作HDFS对象

1. 使用Eclipse工具创建一个maven工程，配置pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>

<groupId>com.sl</groupId>
<artifactId>bigdatatest</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>war</packaging>

<name>bigdatatest</name>
<url>http://maven.apache.org</url>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.6.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.4</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.6.4</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

2. Java操作HDFS对象

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

/**
* @description  Java操作HDFS对象
*   得到DFSClient对象并得到HDFS根目录下所有对象
*   FileSystem对象就是DFSClient对象，也就是描述HDFS文件系统的一个对象。
*/
public class FileSystemOperate {

FileSystem fileSystem;
Configuration conf;

/**
* URI对象通过使用HDFS协议指定HDFS文件系统配置的节点地址
* URI中的ip地址可以换成主机名但是需要配置好ip与主机的映射关系
* @throws URISyntaxException
* @throws IOException
*/
@Before
public void before() throws URISyntaxException, IOException {
//1.得到DFSClient对象
conf = new Configuration();
URI uri = new URI("hdfs://192.168.1.200:9000");
fileSystem = FileSystem.get(uri, conf);
System.out.println(fileSystem);
}

@After
public void after() throws IOException {
fileSystem.close();
}

@Test
public void getFileStatus() throws FileNotFoundException, IllegalArgumentException, IOException {
//2.获取路径根目录下的所有对象信息
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : listStatus) {
System.out.println(fileStatus);
}
}
}

@Test
public void mkdir() throws IllegalArgumentException, IOException {
//3.在根目录下创建目录
boolean mkdirs = fileSystem.mkdirs(new Path("/myhdfs"));//用Java操作hdfs创建的文件夹-目录
System.out.println(mkdirs);
}

@Test
public void delete() throws IllegalArgumentException, IOException {
//4.删除目录
boolean delete = fileSystem.delete(new Path("/myhdfs"), true);
System.out.println(delete);
}

@Test
public void deleteNotExist() throws IllegalArgumentException, IOException {
//删除不存在的目录或文件会返回false
boolean deleteNotExist = fileSystem.delete(new Path("/myhdfs1"), true);
System.out.println(deleteNotExist);
}

@Test
public void uploadData() throws IllegalArgumentException, IOException {
//5.上传文件-将Windows下hosts文件上传
FSDataOutputStream create = fileSystem.create(new Path("/hosts1"));
FileInputStream fsIn = new FileInputStream("C:\\Windows\\System32\\drivers\\etc\\hosts");
IOUtils.copyBytes(fsIn, create, conf, true);
}

@Test
public void readFile() throws IOException {
//6.读取文件
FSDataInputStream open = fileSystem.open(new Path("/hosts1"));
IOUtils.copyBytes(open, System.out, conf, true);
}

@Test
public void readFile2() throws IllegalArgumentException, IOException {
//7.按行读取文件
FSDataInputStream fsin = fileSystem.open(new Path("/input/test1.txt"));
byte RL = '\r';//回车
byte NL = '\n';//换行
byte[] buffer = new byte[1024*64];
int length = fsin.read(buffer);
int start_pos = 0;
int pos = 0;
String str = null;
for (pos = 0;pos < length; pos++) {
//当为回车或者换行符时
if (buffer[pos] == RL || buffer[pos] == NL) {
//根据起始坐标start_pos和当前坐标生成字符串
str = new String(buffer, start_pos, pos);
System.out.println(str);

pos++;//当前坐标加1
start_pos = pos;//将新的位置赋给起始坐标
}
}
}

@Test
public void deleteFile() throws IllegalArgumentException, IOException {
boolean deleteFile = fileSystem.delete(new Path("/hosts1"), true);//删除文件
System.out.println(deleteFile);
}