package cn.itcast.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsUtil {
FileSystem fs = null;
@Before
public void init() throws Exception{
//讀取classpath下的xxx-site.xml 配置文件,並解析其內容,封裝到conf對象中
Configuration conf = new Configuration();
//也可以在代碼中對conf中的配置信息進行手動設置,會覆蓋掉配置文件中的讀取的值
conf.set("fs.defaultFS", "hdfs://weekend110:9000/");
//根據配置信息,去獲取一個具體文件系統的客戶端操作實例對象
fs = FileSystem.get(new URI("hdfs://weekend110:9000/"),conf,"hadoop");
}
/**
* 上傳文件,比較底層的寫法
*
* @throws Exception
*/
@Test
public void upload() throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://weekend110:9000/");
FileSystem fs = FileSystem.get(conf);
Path dst = new Path("hdfs://weekend110:9000/aa/qingshu.txt");
FSDataOutputStream os = fs.create(dst);
FileInputStream is = new FileInputStream("c:/qingshu.txt");
IOUtils.copy(is, os);
}
/**
* 上傳文件,封裝好的寫法
* @throws Exception
* @throws IOException
*/
@Test
public void upload2() throws Exception, IOException{
fs.copyFromLocalFile(new Path("c:/qingshu.txt"), new Path("hdfs://weekend110:9000/aaa/bbb/ccc/qingshu2.txt"));
}
/**
* 下載文件
* @throws Exception
* @throws IllegalArgumentException
*/
@Test
public void download() throws Exception {
fs.copyToLocalFile(new Path("hdfs://weekend110:9000/aa/qingshu2.txt"), new Path("c:/qingshu2.txt"));
}
/**
* 查看文件信息
* @throws IOException
* @throws IllegalArgumentException
* @throws FileNotFoundException
*
*/
@Test
public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
// listFiles列出的是文件信息,而且提供遞歸遍歷
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
while(files.hasNext()){
LocatedFileStatus file = files.next();
Path filePath = file.getPath();
String fileName = filePath.getName();
System.out.println(fileName);
}
System.out.println("---------------------------------");
//listStatus 可以列出文件和文件夾的信息,但是不提供自帶的遞歸遍歷
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for(FileStatus status: listStatus){
String name = status.getPath().getName();
System.out.println(name + (status.isDirectory()?" is dir":" is file"));
}
}
/**
* 創建文件夾
* @throws Exception
* @throws IllegalArgumentException
*/
@Test
public void mkdir() throws IllegalArgumentException, Exception {
fs.mkdirs(new Path("/aaa/bbb/ccc"));
}
/**
* 刪除文件或文件夾
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void rm() throws IllegalArgumentException, IOException {
fs.delete(new Path("/aa"), true);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://weekend110:9000/");
FileSystem fs = FileSystem.get(conf);
FSDataInputStream is = fs.open(new Path("/jdk-7u65-linux-i586.tar.gz"));
FileOutputStream os = new FileOutputStream("c:/jdk7.tgz");
IOUtils.copy(is, os);
}
}
Java操作hdfs的文件增刪改查
發表評論
所有評論
還沒有人評論,想成為第一個評論的人麼? 請在上方評論欄輸入並且點擊發布.