对HDFS API基本操作都是通过 org.apache.hadoop.fs.FileSystem
类进行的,以下是一些常见的操作:
package HdfsAPI;import java.io.BufferedInputStream;import java.io.File;import java.io.FileInputStream;import java.io.IOException;import java.io.InputStream;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.BlockLocation;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.junit.Test;import junit.framework.TestCase;public class HDFSAPI extends TestCase{ public static String hdfsUrl = "hdfs://master:8020"; //创建文件夹 @Test public void testHDFSmkdir() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); Path path = new Path("/liguodong/hdfs"); fs.mkdirs(path); } //创建文件 @Test public void testHDFSmkFile() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); Path path = new Path("/liguodong/hdfs/liguodong.txt"); FSDataOutputStream fdos = fs.create(path); fdos.write("hello hadoop".getBytes()); } //重命名 @Test public void testHDFSRenameFile() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); Path path = new Path("/liguodong/hdfs/liguodong.txt"); Path Renamepath = new Path("/liguodong/hdfs/love.txt"); System.out.println(fs.rename(path, Renamepath)); } //上传一个本地文件 @Test public void testHDFSPutLocalFile1() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); //因为在windows上面调试,所以选择的是windows的文件夹, //假设在Linxu,须要Linux文件夹。 Path srcpath = new Path("g:/liguodong.txt"); Path destpath = new Path("/liguodong/hdfs"); fs.copyFromLocalFile(srcpath, destpath); } @Test public void testHDFSPutLocalFile2() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); //因为在windows上面调试,所以选择的是windows的文件夹, //假设在Linxu,须要Linux文件夹。 String srcpath ="g:/oncemore.txt"; Path destpath = new Path("/liguodong/hdfs/kkk.txt"); InputStream is = new BufferedInputStream( new FileInputStream(new File(srcpath))); FSDataOutputStream fdos = fs.create(destpath); IOUtils.copyBytes(is, fdos, 4094); } //查看某个文件夹以下的全部文件 @Test public void testHDFSListFiles() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); Path path = new Path("/liguodong/hdfs"); FileStatus[] files = fs.listStatus(path); for (FileStatus file : files) { System.out.println(file.getPath().toString()); } } //查看某个文件的数据块信息 @Test public void testHDFSGetBlockInfo() throws IOException{ Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(hdfsUrl),conf); Path path = new Path("/liguodong/hdfs/kkk.txt"); FileStatus filestatus = fs.getFileStatus(path); BlockLocation[] blkLoc = fs.getFileBlockLocations (filestatus, 0, filestatus.getLen()); for (BlockLocation loc : blkLoc) { for (int i = 0; i < loc.getHosts().length; i++) { //获取数据块在哪些主机上 System.out.println(loc.getHosts()[i]);//获取文件块的主机名 //因为这个文件仅仅有一个块。所以输出结果为:slave2、slave1、slave5 } } } }