Hadoop学习笔记-HDFS常用API

HDFS-API

Maven依赖

<dependencies>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-common</artifactId>
        <version>3.3.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>3.3.0</version>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-hdfs</artifactId>
        <version>3.3.0</version>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.13</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.13</version>
        <scope>compile</scope>
    </dependency>
</dependencies>

常用API

1. 建立链接

   public Configuration conf = null;
   public FileSystem fs = null;
   
   @Before
   public void conn() throws IOException, InterruptedException {
       conf = new Configuration(true);
       // fs = FileSystem.get(conf);		//resource目录中放置hadoop配置的文件
       fs = FileSystem.get(URI.create("hdfs://mycluster"),conf,"xun");
   }
   

2. 创建目录

   @Test
   public void mkdir() throws Exception {
       Path dir = new Path("/szx");	//要创建的目录
       if(fs.exists(dir)) {			//如果目录存在就先删除
           fs.delete(dir, true);
       }
       fs.mkdirs(dir);					//创建目录
   }
   

3. 上传文件

   @Test
   public void upload() throws IOException {
       BufferedInputStream input = new BufferedInputStream(new FileInputStream(new File("./data/hello.txt")));		//输入流，将文件放到内存
       Path outfile = new Path("/szx/outfile.txt");	//上传至哪个目录
       FSDataOutputStream output = fs.create(outfile);	//输出流
       IoUtils.copyBytes(input,output,conf,true);
   }
   

4. 获取块信息

   @Test
   public void block() throws IOException {
       Path file = new Path("/user/xun/data.txt");				//文件路径
       FileStatus fss = fs.getFileStatus(file);				//获取文件
       BlockLocation[] blocks = fs.getFileBlockLocations(fss, 0, fss.getLen());	//获取从文件的起始位置和结束为止的内容
       for (BlockLocation block : blocks) {
           System.out.println(block);		//打印的是文件的信息，偏移量，长度，存放在哪些块中
       }
       FSDataInputStream in = fs.open(file);
       in.seek(1048576);					//设置偏移量，直接从1048576字节读取(定位)
       System.out.println((char)in.readByte());	//一个个的输出字符
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
       System.out.println((char)in.readByte());
   }