Java程序操作HDFS

1.新建项目

2.导包

解压hadoop-2.7.3.tar.gz

E:\工具\大数据\大数据提升资料\01-软件资料\06-Hadoop\安装包\Java1.8环境

下编译\hadoop-2.7.3\hadoop-2.7.3\share\hadoop\common

E:\工具\大数据\大数据提升资料\01-软件资料\06-Hadoop\安装包\Java1.8环境

下编译\hadoop-2.7.3\hadoop-2.7.3\share\hadoop\common\lib

E:\工具\大数据\大数据提升资料\01-软件资料\06-Hadoop\安装包\Java1.8环境

下编译\hadoop-2.7.3\hadoop-2.7.3\share\hadoop\hdfs

JUNIT 单元测试

/**

* 单元测试

* 1.修饰符必须是public

* 2.不能有参数

* 3.不能有返回值

*

* 运行：选中方法名来运行

* run as --junit test

*/

@Test

public void m1(){

System.out.println("hello");

}

package com.zy.hdfs;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class test {
    public static void main(String[] args) throws Exception {
        //上传
        
        //1实例化configuration
        Configuration configuration = new Configuration();
        //-------------------
        //设置一些属性
        configuration.set("dfs.replication", "2");//2个副本
        configuration.set("dfs.blocksize", "80m");//按照80m切分
        //----------------------
        //2.获取操作文件系统的客户端实例
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000/"), configuration, "root");
        //3.执行上传操作  (src 本地资源 ,dst 上传目的地)
        fs.copyFromLocalFile(new Path("C:\\Users\\Administrator\\Desktop\\wc.txt"), new Path("/"));//两次上传一样的会被覆盖
        //4.关闭链接
        fs.close();
    }
    
    

    @Test
    public void download() throws Exception{
        //下载
        
        Configuration configuration = new Configuration();
        
        FileSystem fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000"), configuration, "root");
        
        fs.copyToLocalFile(false, new Path("/jdk-8u141-linux-x64.tar.gz"), new Path("E://jdk-8u141-linux-x64.tar.gz"), true);
        
        fs.close();    
        
    }
    //----------------
    FileSystem fs=null;
    @Before//在每次运行junit时先执行
    public void before() throws Exception{
        System.out.println("11111");
        Configuration configuration = new Configuration();    
        fs = FileSystem.get(new URI("hdfs://192.168.64.111:9000/"), configuration, "root");        
    }
    //---------------
    @Test
    public void delete() throws Exception{//删除
        fs.delete(new Path("/弹性表达式.doc"), true);//为true删除文件夹及里面的内容，false只能删除空的文件夹
        fs.close();
        
    }
    @Test
    public void mkdir() throws Exception{//创建目录
        fs.mkdirs(new Path("/a/b/c"));
        fs.close();
        
    }
    
    @Test
    public void list() throws Exception{
        FileStatus[] listStatus = fs.listStatus(new Path("/"));//不会递归进去
        for (FileStatus fileStatus : listStatus) {
            if(fileStatus.isFile()){
                System.out.println("文件名:"+fileStatus.getPath().toString());
                
            }else{
                System.out.println("目录:"+fileStatus.getPath().toString());
                
            }
            
        }
        fs.close();
        
    }
    @Test
    public void list2() throws Exception{//递归遍历出所有文件
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);//为true，递归遍历
        while(listFiles.hasNext()){
            LocatedFileStatus next = listFiles.next();
            System.out.println(next.toString());
            System.out.println("文件块尺寸"+next.getBlockSize());
         
        }

    }
 
}