java实现hdfs文件操作

作者 zhan-bin 日期 2018-07-03
hadoop
java实现hdfs文件操作

java实现hdfs文件操作

作者：Zhan-bin
日期：2018-7-3

1.导包

  • 在项目文件夹右键- Build Path - Add External Archive
    1
  • 将hadoop解压后找到share文件夹进入hadoop文件夹
  • common文件夹里面需要导入的包： hadoop-common-2.9.1.jar和lib里面的所有包
  • hdfs文件夹里面需要导入的包：hadoop-hdfs-2.9.1.jar，hadoop-hdfs-client-2.9.1.jar

2.各种文件操作代码

上传文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
	 * upload the local file to the hds notice that the path is full like
	 * /tmp/test.c
	 */

	public static void uploadLocalFile2HDFS(String s, String d)
			throws IOException {
		Configuration config = new Configuration();
		config.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem hdfs = FileSystem.get(config);
		Path src = new Path(s);
		Path dst = new Path(d);
		hdfs.copyFromLocalFile(src, dst);
		hdfs.close();

	}

创建文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
	 * create a new file in the hdfs. notice that the toCreateFilePath is the
	 * full path and write the content to the hdfs file.
	 */

	public static void createNewHDFSFile(String toCreateFilePath, String content)
			throws IOException {
		Configuration config = new Configuration();
		config.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem hdfs = FileSystem.get(config);
		FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath));
		os.write(content.getBytes("UTF-8"));
		os.close();
		hdfs.close();

	}

删除文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * 
 * delete the hdfs file
 * 
 * notice that the dst is the full path name
 */

public static boolean deleteHDFSFile(String dst) throws IOException {
	Configuration config = new Configuration();
	config.set("fs.default.name", "hdfs://10.21.21.2:9000");
	FileSystem hdfs = FileSystem.get(config);
	Path path = new Path(dst);
	boolean isDeleted = hdfs.delete(path);
	hdfs.close();
	return isDeleted;

}

读取文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
/**
	 * read the hdfs file content
	 * 
	 * notice that the dst is the full path name
	 */

	public static byte[] readHDFSFile(String dst) throws Exception

	{

		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem fs = FileSystem.get(conf);

		// check if the file exists

		Path path = new Path(dst);

		if (fs.exists(path))

		{

			FSDataInputStream is = fs.open(path);

			// get the file info to create the buffer

			FileStatus stat = fs.getFileStatus(path);

			// create the buffer

			byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat
					.getLen()))];

			is.readFully(0, buffer);

			is.close();

			fs.close();

			return buffer;

		}

		else

		{

			throw new Exception("the file is not found .");

		}

	}

创建文件夹（目录）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/**
	 * make a new dir in the hdfs
	 * 
	 * 
	 * 
	 * the dir may like '/tmp/testdir'
	 */

	public static void mkdir(String dir) throws IOException {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem fs = FileSystem.get(conf);
		fs.mkdirs(new Path(dir));
		fs.close();

	}

删除文件夹（目录）

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/**
	 * delete a dir in the hdfs
	 * 
	 * 
	 * 
	 * dir may like '/tmp/testdir'
	 */

	public static void deleteDir(String dir) throws IOException {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem fs = FileSystem.get(conf);
		fs.delete(new Path(dir));
		fs.close();

	}

列出指定目录下的所有文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
public static void listAll(String dir) throws IOException {
		Configuration conf = new Configuration();
		conf.set("fs.default.name", "hdfs://10.21.21.2:9000");
		FileSystem fs = FileSystem.get(conf);
		FileStatus[] stats = fs.listStatus(new Path(dir));
		for (int i = 0; i < stats.length; ++i) {
			if (stats[i].isFile()) {
				// regular file
				System.out.println(stats[i].getPath().toString());
			} else if (stats[i].isDirectory()) {
				// dir
				System.out.println(stats[i].getPath().toString());
			} else if (stats[i].isSymlink()) {
				// is s symlink in linux
				System.out.println(stats[i].getPath().toString());
			}
		}
		fs.close();
	}
FRIENDS