package com.qcloud.cos.demo;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.net.URI;
import java.nio.ByteBuffer;
public class Demo {
private static FileSystem initFS() throws IOException {
Configuration conf = new Configuration();
conf.set("fs.cosn.impl", "org.apache.hadoop.fs.CosFileSystem");
conf.set("fs.AbstractFileSystem.cosn.impl", "org.apache.hadoop.fs.CosN");
conf.set("fs.cosn.userinfo.secretId", "xxxxxx");
conf.set("fs.cosn.userinfo.secretKey", "xxxxxx");
conf.set("fs.cosn.bucket.region", "xxxxxx");
conf.set("fs.cosn.tmp.dir", "/data/chdfs_tmp_cache");
conf.set("fs.cosn.trsf.fs.AbstractFileSystem.ofs.impl", "com.qcloud.chdfs.fs.CHDFSDelegateFSAdapter");
conf.set("fs.cosn.trsf.fs.ofs.impl", "com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter");
conf.set("fs.cosn.trsf.fs.ofs.tmp.cache.dir", "com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter");
conf.set("fs.cosn.trsf.fs.ofs.impl", "com.qcloud.chdfs.fs.CHDFSHadoopFileSystemAdapter");
conf.set("fs.cosn.trsf.fs.ofs.tmp.cache.dir", "/data/chdfs_tmp_cache");
conf.set("fs.cosn.trsf.fs.ofs.user.appid", "1250000000");
conf.set("fs.cosn.trsf.fs.ofs.bucket.region", "ap-beijing");
conf.set("fs.cosn.crc64.checksum.enabled", "true");
String cosHadoopFSUrl = "cosn://examplebucket-12500000000/";
return FileSystem.get(URI.create(cosHadoopFSUrl), conf);
}
private static void mkdir(FileSystem fs, Path filePath) throws IOException {
fs.mkdirs(filePath);
}
private static void createFile(FileSystem fs, Path filePath) throws IOException {
FSDataOutputStream out = fs.create(filePath, true);
try {
String content = "test write file";
out.write(content.getBytes());
} finally {
out.close();
}
}
private static void readFile(FileSystem fs, Path filePath) throws IOException {
FSDataInputStream in = fs.open(filePath);
try {
byte[] buf = new byte[4096];
int readLen = -1;
do {
readLen = in.read(buf);
} while (readLen >= 0);
} finally {
IOUtils.closeQuietly(in);
}
}
private static void queryFileOrDirStatus(FileSystem fs, Path path) throws IOException {
FileStatus fileStatus = fs.getFileStatus(path);
if (fileStatus.isDirectory()) {
System.out.printf("path %s is dir\\n", path);
return;
}
long fileLen = fileStatus.getLen();
long accessTime = fileStatus.getAccessTime();
long modifyTime = fileStatus.getModificationTime();
String owner = fileStatus.getOwner();
String group = fileStatus.getGroup();
System.out.printf("path %s is file, fileLen: %d, accessTime: %d, modifyTime: %d, owner: %s, group: %s\\n",
path, fileLen, accessTime, modifyTime, owner, group);
}
private static void getFileCheckSum(FileSystem fs, Path path) throws IOException {
FileChecksum checksum = fs.getFileChecksum(path);
System.out.printf("path %s, checkSumType: %s, checkSumCrcVal: %d\\n",
path, checksum.getAlgorithmName(), ByteBuffer.wrap(checksum.getBytes()).getInt());
}
private static void copyFileFromLocal(FileSystem fs, Path chdfsPath, Path localPath) throws IOException {
fs.copyFromLocalFile(localPath, chdfsPath);
}
private static void copyFileToLocal(FileSystem fs, Path chdfsPath, Path localPath) throws IOException {
fs.copyToLocalFile(chdfsPath, localPath);
}
private static void renamePath(FileSystem fs, Path oldPath, Path newPath) throws IOException {
fs.rename(oldPath, newPath);
}
private static void listDirPath(FileSystem fs, Path dirPath) throws IOException {
FileStatus[] dirMemberArray = fs.listStatus(dirPath);
for (FileStatus dirMember : dirMemberArray) {
System.out.printf("dirMember path %s, fileLen: %d\\n", dirMember.getPath(), dirMember.getLen());
}
}
private static void deleteFileOrDir(FileSystem fs, Path path, boolean recursive) throws IOException {
fs.delete(path, recursive);
}
private static void closeFileSystem(FileSystem fs) throws IOException {
fs.close();
}
public static void main(String[] args) throws IOException {
FileSystem fs = initFS();
Path chdfsFilePath = new Path("/folder/exampleobject.txt");
createFile(fs, chdfsFilePath);
readFile(fs, chdfsFilePath);
queryFileOrDirStatus(fs, chdfsFilePath);
getFileCheckSum(fs, chdfsFilePath);
Path localFilePath = new Path("file:///home/hadoop/cosn_demo/data/exampleobject.txt");
copyFileFromLocal(fs, chdfsFilePath, localFilePath);
Path localDownFilePath = new Path("file:///home/hadoop/cosn_demo/data/exampleobject.txt");
copyFileToLocal(fs, chdfsFilePath, localDownFilePath);
Path newPath = new Path("/doc/example.txt");
renamePath(fs, chdfsFilePath, newPath);
deleteFileOrDir(fs, newPath, false);
Path dirPath = new Path("/folder");
mkdir(fs, dirPath);
Path subFilePath = new Path("/folder/exampleobject.txt");
createFile(fs, subFilePath);
listDirPath(fs, dirPath);
deleteFileOrDir(fs, dirPath, true);
closeFileSystem(fs);
}
}
Was this page helpful?