ZhouXY108
/
fastdfs-client-java
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support appender file rename to normal file

dev
YuQing 2019-11-13 17:07:01 +08:00
parent 3fb2d4952e
commit 437a993a66
9 changed files with 300 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
HISTORY
View File

@ -1,4 +1,8 @@
Version 1.27 2019-11-11
* StorageClient add function: regenerate_filename for appender file
StorageClient1 add function: regenerate_filename1 for appender file
Version 1.26 2017-04-17
* IniFileReader use getResourceAsStream first
* change charactor encoding to UTF-8

62
src/main/java/org/csource/fastdfs/FileInfo.java
View File

@ -18,6 +18,12 @@ import java.util.Date;
* @version Version 1.23
*/
public class FileInfo {
public static final short FILE_TYPE_NORMAL = 1;
public static final short FILE_TYPE_APPENDER = 2;
public static final short FILE_TYPE_SLAVE = 4;
protected boolean fetch_from_server;
protected short file_type;
protected String source_ip_addr;
protected long file_size;
protected Date create_timestamp;
@ -26,18 +32,60 @@ public class FileInfo {
/**
* Constructor
*
* @param file_size the file size
* @param create_timestamp create timestamp in seconds
* @param crc32 the crc32 signature
* @param source_ip_addr the source storage ip address
* @param fetch_from_server if fetch from server flag
* @param file_type the file type
* @param file_size the file size
* @param create_timestamp create timestamp in seconds
* @param crc32 the crc32 signature
* @param source_ip_addr the source storage ip address
*/
public FileInfo(long file_size, int create_timestamp, int crc32, String source_ip_addr) {
public FileInfo(boolean fetch_from_server, short file_type, long file_size,
int create_timestamp, int crc32, String source_ip_addr)
{
this.fetch_from_server = fetch_from_server;
this.file_type = file_type;
this.file_size = file_size;
this.create_timestamp = new Date(create_timestamp * 1000L);
this.crc32 = crc32;
this.source_ip_addr = source_ip_addr;
}
/**
* get the fetch_from_server flag
*
* @return the fetch_from_server flag
*/
public boolean getFetchFromServer() {
return this.fetch_from_server;
}
/**
* set the fetch_from_server flag
*
* @param fetch_from_server the fetch from server flag
*/
public void setFetchFromServer(boolean fetch_from_server) {
this.fetch_from_server = fetch_from_server;
}
/**
* get the file type
*
* @return the file type
*/
public short getFileType() {
return this.file_type;
}
/**
* set the file type
*
* @param file_type the file type
*/
public void setFileType(short file_type) {
this.file_type = file_type;
}
/**
* get the source ip address of the file uploaded to
*
@ -117,7 +165,9 @@ public class FileInfo {
*/
public String toString() {
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return "source_ip_addr = " + this.source_ip_addr + ", " +
return "fetch_from_server = " + this.fetch_from_server + ", " +
"file_type = " + this.file_type + ", " +
"source_ip_addr = " + this.source_ip_addr + ", " +
"file_size = " + this.file_size + ", " +
"create_timestamp = " + df.format(this.create_timestamp) + ", " +
"crc32 = " + this.crc32;

2
src/main/java/org/csource/fastdfs/ProtoCommon.java
View File

@ -49,6 +49,8 @@ public class ProtoCommon {
public static final byte STORAGE_PROTO_CMD_APPEND_FILE = 24; //append file
public static final byte STORAGE_PROTO_CMD_MODIFY_FILE = 34; //modify appender file
public static final byte STORAGE_PROTO_CMD_TRUNCATE_FILE = 36; //truncate appender file
public static final byte STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME = 38; //rename appender file to normal file
public static final byte STORAGE_PROTO_CMD_RESP = TRACKER_PROTO_CMD_RESP;
public static final byte FDFS_STORAGE_STATUS_INIT = 0;
public static final byte FDFS_STORAGE_STATUS_WAIT_SYNC = 1;

139
src/main/java/org/csource/fastdfs/StorageClient.java
View File

@ -19,9 +19,10 @@ import java.util.Arrays;
/**
* Storage client for 2 fields file id: group name and filename
* Note: the instance of this class is NOT thread safe !!!
* if not necessary, do NOT set storage server instance
*
* @author Happy Fish / YuQing
* @version Version 1.24
* @version Version 1.27
*/
public class StorageClient {
public final static Base64 base64 = new Base64('-', '_', '.', 0);
@ -29,8 +30,6 @@ public class StorageClient {
protected StorageServer storageServer;
protected byte errno;
/**
* constructor using global settings in class ClientGlobal
*/
@ -39,8 +38,19 @@ public class StorageClient {
this.storageServer = null;
}
/**
* constructor with tracker server
*
* @param trackerServer the tracker server, can be null
*/
public StorageClient(TrackerServer trackerServer) {
this.trackerServer = trackerServer;
this.storageServer = null;
}
/**
* constructor with tracker server and storage server
* NOTE: if not necessary, do NOT set storage server instance
*
* @param trackerServer the tracker server, can be null
* @param storageServer the storage server, can be null
@ -601,6 +611,94 @@ public class StorageClient {
modify_size, callback);
}
/**
* regenerate filename for appender file
*
* @param group_name the group name of appender file
* @param appender_filename the appender filename
* @return 2 elements string array if success:<br>
* <ul><li> results[0]: the group name to store the file</li></ul>
* <ul><li> results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] regenerate_appender_filename(String group_name, String appender_filename) throws IOException, MyException {
byte[] header;
boolean bNewConnection;
Socket storageSocket;
byte[] hexLenBytes;
byte[] appenderFilenameBytes;
int offset;
long body_len;
if ((group_name == null || group_name.length() == 0) ||
(appender_filename == null || appender_filename.length() == 0)) {
this.errno = ProtoCommon.ERR_NO_EINVAL;
return null;
}
bNewConnection = this.newUpdatableStorageConnection(group_name, appender_filename);
try {
storageSocket = this.storageServer.getSocket();
appenderFilenameBytes = appender_filename.getBytes(ClientGlobal.g_charset);
body_len = appenderFilenameBytes.length;
header = ProtoCommon.packHeader(ProtoCommon.STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME, body_len, (byte) 0);
byte[] wholePkg = new byte[(int) (header.length + body_len)];
System.arraycopy(header, 0, wholePkg, 0, header.length);
offset = header.length;
System.arraycopy(appenderFilenameBytes, 0, wholePkg, offset, appenderFilenameBytes.length);
offset += appenderFilenameBytes.length;
OutputStream out = storageSocket.getOutputStream();
out.write(wholePkg);
ProtoCommon.RecvPackageInfo pkgInfo = ProtoCommon.recvPackage(storageSocket.getInputStream(),
ProtoCommon.STORAGE_PROTO_CMD_RESP, -1);
this.errno = pkgInfo.errno;
if (pkgInfo.errno != 0) {
return null;
}
if (pkgInfo.body.length <= ProtoCommon.FDFS_GROUP_NAME_MAX_LEN) {
throw new MyException("body length: " + pkgInfo.body.length + " <= " + ProtoCommon.FDFS_GROUP_NAME_MAX_LEN);
}
String new_group_name = new String(pkgInfo.body, 0, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN).trim();
String remote_filename = new String(pkgInfo.body, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN,
pkgInfo.body.length - ProtoCommon.FDFS_GROUP_NAME_MAX_LEN);
String[] results = new String[2];
results[0] = new_group_name;
results[1] = remote_filename;
return results;
} catch (IOException ex) {
if (!bNewConnection) {
try {
this.storageServer.close();
} catch (IOException ex1) {
ex1.printStackTrace();
} finally {
this.storageServer = null;
}
}
throw ex;
} finally {
if (bNewConnection) {
try {
this.storageServer.close();
} catch (IOException ex1) {
ex1.printStackTrace();
} finally {
this.storageServer = null;
}
}
}
}
/**
* upload file to storage server
*
@ -1514,26 +1612,42 @@ public class StorageClient {
byte[] buff = base64.decodeAuto(remote_filename.substring(ProtoCommon.FDFS_FILE_PATH_LEN,
ProtoCommon.FDFS_FILE_PATH_LEN + ProtoCommon.FDFS_FILENAME_BASE64_LENGTH));
short file_type;
long file_size = ProtoCommon.buff2long(buff, 4 * 2);
if (((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) ||
((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) && ((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0))) ||
((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0)) { //slave file or appender file
if (((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0))
{
file_type = FileInfo.FILE_TYPE_APPENDER;
}
else if ((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) ||
((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) &&
((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0)))
{
file_type = FileInfo.FILE_TYPE_SLAVE;
}
else {
file_type = FileInfo.FILE_TYPE_NORMAL;
}
if (file_type == FileInfo.FILE_TYPE_SLAVE ||
file_type == FileInfo.FILE_TYPE_APPENDER)
{ //slave file or appender file
FileInfo fi = this.query_file_info(group_name, remote_filename);
if (fi == null) {
return null;
}
fi.setFileType(file_type);
return fi;
}
FileInfo fileInfo = new FileInfo(file_size, 0, 0, ProtoCommon.getIpAddress(buff, 0));
fileInfo.setCreateTimestamp(ProtoCommon.buff2int(buff, 4));
int create_timestamp = ProtoCommon.buff2int(buff, 4);
if ((file_size >> 63) != 0) {
file_size &= 0xFFFFFFFFL; //low 32 bits is file size
fileInfo.setFileSize(file_size);
}
fileInfo.setCrc32(ProtoCommon.buff2int(buff, 4 * 4));
int crc32 = ProtoCommon.buff2int(buff, 4 * 4);
return fileInfo;
return new FileInfo(false, file_type, file_size, create_timestamp,
crc32, ProtoCommon.getIpAddress(buff, 0));
}
/**
@ -1590,7 +1704,8 @@ public class StorageClient {
int create_timestamp = (int) ProtoCommon.buff2long(pkgInfo.body, ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE);
int crc32 = (int) ProtoCommon.buff2long(pkgInfo.body, 2 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE);
String source_ip_addr = (new String(pkgInfo.body, 3 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE, ProtoCommon.FDFS_IPADDR_SIZE)).trim();
return new FileInfo(file_size, create_timestamp, crc32, source_ip_addr);
return new FileInfo(true, FileInfo.FILE_TYPE_NORMAL, file_size,
create_timestamp, crc32, source_ip_addr);
} catch (IOException ex) {
if (!bNewConnection) {
try {

37
src/main/java/org/csource/fastdfs/StorageClient1.java
View File

@ -15,9 +15,11 @@ import java.io.IOException;
/**
* Storage client for 1 field file id: combined group name and filename
* Note: the instance of this class is NOT thread safe !!!
* if not necessary, do NOT set storage server instance
*
* @author Happy Fish / YuQing
* @version Version 1.21
* @version Version 1.27
*/
public class StorageClient1 extends StorageClient {
public static final String SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR = "/";
@ -30,7 +32,17 @@ public class StorageClient1 extends StorageClient {
}
/**
* constructor
* constructor with trackerServer
*
* @param trackerServer the tracker server, can be null
*/
public StorageClient1(TrackerServer trackerServer) {
super(trackerServer);
}
/**
* constructor with trackerServer and storageServer
* NOTE: if not necessary, do NOT set storage server instance
*
* @param trackerServer the tracker server, can be null
* @param storageServer the storage server, can be null
@ -514,6 +526,27 @@ public class StorageClient1 extends StorageClient {
return this.modify_file(parts[0], parts[1], file_offset, modify_size, callback);
}
/**
* regenerate filename for appender file
*
* @param appender_file_id the appender file id
* @return the regenerated file id, return null if fail
*/
public String regenerate_appender_filename1(String appender_file_id) throws IOException, MyException {
String[] parts = new String[2];
this.errno = this.split_file_id(appender_file_id, parts);
if (this.errno != 0) {
return null;
}
String[] new_parts = this.regenerate_appender_filename(parts[0], parts[1]);
if (new_parts != null) {
return new_parts[0] + SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR + new_parts[1];
} else {
return null;
}
}
/**
* delete file from storage server
*

18
src/main/java/org/csource/fastdfs/TrackerServer.java
View File

@ -81,29 +81,29 @@ public class TrackerServer {
public boolean isConnected(){
boolean isConnected = false;
if (sock != null){
if (sock.isConnected()){
if (sock != null) {
if (sock.isConnected()) {
isConnected = true;
}
}
return isConnected;
}
public boolean isAvaliable(){
if(isConnected()){
if(sock.getPort() == 0){
public boolean isAvaliable() {
if (isConnected()) {
if (sock.getPort() == 0) {
return false;
}
if(sock.getInetAddress() == null){
if (sock.getInetAddress() == null) {
return false;
}
if(sock.getRemoteSocketAddress() == null){
if (sock.getRemoteSocketAddress() == null) {
return false;
}
if(sock.isInputShutdown()){
if (sock.isInputShutdown()) {
return false;
}
if(sock.isOutputShutdown()){
if (sock.isOutputShutdown()) {
return false;
}
return true;

77
src/test/java/org/csource/fastdfs/Test1.java
View File

@ -6,42 +6,57 @@ import org.csource.fastdfs.*;
import java.net.InetSocketAddress;
public class Test1 {
public static void main(String args[]) {
try {
ClientGlobal.init("fdfs_client.conf");
System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms");
System.out.println("charset=" + ClientGlobal.g_charset);
public static void main(String args[]) {
try {
TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("10.0.11.243", 22122)});
TrackerClient tc = new TrackerClient(tg);
if (args.length < 1) {
System.out.println("Usage: 2 parameters, one is config filename, "
+ "the other is the local filename to upload");
return;
}
TrackerServer ts = tc.getConnection();
if (ts == null) {
System.out.println("getConnection return null");
return;
}
System.out.println("java.version=" + System.getProperty("java.version"));
StorageServer ss = tc.getStoreStorage(ts);
if (ss == null) {
System.out.println("getStoreStorage return null");
}
String conf_filename = args[0];
String local_filename;
String ext_name;
if (args.length > 1) {
local_filename = args[1];
ext_name = null;
}
else if (System.getProperty("os.name").equalsIgnoreCase("windows")) {
local_filename = "c:/windows/system32/notepad.exe";
ext_name = "exe";
} else {
local_filename = "/etc/hosts";
ext_name = "";
}
StorageClient1 sc1 = new StorageClient1(ts, ss);
ClientGlobal.init(conf_filename);
System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms");
System.out.println("charset=" + ClientGlobal.g_charset);
NameValuePair[] meta_list = null; //new NameValuePair[0];
String item;
String fileid;
if (System.getProperty("os.name").equalsIgnoreCase("windows")) {
item = "c:/windows/system32/notepad.exe";
fileid = sc1.upload_file1(item, "exe", meta_list);
} else {
item = "/etc/hosts";
fileid = sc1.upload_file1(item, "", meta_list);
}
TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("47.95.221.159", 22122)});
TrackerClient tc = new TrackerClient(tg);
System.out.println("Upload local file " + item + " ok, fileid=" + fileid);
} catch (Exception ex) {
ex.printStackTrace();
TrackerServer ts = tc.getConnection();
if (ts == null) {
System.out.println("getConnection return null");
return;
}
StorageServer ss = tc.getStoreStorage(ts);
if (ss == null) {
System.out.println("getStoreStorage return null");
}
StorageClient1 sc1 = new StorageClient1(ts, ss);
NameValuePair[] meta_list = null; //new NameValuePair[0];
String fileid = sc1.upload_file1(local_filename, ext_name, meta_list);
System.out.println("Upload local file " + local_filename + " ok, fileid: " + fileid);
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
}

11
src/test/java/org/csource/fastdfs/TestAppender.java
View File

@ -168,6 +168,17 @@ public class TestAppender {
System.err.println("append file fail, error no: " + errno);
}
startTime = System.currentTimeMillis();
results = client.regenerate_appender_filename(group_name, appender_filename);
System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms");
if (errno == 0) {
group_name = results[0];
appender_filename = results[1];
System.err.println(client.get_file_info(group_name, appender_filename));
} else {
System.err.println("regenerate_appender_filename fail, error no: " + errno);
}
startTime = System.currentTimeMillis();
errno = client.delete_file(group_name, remote_filename);
System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms");

10
src/test/java/org/csource/fastdfs/TestAppender1.java
View File

@ -160,6 +160,16 @@ public class TestAppender1 {
System.err.println("append file fail, error no: " + errno);
}
startTime = System.currentTimeMillis();
String new_file_id = client.regenerate_appender_filename1(appender_file_id);
System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms");
if (errno == 0) {
appender_file_id = new_file_id;
System.err.println(client.get_file_info1(appender_file_id));
} else {
System.err.println("regenerate_appender_filename fail, error no: " + errno);
}
startTime = System.currentTimeMillis();
errno = client.delete_file1(appender_file_id);
System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms");