From 437a993a6667e2afe645bc37f34c8102e2004ccc Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Wed, 13 Nov 2019 17:07:01 +0800 Subject: [PATCH] support appender file rename to normal file --- HISTORY | 4 + .../java/org/csource/fastdfs/FileInfo.java | 62 +++++++- .../java/org/csource/fastdfs/ProtoCommon.java | 2 + .../org/csource/fastdfs/StorageClient.java | 139 ++++++++++++++++-- .../org/csource/fastdfs/StorageClient1.java | 37 ++++- .../org/csource/fastdfs/TrackerServer.java | 18 +-- src/test/java/org/csource/fastdfs/Test1.java | 77 ++++++---- .../org/csource/fastdfs/TestAppender.java | 11 ++ .../org/csource/fastdfs/TestAppender1.java | 10 ++ 9 files changed, 300 insertions(+), 60 deletions(-) diff --git a/HISTORY b/HISTORY index 93ecb4a..273f971 100644 --- a/HISTORY +++ b/HISTORY @@ -1,4 +1,8 @@ +Version 1.27 2019-11-11 + * StorageClient add function: regenerate_filename for appender file + StorageClient1 add function: regenerate_filename1 for appender file + Version 1.26 2017-04-17 * IniFileReader use getResourceAsStream first * change charactor encoding to UTF-8 diff --git a/src/main/java/org/csource/fastdfs/FileInfo.java b/src/main/java/org/csource/fastdfs/FileInfo.java index faa66fb..2ceb8b5 100644 --- a/src/main/java/org/csource/fastdfs/FileInfo.java +++ b/src/main/java/org/csource/fastdfs/FileInfo.java @@ -18,6 +18,12 @@ import java.util.Date; * @version Version 1.23 */ public class FileInfo { + public static final short FILE_TYPE_NORMAL = 1; + public static final short FILE_TYPE_APPENDER = 2; + public static final short FILE_TYPE_SLAVE = 4; + + protected boolean fetch_from_server; + protected short file_type; protected String source_ip_addr; protected long file_size; protected Date create_timestamp; @@ -26,18 +32,60 @@ public class FileInfo { /** * Constructor * - * @param file_size the file size - * @param create_timestamp create timestamp in seconds - * @param crc32 the crc32 signature - * @param source_ip_addr the source storage ip address + * @param fetch_from_server if fetch from server flag + * @param file_type the file type + * @param file_size the file size + * @param create_timestamp create timestamp in seconds + * @param crc32 the crc32 signature + * @param source_ip_addr the source storage ip address */ - public FileInfo(long file_size, int create_timestamp, int crc32, String source_ip_addr) { + public FileInfo(boolean fetch_from_server, short file_type, long file_size, + int create_timestamp, int crc32, String source_ip_addr) + { + this.fetch_from_server = fetch_from_server; + this.file_type = file_type; this.file_size = file_size; this.create_timestamp = new Date(create_timestamp * 1000L); this.crc32 = crc32; this.source_ip_addr = source_ip_addr; } + /** + * get the fetch_from_server flag + * + * @return the fetch_from_server flag + */ + public boolean getFetchFromServer() { + return this.fetch_from_server; + } + + /** + * set the fetch_from_server flag + * + * @param fetch_from_server the fetch from server flag + */ + public void setFetchFromServer(boolean fetch_from_server) { + this.fetch_from_server = fetch_from_server; + } + + /** + * get the file type + * + * @return the file type + */ + public short getFileType() { + return this.file_type; + } + + /** + * set the file type + * + * @param file_type the file type + */ + public void setFileType(short file_type) { + this.file_type = file_type; + } + /** * get the source ip address of the file uploaded to * @@ -117,7 +165,9 @@ public class FileInfo { */ public String toString() { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - return "source_ip_addr = " + this.source_ip_addr + ", " + + return "fetch_from_server = " + this.fetch_from_server + ", " + + "file_type = " + this.file_type + ", " + + "source_ip_addr = " + this.source_ip_addr + ", " + "file_size = " + this.file_size + ", " + "create_timestamp = " + df.format(this.create_timestamp) + ", " + "crc32 = " + this.crc32; diff --git a/src/main/java/org/csource/fastdfs/ProtoCommon.java b/src/main/java/org/csource/fastdfs/ProtoCommon.java index b194fd7..6705759 100644 --- a/src/main/java/org/csource/fastdfs/ProtoCommon.java +++ b/src/main/java/org/csource/fastdfs/ProtoCommon.java @@ -49,6 +49,8 @@ public class ProtoCommon { public static final byte STORAGE_PROTO_CMD_APPEND_FILE = 24; //append file public static final byte STORAGE_PROTO_CMD_MODIFY_FILE = 34; //modify appender file public static final byte STORAGE_PROTO_CMD_TRUNCATE_FILE = 36; //truncate appender file + public static final byte STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME = 38; //rename appender file to normal file + public static final byte STORAGE_PROTO_CMD_RESP = TRACKER_PROTO_CMD_RESP; public static final byte FDFS_STORAGE_STATUS_INIT = 0; public static final byte FDFS_STORAGE_STATUS_WAIT_SYNC = 1; diff --git a/src/main/java/org/csource/fastdfs/StorageClient.java b/src/main/java/org/csource/fastdfs/StorageClient.java index 6d6be88..d1d2045 100644 --- a/src/main/java/org/csource/fastdfs/StorageClient.java +++ b/src/main/java/org/csource/fastdfs/StorageClient.java @@ -19,9 +19,10 @@ import java.util.Arrays; /** * Storage client for 2 fields file id: group name and filename * Note: the instance of this class is NOT thread safe !!! + * if not necessary, do NOT set storage server instance * * @author Happy Fish / YuQing - * @version Version 1.24 + * @version Version 1.27 */ public class StorageClient { public final static Base64 base64 = new Base64('-', '_', '.', 0); @@ -29,8 +30,6 @@ public class StorageClient { protected StorageServer storageServer; protected byte errno; - - /** * constructor using global settings in class ClientGlobal */ @@ -39,8 +38,19 @@ public class StorageClient { this.storageServer = null; } + /** + * constructor with tracker server + * + * @param trackerServer the tracker server, can be null + */ + public StorageClient(TrackerServer trackerServer) { + this.trackerServer = trackerServer; + this.storageServer = null; + } + /** * constructor with tracker server and storage server + * NOTE: if not necessary, do NOT set storage server instance * * @param trackerServer the tracker server, can be null * @param storageServer the storage server, can be null @@ -601,6 +611,94 @@ public class StorageClient { modify_size, callback); } + /** + * regenerate filename for appender file + * + * @param group_name the group name of appender file + * @param appender_filename the appender filename + * @return 2 elements string array if success:
+ * + * + * return null if fail + */ + public String[] regenerate_appender_filename(String group_name, String appender_filename) throws IOException, MyException { + byte[] header; + boolean bNewConnection; + Socket storageSocket; + byte[] hexLenBytes; + byte[] appenderFilenameBytes; + int offset; + long body_len; + + if ((group_name == null || group_name.length() == 0) || + (appender_filename == null || appender_filename.length() == 0)) { + this.errno = ProtoCommon.ERR_NO_EINVAL; + return null; + } + + bNewConnection = this.newUpdatableStorageConnection(group_name, appender_filename); + + try { + storageSocket = this.storageServer.getSocket(); + + appenderFilenameBytes = appender_filename.getBytes(ClientGlobal.g_charset); + body_len = appenderFilenameBytes.length; + + header = ProtoCommon.packHeader(ProtoCommon.STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME, body_len, (byte) 0); + byte[] wholePkg = new byte[(int) (header.length + body_len)]; + System.arraycopy(header, 0, wholePkg, 0, header.length); + offset = header.length; + + System.arraycopy(appenderFilenameBytes, 0, wholePkg, offset, appenderFilenameBytes.length); + offset += appenderFilenameBytes.length; + + OutputStream out = storageSocket.getOutputStream(); + out.write(wholePkg); + + ProtoCommon.RecvPackageInfo pkgInfo = ProtoCommon.recvPackage(storageSocket.getInputStream(), + ProtoCommon.STORAGE_PROTO_CMD_RESP, -1); + this.errno = pkgInfo.errno; + if (pkgInfo.errno != 0) { + return null; + } + + if (pkgInfo.body.length <= ProtoCommon.FDFS_GROUP_NAME_MAX_LEN) { + throw new MyException("body length: " + pkgInfo.body.length + " <= " + ProtoCommon.FDFS_GROUP_NAME_MAX_LEN); + } + + String new_group_name = new String(pkgInfo.body, 0, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN).trim(); + String remote_filename = new String(pkgInfo.body, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN, + pkgInfo.body.length - ProtoCommon.FDFS_GROUP_NAME_MAX_LEN); + String[] results = new String[2]; + results[0] = new_group_name; + results[1] = remote_filename; + + return results; + } catch (IOException ex) { + if (!bNewConnection) { + try { + this.storageServer.close(); + } catch (IOException ex1) { + ex1.printStackTrace(); + } finally { + this.storageServer = null; + } + } + + throw ex; + } finally { + if (bNewConnection) { + try { + this.storageServer.close(); + } catch (IOException ex1) { + ex1.printStackTrace(); + } finally { + this.storageServer = null; + } + } + } + } + /** * upload file to storage server * @@ -1514,26 +1612,42 @@ public class StorageClient { byte[] buff = base64.decodeAuto(remote_filename.substring(ProtoCommon.FDFS_FILE_PATH_LEN, ProtoCommon.FDFS_FILE_PATH_LEN + ProtoCommon.FDFS_FILENAME_BASE64_LENGTH)); + short file_type; long file_size = ProtoCommon.buff2long(buff, 4 * 2); - if (((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) || - ((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) && ((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0))) || - ((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0)) { //slave file or appender file + if (((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0)) + { + file_type = FileInfo.FILE_TYPE_APPENDER; + } + else if ((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) || + ((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) && + ((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0))) + { + file_type = FileInfo.FILE_TYPE_SLAVE; + } + else { + file_type = FileInfo.FILE_TYPE_NORMAL; + } + + if (file_type == FileInfo.FILE_TYPE_SLAVE || + file_type == FileInfo.FILE_TYPE_APPENDER) + { //slave file or appender file FileInfo fi = this.query_file_info(group_name, remote_filename); if (fi == null) { return null; } + + fi.setFileType(file_type); return fi; } - FileInfo fileInfo = new FileInfo(file_size, 0, 0, ProtoCommon.getIpAddress(buff, 0)); - fileInfo.setCreateTimestamp(ProtoCommon.buff2int(buff, 4)); + int create_timestamp = ProtoCommon.buff2int(buff, 4); if ((file_size >> 63) != 0) { file_size &= 0xFFFFFFFFL; //low 32 bits is file size - fileInfo.setFileSize(file_size); } - fileInfo.setCrc32(ProtoCommon.buff2int(buff, 4 * 4)); + int crc32 = ProtoCommon.buff2int(buff, 4 * 4); - return fileInfo; + return new FileInfo(false, file_type, file_size, create_timestamp, + crc32, ProtoCommon.getIpAddress(buff, 0)); } /** @@ -1590,7 +1704,8 @@ public class StorageClient { int create_timestamp = (int) ProtoCommon.buff2long(pkgInfo.body, ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE); int crc32 = (int) ProtoCommon.buff2long(pkgInfo.body, 2 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE); String source_ip_addr = (new String(pkgInfo.body, 3 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE, ProtoCommon.FDFS_IPADDR_SIZE)).trim(); - return new FileInfo(file_size, create_timestamp, crc32, source_ip_addr); + return new FileInfo(true, FileInfo.FILE_TYPE_NORMAL, file_size, + create_timestamp, crc32, source_ip_addr); } catch (IOException ex) { if (!bNewConnection) { try { diff --git a/src/main/java/org/csource/fastdfs/StorageClient1.java b/src/main/java/org/csource/fastdfs/StorageClient1.java index ec3f6fb..50004eb 100644 --- a/src/main/java/org/csource/fastdfs/StorageClient1.java +++ b/src/main/java/org/csource/fastdfs/StorageClient1.java @@ -15,9 +15,11 @@ import java.io.IOException; /** * Storage client for 1 field file id: combined group name and filename + * Note: the instance of this class is NOT thread safe !!! + * if not necessary, do NOT set storage server instance * * @author Happy Fish / YuQing - * @version Version 1.21 + * @version Version 1.27 */ public class StorageClient1 extends StorageClient { public static final String SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR = "/"; @@ -30,7 +32,17 @@ public class StorageClient1 extends StorageClient { } /** - * constructor + * constructor with trackerServer + * + * @param trackerServer the tracker server, can be null + */ + public StorageClient1(TrackerServer trackerServer) { + super(trackerServer); + } + + /** + * constructor with trackerServer and storageServer + * NOTE: if not necessary, do NOT set storage server instance * * @param trackerServer the tracker server, can be null * @param storageServer the storage server, can be null @@ -514,6 +526,27 @@ public class StorageClient1 extends StorageClient { return this.modify_file(parts[0], parts[1], file_offset, modify_size, callback); } + /** + * regenerate filename for appender file + * + * @param appender_file_id the appender file id + * @return the regenerated file id, return null if fail + */ + public String regenerate_appender_filename1(String appender_file_id) throws IOException, MyException { + String[] parts = new String[2]; + this.errno = this.split_file_id(appender_file_id, parts); + if (this.errno != 0) { + return null; + } + + String[] new_parts = this.regenerate_appender_filename(parts[0], parts[1]); + if (new_parts != null) { + return new_parts[0] + SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR + new_parts[1]; + } else { + return null; + } + } + /** * delete file from storage server * diff --git a/src/main/java/org/csource/fastdfs/TrackerServer.java b/src/main/java/org/csource/fastdfs/TrackerServer.java index 3ea62b5..09c9442 100644 --- a/src/main/java/org/csource/fastdfs/TrackerServer.java +++ b/src/main/java/org/csource/fastdfs/TrackerServer.java @@ -81,29 +81,29 @@ public class TrackerServer { public boolean isConnected(){ boolean isConnected = false; - if (sock != null){ - if (sock.isConnected()){ + if (sock != null) { + if (sock.isConnected()) { isConnected = true; } } return isConnected; } - public boolean isAvaliable(){ - if(isConnected()){ - if(sock.getPort() == 0){ + public boolean isAvaliable() { + if (isConnected()) { + if (sock.getPort() == 0) { return false; } - if(sock.getInetAddress() == null){ + if (sock.getInetAddress() == null) { return false; } - if(sock.getRemoteSocketAddress() == null){ + if (sock.getRemoteSocketAddress() == null) { return false; } - if(sock.isInputShutdown()){ + if (sock.isInputShutdown()) { return false; } - if(sock.isOutputShutdown()){ + if (sock.isOutputShutdown()) { return false; } return true; diff --git a/src/test/java/org/csource/fastdfs/Test1.java b/src/test/java/org/csource/fastdfs/Test1.java index 2573f59..5198afd 100644 --- a/src/test/java/org/csource/fastdfs/Test1.java +++ b/src/test/java/org/csource/fastdfs/Test1.java @@ -6,42 +6,57 @@ import org.csource.fastdfs.*; import java.net.InetSocketAddress; public class Test1 { - public static void main(String args[]) { - try { - ClientGlobal.init("fdfs_client.conf"); - System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms"); - System.out.println("charset=" + ClientGlobal.g_charset); + public static void main(String args[]) { + try { - TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("10.0.11.243", 22122)}); - TrackerClient tc = new TrackerClient(tg); + if (args.length < 1) { + System.out.println("Usage: 2 parameters, one is config filename, " + + "the other is the local filename to upload"); + return; + } - TrackerServer ts = tc.getConnection(); - if (ts == null) { - System.out.println("getConnection return null"); - return; - } + System.out.println("java.version=" + System.getProperty("java.version")); - StorageServer ss = tc.getStoreStorage(ts); - if (ss == null) { - System.out.println("getStoreStorage return null"); - } + String conf_filename = args[0]; + String local_filename; + String ext_name; + if (args.length > 1) { + local_filename = args[1]; + ext_name = null; + } + else if (System.getProperty("os.name").equalsIgnoreCase("windows")) { + local_filename = "c:/windows/system32/notepad.exe"; + ext_name = "exe"; + } else { + local_filename = "/etc/hosts"; + ext_name = ""; + } - StorageClient1 sc1 = new StorageClient1(ts, ss); + ClientGlobal.init(conf_filename); + System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms"); + System.out.println("charset=" + ClientGlobal.g_charset); - NameValuePair[] meta_list = null; //new NameValuePair[0]; - String item; - String fileid; - if (System.getProperty("os.name").equalsIgnoreCase("windows")) { - item = "c:/windows/system32/notepad.exe"; - fileid = sc1.upload_file1(item, "exe", meta_list); - } else { - item = "/etc/hosts"; - fileid = sc1.upload_file1(item, "", meta_list); - } + TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("47.95.221.159", 22122)}); + TrackerClient tc = new TrackerClient(tg); - System.out.println("Upload local file " + item + " ok, fileid=" + fileid); - } catch (Exception ex) { - ex.printStackTrace(); + TrackerServer ts = tc.getConnection(); + if (ts == null) { + System.out.println("getConnection return null"); + return; + } + + StorageServer ss = tc.getStoreStorage(ts); + if (ss == null) { + System.out.println("getStoreStorage return null"); + } + + StorageClient1 sc1 = new StorageClient1(ts, ss); + + NameValuePair[] meta_list = null; //new NameValuePair[0]; + String fileid = sc1.upload_file1(local_filename, ext_name, meta_list); + System.out.println("Upload local file " + local_filename + " ok, fileid: " + fileid); + } catch (Exception ex) { + ex.printStackTrace(); + } } - } } diff --git a/src/test/java/org/csource/fastdfs/TestAppender.java b/src/test/java/org/csource/fastdfs/TestAppender.java index 5eb34b4..ab01926 100644 --- a/src/test/java/org/csource/fastdfs/TestAppender.java +++ b/src/test/java/org/csource/fastdfs/TestAppender.java @@ -168,6 +168,17 @@ public class TestAppender { System.err.println("append file fail, error no: " + errno); } + startTime = System.currentTimeMillis(); + results = client.regenerate_appender_filename(group_name, appender_filename); + System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms"); + if (errno == 0) { + group_name = results[0]; + appender_filename = results[1]; + System.err.println(client.get_file_info(group_name, appender_filename)); + } else { + System.err.println("regenerate_appender_filename fail, error no: " + errno); + } + startTime = System.currentTimeMillis(); errno = client.delete_file(group_name, remote_filename); System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms"); diff --git a/src/test/java/org/csource/fastdfs/TestAppender1.java b/src/test/java/org/csource/fastdfs/TestAppender1.java index c2c5278..0e1fbcd 100644 --- a/src/test/java/org/csource/fastdfs/TestAppender1.java +++ b/src/test/java/org/csource/fastdfs/TestAppender1.java @@ -160,6 +160,16 @@ public class TestAppender1 { System.err.println("append file fail, error no: " + errno); } + startTime = System.currentTimeMillis(); + String new_file_id = client.regenerate_appender_filename1(appender_file_id); + System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms"); + if (errno == 0) { + appender_file_id = new_file_id; + System.err.println(client.get_file_info1(appender_file_id)); + } else { + System.err.println("regenerate_appender_filename fail, error no: " + errno); + } + startTime = System.currentTimeMillis(); errno = client.delete_file1(appender_file_id); System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms");