support appender file rename to normal file

HISTORY

@@ -1,4 +1,8 @@
 
+Version 1.27  2019-11-11
+  * StorageClient add function: regenerate_filename for appender file
+    StorageClient1 add function: regenerate_filename1 for appender file
+
 Version 1.26  2017-04-17
   * IniFileReader use getResourceAsStream first
   * change charactor encoding to UTF-8

src/main/java/org/csource/fastdfs/FileInfo.java

@@ -18,6 +18,12 @@ import java.util.Date;
  * @version Version 1.23
  */
 public class FileInfo {
+  public static final short FILE_TYPE_NORMAL = 1;
+  public static final short FILE_TYPE_APPENDER = 2;
+  public static final short FILE_TYPE_SLAVE = 4;
+
+  protected boolean fetch_from_server;
+  protected short file_type;
   protected String source_ip_addr;
   protected long file_size;
   protected Date create_timestamp;
@@ -26,18 +32,60 @@ public class FileInfo {
   /**
    * Constructor
    *
-   * @param file_size        the file size
+   * @param fetch_from_server if fetch from server flag
-   * @param create_timestamp create timestamp in seconds
+   * @param file_type         the file type
-   * @param crc32            the crc32 signature
+   * @param file_size         the file size
-   * @param source_ip_addr   the source storage ip address
+   * @param create_timestamp  create timestamp in seconds
+   * @param crc32             the crc32 signature
+   * @param source_ip_addr    the source storage ip address
    */
-  public FileInfo(long file_size, int create_timestamp, int crc32, String source_ip_addr) {
+  public FileInfo(boolean fetch_from_server, short file_type, long file_size,
+          int create_timestamp, int crc32, String source_ip_addr)
+  {
+    this.fetch_from_server = fetch_from_server;
+    this.file_type = file_type;
     this.file_size = file_size;
     this.create_timestamp = new Date(create_timestamp * 1000L);
     this.crc32 = crc32;
     this.source_ip_addr = source_ip_addr;
   }
 
+  /**
+   * get the fetch_from_server flag
+   *
+   * @return the fetch_from_server flag
+   */
+  public boolean getFetchFromServer() {
+    return this.fetch_from_server;
+  }
+
+  /**
+   * set the fetch_from_server flag
+   *
+   * @param fetch_from_server the fetch from server flag
+   */
+  public void setFetchFromServer(boolean fetch_from_server) {
+    this.fetch_from_server = fetch_from_server;
+  }
+
+  /**
+   * get the file type
+   *
+   * @return the file type
+   */
+  public short getFileType() {
+    return this.file_type;
+  }
+
+  /**
+   * set the file type
+   *
+   * @param file_type the file type
+   */
+  public void setFileType(short file_type) {
+    this.file_type = file_type;
+  }
+
   /**
    * get the source ip address of the file uploaded to
    *
@@ -117,7 +165,9 @@ public class FileInfo {
    */
   public String toString() {
     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    return "source_ip_addr = " + this.source_ip_addr + ", " +
+    return "fetch_from_server = " + this.fetch_from_server + ", " +
+      "file_type = " + this.file_type + ", " +
+      "source_ip_addr = " + this.source_ip_addr + ", " +
       "file_size = " + this.file_size + ", " +
       "create_timestamp = " + df.format(this.create_timestamp) + ", " +
       "crc32 = " + this.crc32;

src/main/java/org/csource/fastdfs/ProtoCommon.java

@@ -49,6 +49,8 @@ public class ProtoCommon {
   public static final byte STORAGE_PROTO_CMD_APPEND_FILE = 24;  //append file
   public static final byte STORAGE_PROTO_CMD_MODIFY_FILE = 34;  //modify appender file
   public static final byte STORAGE_PROTO_CMD_TRUNCATE_FILE = 36;  //truncate appender file
+  public static final byte STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME = 38;  //rename appender file to normal file
+
   public static final byte STORAGE_PROTO_CMD_RESP = TRACKER_PROTO_CMD_RESP;
   public static final byte FDFS_STORAGE_STATUS_INIT = 0;
   public static final byte FDFS_STORAGE_STATUS_WAIT_SYNC = 1;

src/main/java/org/csource/fastdfs/StorageClient.java

@@ -19,9 +19,10 @@ import java.util.Arrays;
 /**
  * Storage client for 2 fields file id: group name and filename
  * Note: the instance of this class is NOT thread safe !!!
+ *       if not necessary, do NOT set storage server instance
  *
  * @author Happy Fish / YuQing
- * @version Version 1.24
+ * @version Version 1.27
  */
 public class StorageClient {
   public final static Base64 base64 = new Base64('-', '_', '.', 0);
@@ -29,8 +30,6 @@ public class StorageClient {
   protected StorageServer storageServer;
   protected byte errno;
 
-
-
   /**
    * constructor using global settings in class ClientGlobal
    */
@@ -39,8 +38,19 @@ public class StorageClient {
     this.storageServer = null;
   }
 
+  /**
+   * constructor with tracker server
+   *
+   * @param trackerServer the tracker server, can be null
+   */
+  public StorageClient(TrackerServer trackerServer) {
+    this.trackerServer = trackerServer;
+    this.storageServer = null;
+  }
+
   /**
    * constructor with tracker server and storage server
+   * NOTE: if not necessary, do NOT set storage server instance
    *
    * @param trackerServer the tracker server, can be null
    * @param storageServer the storage server, can be null
@@ -601,6 +611,94 @@ public class StorageClient {
       modify_size, callback);
   }
 
+  /**
+   * regenerate filename for appender file
+   *
+   * @param group_name        the group name of appender file
+   * @param appender_filename the appender filename
+   * @return 2 elements string array if success:<br>
+   * <ul><li> results[0]: the group name to store the file</li></ul>
+   * <ul><li> results[1]: the new created filename</li></ul>
+   * return null if fail
+   */
+  public String[] regenerate_appender_filename(String group_name, String appender_filename) throws IOException, MyException {
+    byte[] header;
+    boolean bNewConnection;
+    Socket storageSocket;
+    byte[] hexLenBytes;
+    byte[] appenderFilenameBytes;
+    int offset;
+    long body_len;
+
+    if ((group_name == null || group_name.length() == 0) ||
+      (appender_filename == null || appender_filename.length() == 0)) {
+      this.errno = ProtoCommon.ERR_NO_EINVAL;
+      return null;
+    }
+
+    bNewConnection = this.newUpdatableStorageConnection(group_name, appender_filename);
+
+    try {
+      storageSocket = this.storageServer.getSocket();
+
+      appenderFilenameBytes = appender_filename.getBytes(ClientGlobal.g_charset);
+      body_len = appenderFilenameBytes.length;
+
+      header = ProtoCommon.packHeader(ProtoCommon.STORAGE_PROTO_CMD_REGENERATE_APPENDER_FILENAME, body_len, (byte) 0);
+      byte[] wholePkg = new byte[(int) (header.length + body_len)];
+      System.arraycopy(header, 0, wholePkg, 0, header.length);
+      offset = header.length;
+
+      System.arraycopy(appenderFilenameBytes, 0, wholePkg, offset, appenderFilenameBytes.length);
+      offset += appenderFilenameBytes.length;
+
+      OutputStream out = storageSocket.getOutputStream();
+      out.write(wholePkg);
+
+      ProtoCommon.RecvPackageInfo pkgInfo = ProtoCommon.recvPackage(storageSocket.getInputStream(),
+        ProtoCommon.STORAGE_PROTO_CMD_RESP, -1);
+      this.errno = pkgInfo.errno;
+      if (pkgInfo.errno != 0) {
+        return null;
+      }
+
+      if (pkgInfo.body.length <= ProtoCommon.FDFS_GROUP_NAME_MAX_LEN) {
+        throw new MyException("body length: " + pkgInfo.body.length + " <= " + ProtoCommon.FDFS_GROUP_NAME_MAX_LEN);
+      }
+
+      String new_group_name = new String(pkgInfo.body, 0, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN).trim();
+      String remote_filename = new String(pkgInfo.body, ProtoCommon.FDFS_GROUP_NAME_MAX_LEN,
+              pkgInfo.body.length - ProtoCommon.FDFS_GROUP_NAME_MAX_LEN);
+      String[] results = new String[2];
+      results[0] = new_group_name;
+      results[1] = remote_filename;
+
+      return results;
+    } catch (IOException ex) {
+      if (!bNewConnection) {
+        try {
+          this.storageServer.close();
+        } catch (IOException ex1) {
+          ex1.printStackTrace();
+        } finally {
+          this.storageServer = null;
+        }
+      }
+
+      throw ex;
+    } finally {
+      if (bNewConnection) {
+        try {
+          this.storageServer.close();
+        } catch (IOException ex1) {
+          ex1.printStackTrace();
+        } finally {
+          this.storageServer = null;
+        }
+      }
+    }
+  }
+
   /**
    * upload file to storage server
    *
@@ -1514,26 +1612,42 @@ public class StorageClient {
     byte[] buff = base64.decodeAuto(remote_filename.substring(ProtoCommon.FDFS_FILE_PATH_LEN,
       ProtoCommon.FDFS_FILE_PATH_LEN + ProtoCommon.FDFS_FILENAME_BASE64_LENGTH));
 
+    short file_type;
     long file_size = ProtoCommon.buff2long(buff, 4 * 2);
-    if (((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) ||
+    if (((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0))
-      ((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) && ((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0))) ||
+    {
-      ((file_size & ProtoCommon.APPENDER_FILE_SIZE) != 0)) { //slave file or appender file
+        file_type = FileInfo.FILE_TYPE_APPENDER;
+    }
+    else if ((remote_filename.length() > ProtoCommon.TRUNK_LOGIC_FILENAME_LENGTH) ||
+            ((remote_filename.length() > ProtoCommon.NORMAL_LOGIC_FILENAME_LENGTH) &&
+             ((file_size & ProtoCommon.TRUNK_FILE_MARK_SIZE) == 0)))
+    {
+        file_type = FileInfo.FILE_TYPE_SLAVE;
+    }
+    else {
+        file_type = FileInfo.FILE_TYPE_NORMAL;
+    }
+
+    if (file_type == FileInfo.FILE_TYPE_SLAVE ||
+        file_type == FileInfo.FILE_TYPE_APPENDER)
+    { //slave file or appender file
       FileInfo fi = this.query_file_info(group_name, remote_filename);
       if (fi == null) {
         return null;
       }
+
+      fi.setFileType(file_type);
       return fi;
     }
 
-    FileInfo fileInfo = new FileInfo(file_size, 0, 0, ProtoCommon.getIpAddress(buff, 0));
+    int create_timestamp = ProtoCommon.buff2int(buff, 4);
-    fileInfo.setCreateTimestamp(ProtoCommon.buff2int(buff, 4));
     if ((file_size >> 63) != 0) {
       file_size &= 0xFFFFFFFFL;  //low 32 bits is file size
-      fileInfo.setFileSize(file_size);
     }
-    fileInfo.setCrc32(ProtoCommon.buff2int(buff, 4 * 4));
+    int crc32 = ProtoCommon.buff2int(buff, 4 * 4);
 
-    return fileInfo;
+    return new FileInfo(false, file_type, file_size, create_timestamp,
+            crc32, ProtoCommon.getIpAddress(buff, 0));
   }
 
   /**
@@ -1590,7 +1704,8 @@ public class StorageClient {
       int create_timestamp = (int) ProtoCommon.buff2long(pkgInfo.body, ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE);
       int crc32 = (int) ProtoCommon.buff2long(pkgInfo.body, 2 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE);
       String source_ip_addr = (new String(pkgInfo.body, 3 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE, ProtoCommon.FDFS_IPADDR_SIZE)).trim();
-      return new FileInfo(file_size, create_timestamp, crc32, source_ip_addr);
+      return new FileInfo(true, FileInfo.FILE_TYPE_NORMAL, file_size,
+              create_timestamp, crc32, source_ip_addr);
     } catch (IOException ex) {
       if (!bNewConnection) {
         try {

src/main/java/org/csource/fastdfs/StorageClient1.java

@@ -15,9 +15,11 @@ import java.io.IOException;
 
 /**
  * Storage client for 1 field file id: combined group name and filename
+ * Note: the instance of this class is NOT thread safe !!!
+ *       if not necessary, do NOT set storage server instance
  *
  * @author Happy Fish / YuQing
- * @version Version 1.21
+ * @version Version 1.27
  */
 public class StorageClient1 extends StorageClient {
   public static final String SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR = "/";
@@ -30,7 +32,17 @@ public class StorageClient1 extends StorageClient {
   }
 
   /**
-   * constructor
+   * constructor with trackerServer
+   *
+   * @param trackerServer the tracker server, can be null
+   */
+  public StorageClient1(TrackerServer trackerServer) {
+    super(trackerServer);
+  }
+
+  /**
+   * constructor with trackerServer and storageServer
+   * NOTE: if not necessary, do NOT set storage server instance
    *
    * @param trackerServer the tracker server, can be null
    * @param storageServer the storage server, can be null
@@ -514,6 +526,27 @@ public class StorageClient1 extends StorageClient {
     return this.modify_file(parts[0], parts[1], file_offset, modify_size, callback);
   }
 
+  /**
+   * regenerate filename for appender file
+   *
+   * @param appender_file_id the appender file id
+   * @return the regenerated file id, return null if fail
+   */
+  public String regenerate_appender_filename1(String appender_file_id) throws IOException, MyException {
+    String[] parts = new String[2];
+    this.errno = this.split_file_id(appender_file_id, parts);
+    if (this.errno != 0) {
+      return null;
+    }
+
+    String[] new_parts = this.regenerate_appender_filename(parts[0], parts[1]);
+    if (new_parts != null) {
+      return new_parts[0] + SPLIT_GROUP_NAME_AND_FILENAME_SEPERATOR + new_parts[1];
+    } else {
+      return null;
+    }
+  }
+
   /**
    * delete file from storage server
    *

src/main/java/org/csource/fastdfs/TrackerServer.java

@@ -81,29 +81,29 @@ public class TrackerServer {
 
   public boolean isConnected(){
     boolean isConnected = false;
-    if (sock != null){
+    if (sock != null) {
-      if (sock.isConnected()){
+      if (sock.isConnected()) {
         isConnected = true;
       }
     }
     return isConnected;
   }
 
-  public boolean isAvaliable(){
+  public boolean isAvaliable() {
-    if(isConnected()){
+    if (isConnected()) {
-      if(sock.getPort() == 0){
+      if (sock.getPort() == 0) {
         return false;
       }
-      if(sock.getInetAddress() == null){
+      if (sock.getInetAddress() == null) {
         return false;
       }
-      if(sock.getRemoteSocketAddress() == null){
+      if (sock.getRemoteSocketAddress() == null) {
         return false;
       }
-      if(sock.isInputShutdown()){
+      if (sock.isInputShutdown()) {
         return false;
       }
-      if(sock.isOutputShutdown()){
+      if (sock.isOutputShutdown()) {
         return false;
       }
       return true;

src/test/java/org/csource/fastdfs/Test1.java

@@ -6,42 +6,57 @@ import org.csource.fastdfs.*;
 import java.net.InetSocketAddress;
 
 public class Test1 {
-  public static void main(String args[]) {
+    public static void main(String args[]) {
-    try {
+        try {
-      ClientGlobal.init("fdfs_client.conf");
-      System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms");
-      System.out.println("charset=" + ClientGlobal.g_charset);
 
-      TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("10.0.11.243", 22122)});
+            if (args.length < 1) {
-      TrackerClient tc = new TrackerClient(tg);
+                System.out.println("Usage: 2 parameters, one is config filename, "
+                        + "the other is the local filename to upload");
+                return;
+            }
 
-      TrackerServer ts = tc.getConnection();
+            System.out.println("java.version=" + System.getProperty("java.version"));
-      if (ts == null) {
-        System.out.println("getConnection return null");
-        return;
-      }
 
-      StorageServer ss = tc.getStoreStorage(ts);
+            String conf_filename = args[0];
-      if (ss == null) {
+            String local_filename;
-        System.out.println("getStoreStorage return null");
+            String ext_name;
-      }
+            if (args.length > 1) {
+                local_filename = args[1];
+                ext_name = null;
+            }
+            else if (System.getProperty("os.name").equalsIgnoreCase("windows")) {
+                local_filename = "c:/windows/system32/notepad.exe";
+                ext_name = "exe";
+            } else {
+                local_filename = "/etc/hosts";
+                ext_name = "";
+            }
 
-      StorageClient1 sc1 = new StorageClient1(ts, ss);
+            ClientGlobal.init(conf_filename);
+            System.out.println("network_timeout=" + ClientGlobal.g_network_timeout + "ms");
+            System.out.println("charset=" + ClientGlobal.g_charset);
 
-      NameValuePair[] meta_list = null;  //new NameValuePair[0];
+            TrackerGroup tg = new TrackerGroup(new InetSocketAddress[]{new InetSocketAddress("47.95.221.159", 22122)});
-      String item;
+            TrackerClient tc = new TrackerClient(tg);
-      String fileid;
-      if (System.getProperty("os.name").equalsIgnoreCase("windows")) {
-        item = "c:/windows/system32/notepad.exe";
-        fileid = sc1.upload_file1(item, "exe", meta_list);
-      } else {
-        item = "/etc/hosts";
-        fileid = sc1.upload_file1(item, "", meta_list);
-      }
 
-      System.out.println("Upload local file " + item + " ok, fileid=" + fileid);
+            TrackerServer ts = tc.getConnection();
-    } catch (Exception ex) {
+            if (ts == null) {
-      ex.printStackTrace();
+                System.out.println("getConnection return null");
+                return;
+            }
+
+            StorageServer ss = tc.getStoreStorage(ts);
+            if (ss == null) {
+                System.out.println("getStoreStorage return null");
+            }
+
+            StorageClient1 sc1 = new StorageClient1(ts, ss);
+
+            NameValuePair[] meta_list = null;  //new NameValuePair[0];
+            String fileid = sc1.upload_file1(local_filename, ext_name, meta_list);
+            System.out.println("Upload local file " + local_filename + " ok, fileid: " + fileid);
+        } catch (Exception ex) {
+            ex.printStackTrace();
+        }
     }
-  }
 }

src/test/java/org/csource/fastdfs/TestAppender.java

@@ -168,6 +168,17 @@ public class TestAppender {
           System.err.println("append file fail, error no: " + errno);
         }
 
+        startTime = System.currentTimeMillis();
+        results  = client.regenerate_appender_filename(group_name, appender_filename);
+        System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms");
+        if (errno == 0) {
+          group_name = results[0];
+          appender_filename = results[1];
+          System.err.println(client.get_file_info(group_name, appender_filename));
+        } else {
+          System.err.println("regenerate_appender_filename fail, error no: " + errno);
+        }
+
         startTime = System.currentTimeMillis();
         errno = client.delete_file(group_name, remote_filename);
         System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms");

src/test/java/org/csource/fastdfs/TestAppender1.java

@@ -160,6 +160,16 @@ public class TestAppender1 {
           System.err.println("append file fail, error no: " + errno);
         }
 
+        startTime = System.currentTimeMillis();
+        String new_file_id = client.regenerate_appender_filename1(appender_file_id);
+        System.out.println("regenerate_appender_filename time used: " + (System.currentTimeMillis() - startTime) + " ms");
+        if (errno == 0) {
+          appender_file_id = new_file_id;
+          System.err.println(client.get_file_info1(appender_file_id));
+        } else {
+          System.err.println("regenerate_appender_filename fail, error no: " + errno);
+        }
+
         startTime = System.currentTimeMillis();
         errno = client.delete_file1(appender_file_id);
         System.out.println("delete_file time used: " + (System.currentTimeMillis() - startTime) + " ms");