Compare commits

...

5 Commits
master ... dev

4 changed files with 214 additions and 147 deletions

2
.gitignore vendored
View File

@ -14,4 +14,6 @@ target
.idea
*.conf
*.PNG
.vscode/
*.class

View File

@ -14,8 +14,8 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.test.failure.ignore>true</maven.test.failure.ignore>
<maven.test.skip>true</maven.test.skip>
<jdk.version>1.6</jdk.version>
<slf4j.version>1.7.26</slf4j.version>
<jdk.version>1.8</jdk.version>
<slf4j.version>1.7.36</slf4j.version>
</properties>
<dependencies>

View File

@ -16,6 +16,7 @@ package org.csource.common;
*/
public class MyException extends Exception {
public MyException(String s, Exception e) {
super(s, e);
}
public MyException(String message) {

View File

@ -73,11 +73,14 @@ public class StorageClient {
* upload file to storage server (by file name)
*
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String local_filename, String file_ext_name,
@ -91,11 +94,14 @@ public class StorageClient {
*
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
protected String[] upload_file(String group_name, String local_filename, String file_ext_name,
@ -110,11 +116,14 @@ public class StorageClient {
* @param cmd the command
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
protected String[] upload_file(byte cmd, String group_name, String local_filename, String file_ext_name,
@ -146,8 +155,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(byte[] file_buff, int offset, int length, String file_ext_name,
@ -166,8 +177,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, byte[] file_buff, int offset, int length,
@ -183,8 +196,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(byte[] file_buff, String file_ext_name,
@ -201,8 +216,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, byte[] file_buff,
@ -220,8 +237,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, long file_size, UploadCallback callback,
@ -240,11 +259,14 @@ public class StorageClient {
* @param master_filename the master file name to generate the slave file
* @param prefix_name the prefix name to generate the slave file
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -266,8 +288,8 @@ public class StorageClient {
}
try {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, f.length(), new UploadStream(fis, f.length()), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, f.length(), new UploadStream(fis, f.length()), meta_list);
} finally {
fis.close();
}
@ -283,8 +305,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -295,8 +319,9 @@ public class StorageClient {
throw new MyException("invalid arguement");
}
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length),
meta_list);
}
/**
@ -311,8 +336,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -324,8 +351,8 @@ public class StorageClient {
throw new MyException("invalid arguement");
}
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
}
/**
@ -339,26 +366,31 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename,
String prefix_name, long file_size, UploadCallback callback,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, file_size, callback, meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, file_size, callback, meta_list);
}
/**
* upload appender file to storage server (by file name)
*
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(String local_filename, String file_ext_name,
@ -372,11 +404,14 @@ public class StorageClient {
*
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
protected String[] upload_appender_file(String group_name, String local_filename, String file_ext_name,
@ -394,8 +429,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(byte[] file_buff, int offset, int length, String file_ext_name,
@ -414,14 +451,16 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, byte[] file_buff, int offset, int length,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null, file_ext_name,
length, new UploadBuff(file_buff, offset, length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null,
file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
}
/**
@ -431,8 +470,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(byte[] file_buff, String file_ext_name,
@ -449,14 +490,16 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, byte[] file_buff,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null, file_ext_name,
file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null,
file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
}
/**
@ -468,8 +511,10 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, long file_size, UploadCallback callback,
@ -477,7 +522,8 @@ public class StorageClient {
final String master_filename = null;
final String prefix_name = null;
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, master_filename, prefix_name,
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, master_filename,
prefix_name,
file_ext_name, file_size, callback, meta_list);
}
@ -489,7 +535,8 @@ public class StorageClient {
* @param local_filename local filename to append
* @return 0 for success, != 0 for error (error no)
*/
public int append_file(String group_name, String appender_filename, String local_filename) throws IOException, MyException {
public int append_file(String group_name, String appender_filename, String local_filename)
throws IOException, MyException {
File f = new File(local_filename);
FileInputStream fis = new FileInputStream(f);
@ -508,8 +555,10 @@ public class StorageClient {
* @param file_buff file content/buff
* @return 0 for success, != 0 for error (error no)
*/
public int append_file(String group_name, String appender_filename, byte[] file_buff) throws IOException, MyException {
return this.do_append_file(group_name, appender_filename, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length));
public int append_file(String group_name, String appender_filename, byte[] file_buff)
throws IOException, MyException {
return this.do_append_file(group_name, appender_filename, file_buff.length,
new UploadBuff(file_buff, 0, file_buff.length));
}
/**
@ -617,11 +666,14 @@ public class StorageClient {
* @param group_name the group name of appender file
* @param appender_filename the appender filename
* @return 2 elements string array if success:<br>
* <ul><li> results[0]: the group name to store the file</li></ul>
* <ul><li> results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
public String[] regenerate_appender_filename(String group_name, String appender_filename) throws IOException, MyException {
public String[] regenerate_appender_filename(String group_name, String appender_filename)
throws IOException, MyException {
byte[] header;
boolean bNewStorageServer;
Connection connection = null;
@ -700,8 +752,10 @@ public class StorageClient {
* @param callback the write data callback object
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul><li> results[0]: the group name to store the file</li></ul>
* <ul><li> results[1]: the new created filename</li></ul>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* return null if fail
*/
protected String[] do_upload_file(byte cmd, String group_name, String master_filename,
@ -746,8 +800,8 @@ public class StorageClient {
masterFilenameBytes = master_filename.getBytes(ClientGlobal.g_charset);
sizeBytes = new byte[2 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE];
body_len = sizeBytes.length + ProtoCommon.FDFS_FILE_PREFIX_MAX_LEN + ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN
+ masterFilenameBytes.length + file_size;
body_len = sizeBytes.length + ProtoCommon.FDFS_FILE_PREFIX_MAX_LEN
+ ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN + masterFilenameBytes.length + file_size;
hexLenBytes = ProtoCommon.long2buff(master_filename.length());
System.arraycopy(hexLenBytes, 0, sizeBytes, 0, hexLenBytes.length);
@ -1163,7 +1217,8 @@ public class StorageClient {
* @param download_bytes download bytes, 0 for remain bytes from offset
* @return file content/buff, return null if fail
*/
public byte[] download_file(String group_name, String remote_filename, long file_offset, long download_bytes) throws IOException, MyException {
public byte[] download_file(String group_name, String remote_filename, long file_offset, long download_bytes)
throws IOException, MyException {
boolean bNewStorageServer = this.newReadableStorageConnection(group_name, remote_filename);
Connection connection = this.storageServer.getConnection();
@ -1243,17 +1298,18 @@ public class StorageClient {
long remainBytes = header.body_len;
int bytes;
//System.out.println("expect_body_len=" + header.body_len);
// System.out.println("expect_body_len=" + header.body_len);
while (remainBytes > 0) {
if ((bytes = in.read(buff, 0, remainBytes > buff.length ? buff.length : (int) remainBytes)) < 0) {
throw new IOException("recv package size " + (header.body_len - remainBytes) + " != " + header.body_len);
throw new IOException(
"recv package size " + (header.body_len - remainBytes) + " != " + header.body_len);
}
out.write(buff, 0, bytes);
remainBytes -= bytes;
//System.out.println("totalBytes=" + (header.body_len - remainBytes));
// System.out.println("totalBytes=" + (header.body_len - remainBytes));
}
return 0;
@ -1331,7 +1387,7 @@ public class StorageClient {
long remainBytes = header.body_len;
int bytes;
//System.out.println("expect_body_len=" + header.body_len);
// System.out.println("expect_body_len=" + header.body_len);
while (remainBytes > 0) {
if ((bytes = in.read(buff, 0, remainBytes > buff.length ? buff.length : (int) remainBytes)) < 0) {
@ -1344,7 +1400,7 @@ public class StorageClient {
}
remainBytes -= bytes;
//System.out.println("totalBytes=" + (header.body_len - remainBytes));
// System.out.println("totalBytes=" + (header.body_len - remainBytes));
}
return 0;
@ -1406,10 +1462,14 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @param meta_list meta item array
* @param op_flag flag, can be one of following values: <br>
* <ul><li> ProtoCommon.STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite all old
* metadata items</li></ul>
* <ul><li> ProtoCommon.STORAGE_SET_METADATA_FLAG_MERGE: merge, insert when
* the metadata item not exist, otherwise update it</li></ul>
* <ul>
* <li>ProtoCommon.STORAGE_SET_METADATA_FLAG_OVERWRITE:
* overwrite all old
* metadata items</li>
* <li>ProtoCommon.STORAGE_SET_METADATA_FLAG_MERGE:
* merge, insert when
* the metadata item not exist, otherwise update it</li>
* </ul>
* @return 0 for success, !=0 fail (error code)
*/
public int set_metadata(String group_name, String remote_filename,
@ -1516,7 +1576,7 @@ public class StorageClient {
}
if (file_type == FileInfo.FILE_TYPE_SLAVE ||
file_type == FileInfo.FILE_TYPE_APPENDER) { //slave file or appender file
file_type == FileInfo.FILE_TYPE_APPENDER) { // slave file or appender file
FileInfo fi = this.query_file_info(group_name, remote_filename);
if (fi == null) {
return null;
@ -1528,7 +1588,7 @@ public class StorageClient {
int create_timestamp = ProtoCommon.buff2int(buff, 4);
if ((file_size >> 63) != 0) {
file_size &= 0xFFFFFFFFL; //low 32 bits is file size
file_size &= 0xFFFFFFFFL; // low 32 bits is file size
}
int crc32 = ProtoCommon.buff2int(buff, 4 * 4);
@ -1618,7 +1678,7 @@ public class StorageClient {
TrackerClient tracker = new TrackerClient();
this.storageServer = tracker.getStoreStorage(this.trackerServer, group_name);
if (this.storageServer == null) {
throw new MyException("getStoreStorage fail, errno code: " + tracker.getErrorCode());
throw new MyException("getStoreStorage fail, error code: " + tracker.getErrorCode());
}
return true;
}
@ -1631,7 +1691,8 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @return true if create a new connection
*/
protected boolean newReadableStorageConnection(String group_name, String remote_filename) throws IOException, MyException {
protected boolean newReadableStorageConnection(String group_name, String remote_filename)
throws IOException, MyException {
if (this.storageServer != null) {
return false;
} else {
@ -1651,7 +1712,8 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @return true if create a new connection
*/
protected boolean newUpdatableStorageConnection(String group_name, String remote_filename) throws IOException, MyException {
protected boolean newUpdatableStorageConnection(String group_name, String remote_filename)
throws IOException, MyException {
if (this.storageServer != null) {
return false;
} else {
@ -1671,7 +1733,8 @@ public class StorageClient {
* @param group_name the group name of storage server
* @param remote_filename filename on storage server
*/
protected void send_package(byte cmd, String group_name, String remote_filename, Connection connection) throws IOException {
protected void send_package(byte cmd, String group_name, String remote_filename, Connection connection)
throws IOException {
byte[] header;
byte[] groupBytes;
byte[] filenameBytes;
@ -1792,7 +1855,8 @@ public class StorageClient {
}
/**
* send file content callback function, be called only once when the file uploaded
* send file content callback function, be called only once when the file
* uploaded
*
* @param out output stream for writing file content
* @return 0 success, return none zero(errno) if fail