Compare commits

..

No commits in common. "dev" and "master" have entirely different histories.
dev ... master

4 changed files with 147 additions and 214 deletions

2
.gitignore vendored
View File

@ -14,6 +14,4 @@ target
.idea
*.conf
*.PNG
.vscode/
*.class

View File

@ -14,8 +14,8 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.test.failure.ignore>true</maven.test.failure.ignore>
<maven.test.skip>true</maven.test.skip>
<jdk.version>1.8</jdk.version>
<slf4j.version>1.7.36</slf4j.version>
<jdk.version>1.6</jdk.version>
<slf4j.version>1.7.26</slf4j.version>
</properties>
<dependencies>

View File

@ -16,7 +16,6 @@ package org.csource.common;
*/
public class MyException extends Exception {
public MyException(String s, Exception e) {
super(s, e);
}
public MyException(String message) {

View File

@ -73,14 +73,11 @@ public class StorageClient {
* upload file to storage server (by file name)
*
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String local_filename, String file_ext_name,
@ -94,14 +91,11 @@ public class StorageClient {
*
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
protected String[] upload_file(String group_name, String local_filename, String file_ext_name,
@ -116,14 +110,11 @@ public class StorageClient {
* @param cmd the command
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
protected String[] upload_file(byte cmd, String group_name, String local_filename, String file_ext_name,
@ -155,10 +146,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(byte[] file_buff, int offset, int length, String file_ext_name,
@ -177,10 +166,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, byte[] file_buff, int offset, int length,
@ -196,10 +183,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(byte[] file_buff, String file_ext_name,
@ -216,10 +201,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, byte[] file_buff,
@ -237,10 +220,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, long file_size, UploadCallback callback,
@ -259,14 +240,11 @@ public class StorageClient {
* @param master_filename the master file name to generate the slave file
* @param prefix_name the prefix name to generate the slave file
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -288,8 +266,8 @@ public class StorageClient {
}
try {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, f.length(), new UploadStream(fis, f.length()), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, f.length(), new UploadStream(fis, f.length()), meta_list);
} finally {
fis.close();
}
@ -305,10 +283,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -319,9 +295,8 @@ public class StorageClient {
throw new MyException("invalid arguement");
}
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length),
meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
}
/**
@ -336,10 +311,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename, String prefix_name,
@ -351,8 +324,8 @@ public class StorageClient {
throw new MyException("invalid arguement");
}
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
}
/**
@ -366,31 +339,26 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_file(String group_name, String master_filename,
String prefix_name, long file_size, UploadCallback callback,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename,
prefix_name, file_ext_name, file_size, callback, meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_SLAVE_FILE, group_name, master_filename, prefix_name,
file_ext_name, file_size, callback, meta_list);
}
/**
* upload appender file to storage server (by file name)
*
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(String local_filename, String file_ext_name,
@ -404,14 +372,11 @@ public class StorageClient {
*
* @param group_name the group name to upload file to, can be empty
* @param local_filename local filename to upload
* @param file_ext_name file ext name, do not include dot(.), null to extract
* ext name from the local filename
* @param file_ext_name file ext name, do not include dot(.), null to extract ext name from the local filename
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file </li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
protected String[] upload_appender_file(String group_name, String local_filename, String file_ext_name,
@ -429,10 +394,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(byte[] file_buff, int offset, int length, String file_ext_name,
@ -451,16 +414,14 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, byte[] file_buff, int offset, int length,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null,
file_ext_name, length, new UploadBuff(file_buff, offset, length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null, file_ext_name,
length, new UploadBuff(file_buff, offset, length), meta_list);
}
/**
@ -470,10 +431,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(byte[] file_buff, String file_ext_name,
@ -490,16 +449,14 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, byte[] file_buff,
String file_ext_name, NameValuePair[] meta_list) throws IOException, MyException {
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null,
file_ext_name, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, null, null, file_ext_name,
file_buff.length, new UploadBuff(file_buff, 0, file_buff.length), meta_list);
}
/**
@ -511,10 +468,8 @@ public class StorageClient {
* @param file_ext_name file ext name, do not include dot(.)
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li>results[0]: the group name to store the file</li></ul>
* <ul><li>results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] upload_appender_file(String group_name, long file_size, UploadCallback callback,
@ -522,8 +477,7 @@ public class StorageClient {
final String master_filename = null;
final String prefix_name = null;
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, master_filename,
prefix_name,
return this.do_upload_file(ProtoCommon.STORAGE_PROTO_CMD_UPLOAD_APPENDER_FILE, group_name, master_filename, prefix_name,
file_ext_name, file_size, callback, meta_list);
}
@ -535,8 +489,7 @@ public class StorageClient {
* @param local_filename local filename to append
* @return 0 for success, != 0 for error (error no)
*/
public int append_file(String group_name, String appender_filename, String local_filename)
throws IOException, MyException {
public int append_file(String group_name, String appender_filename, String local_filename) throws IOException, MyException {
File f = new File(local_filename);
FileInputStream fis = new FileInputStream(f);
@ -555,10 +508,8 @@ public class StorageClient {
* @param file_buff file content/buff
* @return 0 for success, != 0 for error (error no)
*/
public int append_file(String group_name, String appender_filename, byte[] file_buff)
throws IOException, MyException {
return this.do_append_file(group_name, appender_filename, file_buff.length,
new UploadBuff(file_buff, 0, file_buff.length));
public int append_file(String group_name, String appender_filename, byte[] file_buff) throws IOException, MyException {
return this.do_append_file(group_name, appender_filename, file_buff.length, new UploadBuff(file_buff, 0, file_buff.length));
}
/**
@ -666,14 +617,11 @@ public class StorageClient {
* @param group_name the group name of appender file
* @param appender_filename the appender filename
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li> results[0]: the group name to store the file</li></ul>
* <ul><li> results[1]: the new created filename</li></ul>
* return null if fail
*/
public String[] regenerate_appender_filename(String group_name, String appender_filename)
throws IOException, MyException {
public String[] regenerate_appender_filename(String group_name, String appender_filename) throws IOException, MyException {
byte[] header;
boolean bNewStorageServer;
Connection connection = null;
@ -752,10 +700,8 @@ public class StorageClient {
* @param callback the write data callback object
* @param meta_list meta info array
* @return 2 elements string array if success:<br>
* <ul>
* <li>results[0]: the group name to store the file</li>
* <li>results[1]: the new created filename</li>
* </ul>
* <ul><li> results[0]: the group name to store the file</li></ul>
* <ul><li> results[1]: the new created filename</li></ul>
* return null if fail
*/
protected String[] do_upload_file(byte cmd, String group_name, String master_filename,
@ -800,8 +746,8 @@ public class StorageClient {
masterFilenameBytes = master_filename.getBytes(ClientGlobal.g_charset);
sizeBytes = new byte[2 * ProtoCommon.FDFS_PROTO_PKG_LEN_SIZE];
body_len = sizeBytes.length + ProtoCommon.FDFS_FILE_PREFIX_MAX_LEN
+ ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN + masterFilenameBytes.length + file_size;
body_len = sizeBytes.length + ProtoCommon.FDFS_FILE_PREFIX_MAX_LEN + ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN
+ masterFilenameBytes.length + file_size;
hexLenBytes = ProtoCommon.long2buff(master_filename.length());
System.arraycopy(hexLenBytes, 0, sizeBytes, 0, hexLenBytes.length);
@ -1217,8 +1163,7 @@ public class StorageClient {
* @param download_bytes download bytes, 0 for remain bytes from offset
* @return file content/buff, return null if fail
*/
public byte[] download_file(String group_name, String remote_filename, long file_offset, long download_bytes)
throws IOException, MyException {
public byte[] download_file(String group_name, String remote_filename, long file_offset, long download_bytes) throws IOException, MyException {
boolean bNewStorageServer = this.newReadableStorageConnection(group_name, remote_filename);
Connection connection = this.storageServer.getConnection();
@ -1298,18 +1243,17 @@ public class StorageClient {
long remainBytes = header.body_len;
int bytes;
// System.out.println("expect_body_len=" + header.body_len);
//System.out.println("expect_body_len=" + header.body_len);
while (remainBytes > 0) {
if ((bytes = in.read(buff, 0, remainBytes > buff.length ? buff.length : (int) remainBytes)) < 0) {
throw new IOException(
"recv package size " + (header.body_len - remainBytes) + " != " + header.body_len);
throw new IOException("recv package size " + (header.body_len - remainBytes) + " != " + header.body_len);
}
out.write(buff, 0, bytes);
remainBytes -= bytes;
// System.out.println("totalBytes=" + (header.body_len - remainBytes));
//System.out.println("totalBytes=" + (header.body_len - remainBytes));
}
return 0;
@ -1387,7 +1331,7 @@ public class StorageClient {
long remainBytes = header.body_len;
int bytes;
// System.out.println("expect_body_len=" + header.body_len);
//System.out.println("expect_body_len=" + header.body_len);
while (remainBytes > 0) {
if ((bytes = in.read(buff, 0, remainBytes > buff.length ? buff.length : (int) remainBytes)) < 0) {
@ -1400,7 +1344,7 @@ public class StorageClient {
}
remainBytes -= bytes;
// System.out.println("totalBytes=" + (header.body_len - remainBytes));
//System.out.println("totalBytes=" + (header.body_len - remainBytes));
}
return 0;
@ -1462,14 +1406,10 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @param meta_list meta item array
* @param op_flag flag, can be one of following values: <br>
* <ul>
* <li>ProtoCommon.STORAGE_SET_METADATA_FLAG_OVERWRITE:
* overwrite all old
* metadata items</li>
* <li>ProtoCommon.STORAGE_SET_METADATA_FLAG_MERGE:
* merge, insert when
* the metadata item not exist, otherwise update it</li>
* </ul>
* <ul><li> ProtoCommon.STORAGE_SET_METADATA_FLAG_OVERWRITE: overwrite all old
* metadata items</li></ul>
* <ul><li> ProtoCommon.STORAGE_SET_METADATA_FLAG_MERGE: merge, insert when
* the metadata item not exist, otherwise update it</li></ul>
* @return 0 for success, !=0 fail (error code)
*/
public int set_metadata(String group_name, String remote_filename,
@ -1576,7 +1516,7 @@ public class StorageClient {
}
if (file_type == FileInfo.FILE_TYPE_SLAVE ||
file_type == FileInfo.FILE_TYPE_APPENDER) { // slave file or appender file
file_type == FileInfo.FILE_TYPE_APPENDER) { //slave file or appender file
FileInfo fi = this.query_file_info(group_name, remote_filename);
if (fi == null) {
return null;
@ -1588,7 +1528,7 @@ public class StorageClient {
int create_timestamp = ProtoCommon.buff2int(buff, 4);
if ((file_size >> 63) != 0) {
file_size &= 0xFFFFFFFFL; // low 32 bits is file size
file_size &= 0xFFFFFFFFL; //low 32 bits is file size
}
int crc32 = ProtoCommon.buff2int(buff, 4 * 4);
@ -1678,7 +1618,7 @@ public class StorageClient {
TrackerClient tracker = new TrackerClient();
this.storageServer = tracker.getStoreStorage(this.trackerServer, group_name);
if (this.storageServer == null) {
throw new MyException("getStoreStorage fail, error code: " + tracker.getErrorCode());
throw new MyException("getStoreStorage fail, errno code: " + tracker.getErrorCode());
}
return true;
}
@ -1691,8 +1631,7 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @return true if create a new connection
*/
protected boolean newReadableStorageConnection(String group_name, String remote_filename)
throws IOException, MyException {
protected boolean newReadableStorageConnection(String group_name, String remote_filename) throws IOException, MyException {
if (this.storageServer != null) {
return false;
} else {
@ -1712,8 +1651,7 @@ public class StorageClient {
* @param remote_filename filename on storage server
* @return true if create a new connection
*/
protected boolean newUpdatableStorageConnection(String group_name, String remote_filename)
throws IOException, MyException {
protected boolean newUpdatableStorageConnection(String group_name, String remote_filename) throws IOException, MyException {
if (this.storageServer != null) {
return false;
} else {
@ -1733,8 +1671,7 @@ public class StorageClient {
* @param group_name the group name of storage server
* @param remote_filename filename on storage server
*/
protected void send_package(byte cmd, String group_name, String remote_filename, Connection connection)
throws IOException {
protected void send_package(byte cmd, String group_name, String remote_filename, Connection connection) throws IOException {
byte[] header;
byte[] groupBytes;
byte[] filenameBytes;
@ -1855,8 +1792,7 @@ public class StorageClient {
}
/**
* send file content callback function, be called only once when the file
* uploaded
* send file content callback function, be called only once when the file uploaded
*
* @param out output stream for writing file content
* @return 0 success, return none zero(errno) if fail