From 8d966af4a4abcc874a93e8af0f8bf6ba78a25ae1 Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sat, 28 Jan 2023 15:02:40 +0800 Subject: [PATCH 01/11] feature:add tracker server fail over --- .../org/csource/fastdfs/ClientGlobal.java | 12 ++ .../org/csource/fastdfs/StorageClient.java | 2 +- .../org/csource/fastdfs/TrackerClient.java | 88 ++++++------ .../org/csource/fastdfs/TrackerGroup.java | 133 +++++++++--------- .../fastdfs-client.properties.sample | 4 +- src/main/resources/fdfs_client.conf.sample | 4 +- .../java/org/csource/fastdfs/FdfsTest.java | 4 +- 7 files changed, 131 insertions(+), 116 deletions(-) diff --git a/src/main/java/org/csource/fastdfs/ClientGlobal.java b/src/main/java/org/csource/fastdfs/ClientGlobal.java index 382e2a5..0fc3450 100644 --- a/src/main/java/org/csource/fastdfs/ClientGlobal.java +++ b/src/main/java/org/csource/fastdfs/ClientGlobal.java @@ -48,6 +48,8 @@ public class ClientGlobal { public static final String PROP_KEY_CONNECTION_POOL_MAX_IDLE_TIME = "fastdfs.connection_pool.max_idle_time"; public static final String PROP_KEY_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS = "fastdfs.connection_pool.max_wait_time_in_ms"; + public static final String PROP_KEY_FAIL_OVER_RETRY_COUNT = "fastdfs.fail_over_retry_count"; + public static final int DEFAULT_CONNECT_TIMEOUT = 5; //second public static final int DEFAULT_NETWORK_TIMEOUT = 30; //second @@ -61,6 +63,8 @@ public class ClientGlobal { public static final int DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME = 3600 ;//second public static final int DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS = 1000 ;//millisecond + public static final int DEFAULT_FAIL_OVER_RETRY_COUNT = 1; + public static int g_connect_timeout = DEFAULT_CONNECT_TIMEOUT * 1000; //millisecond public static int g_network_timeout = DEFAULT_NETWORK_TIMEOUT * 1000; //millisecond public static String g_charset = DEFAULT_CHARSET; @@ -73,6 +77,8 @@ public class ClientGlobal { public static int g_connection_pool_max_idle_time = DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME * 1000; //millisecond public static int g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; //millisecond + public static int g_fail_over_retry_count = DEFAULT_FAIL_OVER_RETRY_COUNT ;//get connection retry count when fail + public static TrackerGroup g_tracker_group; private ClientGlobal() { @@ -139,6 +145,7 @@ public class ClientGlobal { if (g_connection_pool_max_wait_time_in_ms < 0) { g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; } + g_fail_over_retry_count = iniReader.getIntValue("fail_over_retry_count", DEFAULT_FAIL_OVER_RETRY_COUNT); } /** @@ -180,6 +187,7 @@ public class ClientGlobal { String poolMaxCountPerEntry = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_COUNT_PER_ENTRY); String poolMaxIdleTime = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_IDLE_TIME); String poolMaxWaitTimeInMS = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS); + String failOverRetryCount = props.getProperty(PROP_KEY_FAIL_OVER_RETRY_COUNT); if (connectTimeoutInSecondsConf != null && connectTimeoutInSecondsConf.trim().length() != 0) { g_connect_timeout = Integer.parseInt(connectTimeoutInSecondsConf.trim()) * 1000; @@ -211,6 +219,9 @@ public class ClientGlobal { if (poolMaxWaitTimeInMS != null && poolMaxWaitTimeInMS.trim().length() != 0) { g_connection_pool_max_wait_time_in_ms = Integer.parseInt(poolMaxWaitTimeInMS); } + if (failOverRetryCount != null && failOverRetryCount.trim().length() != 0) { + g_fail_over_retry_count = Integer.parseInt(failOverRetryCount); + } } /** @@ -352,6 +363,7 @@ public class ClientGlobal { + "\n g_connection_pool_max_count_per_entry = " + g_connection_pool_max_count_per_entry + "\n g_connection_pool_max_idle_time(ms) = " + g_connection_pool_max_idle_time + "\n g_connection_pool_max_wait_time_in_ms(ms) = " + g_connection_pool_max_wait_time_in_ms + + "\n g_fail_over_retry_count = " + g_fail_over_retry_count + "\n trackerServers = " + trackerServers + "\n}"; } diff --git a/src/main/java/org/csource/fastdfs/StorageClient.java b/src/main/java/org/csource/fastdfs/StorageClient.java index 89f2798..5e11726 100644 --- a/src/main/java/org/csource/fastdfs/StorageClient.java +++ b/src/main/java/org/csource/fastdfs/StorageClient.java @@ -28,6 +28,7 @@ public class StorageClient { public final static Base64 base64 = new Base64('-', '_', '.', 0); protected TrackerServer trackerServer; protected StorageServer storageServer; + protected TrackerClient trackerClient; protected byte errno; /** @@ -731,7 +732,6 @@ public class StorageClient { try { connection = this.storageServer.getConnection(); - ext_name_bs = new byte[ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN]; Arrays.fill(ext_name_bs, (byte) 0); if (file_ext_name != null && file_ext_name.length() > 0) { diff --git a/src/main/java/org/csource/fastdfs/TrackerClient.java b/src/main/java/org/csource/fastdfs/TrackerClient.java index 22c4bba..5d4e4fe 100644 --- a/src/main/java/org/csource/fastdfs/TrackerClient.java +++ b/src/main/java/org/csource/fastdfs/TrackerClient.java @@ -71,6 +71,45 @@ public class TrackerClient { return this.getStoreStorage(trackerServer, groupName); } + public Connection getConnection(TrackerServer trackerServer) throws IOException, MyException { + if (ClientGlobal.g_fail_over_retry_count > 0) { + int retryCount = 0; + do { + try { + trackerServer = getTrackerServer(); + if (trackerServer == null) { + throw new MyException("tracker server is empty!"); + } + return trackerServer.getConnection(); + } catch (IOException e) { + if (retryCount <= ClientGlobal.g_fail_over_retry_count) { + //allow retry ignore exception + System.err.println("trackerServer get connection error, get connection from next tracker, emsg:" + e.getMessage()); + } else { + throw e; + } + } catch (MyException e) { + if (retryCount <= ClientGlobal.g_fail_over_retry_count) { + System.err.println("trackerServer get connection error, get connection from next tracker, emsg:" + e.getMessage()); + //allow retry ignore exception + } else { + throw e; + } + } + } while (retryCount++ <= ClientGlobal.g_fail_over_retry_count); + } else { + if (trackerServer == null) { + trackerServer = getTrackerServer(); + if (trackerServer == null) { + throw new MyException("tracker server is empty!"); + } + } + trackerServer.getConnection(); + } + return null; + } + + /** * query storage server to upload file * @@ -85,12 +124,7 @@ public class TrackerClient { byte cmd; int out_len; byte store_path; - Connection connection; - - if (trackerServer == null) { - trackerServer = getTrackerServer(); - } - connection = trackerServer.getConnection(); + Connection connection = getConnection(trackerServer); OutputStream out = connection.getOutputStream(); try { @@ -170,16 +204,7 @@ public class TrackerClient { int port; byte cmd; int out_len; - Connection connection; - - if (trackerServer == null) { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - return null; - } - } - - connection = trackerServer.getConnection(); + Connection connection = getConnection(trackerServer); OutputStream out = connection.getOutputStream(); try { @@ -343,15 +368,7 @@ public class TrackerClient { int len; String ip_addr; int port; - Connection connection; - - if (trackerServer == null) { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - return null; - } - } - connection = trackerServer.getConnection(); + Connection connection = getConnection(trackerServer); OutputStream out = connection.getOutputStream(); try { @@ -473,16 +490,7 @@ public class TrackerClient { byte cmd; int out_len; byte store_path; - Connection connection; - - if (trackerServer == null) { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - return null; - } - } - - connection = trackerServer.getConnection(); + Connection connection = getConnection(trackerServer); OutputStream out = connection.getOutputStream(); try { @@ -548,15 +556,7 @@ public class TrackerClient { byte[] bGroupName; byte[] bs; int len; - Connection connection; - - if (trackerServer == null) { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - return null; - } - } - connection = trackerServer.getConnection(); + Connection connection = getConnection(trackerServer); OutputStream out = connection.getOutputStream(); try { diff --git a/src/main/java/org/csource/fastdfs/TrackerGroup.java b/src/main/java/org/csource/fastdfs/TrackerGroup.java index dfcfd74..d9ae475 100644 --- a/src/main/java/org/csource/fastdfs/TrackerGroup.java +++ b/src/main/java/org/csource/fastdfs/TrackerGroup.java @@ -18,84 +18,83 @@ import java.net.InetSocketAddress; * @version Version 1.17 */ public class TrackerGroup { - public int tracker_server_index; - public InetSocketAddress[] tracker_servers; - protected Integer lock; + public int tracker_server_index; + public InetSocketAddress[] tracker_servers; + protected Integer lock; - /** - * Constructor - * - * @param tracker_servers tracker servers - */ - public TrackerGroup(InetSocketAddress[] tracker_servers) { - this.tracker_servers = tracker_servers; - this.lock = new Integer(0); - this.tracker_server_index = 0; - } - - /** - * return connected tracker server - * - * @return connected tracker server, null for fail - */ - public TrackerServer getTrackerServer(int serverIndex) throws IOException { - return new TrackerServer(this.tracker_servers[serverIndex]); - } - - /** - * return connected tracker server - * - * @return connected tracker server, null for fail - */ - public TrackerServer getTrackerServer() throws IOException { - int current_index; - - synchronized (this.lock) { - this.tracker_server_index++; - if (this.tracker_server_index >= this.tracker_servers.length) { + /** + * Constructor + * + * @param tracker_servers tracker servers + */ + public TrackerGroup(InetSocketAddress[] tracker_servers) { + this.tracker_servers = tracker_servers; + this.lock = new Integer(0); this.tracker_server_index = 0; - } - - current_index = this.tracker_server_index; } - try { - return this.getTrackerServer(current_index); - } catch (IOException ex) { - System.err.println("connect to server " + this.tracker_servers[current_index].getAddress().getHostAddress() + ":" + this.tracker_servers[current_index].getPort() + " fail"); - ex.printStackTrace(System.err); + /** + * return connected tracker server + * + * @return connected tracker server, null for fail + */ + public TrackerServer getTrackerServer(int serverIndex) throws IOException { + return new TrackerServer(this.tracker_servers[serverIndex]); } - - for (int i = 0; i < this.tracker_servers.length; i++) { - if (i == current_index) { - continue; - } - - try { - TrackerServer trackerServer = this.getTrackerServer(i); + /** + * return connected tracker server + * + * @return connected tracker server, null for fail + */ + public TrackerServer getTrackerServer() throws IOException { + int current_index; synchronized (this.lock) { - if (this.tracker_server_index == current_index) { - this.tracker_server_index = i; - } + this.tracker_server_index++; + if (this.tracker_server_index >= this.tracker_servers.length) { + this.tracker_server_index = 0; + } + + current_index = this.tracker_server_index; } - return trackerServer; - } catch (IOException ex) { - System.err.println("connect to server " + this.tracker_servers[i].getAddress().getHostAddress() + ":" + this.tracker_servers[i].getPort() + " fail"); - ex.printStackTrace(System.err); - } + try { + return this.getTrackerServer(current_index); + } catch (IOException ex) { + System.err.println("connect to server " + this.tracker_servers[current_index].getAddress().getHostAddress() + ":" + this.tracker_servers[current_index].getPort() + " fail"); + ex.printStackTrace(System.err); + } + + for (int i = 0; i < this.tracker_servers.length; i++) { + if (i == current_index) { + continue; + } + + try { + TrackerServer trackerServer = this.getTrackerServer(i); + + synchronized (this.lock) { + if (this.tracker_server_index == current_index) { + this.tracker_server_index = i; + } + } + + return trackerServer; + } catch (IOException ex) { + System.err.println("connect to server " + this.tracker_servers[i].getAddress().getHostAddress() + ":" + this.tracker_servers[i].getPort() + " fail"); + ex.printStackTrace(System.err); + } + } + + return null; } - return null; - } + public Object clone() { + InetSocketAddress[] trackerServers = new InetSocketAddress[this.tracker_servers.length]; + for (int i = 0; i < trackerServers.length; i++) { + trackerServers[i] = new InetSocketAddress(this.tracker_servers[i].getAddress().getHostAddress(), this.tracker_servers[i].getPort()); + } - public Object clone() { - InetSocketAddress[] trackerServers = new InetSocketAddress[this.tracker_servers.length]; - for (int i = 0; i < trackerServers.length; i++) { - trackerServers[i] = new InetSocketAddress(this.tracker_servers[i].getAddress().getHostAddress(), this.tracker_servers[i].getPort()); + return new TrackerGroup(trackerServers); } - - return new TrackerGroup(trackerServers); - } } diff --git a/src/main/resources/fastdfs-client.properties.sample b/src/main/resources/fastdfs-client.properties.sample index dd33b50..32f5a87 100644 --- a/src/main/resources/fastdfs-client.properties.sample +++ b/src/main/resources/fastdfs-client.properties.sample @@ -21,4 +21,6 @@ fastdfs.connection_pool.max_count_per_entry = 500 fastdfs.connection_pool.max_idle_time = 3600 ## Maximum waiting time when the maximum number of connections is reached, unit: millisecond, default value is 1000 -fastdfs.connection_pool.max_wait_time_in_ms = 1000 \ No newline at end of file +fastdfs.connection_pool.max_wait_time_in_ms = 1000 + +fastdfs.fail_over_retry_count = 0 \ No newline at end of file diff --git a/src/main/resources/fdfs_client.conf.sample b/src/main/resources/fdfs_client.conf.sample index 741218c..7db129b 100644 --- a/src/main/resources/fdfs_client.conf.sample +++ b/src/main/resources/fdfs_client.conf.sample @@ -11,4 +11,6 @@ tracker_server = 10.0.11.244:22122 connection_pool.enabled = true connection_pool.max_count_per_entry = 500 connection_pool.max_idle_time = 3600 -connection_pool.max_wait_time_in_ms = 1000 \ No newline at end of file +connection_pool.max_wait_time_in_ms = 1000 + +fail_over_retry_count = 1 \ No newline at end of file diff --git a/src/test/java/org/csource/fastdfs/FdfsTest.java b/src/test/java/org/csource/fastdfs/FdfsTest.java index 6a23441..4ad05f0 100644 --- a/src/test/java/org/csource/fastdfs/FdfsTest.java +++ b/src/test/java/org/csource/fastdfs/FdfsTest.java @@ -98,9 +98,9 @@ public class FdfsTest { @Test public void testUploadDownload() throws Exception { NameValuePair[] metaList = new NameValuePair[1]; - String local_filename = "build.PNG"; + String local_filename = "commitment.d2f57e10 (2).jpg"; metaList[0] = new NameValuePair("fileName", local_filename); - File file = new File("C:/Users/chengdu/Desktop/build.PNG"); + File file = new File("/Users/iyw/Downloads/commitment.d2f57e10 (2).jpg"); InputStream inputStream = new FileInputStream(file); int length = inputStream.available(); byte[] bytes = new byte[length]; From 0b2ec3aa10023919db5fc0964cd507754390b483 Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sat, 28 Jan 2023 15:27:19 +0800 Subject: [PATCH 02/11] fix:remove not use code --- src/main/java/org/csource/fastdfs/StorageClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/csource/fastdfs/StorageClient.java b/src/main/java/org/csource/fastdfs/StorageClient.java index 5e11726..5f0d61e 100644 --- a/src/main/java/org/csource/fastdfs/StorageClient.java +++ b/src/main/java/org/csource/fastdfs/StorageClient.java @@ -28,7 +28,6 @@ public class StorageClient { public final static Base64 base64 = new Base64('-', '_', '.', 0); protected TrackerServer trackerServer; protected StorageServer storageServer; - protected TrackerClient trackerClient; protected byte errno; /** From 77946b2382c2d5e45790e32e03abb55c08241ab7 Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sat, 28 Jan 2023 16:51:33 +0800 Subject: [PATCH 03/11] feature: set default failover size --- .../org/csource/fastdfs/ClientGlobal.java | 16 ++++- .../org/csource/fastdfs/TrackerClient.java | 70 ++++++++++++------- .../fastdfs-client.properties.sample | 3 +- src/main/resources/fdfs_client.conf.sample | 2 +- .../java/org/csource/fastdfs/FdfsTest.java | 6 +- 5 files changed, 63 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/csource/fastdfs/ClientGlobal.java b/src/main/java/org/csource/fastdfs/ClientGlobal.java index 0fc3450..9d36104 100644 --- a/src/main/java/org/csource/fastdfs/ClientGlobal.java +++ b/src/main/java/org/csource/fastdfs/ClientGlobal.java @@ -63,7 +63,7 @@ public class ClientGlobal { public static final int DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME = 3600 ;//second public static final int DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS = 1000 ;//millisecond - public static final int DEFAULT_FAIL_OVER_RETRY_COUNT = 1; + public static final int DEFAULT_FAIL_OVER_RETRY_COUNT = -1; public static int g_connect_timeout = DEFAULT_CONNECT_TIMEOUT * 1000; //millisecond public static int g_network_timeout = DEFAULT_NETWORK_TIMEOUT * 1000; //millisecond @@ -77,7 +77,7 @@ public class ClientGlobal { public static int g_connection_pool_max_idle_time = DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME * 1000; //millisecond public static int g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; //millisecond - public static int g_fail_over_retry_count = DEFAULT_FAIL_OVER_RETRY_COUNT ;//get connection retry count when fail + public static int g_fail_over_retry_count = 0;//get connection retry count when fail public static TrackerGroup g_tracker_group; @@ -146,6 +146,12 @@ public class ClientGlobal { g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; } g_fail_over_retry_count = iniReader.getIntValue("fail_over_retry_count", DEFAULT_FAIL_OVER_RETRY_COUNT); + if (g_fail_over_retry_count == DEFAULT_FAIL_OVER_RETRY_COUNT) { + //缺省值为tracker server数量 -1 + if (tracker_servers.length > 1) { + g_fail_over_retry_count = tracker_servers.length -1; + } + } } /** @@ -221,6 +227,12 @@ public class ClientGlobal { } if (failOverRetryCount != null && failOverRetryCount.trim().length() != 0) { g_fail_over_retry_count = Integer.parseInt(failOverRetryCount); + if(g_fail_over_retry_count == DEFAULT_FAIL_OVER_RETRY_COUNT) { + int trackerLength = g_tracker_group.tracker_servers.length; + if (trackerLength > 1) { + g_fail_over_retry_count = trackerLength - 1; + } + } } } diff --git a/src/main/java/org/csource/fastdfs/TrackerClient.java b/src/main/java/org/csource/fastdfs/TrackerClient.java index 5d4e4fe..c11f131 100644 --- a/src/main/java/org/csource/fastdfs/TrackerClient.java +++ b/src/main/java/org/csource/fastdfs/TrackerClient.java @@ -72,39 +72,55 @@ public class TrackerClient { } public Connection getConnection(TrackerServer trackerServer) throws IOException, MyException { - if (ClientGlobal.g_fail_over_retry_count > 0) { - int retryCount = 0; - do { - try { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - throw new MyException("tracker server is empty!"); - } - return trackerServer.getConnection(); - } catch (IOException e) { - if (retryCount <= ClientGlobal.g_fail_over_retry_count) { - //allow retry ignore exception - System.err.println("trackerServer get connection error, get connection from next tracker, emsg:" + e.getMessage()); - } else { - throw e; - } - } catch (MyException e) { - if (retryCount <= ClientGlobal.g_fail_over_retry_count) { - System.err.println("trackerServer get connection error, get connection from next tracker, emsg:" + e.getMessage()); - //allow retry ignore exception - } else { - throw e; - } - } - } while (retryCount++ <= ClientGlobal.g_fail_over_retry_count); - } else { + Connection connection = null; + boolean failOver = ClientGlobal.g_fail_over_retry_count > 0 && trackerServer == null; + try { if (trackerServer == null) { trackerServer = getTrackerServer(); if (trackerServer == null) { throw new MyException("tracker server is empty!"); } } - trackerServer.getConnection(); + connection = trackerServer.getConnection(); + } catch (IOException e) { + if (failOver) { + System.err.println("default trackerServer get connection error, emsg:" + e.getMessage()); + } else { + throw e; + } + } catch (MyException e) { + if (failOver) { + System.err.println("default trackerServer get connection error, emsg:" + e.getMessage()); + } else { + throw e; + } + } + if (connection != null || !failOver) { + return connection; + } + int retryCount = 0; + while (retryCount++ <= ClientGlobal.g_fail_over_retry_count) { + try { + trackerServer = getTrackerServer(); + if (trackerServer == null) { + throw new MyException("tracker server is empty!"); + } + return trackerServer.getConnection(); + } catch (IOException e) { + if (retryCount <= ClientGlobal.g_fail_over_retry_count) { + //allow retry ignore exception + System.err.println("retry trackerServer get connection error, get connection from next tracker, retryCount:" + retryCount + ",+ emsg:" + e.getMessage()); + } else { + throw e; + } + } catch (MyException e) { + if (retryCount <= ClientGlobal.g_fail_over_retry_count) { + System.err.println("trackerServer get connection error, get connection from next tracker, retryCount:" + retryCount + ", emsg:" + e.getMessage()); + //allow retry ignore exception + } else { + throw e; + } + } } return null; } diff --git a/src/main/resources/fastdfs-client.properties.sample b/src/main/resources/fastdfs-client.properties.sample index 32f5a87..72d1a4a 100644 --- a/src/main/resources/fastdfs-client.properties.sample +++ b/src/main/resources/fastdfs-client.properties.sample @@ -23,4 +23,5 @@ fastdfs.connection_pool.max_idle_time = 3600 ## Maximum waiting time when the maximum number of connections is reached, unit: millisecond, default value is 1000 fastdfs.connection_pool.max_wait_time_in_ms = 1000 -fastdfs.fail_over_retry_count = 0 \ No newline at end of file +## The missing value is - 1, and the acquisition times are the number of tracker servers minus 1 +fastdfs.fail_over_retry_count = -1 \ No newline at end of file diff --git a/src/main/resources/fdfs_client.conf.sample b/src/main/resources/fdfs_client.conf.sample index 7db129b..e7df694 100644 --- a/src/main/resources/fdfs_client.conf.sample +++ b/src/main/resources/fdfs_client.conf.sample @@ -13,4 +13,4 @@ connection_pool.max_count_per_entry = 500 connection_pool.max_idle_time = 3600 connection_pool.max_wait_time_in_ms = 1000 -fail_over_retry_count = 1 \ No newline at end of file +fail_over_retry_count = -1 \ No newline at end of file diff --git a/src/test/java/org/csource/fastdfs/FdfsTest.java b/src/test/java/org/csource/fastdfs/FdfsTest.java index 4ad05f0..1cb2702 100644 --- a/src/test/java/org/csource/fastdfs/FdfsTest.java +++ b/src/test/java/org/csource/fastdfs/FdfsTest.java @@ -33,7 +33,7 @@ public class FdfsTest { TrackerClient tracker = new TrackerClient(); trackerServer = tracker.getTrackerServer(); StorageServer storageServer = null; - storageClient = new StorageClient(trackerServer, storageServer); + storageClient = new StorageClient(null, storageServer); } @After @@ -106,9 +106,9 @@ public class FdfsTest { byte[] bytes = new byte[length]; inputStream.read(bytes); String[] result = storageClient.upload_file(bytes, null, metaList); - Assert.assertTrue(storageClient.isConnected()); + //Assert.assertTrue(storageClient.isConnected()); // pool testOnborrow isAvaliable - Assert.assertTrue(storageClient.isAvaliable()); + // Assert.assertTrue(storageClient.isAvaliable()); LOGGER.info("result {}", Arrays.asList(result)); byte[] resultbytes = storageClient.download_file(result[0], result[1]); writeByteToFile(resultbytes, local_filename); From 0087216c0addc06811252f24ee2f85b424a6d244 Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sat, 28 Jan 2023 19:53:06 +0800 Subject: [PATCH 04/11] feature: fail over from index --- .../org/csource/fastdfs/ClientGlobal.java | 23 -------- .../org/csource/fastdfs/TrackerClient.java | 57 +++++++++++-------- .../org/csource/fastdfs/TrackerGroup.java | 2 +- .../org/csource/fastdfs/TrackerServer.java | 14 +++++ .../fastdfs-client.properties.sample | 5 +- src/main/resources/fdfs_client.conf.sample | 4 +- 6 files changed, 50 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/csource/fastdfs/ClientGlobal.java b/src/main/java/org/csource/fastdfs/ClientGlobal.java index 9d36104..e8377c5 100644 --- a/src/main/java/org/csource/fastdfs/ClientGlobal.java +++ b/src/main/java/org/csource/fastdfs/ClientGlobal.java @@ -48,9 +48,6 @@ public class ClientGlobal { public static final String PROP_KEY_CONNECTION_POOL_MAX_IDLE_TIME = "fastdfs.connection_pool.max_idle_time"; public static final String PROP_KEY_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS = "fastdfs.connection_pool.max_wait_time_in_ms"; - public static final String PROP_KEY_FAIL_OVER_RETRY_COUNT = "fastdfs.fail_over_retry_count"; - - public static final int DEFAULT_CONNECT_TIMEOUT = 5; //second public static final int DEFAULT_NETWORK_TIMEOUT = 30; //second public static final String DEFAULT_CHARSET = "UTF-8"; @@ -63,8 +60,6 @@ public class ClientGlobal { public static final int DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME = 3600 ;//second public static final int DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS = 1000 ;//millisecond - public static final int DEFAULT_FAIL_OVER_RETRY_COUNT = -1; - public static int g_connect_timeout = DEFAULT_CONNECT_TIMEOUT * 1000; //millisecond public static int g_network_timeout = DEFAULT_NETWORK_TIMEOUT * 1000; //millisecond public static String g_charset = DEFAULT_CHARSET; @@ -145,13 +140,6 @@ public class ClientGlobal { if (g_connection_pool_max_wait_time_in_ms < 0) { g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; } - g_fail_over_retry_count = iniReader.getIntValue("fail_over_retry_count", DEFAULT_FAIL_OVER_RETRY_COUNT); - if (g_fail_over_retry_count == DEFAULT_FAIL_OVER_RETRY_COUNT) { - //缺省值为tracker server数量 -1 - if (tracker_servers.length > 1) { - g_fail_over_retry_count = tracker_servers.length -1; - } - } } /** @@ -193,8 +181,6 @@ public class ClientGlobal { String poolMaxCountPerEntry = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_COUNT_PER_ENTRY); String poolMaxIdleTime = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_IDLE_TIME); String poolMaxWaitTimeInMS = props.getProperty(PROP_KEY_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS); - String failOverRetryCount = props.getProperty(PROP_KEY_FAIL_OVER_RETRY_COUNT); - if (connectTimeoutInSecondsConf != null && connectTimeoutInSecondsConf.trim().length() != 0) { g_connect_timeout = Integer.parseInt(connectTimeoutInSecondsConf.trim()) * 1000; } @@ -225,15 +211,6 @@ public class ClientGlobal { if (poolMaxWaitTimeInMS != null && poolMaxWaitTimeInMS.trim().length() != 0) { g_connection_pool_max_wait_time_in_ms = Integer.parseInt(poolMaxWaitTimeInMS); } - if (failOverRetryCount != null && failOverRetryCount.trim().length() != 0) { - g_fail_over_retry_count = Integer.parseInt(failOverRetryCount); - if(g_fail_over_retry_count == DEFAULT_FAIL_OVER_RETRY_COUNT) { - int trackerLength = g_tracker_group.tracker_servers.length; - if (trackerLength > 1) { - g_fail_over_retry_count = trackerLength - 1; - } - } - } } /** diff --git a/src/main/java/org/csource/fastdfs/TrackerClient.java b/src/main/java/org/csource/fastdfs/TrackerClient.java index c11f131..b8fe088 100644 --- a/src/main/java/org/csource/fastdfs/TrackerClient.java +++ b/src/main/java/org/csource/fastdfs/TrackerClient.java @@ -73,7 +73,8 @@ public class TrackerClient { public Connection getConnection(TrackerServer trackerServer) throws IOException, MyException { Connection connection = null; - boolean failOver = ClientGlobal.g_fail_over_retry_count > 0 && trackerServer == null; + int length = this.tracker_group.tracker_servers.length; + boolean failOver = length > 1 && trackerServer == null; try { if (trackerServer == null) { trackerServer = getTrackerServer(); @@ -84,42 +85,50 @@ public class TrackerClient { connection = trackerServer.getConnection(); } catch (IOException e) { if (failOver) { - System.err.println("default trackerServer get connection error, emsg:" + e.getMessage()); + System.err.println("trackerServer get connection error, emsg:" + e.getMessage()); } else { throw e; } } catch (MyException e) { if (failOver) { - System.err.println("default trackerServer get connection error, emsg:" + e.getMessage()); + System.err.println("trackerServer get connection error, emsg:" + e.getMessage()); } else { throw e; } } - if (connection != null || !failOver) { + if (connection != null && !failOver) { return connection; } - int retryCount = 0; - while (retryCount++ <= ClientGlobal.g_fail_over_retry_count) { - try { - trackerServer = getTrackerServer(); - if (trackerServer == null) { - throw new MyException("tracker server is empty!"); + //do fail over + if (length > 1) { + int currentIndex = trackerServer.getIndex(); + int failOverCount = 0; + while (failOverCount < length - 1) { + failOverCount++; + currentIndex++; + if (currentIndex >= length) { + currentIndex = 0; } - return trackerServer.getConnection(); - } catch (IOException e) { - if (retryCount <= ClientGlobal.g_fail_over_retry_count) { - //allow retry ignore exception - System.err.println("retry trackerServer get connection error, get connection from next tracker, retryCount:" + retryCount + ",+ emsg:" + e.getMessage()); - } else { - throw e; - } - } catch (MyException e) { - if (retryCount <= ClientGlobal.g_fail_over_retry_count) { - System.err.println("trackerServer get connection error, get connection from next tracker, retryCount:" + retryCount + ", emsg:" + e.getMessage()); - //allow retry ignore exception - } else { - throw e; + try { + trackerServer = this.tracker_group.getTrackerServer(currentIndex); + if (trackerServer == null) { + throw new MyException("tracker server is empty!"); + } + return trackerServer.getConnection(); + } catch (IOException e) { + System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + "," + e.getMessage()); + if (failOverCount == length - 1) { + throw e; + } + + } catch (MyException e) { + System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + ", " + e.getMessage()); + if (failOverCount == length - 1) { + throw e; + } } + + } } return null; diff --git a/src/main/java/org/csource/fastdfs/TrackerGroup.java b/src/main/java/org/csource/fastdfs/TrackerGroup.java index d9ae475..b79f706 100644 --- a/src/main/java/org/csource/fastdfs/TrackerGroup.java +++ b/src/main/java/org/csource/fastdfs/TrackerGroup.java @@ -39,7 +39,7 @@ public class TrackerGroup { * @return connected tracker server, null for fail */ public TrackerServer getTrackerServer(int serverIndex) throws IOException { - return new TrackerServer(this.tracker_servers[serverIndex]); + return new TrackerServer(this.tracker_servers[serverIndex], serverIndex); } /** * return connected tracker server diff --git a/src/main/java/org/csource/fastdfs/TrackerServer.java b/src/main/java/org/csource/fastdfs/TrackerServer.java index a9875ed..6ff7834 100644 --- a/src/main/java/org/csource/fastdfs/TrackerServer.java +++ b/src/main/java/org/csource/fastdfs/TrackerServer.java @@ -25,11 +25,18 @@ import java.net.InetSocketAddress; public class TrackerServer { protected InetSocketAddress inetSockAddr; + protected int index; + public TrackerServer(InetSocketAddress inetSockAddr) throws IOException { this.inetSockAddr = inetSockAddr; } + public TrackerServer(InetSocketAddress inetSockAddr, int index) { + this.inetSockAddr = inetSockAddr; + this.index = index; + } + public Connection getConnection() throws MyException, IOException { Connection connection; if (ClientGlobal.g_connection_pool_enabled) { @@ -48,4 +55,11 @@ public class TrackerServer { return this.inetSockAddr; } + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } } diff --git a/src/main/resources/fastdfs-client.properties.sample b/src/main/resources/fastdfs-client.properties.sample index 72d1a4a..dd33b50 100644 --- a/src/main/resources/fastdfs-client.properties.sample +++ b/src/main/resources/fastdfs-client.properties.sample @@ -21,7 +21,4 @@ fastdfs.connection_pool.max_count_per_entry = 500 fastdfs.connection_pool.max_idle_time = 3600 ## Maximum waiting time when the maximum number of connections is reached, unit: millisecond, default value is 1000 -fastdfs.connection_pool.max_wait_time_in_ms = 1000 - -## The missing value is - 1, and the acquisition times are the number of tracker servers minus 1 -fastdfs.fail_over_retry_count = -1 \ No newline at end of file +fastdfs.connection_pool.max_wait_time_in_ms = 1000 \ No newline at end of file diff --git a/src/main/resources/fdfs_client.conf.sample b/src/main/resources/fdfs_client.conf.sample index e7df694..741218c 100644 --- a/src/main/resources/fdfs_client.conf.sample +++ b/src/main/resources/fdfs_client.conf.sample @@ -11,6 +11,4 @@ tracker_server = 10.0.11.244:22122 connection_pool.enabled = true connection_pool.max_count_per_entry = 500 connection_pool.max_idle_time = 3600 -connection_pool.max_wait_time_in_ms = 1000 - -fail_over_retry_count = -1 \ No newline at end of file +connection_pool.max_wait_time_in_ms = 1000 \ No newline at end of file From 4de15ae592c73b047714d9ecedbe27d211e0b85d Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sat, 28 Jan 2023 21:37:29 +0800 Subject: [PATCH 05/11] fix:fail over --- src/main/java/org/csource/fastdfs/TrackerClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/csource/fastdfs/TrackerClient.java b/src/main/java/org/csource/fastdfs/TrackerClient.java index b8fe088..529e945 100644 --- a/src/main/java/org/csource/fastdfs/TrackerClient.java +++ b/src/main/java/org/csource/fastdfs/TrackerClient.java @@ -96,7 +96,7 @@ public class TrackerClient { throw e; } } - if (connection != null && !failOver) { + if (connection != null || !failOver) { return connection; } //do fail over From 2b4693643ab19f12a0ff53f0a63751a07c0ffaab Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sun, 29 Jan 2023 09:45:14 +0800 Subject: [PATCH 06/11] fix:not use code --- .../org/csource/fastdfs/TrackerClient.java | 59 ++++++++++--------- .../java/org/csource/fastdfs/FdfsTest.java | 6 +- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/csource/fastdfs/TrackerClient.java b/src/main/java/org/csource/fastdfs/TrackerClient.java index 529e945..8b8bd56 100644 --- a/src/main/java/org/csource/fastdfs/TrackerClient.java +++ b/src/main/java/org/csource/fastdfs/TrackerClient.java @@ -100,36 +100,37 @@ public class TrackerClient { return connection; } //do fail over - if (length > 1) { - int currentIndex = trackerServer.getIndex(); - int failOverCount = 0; - while (failOverCount < length - 1) { - failOverCount++; - currentIndex++; - if (currentIndex >= length) { - currentIndex = 0; - } - try { - trackerServer = this.tracker_group.getTrackerServer(currentIndex); - if (trackerServer == null) { - throw new MyException("tracker server is empty!"); - } - return trackerServer.getConnection(); - } catch (IOException e) { - System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + "," + e.getMessage()); - if (failOverCount == length - 1) { - throw e; - } - - } catch (MyException e) { - System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + ", " + e.getMessage()); - if (failOverCount == length - 1) { - throw e; - } - } - - + int currentIndex = 0; + if (trackerServer != null) { + currentIndex = trackerServer.getIndex(); + } + int failOverCount = 0; + while (failOverCount < length - 1) { + failOverCount++; + currentIndex++; + if (currentIndex >= length) { + currentIndex = 0; } + try { + trackerServer = this.tracker_group.getTrackerServer(currentIndex); + if (trackerServer == null) { + throw new MyException("tracker server is empty!"); + } + return trackerServer.getConnection(); + } catch (IOException e) { + System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + "," + e.getMessage()); + if (failOverCount == length - 1) { + throw e; + } + + } catch (MyException e) { + System.err.println("fail over trackerServer get connection error, failOverCount:" + failOverCount + ", " + e.getMessage()); + if (failOverCount == length - 1) { + throw e; + } + } + + } return null; } diff --git a/src/test/java/org/csource/fastdfs/FdfsTest.java b/src/test/java/org/csource/fastdfs/FdfsTest.java index 1cb2702..74f3e6e 100644 --- a/src/test/java/org/csource/fastdfs/FdfsTest.java +++ b/src/test/java/org/csource/fastdfs/FdfsTest.java @@ -33,7 +33,7 @@ public class FdfsTest { TrackerClient tracker = new TrackerClient(); trackerServer = tracker.getTrackerServer(); StorageServer storageServer = null; - storageClient = new StorageClient(null, storageServer); + storageClient = new StorageClient(trackerServer, storageServer); } @After @@ -87,9 +87,9 @@ public class FdfsTest { @Test public void download() throws Exception { - String[] uploadresult = {"group1", "M00/00/00/wKgBZV0phl2ASV1nAACk1tFxwrM3814331"}; + String[] uploadresult = {"group1", "M00/00/00/J2fL12PVypeAWiGcAAM_gDeWVyw5817085"}; byte[] result = storageClient.download_file(uploadresult[0], uploadresult[1]); - String local_filename = "build.PNG"; + String local_filename = "commitment.d2f57e10.jpg"; writeByteToFile(result, local_filename); File file = new File(local_filename); Assert.assertTrue(file.isFile()); From a1df79c50eeaf047f0995f5eb0c26745c68f9457 Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sun, 29 Jan 2023 09:49:29 +0800 Subject: [PATCH 07/11] fix:upgrade pom version 1.30 --- pom.xml | 2 +- src/main/java/org/csource/fastdfs/ClientGlobal.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index ac8c219..65f6ea2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ org.csource fastdfs-client-java - 1.29-SNAPSHOT + 1.30-SNAPSHOT fastdfs-client-java fastdfs client for java jar diff --git a/src/main/java/org/csource/fastdfs/ClientGlobal.java b/src/main/java/org/csource/fastdfs/ClientGlobal.java index e8377c5..af00000 100644 --- a/src/main/java/org/csource/fastdfs/ClientGlobal.java +++ b/src/main/java/org/csource/fastdfs/ClientGlobal.java @@ -72,8 +72,6 @@ public class ClientGlobal { public static int g_connection_pool_max_idle_time = DEFAULT_CONNECTION_POOL_MAX_IDLE_TIME * 1000; //millisecond public static int g_connection_pool_max_wait_time_in_ms = DEFAULT_CONNECTION_POOL_MAX_WAIT_TIME_IN_MS; //millisecond - public static int g_fail_over_retry_count = 0;//get connection retry count when fail - public static TrackerGroup g_tracker_group; private ClientGlobal() { @@ -352,7 +350,6 @@ public class ClientGlobal { + "\n g_connection_pool_max_count_per_entry = " + g_connection_pool_max_count_per_entry + "\n g_connection_pool_max_idle_time(ms) = " + g_connection_pool_max_idle_time + "\n g_connection_pool_max_wait_time_in_ms(ms) = " + g_connection_pool_max_wait_time_in_ms - + "\n g_fail_over_retry_count = " + g_fail_over_retry_count + "\n trackerServers = " + trackerServers + "\n}"; } From 41655defd87e3b991f0afa7db834a81e81568a8f Mon Sep 17 00:00:00 2001 From: tanyawen01770 Date: Sun, 29 Jan 2023 11:46:55 +0800 Subject: [PATCH 08/11] feature:add history --- HISTORY | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY b/HISTORY index 1d26881..b1a12b7 100644 --- a/HISTORY +++ b/HISTORY @@ -1,3 +1,8 @@ +Version 1.30 2023-01-29 + * support tracker server fail over + If the tracker server is not specified, when the tracker server fails to + get the connection, it will try to get the connection from other tracker servers. + The maximum number of attempts is the number of tracker servers minus 1 Version 1.29 2020-01-03 * support active test for connection pool. From 1b263e64362efca91bad0624cad1e8ae58bc4cc4 Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Sun, 29 Jan 2023 14:16:17 +0800 Subject: [PATCH 09/11] add *.class to .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fc1e3bd..76d9377 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,6 @@ target *.iws *.log .idea - *.conf *.PNG +*.class From 42132c694ccf2da8363d09ccf3622a4bb86a474e Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Sun, 29 Jan 2023 14:47:09 +0800 Subject: [PATCH 10/11] add @SuppressWarnings(unchecked) --- src/main/java/org/csource/common/IniFileReader.java | 3 +-- src/main/java/org/csource/fastdfs/ClientGlobal.java | 2 +- src/main/java/org/csource/fastdfs/ProtoStructDecoder.java | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/csource/common/IniFileReader.java b/src/main/java/org/csource/common/IniFileReader.java index e1c609c..3146635 100644 --- a/src/main/java/org/csource/common/IniFileReader.java +++ b/src/main/java/org/csource/common/IniFileReader.java @@ -155,13 +155,13 @@ public class IniFileReader { } finally { try { if (in != null) in.close(); - //System.out.println("loadFrom...finally...in.close(); done"); } catch (Exception ex) { ex.printStackTrace(); } } } + @SuppressWarnings("unchecked") private void readToParamTable(InputStream in) throws IOException { this.paramTable = new Hashtable(); if (in == null) return; @@ -206,7 +206,6 @@ public class IniFileReader { try { if (bufferedReader != null) bufferedReader.close(); if (inReader != null) inReader.close(); - //System.out.println("readToParamTable...finally...bufferedReader.close();inReader.close(); done"); } catch (Exception ex) { ex.printStackTrace(); } diff --git a/src/main/java/org/csource/fastdfs/ClientGlobal.java b/src/main/java/org/csource/fastdfs/ClientGlobal.java index af00000..98e763d 100644 --- a/src/main/java/org/csource/fastdfs/ClientGlobal.java +++ b/src/main/java/org/csource/fastdfs/ClientGlobal.java @@ -219,7 +219,7 @@ public class ClientGlobal { * server之间用逗号','分隔 */ public static void initByTrackers(String trackerServers) throws IOException, MyException { - List list = new ArrayList(); + List list = new ArrayList(); String spr1 = ","; String spr2 = ":"; String[] arr1 = trackerServers.trim().split(spr1); diff --git a/src/main/java/org/csource/fastdfs/ProtoStructDecoder.java b/src/main/java/org/csource/fastdfs/ProtoStructDecoder.java index f6b9d9f..07dec45 100644 --- a/src/main/java/org/csource/fastdfs/ProtoStructDecoder.java +++ b/src/main/java/org/csource/fastdfs/ProtoStructDecoder.java @@ -27,6 +27,7 @@ public class ProtoStructDecoder { /** * decode byte buffer */ + @SuppressWarnings("unchecked") public T[] decode(byte[] bs, Class clazz, int fieldsTotalSize) throws Exception { if (bs.length % fieldsTotalSize != 0) { throw new IOException("byte array length: " + bs.length + " is invalid!"); From 45d9a89e2583297a47870a47694184dffb265dc7 Mon Sep 17 00:00:00 2001 From: YuQing <384681@qq.com> Date: Mon, 30 Jan 2023 09:27:21 +0800 Subject: [PATCH 11/11] upgrade version to v1.30 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21782b6..b69c30f 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ mvn install:install-file -DgroupId=org.csource -DartifactId=fastdfs-client-java org.csource fastdfs-client-java - 1.29-SNAPSHOT + 1.30-SNAPSHOT ```