7.0

- Some Smart Proxy tunning (excluded proxy list)

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.tonikelope</groupId>
     <artifactId>MegaBasterd</artifactId>
-    <version>6.99</version>
+    <version>7.0</version>
     <packaging>jar</packaging>
     <dependencies>
         <dependency>

src/main/java/com/tonikelope/megabasterd/ChunkDownloader.java

@@ -12,6 +12,7 @@ import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.SocketTimeoutException;
 import java.net.URL;
+import java.util.LinkedHashMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -31,6 +32,7 @@ public class ChunkDownloader implements Runnable, SecureSingleThreadNotifiable {
     private volatile boolean _error_wait;
     private volatile boolean _chunk_exception;
     private boolean _notified;
+    private final LinkedHashMap<String, Long> _excluded_proxy_list;
 
     private String _current_smart_proxy;
 
@@ -42,6 +44,7 @@ public class ChunkDownloader implements Runnable, SecureSingleThreadNotifiable {
         _id = id;
         _download = download;
         _current_smart_proxy = null;
+        _excluded_proxy_list = new LinkedHashMap<>();
         _error_wait = false;
 
     }
@@ -129,7 +132,7 @@ public class ChunkDownloader implements Runnable, SecureSingleThreadNotifiable {
 
             if (FORCE_SMART_PROXY) {
 
-                _current_smart_proxy = proxy_manager.getProxy();
+                _current_smart_proxy = proxy_manager.getProxy(_excluded_proxy_list);
 
                 if (!getDownload().isTurbo()) {
                     getDownload().enableTurboMode();
@@ -165,15 +168,21 @@ public class ChunkDownloader implements Runnable, SecureSingleThreadNotifiable {
 
                     if (_current_smart_proxy != null && (slow_proxy || chunk_error)) {
 
-                        proxy_manager.blockProxy(_current_smart_proxy);
+                        if (http_error == 509) {
+                            proxy_manager.blockProxy(_current_smart_proxy);
+                        }
 
-                        _current_smart_proxy = proxy_manager.getProxy();
+                        _excluded_proxy_list.put(_current_smart_proxy, System.currentTimeMillis() + SmartMegaProxyManager.BLOCK_TIME * 1000);
+
+                        SmartMegaProxyManager.purgeExcludedProxyList(_excluded_proxy_list);
+
+                        _current_smart_proxy = proxy_manager.getProxy(_excluded_proxy_list);
 
                         Logger.getLogger(MiscTools.class.getName()).log(Level.WARNING, "{0}: worker {1} excluding proxy -> {2} {3}", new Object[]{Thread.currentThread().getName(), _id, _current_smart_proxy, _download.getFile_name()});
 
                     } else if (_current_smart_proxy == null) {
 
-                        _current_smart_proxy = proxy_manager.getProxy();
+                        _current_smart_proxy = proxy_manager.getProxy(_excluded_proxy_list);
 
                         if (!getDownload().isTurbo()) {
                             getDownload().enableTurboMode();

src/main/java/com/tonikelope/megabasterd/MainPanel.java

@@ -54,7 +54,7 @@ import javax.swing.UIManager;
  */
 public final class MainPanel {
 
-    public static final String VERSION = "6.99";
+    public static final String VERSION = "7.0";
     public static final int THROTTLE_SLICE_SIZE = 16 * 1024;
     public static final int DEFAULT_BYTE_BUFFER_SIZE = 16 * 1024;
     public static final int STREAMER_PORT = 1337;

src/main/java/com/tonikelope/megabasterd/MegaAPI.java

@@ -16,6 +16,7 @@ import java.net.Proxy;
 import java.net.URL;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Random;
 import java.util.logging.Level;
@@ -359,21 +360,33 @@ public class MegaAPI implements Serializable {
 
         HttpsURLConnection con = null;
 
+        LinkedHashMap<String, Long> excluded_proxy_list = new LinkedHashMap<>();
+
+        SmartMegaProxyManager proxy_manager = MainPanel.getProxy_manager();
+
         do {
 
             try {
 
-                if (http_error == 509 && !MainPanel.isUse_proxy()) {
+                if ((current_smart_proxy != null || http_error == 509) && MainPanel.isUse_smart_proxy() && !MainPanel.isUse_proxy()) {
 
-                    if (current_smart_proxy != null) {
+                    if (current_smart_proxy != null && (http_error != 0 || mega_error != 0)) {
 
-                        Logger.getLogger(MiscTools.class.getName()).log(Level.WARNING, "{0}: excluding proxy -> {1}", new Object[]{Thread.currentThread().getName(), current_smart_proxy});
+                        if (http_error == 509) {
+                            proxy_manager.blockProxy(current_smart_proxy);
+                        }
 
-                        MainPanel.getProxy_manager().blockProxy(current_smart_proxy);
+                        excluded_proxy_list.put(current_smart_proxy, System.currentTimeMillis() + SmartMegaProxyManager.BLOCK_TIME * 1000);
+
+                        SmartMegaProxyManager.purgeExcludedProxyList(excluded_proxy_list);
+
+                        current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
+
+                    } else if (current_smart_proxy == null) {
+
+                        current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
                     }
 
-                    current_smart_proxy = MainPanel.getProxy_manager().getProxy();
-
                     if (current_smart_proxy != null) {
 
                         String[] proxy_info = current_smart_proxy.split(":");

src/main/java/com/tonikelope/megabasterd/MiscTools.java

@@ -49,6 +49,7 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.Enumeration;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.Callable;
@@ -849,15 +850,17 @@ public class MiscTools {
             return false;
         }
 
-        boolean url_ok = false;
-
         HttpURLConnection con = null;
 
-        boolean error = false, error509 = false, error403 = false;
+        boolean error = false;
+
+        int http_status = 0, http_error = 0;
 
         SmartMegaProxyManager proxy_manager = MainPanel.getProxy_manager();
 
-        String current_proxy = null;
+        String current_smart_proxy = null;
+
+        LinkedHashMap<String, Long> excluded_proxy_list = new LinkedHashMap<>();
 
         do {
 
@@ -867,26 +870,28 @@ public class MiscTools {
 
                 if (con == null || error) {
 
-                    if (error509 && MainPanel.isUse_smart_proxy() && !MainPanel.isUse_proxy()) {
+                    if ((current_smart_proxy != null || http_error == 509) && MainPanel.isUse_smart_proxy() && !MainPanel.isUse_proxy()) {
 
-                        if (MainPanel.isUse_smart_proxy() && proxy_manager == null) {
+                        if (current_smart_proxy != null && error) {
 
-                            proxy_manager = MainPanel.getProxy_manager();
+                            if (http_error == 509) {
+                                proxy_manager.blockProxy(current_smart_proxy);
+                            }
 
+                            excluded_proxy_list.put(current_smart_proxy, System.currentTimeMillis() + SmartMegaProxyManager.BLOCK_TIME * 1000);
+
+                            SmartMegaProxyManager.purgeExcludedProxyList(excluded_proxy_list);
+
+                            current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
+
+                        } else if (current_smart_proxy == null) {
+
+                            current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
                         }
 
-                        if (error && !error403 && current_proxy != null) {
+                        if (current_smart_proxy != null) {
 
-                            proxy_manager.blockProxy(current_proxy);
+                            String[] proxy_info = current_smart_proxy.split(":");
-                            Logger.getLogger(MiscTools.class.getName()).log(Level.WARNING, "{0}: excluding proxy -> {1}", new Object[]{Thread.currentThread().getName(), current_proxy});
-
-                        }
-
-                        current_proxy = proxy_manager.getProxy();
-
-                        if (current_proxy != null) {
-
-                            String[] proxy_info = current_proxy.split(":");
 
                             Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_info[0], Integer.parseInt(proxy_info[1])));
 
@@ -926,7 +931,7 @@ public class MiscTools {
 
                 }
 
-                if (current_proxy != null) {
+                if (current_smart_proxy != null) {
                     con.setConnectTimeout(Transference.HTTP_PROXY_CONNECT_TIMEOUT);
                     con.setReadTimeout(Transference.HTTP_PROXY_READ_TIMEOUT);
                 }
@@ -935,12 +940,10 @@ public class MiscTools {
 
                 con.setRequestProperty("User-Agent", MainPanel.DEFAULT_USER_AGENT);
 
-                int http_status = con.getResponseCode();
+                http_status = con.getResponseCode();
-
-                if (http_status != 403) {
-
-                    url_ok = true;
 
+                if (http_status != 200) {
+                    http_error = http_status;
                 }
 
             } catch (IOException ex) {
@@ -952,9 +955,9 @@ public class MiscTools {
                 }
             }
 
-        } while (error509);
+        } while (http_error == 509);
 
-        return url_ok;
+        return http_status != 403;
     }
 
     public static String getMyPublicIP() {

src/main/java/com/tonikelope/megabasterd/SmartMegaProxyManager.java

@@ -9,7 +9,9 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
+import java.util.Map;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -34,12 +36,28 @@ public final class SmartMegaProxyManager {
         refreshProxyList();
     }
 
+    public static void purgeExcludedProxyList(LinkedHashMap<String, Long> excluded) {
+
+        Long current_time = System.currentTimeMillis();
+
+        Iterator<Map.Entry<String, Long>> iterator = excluded.entrySet().iterator();
+
+        while (iterator.hasNext()) {
+
+            Map.Entry<String, Long> entry = iterator.next();
+
+            if (entry.getValue() < current_time) {
+                iterator.remove();
+            }
+        }
+    }
+
     public synchronized int getProxyCount() {
 
         return _proxy_list.size();
     }
 
-    public synchronized String getProxy() {
+    public synchronized String getProxy(LinkedHashMap<String, Long> excluded) {
 
         if (_proxy_list.size() > 0) {
 
@@ -49,7 +67,7 @@ public final class SmartMegaProxyManager {
 
             for (String k : keys) {
 
-                if (_proxy_list.get(k) < current_time) {
+                if (_proxy_list.get(k) < current_time && (excluded == null || (excluded.containsKey(k) && excluded.get(k) < current_time))) {
 
                     return k;
                 }
@@ -60,7 +78,7 @@ public final class SmartMegaProxyManager {
 
         refreshProxyList();
 
-        return getProxyCount() > 0 ? getProxy() : null;
+        return getProxyCount() > 0 ? getProxy(excluded) : null;
     }
 
     public synchronized void blockProxy(String proxy) {

src/main/java/com/tonikelope/megabasterd/StreamChunkDownloader.java

@@ -8,6 +8,7 @@ import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.LinkedHashMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -52,6 +53,8 @@ public class StreamChunkDownloader implements Runnable {
 
             SmartMegaProxyManager proxy_manager = MainPanel.getProxy_manager();
 
+            LinkedHashMap<String, Long> excluded_proxy_list = new LinkedHashMap<>();
+
             while (!_exit && !_chunkmanager.isExit()) {
 
                 while (!_exit && !_chunkmanager.isExit() && _chunkmanager.getChunk_queue().size() >= StreamChunkManager.BUFFER_CHUNKS_SIZE) {
@@ -74,15 +77,28 @@ public class StreamChunkDownloader implements Runnable {
 
                     StreamChunk chunk_stream = new StreamChunk(offset, _chunkmanager.calculateChunkSize(offset), url);
 
-                    if (http_error == 509 && MainPanel.isUse_smart_proxy() && !MainPanel.isUse_proxy()) {
+                    if ((current_smart_proxy != null || http_error == 509) && MainPanel.isUse_smart_proxy() && !MainPanel.isUse_proxy()) {
 
-                        if (current_smart_proxy != null) {
+                        if (current_smart_proxy != null && http_error != 0) {
+
+                            if (http_error == 509) {
+                                proxy_manager.blockProxy(current_smart_proxy);
+                            }
+
+                            excluded_proxy_list.put(current_smart_proxy, System.currentTimeMillis() + SmartMegaProxyManager.BLOCK_TIME * 1000);
+
+                            SmartMegaProxyManager.purgeExcludedProxyList(excluded_proxy_list);
+
+                            current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
+
+                            Logger.getLogger(MiscTools.class.getName()).log(Level.WARNING, "{0}: worker {1} excluding proxy -> {2}", new Object[]{Thread.currentThread().getName(), _id, current_smart_proxy});
+
+                        } else if (current_smart_proxy == null) {
+
+                            current_smart_proxy = proxy_manager.getProxy(excluded_proxy_list);
 
-                            proxy_manager.blockProxy(current_smart_proxy);
                         }
 
-                        current_smart_proxy = proxy_manager.getProxy();
-
                         if (current_smart_proxy != null) {
 
                             String[] proxy_info = current_smart_proxy.split(":");
@@ -177,6 +193,8 @@ public class StreamChunkDownloader implements Runnable {
                                         _chunkmanager.getChunk_queue().put(chunk_stream.getOffset(), chunk_stream);
 
                                         _chunkmanager.secureNotifyAll();
+
+                                        current_smart_proxy = null;
                                     }
                                 }
                             }