Optimize network hosts

internal/app.go

@@ -52,16 +52,17 @@ func MainApp(configPath string) {
 		os.Exit(1)
 	}
 
-	repoClient4 := http.NewHTTPClient("", 0, 1, false, config, log.Named("network_test"))
-	repoClient6 := http.NewHTTPClient("", 0, 1, true, config, log.Named("network_test"))
+	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, config, log.Named("network_test"))
+	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, config, log.Named("network_test"))
 	repo := http.NewIPRepository(repoClient4, repoClient6, log.Named("network_test"))
-	repo.NetworkTest(true)
+	repo.NetworkTest(false)
 
 	apiClient := http.NewHTTPClient(
 		config.GetToken(),
 		config.GetRetriesUntilFailed(), // default retries = 2
 		config.GetApiTimeoutSecs(),     // default api timeout = 60
 		false,                          // no need for ipv6 support
+		[]string{},                     // no optimal hosts needed
 		config,
 		log.Named("api_client"),
 	)
@@ -71,15 +72,20 @@ func MainApp(configPath string) {
 		config.GetRetriesUntilFailed(),  // default retries = 2
 		config.GetDownloadTimeoutSecs(), // default download timeout = 10
 		false,                           // no need for ipv6 support
+		[]string{},                      // no optimal hosts needed
 		config,
 		log.Named("unrestrict_client"),
 	)
 
+	hosts := repo.GetOptimalHosts(config.ShouldForceIPv6())
+	zurglog.Debugf("Optimal hosts (%d): %v", len(hosts), hosts)
+
 	downloadClient := http.NewHTTPClient(
 		"",
 		config.GetRetriesUntilFailed(),
 		config.GetDownloadTimeoutSecs(),
 		config.ShouldForceIPv6(),
+		hosts,
 		config,
 		log.Named("download_client"),
 	)

pkg/http/client.go

@@ -23,15 +23,15 @@ import (
 )
 
 type HTTPClient struct {
-	client      *http.Client
-	maxRetries  int
-	timeoutSecs int
-	backoff     func(attempt int) time.Duration
-	bearerToken string
-	cfg         config.ConfigInterface
-	dnsCache    cmap.ConcurrentMap[string, string]
-	ipv6Hosts   []string
-	log         *logutil.Logger
+	client       *http.Client
+	maxRetries   int
+	timeoutSecs  int
+	backoff      func(attempt int) time.Duration
+	bearerToken  string
+	cfg          config.ConfigInterface
+	dnsCache     cmap.ConcurrentMap[string, string]
+	optimalHosts []string
+	log          *logutil.Logger
 }
 
 type ApiErrorResponse struct {
@@ -48,19 +48,20 @@ func NewHTTPClient(
 	maxRetries int,
 	timeoutSecs int,
 	forceIPv6 bool,
+	optimalHosts []string,
 	cfg config.ConfigInterface,
 	log *logutil.Logger,
 ) *HTTPClient {
 	client := HTTPClient{
-		bearerToken: token,
-		client:      &http.Client{},
-		maxRetries:  maxRetries,
-		timeoutSecs: timeoutSecs,
-		backoff:     backoffFunc,
-		cfg:         cfg,
-		dnsCache:    cmap.New[string](),
-		ipv6Hosts:   []string{},
-		log:         log,
+		bearerToken:  token,
+		client:       &http.Client{},
+		maxRetries:   maxRetries,
+		timeoutSecs:  timeoutSecs,
+		backoff:      backoffFunc,
+		cfg:          cfg,
+		dnsCache:     cmap.New[string](),
+		optimalHosts: optimalHosts,
+		log:          log,
 	}
 
 	var dialer proxy.Dialer = &net.Dialer{
@@ -148,7 +149,9 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			resp.Body.Close()
 		}
 
-		// r.optimizeHost(req)
+		if len(r.optimalHosts) > 0 {
+			r.optimizeHost(req)
+		}
 
 		resp, err = r.client.Do(req)
 
@@ -188,23 +191,11 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 }
 
 func (r *HTTPClient) optimizeHost(req *http.Request) {
-	if !strings.HasSuffix(req.Host, ".download.real-debrid.com") {
+	if !strings.Contains(req.Host, ".download.real-debrid.") {
 		return
 	}
-
-	req.Host = strings.Replace(req.Host, ".com", ".cloud", 1)
+	req.Host = r.optimalHosts[rand.Intn(len(r.optimalHosts))]
 	req.URL.Host = req.Host
-	// check if this host is in the list of IPv6 hosts
-	for _, h := range r.ipv6Hosts {
-		if h == req.Host {
-			return
-		}
-	}
-
-	// if not then just assign a random IPv6 host
-	req.Host = r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))]
-	req.URL.Host = req.Host
-	r.log.Debugf("Host %s is not a valid IPv6 host, assigning a random IPv6 host: %s", req.URL.Host, req.Host)
 }
 
 func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) {

pkg/http/ip.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
+	"net"
 	"net/http"
 	"os"
+	"sort"
 	"time"
 
 	"github.com/debridmediamanager/zurg/pkg/logutil"
@@ -37,25 +39,67 @@ func (r *IPRepository) NetworkTest(forceRun bool) {
 	ipv4latencyFile := "data/latency4.json"
 	ipv6latencyFile := "data/latency6.json"
 	if !forceRun {
+		ipv4Loaded := false
+		ipv6Loaded := false
 		latencyData := r.readLatencyFile(ipv4latencyFile)
 		if latencyData != nil {
 			r.ipv4latencyMap = *latencyData
+			ipv4Loaded = true
 		}
 		latencyData = r.readLatencyFile(ipv6latencyFile)
 		if latencyData != nil {
 			r.ipv6latencyMap = *latencyData
+			ipv6Loaded = true
+		}
+		if ipv4Loaded && ipv6Loaded {
+			return
 		}
 	}
 
-	r.log.Info("Network test will start now. IGNORE THE WARNINGS!")
+	r.log.Info("Network test will start now (this will only run once). IGNORE THE WARNINGS!")
 	r.runLatencyTest()
 	r.log.Infof("Network test completed. Saving the results to %s and %s", ipv4latencyFile, ipv6latencyFile)
-	r.log.Debugf("ipv4 %v", r.ipv4latencyMap)
-	r.log.Debugf("ipv6 %v", r.ipv6latencyMap)
 	r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap)
 	r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap)
 }
 
+func (r *IPRepository) GetOptimalHosts(ipv6 bool) []string {
+	latencyMap := r.ipv4latencyMap
+	if ipv6 {
+		latencyMap = r.ipv6latencyMap
+	}
+
+	// Convert the latency map to a slice of key-value pairs
+	type kv struct {
+		Key   string
+		Value float64
+	}
+
+	var kvList []kv
+	for k, v := range latencyMap {
+		kvList = append(kvList, kv{k, v})
+	}
+
+	// Sort the slice by latency values
+	sort.Slice(kvList, func(i, j int) bool {
+		return kvList[i].Value < kvList[j].Value
+	})
+
+	// Calculate the number of hosts to return (top 50%)
+	n := len(kvList) / 5
+	if len(kvList)%5 != 0 {
+		n++
+	}
+
+	// Collect the keys of the top 50% hosts
+	var optimalHosts []string
+	for i := 0; i < n; i++ {
+		optimalHosts = append(optimalHosts, kvList[i].Key)
+	}
+
+	return optimalHosts
+}
+
 func (r *IPRepository) runLatencyTest() {
 	limit := 99
 	start := 0
@@ -63,10 +107,10 @@ func (r *IPRepository) runLatencyTest() {
 		lastDomainsWorked := false
 		for i := start; i <= limit; i++ {
 			domain := fmt.Sprintf("%d.download.real-debrid.com", i)
-			// ips, err := net.LookupIP(domain)
-			// if err != nil || len(ips) == 0 {
-			// 	continue
-			// }
+			ips, err := net.LookupIP(domain)
+			if err != nil || len(ips) == 0 {
+				continue
+			}
 
 			latency, err := r.testDomainLatency(r.ipv4client, domain)
 			if err == nil {
@@ -87,10 +131,10 @@ func (r *IPRepository) runLatencyTest() {
 			}
 
 			domain = fmt.Sprintf("%d.download.real-debrid.cloud", i)
-			// ips, err = net.LookupIP(domain)
-			// if err != nil || len(ips) == 0 {
-			// 	continue
-			// }
+			ips, err = net.LookupIP(domain)
+			if err != nil || len(ips) == 0 {
+				continue
+			}
 
 			latency, err = r.testDomainLatency(r.ipv4client, domain)
 			if err == nil {