diff --git a/internal/app.go b/internal/app.go index 69cd3c9..831dd6b 100644 --- a/internal/app.go +++ b/internal/app.go @@ -52,16 +52,17 @@ func MainApp(configPath string) { os.Exit(1) } - repoClient4 := http.NewHTTPClient("", 0, 1, false, config, log.Named("network_test")) - repoClient6 := http.NewHTTPClient("", 0, 1, true, config, log.Named("network_test")) + repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, config, log.Named("network_test")) + repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, config, log.Named("network_test")) repo := http.NewIPRepository(repoClient4, repoClient6, log.Named("network_test")) - repo.NetworkTest(true) + repo.NetworkTest(false) apiClient := http.NewHTTPClient( config.GetToken(), config.GetRetriesUntilFailed(), // default retries = 2 config.GetApiTimeoutSecs(), // default api timeout = 60 false, // no need for ipv6 support + []string{}, // no optimal hosts needed config, log.Named("api_client"), ) @@ -71,15 +72,20 @@ func MainApp(configPath string) { config.GetRetriesUntilFailed(), // default retries = 2 config.GetDownloadTimeoutSecs(), // default download timeout = 10 false, // no need for ipv6 support + []string{}, // no optimal hosts needed config, log.Named("unrestrict_client"), ) + hosts := repo.GetOptimalHosts(config.ShouldForceIPv6()) + zurglog.Debugf("Optimal hosts (%d): %v", len(hosts), hosts) + downloadClient := http.NewHTTPClient( "", config.GetRetriesUntilFailed(), config.GetDownloadTimeoutSecs(), config.ShouldForceIPv6(), + hosts, config, log.Named("download_client"), ) diff --git a/pkg/http/client.go b/pkg/http/client.go index 70e3385..615614d 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -23,15 +23,15 @@ import ( ) type HTTPClient struct { - client *http.Client - maxRetries int - timeoutSecs int - backoff func(attempt int) time.Duration - bearerToken string - cfg config.ConfigInterface - dnsCache cmap.ConcurrentMap[string, string] - ipv6Hosts []string - log *logutil.Logger + client *http.Client + maxRetries int + timeoutSecs int + backoff func(attempt int) time.Duration + bearerToken string + cfg config.ConfigInterface + dnsCache cmap.ConcurrentMap[string, string] + optimalHosts []string + log *logutil.Logger } type ApiErrorResponse struct { @@ -48,19 +48,20 @@ func NewHTTPClient( maxRetries int, timeoutSecs int, forceIPv6 bool, + optimalHosts []string, cfg config.ConfigInterface, log *logutil.Logger, ) *HTTPClient { client := HTTPClient{ - bearerToken: token, - client: &http.Client{}, - maxRetries: maxRetries, - timeoutSecs: timeoutSecs, - backoff: backoffFunc, - cfg: cfg, - dnsCache: cmap.New[string](), - ipv6Hosts: []string{}, - log: log, + bearerToken: token, + client: &http.Client{}, + maxRetries: maxRetries, + timeoutSecs: timeoutSecs, + backoff: backoffFunc, + cfg: cfg, + dnsCache: cmap.New[string](), + optimalHosts: optimalHosts, + log: log, } var dialer proxy.Dialer = &net.Dialer{ @@ -148,7 +149,9 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { resp.Body.Close() } - // r.optimizeHost(req) + if len(r.optimalHosts) > 0 { + r.optimizeHost(req) + } resp, err = r.client.Do(req) @@ -188,23 +191,11 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { } func (r *HTTPClient) optimizeHost(req *http.Request) { - if !strings.HasSuffix(req.Host, ".download.real-debrid.com") { + if !strings.Contains(req.Host, ".download.real-debrid.") { return } - - req.Host = strings.Replace(req.Host, ".com", ".cloud", 1) + req.Host = r.optimalHosts[rand.Intn(len(r.optimalHosts))] req.URL.Host = req.Host - // check if this host is in the list of IPv6 hosts - for _, h := range r.ipv6Hosts { - if h == req.Host { - return - } - } - - // if not then just assign a random IPv6 host - req.Host = r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))] - req.URL.Host = req.Host - r.log.Debugf("Host %s is not a valid IPv6 host, assigning a random IPv6 host: %s", req.URL.Host, req.Host) } func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) { diff --git a/pkg/http/ip.go b/pkg/http/ip.go index 92faab5..d54ed1b 100644 --- a/pkg/http/ip.go +++ b/pkg/http/ip.go @@ -6,8 +6,10 @@ import ( "fmt" "io" "math/rand" + "net" "net/http" "os" + "sort" "time" "github.com/debridmediamanager/zurg/pkg/logutil" @@ -37,25 +39,67 @@ func (r *IPRepository) NetworkTest(forceRun bool) { ipv4latencyFile := "data/latency4.json" ipv6latencyFile := "data/latency6.json" if !forceRun { + ipv4Loaded := false + ipv6Loaded := false latencyData := r.readLatencyFile(ipv4latencyFile) if latencyData != nil { r.ipv4latencyMap = *latencyData + ipv4Loaded = true } latencyData = r.readLatencyFile(ipv6latencyFile) if latencyData != nil { r.ipv6latencyMap = *latencyData + ipv6Loaded = true + } + if ipv4Loaded && ipv6Loaded { + return } } - r.log.Info("Network test will start now. IGNORE THE WARNINGS!") + r.log.Info("Network test will start now (this will only run once). IGNORE THE WARNINGS!") r.runLatencyTest() r.log.Infof("Network test completed. Saving the results to %s and %s", ipv4latencyFile, ipv6latencyFile) - r.log.Debugf("ipv4 %v", r.ipv4latencyMap) - r.log.Debugf("ipv6 %v", r.ipv6latencyMap) r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap) r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap) } +func (r *IPRepository) GetOptimalHosts(ipv6 bool) []string { + latencyMap := r.ipv4latencyMap + if ipv6 { + latencyMap = r.ipv6latencyMap + } + + // Convert the latency map to a slice of key-value pairs + type kv struct { + Key string + Value float64 + } + + var kvList []kv + for k, v := range latencyMap { + kvList = append(kvList, kv{k, v}) + } + + // Sort the slice by latency values + sort.Slice(kvList, func(i, j int) bool { + return kvList[i].Value < kvList[j].Value + }) + + // Calculate the number of hosts to return (top 50%) + n := len(kvList) / 5 + if len(kvList)%5 != 0 { + n++ + } + + // Collect the keys of the top 50% hosts + var optimalHosts []string + for i := 0; i < n; i++ { + optimalHosts = append(optimalHosts, kvList[i].Key) + } + + return optimalHosts +} + func (r *IPRepository) runLatencyTest() { limit := 99 start := 0 @@ -63,10 +107,10 @@ func (r *IPRepository) runLatencyTest() { lastDomainsWorked := false for i := start; i <= limit; i++ { domain := fmt.Sprintf("%d.download.real-debrid.com", i) - // ips, err := net.LookupIP(domain) - // if err != nil || len(ips) == 0 { - // continue - // } + ips, err := net.LookupIP(domain) + if err != nil || len(ips) == 0 { + continue + } latency, err := r.testDomainLatency(r.ipv4client, domain) if err == nil { @@ -87,10 +131,10 @@ func (r *IPRepository) runLatencyTest() { } domain = fmt.Sprintf("%d.download.real-debrid.cloud", i) - // ips, err = net.LookupIP(domain) - // if err != nil || len(ips) == 0 { - // continue - // } + ips, err = net.LookupIP(domain) + if err != nil || len(ips) == 0 { + continue + } latency, err = r.testDomainLatency(r.ipv4client, domain) if err == nil {