diff --git a/internal/app.go b/internal/app.go index b2b0376..8522515 100644 --- a/internal/app.go +++ b/internal/app.go @@ -64,7 +64,7 @@ func MainApp(configPath string) { repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test")) repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test")) repo := http.NewIPRepository(repoClient4, repoClient6, "", log.Named("network_test")) - repo.NetworkTest(false) + repo.NetworkTest(false, config.ShouldCacheNetworkTestResults()) apiClient := http.NewHTTPClient( config.GetToken(), @@ -86,14 +86,11 @@ func MainApp(configPath string) { log.Named("unrestrict_client"), ) - hosts := repo.GetOptimalHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6()) + hosts := repo.GetHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6()) if len(hosts) == 0 { - zurglog.Fatal("No optimal hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)") + zurglog.Fatal("No reachable hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)") } - zurglog.Debugf("Optimal hosts (%d): %v", len(hosts), hosts) - // help message - zurglog.Debug("To reset optimal hosts, run 'zurg network-test' (Using docker compose? 'docker compose exec zurg ./zurg network-test')") - zurglog.Debug("To run network-test with a proxy, set the PROXY environment variable 'PROXY=http://xyz:123 zurg network-test'") + zurglog.Debugf("Reachable hosts (%d): %v", len(hosts), hosts) downloadClient := http.NewHTTPClient( "", diff --git a/internal/commands.go b/internal/commands.go index 60a8775..54cb35e 100644 --- a/internal/commands.go +++ b/internal/commands.go @@ -38,7 +38,7 @@ func NetworkTest(testURL string) { repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test")) repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test")) repo := http.NewIPRepository(repoClient4, repoClient6, testURL, log.Named("network_test")) - repo.NetworkTest(true) + repo.NetworkTest(true, true) } func ClearDownloads() { diff --git a/internal/config/types.go b/internal/config/types.go index 0dc6ce2..18f99bc 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -34,6 +34,7 @@ type ConfigInterface interface { GetVersion() string MeetsConditions(directory, torrentName string, torrentSize int64, torrentIDs, fileNames []string, fileSizes []int64, mediaInfos []*ffprobe.ProbeData) bool ShouldAutoAnalyzeNewTorrents() bool + ShouldCacheNetworkTestResults() bool ShouldForceIPv6() bool ShouldIgnoreRenames() bool ShouldServeFromRclone() bool @@ -45,6 +46,7 @@ type ZurgConfig struct { ApiTimeoutSecs int `yaml:"api_timeout_secs" json:"api_timeout_secs"` AutoAnalyzeNewTorrents bool `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"` + CacheNetworkTestResults bool `yaml:"cache_network_test_results" json:"cache_network_test_results"` CanRepair bool `yaml:"enable_repair" json:"enable_repair"` DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"` DownloadTimeoutSecs int `yaml:"download_timeout_secs" json:"download_timeout_secs"` @@ -217,12 +219,13 @@ func (z *ZurgConfig) GetProxy() string { } func (z *ZurgConfig) GetNumberOfHosts() int { - if z.NumberOfHosts == 0 { - return 20 - } return z.NumberOfHosts } func (z *ZurgConfig) ShouldAutoAnalyzeNewTorrents() bool { return z.AutoAnalyzeNewTorrents } + +func (z *ZurgConfig) ShouldCacheNetworkTestResults() bool { + return z.CacheNetworkTestResults +} diff --git a/pkg/http/client.go b/pkg/http/client.go index 7d09e09..56f425b 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -29,7 +29,7 @@ type HTTPClient struct { backoff func(attempt int) time.Duration bearerToken string dnsCache cmap.ConcurrentMap[string, string] - optimalHosts []string + hosts []string log *logutil.Logger } @@ -56,7 +56,7 @@ func NewHTTPClient( maxRetries int, timeoutSecs int, forceIPv6 bool, - optimalHosts []string, + hosts []string, proxyURL string, log *logutil.Logger, ) *HTTPClient { @@ -68,7 +68,7 @@ func NewHTTPClient( rateLimitSleepSecs: 4, backoff: backoffFunc, dnsCache: cmap.New[string](), - optimalHosts: optimalHosts, + hosts: hosts, log: log, } @@ -152,8 +152,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { resp.Body.Close() } - if len(r.optimalHosts) > 0 { - r.optimizeHost(req) + if len(r.hosts) > 0 { + r.ensureReachableHost(req) } resp, err = r.client.Do(req) @@ -201,14 +201,44 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { return resp, err } -func (r *HTTPClient) optimizeHost(req *http.Request) { +func (r *HTTPClient) ensureReachableHost(req *http.Request) { if !strings.Contains(req.Host, ".download.real-debrid.") { return } - req.Host = r.optimalHosts[rand.Intn(len(r.optimalHosts))] + if req.Host[0] >= 'a' && req.Host[0] <= 'z' { + return + } + + // check if req.Host is in r.hosts + if r.CheckIfHostIsReachable(req.Host) { + return + } + // replace prefix of req.Host from .com to .cloud or vice versa + var newHost string + if strings.HasSuffix(req.Host, ".com") { + newHost = strings.Replace(req.Host, ".com", ".cloud", 1) + } else if strings.HasSuffix(req.Host, ".cloud") { + newHost = strings.Replace(req.Host, ".cloud", ".com", 1) + } + if r.CheckIfHostIsReachable(newHost) { + req.Host = newHost + req.URL.Host = req.Host + return + } + + req.Host = r.hosts[rand.Intn(len(r.hosts))] req.URL.Host = req.Host } +func (r *HTTPClient) CheckIfHostIsReachable(reqHost string) bool { + for _, host := range r.hosts { + if reqHost == host { + return true + } + } + return false +} + func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) { if proxyURL.Scheme == "http" || proxyURL.Scheme == "https" { httpProxyDialer := http_dialer.New(proxyURL, http_dialer.WithConnectionTimeout(time.Duration(r.timeoutSecs)*time.Second)) diff --git a/pkg/http/ip.go b/pkg/http/ip.go index 11c2896..6e0b10b 100644 --- a/pkg/http/ip.go +++ b/pkg/http/ip.go @@ -38,18 +38,18 @@ func NewIPRepository(ipv4client *HTTPClient, ipv6client *HTTPClient, testURL str return repo } -func (r *IPRepository) NetworkTest(forceRun bool) { - ipv4latencyFile := "data/latency4.json" - ipv6latencyFile := "data/latency6.json" +func (r *IPRepository) NetworkTest(forceRun bool, persist bool) { + ipv4HostsFile := "data/ipv4-hosts.json" + ipv6HostsFile := "data/ipv6-hosts.json" if !forceRun { ipv4Loaded := false ipv6Loaded := false - latencyData := r.readLatencyFile(ipv4latencyFile) + latencyData := r.readLatencyFile(ipv4HostsFile) if latencyData != nil { r.ipv4latencyMap = *latencyData ipv4Loaded = true } - latencyData = r.readLatencyFile(ipv6latencyFile) + latencyData = r.readLatencyFile(ipv6HostsFile) if latencyData != nil { r.ipv6latencyMap = *latencyData ipv6Loaded = true @@ -57,19 +57,23 @@ func (r *IPRepository) NetworkTest(forceRun bool) { if ipv4Loaded && ipv6Loaded { return } else { - r.log.Warn("Network test files not found") + r.log.Warn("Network test files not found, running network test") } } - r.log.Info("Network test will start now. IGNORE THE WARNINGS!") + r.log.Info("zurg will check for all reachable download servers. You can set 'cache_network_test_results: true' in your config to skip this test in the future.") + r.log.Warn("IGNORE THE WARNINGS!") r.runLatencyTest() r.log.Info("Network test completed!") - r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4latencyFile, ipv6latencyFile) - r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap) - r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap) + + if persist { + r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4HostsFile, ipv6HostsFile) + r.writeLatencyFile(ipv4HostsFile, r.ipv4latencyMap) + r.writeLatencyFile(ipv6HostsFile, r.ipv6latencyMap) + } } -func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string { +func (r *IPRepository) GetHosts(numberOfHosts int, ipv6 bool) []string { latencyMap := r.ipv4latencyMap if ipv6 { latencyMap = r.ipv6latencyMap @@ -90,6 +94,9 @@ func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string { }) var optimalHosts []string + if numberOfHosts == 0 { + numberOfHosts = len(kvList) + } for i := 0; i < numberOfHosts && i < len(kvList); i++ { optimalHosts = append(optimalHosts, kvList[i].Key) } @@ -161,7 +168,6 @@ func (r *IPRepository) runLatencyTest() { } func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (float64, error) { - const testFileSize = 1 // byte const iterations = 3 testURL := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64()) if r.testURL != "" { @@ -177,17 +183,13 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, testURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodHead, testURL, nil) if err != nil { r.log.Warnf("Failed to create request for %s: %v", domain, err) retErr = err break } - headers := make(http.Header) - headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1)) - req.Header = headers - start := time.Now() resp, err := client.Do(req) if err != nil { @@ -195,23 +197,11 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo retErr = err break } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + if resp.StatusCode != http.StatusOK { r.log.Warnf("Failed to download from %s: %s", domain, resp.Status) retErr = fmt.Errorf("status code: %s", resp.Status) break } - - limitedReader := io.LimitReader(resp.Body, testFileSize) - - _, err = io.Copy(io.Discard, limitedReader) - resp.Body.Close() - - if err != nil && err != io.EOF { - r.log.Warnf("Failed to read from %s: %v", domain, err) - retErr = err - break - } - duration := time.Since(start).Seconds() totalDuration += duration }