Reachable hosts

internal/app.go

@@ -64,7 +64,7 @@ func MainApp(configPath string) {
 	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
 	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
 	repo := http.NewIPRepository(repoClient4, repoClient6, "", log.Named("network_test"))
-	repo.NetworkTest(false)
+	repo.NetworkTest(false, config.ShouldCacheNetworkTestResults())
 
 	apiClient := http.NewHTTPClient(
 		config.GetToken(),
@@ -86,14 +86,11 @@ func MainApp(configPath string) {
 		log.Named("unrestrict_client"),
 	)
 
-	hosts := repo.GetOptimalHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6())
+	hosts := repo.GetHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6())
 	if len(hosts) == 0 {
-		zurglog.Fatal("No optimal hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)")
+		zurglog.Fatal("No reachable hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)")
 	}
-	zurglog.Debugf("Optimal hosts (%d): %v", len(hosts), hosts)
+	zurglog.Debugf("Reachable hosts (%d): %v", len(hosts), hosts)
-	// help message
-	zurglog.Debug("To reset optimal hosts, run 'zurg network-test' (Using docker compose? 'docker compose exec zurg ./zurg network-test')")
-	zurglog.Debug("To run network-test with a proxy, set the PROXY environment variable 'PROXY=http://xyz:123 zurg network-test'")
 
 	downloadClient := http.NewHTTPClient(
 		"",

internal/commands.go

@@ -38,7 +38,7 @@ func NetworkTest(testURL string) {
 	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
 	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
 	repo := http.NewIPRepository(repoClient4, repoClient6, testURL, log.Named("network_test"))
-	repo.NetworkTest(true)
+	repo.NetworkTest(true, true)
 }
 
 func ClearDownloads() {

internal/config/types.go

@@ -34,6 +34,7 @@ type ConfigInterface interface {
 	GetVersion() string
 	MeetsConditions(directory, torrentName string, torrentSize int64, torrentIDs, fileNames []string, fileSizes []int64, mediaInfos []*ffprobe.ProbeData) bool
 	ShouldAutoAnalyzeNewTorrents() bool
+	ShouldCacheNetworkTestResults() bool
 	ShouldForceIPv6() bool
 	ShouldIgnoreRenames() bool
 	ShouldServeFromRclone() bool
@@ -45,6 +46,7 @@ type ZurgConfig struct {
 
 	ApiTimeoutSecs            int      `yaml:"api_timeout_secs" json:"api_timeout_secs"`
 	AutoAnalyzeNewTorrents    bool     `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"`
+	CacheNetworkTestResults   bool     `yaml:"cache_network_test_results" json:"cache_network_test_results"`
 	CanRepair                 bool     `yaml:"enable_repair" json:"enable_repair"`
 	DownloadsEveryMins        int      `yaml:"downloads_every_mins" json:"downloads_every_mins"`
 	DownloadTimeoutSecs       int      `yaml:"download_timeout_secs" json:"download_timeout_secs"`
@@ -217,12 +219,13 @@ func (z *ZurgConfig) GetProxy() string {
 }
 
 func (z *ZurgConfig) GetNumberOfHosts() int {
-	if z.NumberOfHosts == 0 {
-		return 20
-	}
 	return z.NumberOfHosts
 }
 
 func (z *ZurgConfig) ShouldAutoAnalyzeNewTorrents() bool {
 	return z.AutoAnalyzeNewTorrents
 }
+
+func (z *ZurgConfig) ShouldCacheNetworkTestResults() bool {
+	return z.CacheNetworkTestResults
+}

pkg/http/client.go

@@ -29,7 +29,7 @@ type HTTPClient struct {
 	backoff            func(attempt int) time.Duration
 	bearerToken        string
 	dnsCache           cmap.ConcurrentMap[string, string]
-	optimalHosts       []string
+	hosts              []string
 	log                *logutil.Logger
 }
 
@@ -56,7 +56,7 @@ func NewHTTPClient(
 	maxRetries int,
 	timeoutSecs int,
 	forceIPv6 bool,
-	optimalHosts []string,
+	hosts []string,
 	proxyURL string,
 	log *logutil.Logger,
 ) *HTTPClient {
@@ -68,7 +68,7 @@ func NewHTTPClient(
 		rateLimitSleepSecs: 4,
 		backoff:            backoffFunc,
 		dnsCache:           cmap.New[string](),
-		optimalHosts:       optimalHosts,
+		hosts:              hosts,
 		log:                log,
 	}
 
@@ -152,8 +152,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			resp.Body.Close()
 		}
 
-		if len(r.optimalHosts) > 0 {
+		if len(r.hosts) > 0 {
-			r.optimizeHost(req)
+			r.ensureReachableHost(req)
 		}
 
 		resp, err = r.client.Do(req)
@@ -201,14 +201,44 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, err
 }
 
-func (r *HTTPClient) optimizeHost(req *http.Request) {
+func (r *HTTPClient) ensureReachableHost(req *http.Request) {
 	if !strings.Contains(req.Host, ".download.real-debrid.") {
 		return
 	}
-	req.Host = r.optimalHosts[rand.Intn(len(r.optimalHosts))]
+	if req.Host[0] >= 'a' && req.Host[0] <= 'z' {
+		return
+	}
+
+	// check if req.Host is in r.hosts
+	if r.CheckIfHostIsReachable(req.Host) {
+		return
+	}
+	// replace prefix of req.Host from .com to .cloud or vice versa
+	var newHost string
+	if strings.HasSuffix(req.Host, ".com") {
+		newHost = strings.Replace(req.Host, ".com", ".cloud", 1)
+	} else if strings.HasSuffix(req.Host, ".cloud") {
+		newHost = strings.Replace(req.Host, ".cloud", ".com", 1)
+	}
+	if r.CheckIfHostIsReachable(newHost) {
+		req.Host = newHost
+		req.URL.Host = req.Host
+		return
+	}
+
+	req.Host = r.hosts[rand.Intn(len(r.hosts))]
 	req.URL.Host = req.Host
 }
 
+func (r *HTTPClient) CheckIfHostIsReachable(reqHost string) bool {
+	for _, host := range r.hosts {
+		if reqHost == host {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) {
 	if proxyURL.Scheme == "http" || proxyURL.Scheme == "https" {
 		httpProxyDialer := http_dialer.New(proxyURL, http_dialer.WithConnectionTimeout(time.Duration(r.timeoutSecs)*time.Second))

pkg/http/ip.go

@@ -38,18 +38,18 @@ func NewIPRepository(ipv4client *HTTPClient, ipv6client *HTTPClient, testURL str
 	return repo
 }
 
-func (r *IPRepository) NetworkTest(forceRun bool) {
+func (r *IPRepository) NetworkTest(forceRun bool, persist bool) {
-	ipv4latencyFile := "data/latency4.json"
+	ipv4HostsFile := "data/ipv4-hosts.json"
-	ipv6latencyFile := "data/latency6.json"
+	ipv6HostsFile := "data/ipv6-hosts.json"
 	if !forceRun {
 		ipv4Loaded := false
 		ipv6Loaded := false
-		latencyData := r.readLatencyFile(ipv4latencyFile)
+		latencyData := r.readLatencyFile(ipv4HostsFile)
 		if latencyData != nil {
 			r.ipv4latencyMap = *latencyData
 			ipv4Loaded = true
 		}
-		latencyData = r.readLatencyFile(ipv6latencyFile)
+		latencyData = r.readLatencyFile(ipv6HostsFile)
 		if latencyData != nil {
 			r.ipv6latencyMap = *latencyData
 			ipv6Loaded = true
@@ -57,19 +57,23 @@ func (r *IPRepository) NetworkTest(forceRun bool) {
 		if ipv4Loaded && ipv6Loaded {
 			return
 		} else {
-			r.log.Warn("Network test files not found")
+			r.log.Warn("Network test files not found, running network test")
 		}
 	}
 
-	r.log.Info("Network test will start now. IGNORE THE WARNINGS!")
+	r.log.Info("zurg will check for all reachable download servers. You can set 'cache_network_test_results: true' in your config to skip this test in the future.")
+	r.log.Warn("IGNORE THE WARNINGS!")
 	r.runLatencyTest()
 	r.log.Info("Network test completed!")
-	r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4latencyFile, ipv6latencyFile)
+
-	r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap)
+	if persist {
-	r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap)
+		r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4HostsFile, ipv6HostsFile)
+		r.writeLatencyFile(ipv4HostsFile, r.ipv4latencyMap)
+		r.writeLatencyFile(ipv6HostsFile, r.ipv6latencyMap)
+	}
 }
 
-func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string {
+func (r *IPRepository) GetHosts(numberOfHosts int, ipv6 bool) []string {
 	latencyMap := r.ipv4latencyMap
 	if ipv6 {
 		latencyMap = r.ipv6latencyMap
@@ -90,6 +94,9 @@ func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string {
 	})
 
 	var optimalHosts []string
+	if numberOfHosts == 0 {
+		numberOfHosts = len(kvList)
+	}
 	for i := 0; i < numberOfHosts && i < len(kvList); i++ {
 		optimalHosts = append(optimalHosts, kvList[i].Key)
 	}
@@ -161,7 +168,6 @@ func (r *IPRepository) runLatencyTest() {
 }
 
 func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (float64, error) {
-	const testFileSize = 1 // byte
 	const iterations = 3
 	testURL := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64())
 	if r.testURL != "" {
@@ -177,17 +183,13 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo
 		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
 
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, testURL, nil)
+		req, err := http.NewRequestWithContext(ctx, http.MethodHead, testURL, nil)
 		if err != nil {
 			r.log.Warnf("Failed to create request for %s: %v", domain, err)
 			retErr = err
 			break
 		}
 
-		headers := make(http.Header)
-		headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1))
-		req.Header = headers
-
 		start := time.Now()
 		resp, err := client.Do(req)
 		if err != nil {
@@ -195,23 +197,11 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo
 			retErr = err
 			break
 		}
-		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
+		if resp.StatusCode != http.StatusOK {
 			r.log.Warnf("Failed to download from %s: %s", domain, resp.Status)
 			retErr = fmt.Errorf("status code: %s", resp.Status)
 			break
 		}
-
-		limitedReader := io.LimitReader(resp.Body, testFileSize)
-
-		_, err = io.Copy(io.Discard, limitedReader)
-		resp.Body.Close()
-
-		if err != nil && err != io.EOF {
-			r.log.Warnf("Failed to read from %s: %v", domain, err)
-			retErr = err
-			break
-		}
-
 		duration := time.Since(start).Seconds()
 		totalDuration += duration
 	}