ipv6 network test

internal/app.go

@@ -52,22 +52,16 @@ func MainApp(configPath string) {
 		os.Exit(1)
 	}
 
-	repo := http.NewIPRepository(log.Named("network_test"))
-	repoClient := http.NewHTTPClient(
-		"",
-		0,
-		1,
-		true,
-		config,
-		log.Named("network_test"),
-	)
-	repo.NetworkTest(repoClient, false)
+	repoClient4 := http.NewHTTPClient("", 0, 1, false, config, log.Named("network_test"))
+	repoClient6 := http.NewHTTPClient("", 0, 1, true, config, log.Named("network_test"))
+	repo := http.NewIPRepository(repoClient4, repoClient6, log.Named("network_test"))
+	repo.NetworkTest(true)
 
 	apiClient := http.NewHTTPClient(
 		config.GetToken(),
 		config.GetRetriesUntilFailed(), // default retries = 2
 		config.GetApiTimeoutSecs(),     // default api timeout = 60
-		false,                          // ipv6 support is not needed for api client
+		false,                          // no need for ipv6 support
 		config,
 		log.Named("api_client"),
 	)
@@ -76,16 +70,16 @@ func MainApp(configPath string) {
 		config.GetToken(),
 		config.GetRetriesUntilFailed(),  // default retries = 2
 		config.GetDownloadTimeoutSecs(), // default download timeout = 10
-		false,                           // this is also api client, so no ipv6 support
+		false,                           // no need for ipv6 support
 		config,
 		log.Named("unrestrict_client"),
 	)
 
 	downloadClient := http.NewHTTPClient(
-		"",                              // no token required for download client
-		config.GetRetriesUntilFailed(),  //
-		config.GetDownloadTimeoutSecs(), //
-		true,                            // set as download client
+		"",
+		config.GetRetriesUntilFailed(),
+		config.GetDownloadTimeoutSecs(),
+		config.ShouldForceIPv6(),
 		config,
 		log.Named("download_client"),
 	)

pkg/http/client.go

@@ -15,7 +15,6 @@ import (
 	"time"
 
 	"github.com/debridmediamanager/zurg/internal/config"
-	"github.com/debridmediamanager/zurg/pkg/hosts"
 	"github.com/debridmediamanager/zurg/pkg/logutil"
 	http_dialer "github.com/mwitkow/go-http-dialer"
 	"golang.org/x/net/proxy"
@@ -24,16 +23,15 @@ import (
 )
 
 type HTTPClient struct {
-	client           *http.Client
-	maxRetries       int
-	timeoutSecs      int
-	backoff          func(attempt int) time.Duration
-	bearerToken      string
-	isDownloadClient bool
-	cfg              config.ConfigInterface
-	ipv6             cmap.ConcurrentMap[string, string]
-	ipv6Hosts        []string
-	log              *logutil.Logger
+	client      *http.Client
+	maxRetries  int
+	timeoutSecs int
+	backoff     func(attempt int) time.Duration
+	bearerToken string
+	cfg         config.ConfigInterface
+	dnsCache    cmap.ConcurrentMap[string, string]
+	ipv6Hosts   []string
+	log         *logutil.Logger
 }
 
 type ApiErrorResponse struct {
@@ -49,20 +47,20 @@ func NewHTTPClient(
 	token string,
 	maxRetries int,
 	timeoutSecs int,
-	isDownloadClient bool,
+	forceIPv6 bool,
 	cfg config.ConfigInterface,
 	log *logutil.Logger,
 ) *HTTPClient {
 	client := HTTPClient{
-		bearerToken:      token,
-		client:           &http.Client{},
-		maxRetries:       maxRetries,
-		timeoutSecs:      timeoutSecs,
-		backoff:          backoffFunc,
-		isDownloadClient: isDownloadClient,
-		cfg:              cfg,
-		ipv6:             cmap.New[string](),
-		log:              log,
+		bearerToken: token,
+		client:      &http.Client{},
+		maxRetries:  maxRetries,
+		timeoutSecs: timeoutSecs,
+		backoff:     backoffFunc,
+		cfg:         cfg,
+		dnsCache:    cmap.New[string](),
+		ipv6Hosts:   []string{},
+		log:         log,
 	}
 
 	var dialer proxy.Dialer = &net.Dialer{
@@ -96,21 +94,11 @@ func NewHTTPClient(
 		},
 	}
 
-	if cfg.ShouldForceIPv6() {
-		// fetch IPv6 hosts
-		ipv6List, err := hosts.FetchHosts(hosts.IPV6)
-		if err != nil {
-			log.Warnf("Failed to fetch IPv6 hosts: %v", err)
-			// Decide if you want to return nil here or continue without IPv6
-		} else {
-			client.ipv6Hosts = ipv6List
-			log.Debugf("Fetched %d IPv6 hosts", len(ipv6List))
-		}
-
+	if forceIPv6 {
 		// replace the default dialer with a custom one that resolves hostnames to IPv6 addresses
 		client.client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
 			// if address is already cached, use it
-			if ipv6Address, ok := client.ipv6.Get(address); ok {
+			if ipv6Address, ok := client.dnsCache.Get(address); ok {
 				return dialer.Dial(network, ipv6Address)
 			}
 
@@ -126,24 +114,13 @@ func NewHTTPClient(
 			for _, ip := range ips {
 				if ip.IP.To4() == nil { // IPv6 address found
 					ipv6Address := net.JoinHostPort(ip.IP.String(), port)
-					client.ipv6.Set(address, ipv6Address)
+					client.dnsCache.Set(address, ipv6Address)
 					return dialer.Dial(network, ipv6Address)
 				}
 			}
 
-			// no IPv6 address found, use the original address
-			log.Warnf("No IPv6 address found for host %s", host)
-			for _, ip := range ips {
-				if ip.IP.To4() != nil { // IPv4 address found
-					ipV4Address := net.JoinHostPort(ip.IP.String(), port)
-					client.ipv6.Set(address, ipV4Address)
-					return dialer.Dial(network, ipV4Address)
-				}
-			}
-
-			return dialer.Dial(network, address)
+			return nil, fmt.Errorf("no ipv6 address found")
 		}
-
 	}
 
 	return &client
@@ -171,7 +148,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			resp.Body.Close()
 		}
 
-		r.replaceWithIPv6Host(req) // needed for ipv6
+		// r.optimizeHost(req)
 
 		resp, err = r.client.Do(req)
 
@@ -210,12 +187,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, err
 }
 
-func (r *HTTPClient) replaceWithIPv6Host(req *http.Request) {
-	// don't replace host if IPv6 is not supported or not forced
-	if !r.isDownloadClient || !r.cfg.ShouldForceIPv6() {
-		return
-	}
-	// this host should be replaced
+func (r *HTTPClient) optimizeHost(req *http.Request) {
 	if !strings.HasSuffix(req.Host, ".download.real-debrid.com") {
 		return
 	}
@@ -233,7 +205,6 @@ func (r *HTTPClient) replaceWithIPv6Host(req *http.Request) {
 	req.Host = r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))]
 	req.URL.Host = req.Host
 	r.log.Debugf("Host %s is not a valid IPv6 host, assigning a random IPv6 host: %s", req.URL.Host, req.Host)
-	// if !found && !r.CanFetchFirstByte(req.URL.String()) {
 }
 
 func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) {

pkg/http/ip.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"net"
 	"net/http"
 	"os"
 	"time"
@@ -15,150 +14,161 @@ import (
 )
 
 type IPRepository struct {
-	ipv4       []string
-	ipv6       []string
-	latencyMap map[string]float64
-	log        *logutil.Logger
+	ipv4client     *HTTPClient
+	ipv6client     *HTTPClient
+	ipv4latencyMap map[string]float64
+	ipv6latencyMap map[string]float64
+	log            *logutil.Logger
 }
 
-func NewIPRepository(log *logutil.Logger) *IPRepository {
+func NewIPRepository(ipv4client *HTTPClient, ipv6client *HTTPClient, log *logutil.Logger) *IPRepository {
 	repo := &IPRepository{
-		ipv4:       []string{},
-		ipv6:       []string{},
-		latencyMap: make(map[string]float64),
-		log:        log,
+		ipv4client:     ipv4client,
+		ipv6client:     ipv6client,
+		ipv4latencyMap: make(map[string]float64),
+		ipv6latencyMap: make(map[string]float64),
+		log:            log,
 	}
 
-	repo.lookupDomains()
-
 	return repo
 }
 
-func (r *IPRepository) NetworkTest(downloadClient *HTTPClient, forceRun bool) {
-	latencyFile := "data/latency.json"
+func (r *IPRepository) NetworkTest(forceRun bool) {
+	ipv4latencyFile := "data/latency4.json"
+	ipv6latencyFile := "data/latency6.json"
 	if !forceRun {
-		latencyData := r.readLatencyFile(latencyFile)
+		latencyData := r.readLatencyFile(ipv4latencyFile)
 		if latencyData != nil {
-			r.latencyMap = *latencyData
-			return
+			r.ipv4latencyMap = *latencyData
+		}
+		latencyData = r.readLatencyFile(ipv6latencyFile)
+		if latencyData != nil {
+			r.ipv6latencyMap = *latencyData
 		}
 	}
 
-	r.log.Info("Network test will start now. Note that it will only run once and record the latency of each domain for future use.")
-	r.latencyTest(downloadClient)
-	r.log.Infof("Network test completed. Saving the results to %s", latencyFile)
-	r.writeLatencyFile(latencyFile)
+	r.log.Info("Network test will start now. IGNORE THE WARNINGS!")
+	r.runLatencyTest()
+	r.log.Infof("Network test completed. Saving the results to %s and %s", ipv4latencyFile, ipv6latencyFile)
+	r.log.Debugf("ipv4 %v", r.ipv4latencyMap)
+	r.log.Debugf("ipv6 %v", r.ipv6latencyMap)
+	r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap)
+	r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap)
 }
 
-func (r *IPRepository) lookupDomains() {
+func (r *IPRepository) runLatencyTest() {
 	limit := 99
-	increment := 10
 	start := 0
 	for {
-		lastDomainWorked := false
+		lastDomainsWorked := false
 		for i := start; i <= limit; i++ {
 			domain := fmt.Sprintf("%d.download.real-debrid.com", i)
-			ips, err := net.LookupIP(domain)
-			if err == nil && len(ips) > 0 {
-				hasIPv6 := false
-				for _, ip := range ips {
-					if ip.To4() == nil {
-						hasIPv6 = true
-					}
-				}
-				// assume it always has ipv4
-				r.ipv4 = append(r.ipv4, domain)
-				if hasIPv6 {
-					r.ipv6 = append(r.ipv6, domain)
-				}
-				if i == limit {
-					lastDomainWorked = true
+			// ips, err := net.LookupIP(domain)
+			// if err != nil || len(ips) == 0 {
+			// 	continue
+			// }
+
+			latency, err := r.testDomainLatency(r.ipv4client, domain)
+			if err == nil {
+				r.ipv4latencyMap[domain] = latency
+				r.log.Debugf("Latency from ipv4 %s: %.5f seconds", domain, latency)
+				if i >= limit-2 {
+					lastDomainsWorked = true
 				}
 			}
 
-			domain2 := fmt.Sprintf("%d.download.real-debrid.cloud", i)
-			ips2, err := net.LookupIP(domain2)
-			if err == nil && len(ips2) > 0 {
-				hasIPv6 := false
-				for _, ip := range ips {
-					if ip.To4() == nil {
-						hasIPv6 = true
-					}
+			latency, err = r.testDomainLatency(r.ipv6client, domain)
+			if err == nil {
+				r.ipv6latencyMap[domain] = latency
+				r.log.Debugf("Latency from ipv6 %s: %.5f seconds", domain, latency)
+				if i >= limit-2 {
+					lastDomainsWorked = true
 				}
-				r.ipv4 = append(r.ipv4, domain2)
-				if hasIPv6 {
-					r.ipv6 = append(r.ipv6, domain2)
+			}
+
+			domain = fmt.Sprintf("%d.download.real-debrid.cloud", i)
+			// ips, err = net.LookupIP(domain)
+			// if err != nil || len(ips) == 0 {
+			// 	continue
+			// }
+
+			latency, err = r.testDomainLatency(r.ipv4client, domain)
+			if err == nil {
+				r.ipv4latencyMap[domain] = latency
+				r.log.Debugf("Latency from ipv4 %s: %.5f seconds", domain, latency)
+				if i >= limit-2 {
+					lastDomainsWorked = true
 				}
-				if i == limit {
-					lastDomainWorked = true
+			}
+
+			latency, err = r.testDomainLatency(r.ipv6client, domain)
+			if err == nil {
+				r.ipv6latencyMap[domain] = latency
+				r.log.Debugf("Latency from ipv6 %s: %.5f seconds", domain, latency)
+				if i >= limit-2 {
+					lastDomainsWorked = true
 				}
 			}
 		}
-		if lastDomainWorked {
+		if lastDomainsWorked {
 			start = limit + 1
-			limit += increment
+			limit += 10
 		} else {
 			break
 		}
 	}
-
-	r.log.Infof("Found %d IPv4 domains and %d IPv6 domains", len(r.ipv4), len(r.ipv6))
 }
 
-func (r *IPRepository) latencyTest(downloadClient *HTTPClient) {
-	const testFileSize = 1
+func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (float64, error) {
+	const testFileSize = 1 // byte
 	const iterations = 3
+	url := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64())
 
-	for _, domain := range r.ipv4 {
-		url := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64())
+	var totalDuration float64
+	var retErr error
+	for i := 0; i < iterations; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+		defer cancel()
 
-		var totalDuration float64
-		hasError := false
-		for i := 0; i < iterations; i++ {
-			ctx, cancel := context.WithTimeout(context.Background(), iterations*time.Second)
-			defer cancel()
-
-			req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-			if err != nil {
-				r.log.Warnf("Failed to create request for %s: %v", domain, err)
-				hasError = true
-				break
-			}
-
-			headers := make(http.Header)
-			headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1))
-			req.Header = headers
-
-			start := time.Now()
-			resp, err := downloadClient.Do(req)
-			if err != nil {
-				r.log.Warnf("Failed to download from %s: %v", domain, err)
-				hasError = true
-				break
-			}
-
-			limitedReader := io.LimitReader(resp.Body, testFileSize)
-
-			_, err = io.Copy(io.Discard, limitedReader)
-			resp.Body.Close()
-
-			if err != nil && err != io.EOF {
-				r.log.Warnf("Failed to read from %s: %v", domain, err)
-				hasError = true
-				break
-			}
-
-			duration := time.Since(start).Seconds()
-			totalDuration += duration
-		}
-		if hasError {
-			continue
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+		if err != nil {
+			r.log.Warnf("Failed to create request for %s: %v", domain, err)
+			retErr = err
+			break
 		}
 
-		r.latencyMap[domain] = totalDuration / 3
+		headers := make(http.Header)
+		headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1))
+		req.Header = headers
 
-		r.log.Debugf("Latency from %s: %.5f seconds", domain, r.latencyMap[domain])
+		start := time.Now()
+		resp, err := client.Do(req)
+		if err != nil {
+			r.log.Warnf("Failed to download from %s: %v", domain, err)
+			retErr = err
+			break
+		}
+
+		limitedReader := io.LimitReader(resp.Body, testFileSize)
+
+		_, err = io.Copy(io.Discard, limitedReader)
+		resp.Body.Close()
+
+		if err != nil && err != io.EOF {
+			r.log.Warnf("Failed to read from %s: %v", domain, err)
+			retErr = err
+			break
+		}
+
+		duration := time.Since(start).Seconds()
+		totalDuration += duration
 	}
+	if retErr != nil {
+		return 0, retErr
+	}
+
+	avgDuration := totalDuration / 3
+	return avgDuration, nil
 }
 
 func (r *IPRepository) readLatencyFile(latencyFile string) *map[string]float64 {
@@ -172,16 +182,16 @@ func (r *IPRepository) readLatencyFile(latencyFile string) *map[string]float64 {
 		if err != nil {
 			return nil
 		}
-		var latencyMap map[string]float64
-		if err := json.Unmarshal(jsonData, &latencyMap); err != nil {
+		var ipv4latencyMap map[string]float64
+		if err := json.Unmarshal(jsonData, &ipv4latencyMap); err != nil {
 			return nil
 		}
-		return &latencyMap
+		return &ipv4latencyMap
 	}
 	return nil
 }
 
-func (r *IPRepository) writeLatencyFile(latencyFile string) {
+func (r *IPRepository) writeLatencyFile(latencyFile string, data interface{}) {
 	file, err := os.Create(latencyFile)
 	if err != nil {
 		r.log.Warnf("Cannot create latency file %s: %v", latencyFile, err)
@@ -189,7 +199,7 @@ func (r *IPRepository) writeLatencyFile(latencyFile string) {
 	}
 	defer file.Close()
 
-	jsonData, err := json.Marshal(r.latencyMap)
+	jsonData, err := json.Marshal(data)
 	if err != nil {
 		r.log.Warnf("Cannot marshal latency map: %v", err)
 		return