From 5aea569be70217727a4b2fb804197a9e1e06a3a4 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Sun, 16 Jun 2024 06:36:44 +0000 Subject: [PATCH] ipv6 network test --- internal/app.go | 26 ++---- pkg/http/client.go | 79 +++++----------- pkg/http/ip.go | 226 +++++++++++++++++++++++---------------------- 3 files changed, 153 insertions(+), 178 deletions(-) diff --git a/internal/app.go b/internal/app.go index 5ec5b17..69cd3c9 100644 --- a/internal/app.go +++ b/internal/app.go @@ -52,22 +52,16 @@ func MainApp(configPath string) { os.Exit(1) } - repo := http.NewIPRepository(log.Named("network_test")) - repoClient := http.NewHTTPClient( - "", - 0, - 1, - true, - config, - log.Named("network_test"), - ) - repo.NetworkTest(repoClient, false) + repoClient4 := http.NewHTTPClient("", 0, 1, false, config, log.Named("network_test")) + repoClient6 := http.NewHTTPClient("", 0, 1, true, config, log.Named("network_test")) + repo := http.NewIPRepository(repoClient4, repoClient6, log.Named("network_test")) + repo.NetworkTest(true) apiClient := http.NewHTTPClient( config.GetToken(), config.GetRetriesUntilFailed(), // default retries = 2 config.GetApiTimeoutSecs(), // default api timeout = 60 - false, // ipv6 support is not needed for api client + false, // no need for ipv6 support config, log.Named("api_client"), ) @@ -76,16 +70,16 @@ func MainApp(configPath string) { config.GetToken(), config.GetRetriesUntilFailed(), // default retries = 2 config.GetDownloadTimeoutSecs(), // default download timeout = 10 - false, // this is also api client, so no ipv6 support + false, // no need for ipv6 support config, log.Named("unrestrict_client"), ) downloadClient := http.NewHTTPClient( - "", // no token required for download client - config.GetRetriesUntilFailed(), // - config.GetDownloadTimeoutSecs(), // - true, // set as download client + "", + config.GetRetriesUntilFailed(), + config.GetDownloadTimeoutSecs(), + config.ShouldForceIPv6(), config, log.Named("download_client"), ) diff --git a/pkg/http/client.go b/pkg/http/client.go index 08e7cb7..70e3385 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -15,7 +15,6 @@ import ( "time" "github.com/debridmediamanager/zurg/internal/config" - "github.com/debridmediamanager/zurg/pkg/hosts" "github.com/debridmediamanager/zurg/pkg/logutil" http_dialer "github.com/mwitkow/go-http-dialer" "golang.org/x/net/proxy" @@ -24,16 +23,15 @@ import ( ) type HTTPClient struct { - client *http.Client - maxRetries int - timeoutSecs int - backoff func(attempt int) time.Duration - bearerToken string - isDownloadClient bool - cfg config.ConfigInterface - ipv6 cmap.ConcurrentMap[string, string] - ipv6Hosts []string - log *logutil.Logger + client *http.Client + maxRetries int + timeoutSecs int + backoff func(attempt int) time.Duration + bearerToken string + cfg config.ConfigInterface + dnsCache cmap.ConcurrentMap[string, string] + ipv6Hosts []string + log *logutil.Logger } type ApiErrorResponse struct { @@ -49,20 +47,20 @@ func NewHTTPClient( token string, maxRetries int, timeoutSecs int, - isDownloadClient bool, + forceIPv6 bool, cfg config.ConfigInterface, log *logutil.Logger, ) *HTTPClient { client := HTTPClient{ - bearerToken: token, - client: &http.Client{}, - maxRetries: maxRetries, - timeoutSecs: timeoutSecs, - backoff: backoffFunc, - isDownloadClient: isDownloadClient, - cfg: cfg, - ipv6: cmap.New[string](), - log: log, + bearerToken: token, + client: &http.Client{}, + maxRetries: maxRetries, + timeoutSecs: timeoutSecs, + backoff: backoffFunc, + cfg: cfg, + dnsCache: cmap.New[string](), + ipv6Hosts: []string{}, + log: log, } var dialer proxy.Dialer = &net.Dialer{ @@ -96,21 +94,11 @@ func NewHTTPClient( }, } - if cfg.ShouldForceIPv6() { - // fetch IPv6 hosts - ipv6List, err := hosts.FetchHosts(hosts.IPV6) - if err != nil { - log.Warnf("Failed to fetch IPv6 hosts: %v", err) - // Decide if you want to return nil here or continue without IPv6 - } else { - client.ipv6Hosts = ipv6List - log.Debugf("Fetched %d IPv6 hosts", len(ipv6List)) - } - + if forceIPv6 { // replace the default dialer with a custom one that resolves hostnames to IPv6 addresses client.client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { // if address is already cached, use it - if ipv6Address, ok := client.ipv6.Get(address); ok { + if ipv6Address, ok := client.dnsCache.Get(address); ok { return dialer.Dial(network, ipv6Address) } @@ -126,24 +114,13 @@ func NewHTTPClient( for _, ip := range ips { if ip.IP.To4() == nil { // IPv6 address found ipv6Address := net.JoinHostPort(ip.IP.String(), port) - client.ipv6.Set(address, ipv6Address) + client.dnsCache.Set(address, ipv6Address) return dialer.Dial(network, ipv6Address) } } - // no IPv6 address found, use the original address - log.Warnf("No IPv6 address found for host %s", host) - for _, ip := range ips { - if ip.IP.To4() != nil { // IPv4 address found - ipV4Address := net.JoinHostPort(ip.IP.String(), port) - client.ipv6.Set(address, ipV4Address) - return dialer.Dial(network, ipV4Address) - } - } - - return dialer.Dial(network, address) + return nil, fmt.Errorf("no ipv6 address found") } - } return &client @@ -171,7 +148,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { resp.Body.Close() } - r.replaceWithIPv6Host(req) // needed for ipv6 + // r.optimizeHost(req) resp, err = r.client.Do(req) @@ -210,12 +187,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { return resp, err } -func (r *HTTPClient) replaceWithIPv6Host(req *http.Request) { - // don't replace host if IPv6 is not supported or not forced - if !r.isDownloadClient || !r.cfg.ShouldForceIPv6() { - return - } - // this host should be replaced +func (r *HTTPClient) optimizeHost(req *http.Request) { if !strings.HasSuffix(req.Host, ".download.real-debrid.com") { return } @@ -233,7 +205,6 @@ func (r *HTTPClient) replaceWithIPv6Host(req *http.Request) { req.Host = r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))] req.URL.Host = req.Host r.log.Debugf("Host %s is not a valid IPv6 host, assigning a random IPv6 host: %s", req.URL.Host, req.Host) - // if !found && !r.CanFetchFirstByte(req.URL.String()) { } func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) { diff --git a/pkg/http/ip.go b/pkg/http/ip.go index b77356e..92faab5 100644 --- a/pkg/http/ip.go +++ b/pkg/http/ip.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "math/rand" - "net" "net/http" "os" "time" @@ -15,150 +14,161 @@ import ( ) type IPRepository struct { - ipv4 []string - ipv6 []string - latencyMap map[string]float64 - log *logutil.Logger + ipv4client *HTTPClient + ipv6client *HTTPClient + ipv4latencyMap map[string]float64 + ipv6latencyMap map[string]float64 + log *logutil.Logger } -func NewIPRepository(log *logutil.Logger) *IPRepository { +func NewIPRepository(ipv4client *HTTPClient, ipv6client *HTTPClient, log *logutil.Logger) *IPRepository { repo := &IPRepository{ - ipv4: []string{}, - ipv6: []string{}, - latencyMap: make(map[string]float64), - log: log, + ipv4client: ipv4client, + ipv6client: ipv6client, + ipv4latencyMap: make(map[string]float64), + ipv6latencyMap: make(map[string]float64), + log: log, } - repo.lookupDomains() - return repo } -func (r *IPRepository) NetworkTest(downloadClient *HTTPClient, forceRun bool) { - latencyFile := "data/latency.json" +func (r *IPRepository) NetworkTest(forceRun bool) { + ipv4latencyFile := "data/latency4.json" + ipv6latencyFile := "data/latency6.json" if !forceRun { - latencyData := r.readLatencyFile(latencyFile) + latencyData := r.readLatencyFile(ipv4latencyFile) if latencyData != nil { - r.latencyMap = *latencyData - return + r.ipv4latencyMap = *latencyData + } + latencyData = r.readLatencyFile(ipv6latencyFile) + if latencyData != nil { + r.ipv6latencyMap = *latencyData } } - r.log.Info("Network test will start now. Note that it will only run once and record the latency of each domain for future use.") - r.latencyTest(downloadClient) - r.log.Infof("Network test completed. Saving the results to %s", latencyFile) - r.writeLatencyFile(latencyFile) + r.log.Info("Network test will start now. IGNORE THE WARNINGS!") + r.runLatencyTest() + r.log.Infof("Network test completed. Saving the results to %s and %s", ipv4latencyFile, ipv6latencyFile) + r.log.Debugf("ipv4 %v", r.ipv4latencyMap) + r.log.Debugf("ipv6 %v", r.ipv6latencyMap) + r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap) + r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap) } -func (r *IPRepository) lookupDomains() { +func (r *IPRepository) runLatencyTest() { limit := 99 - increment := 10 start := 0 for { - lastDomainWorked := false + lastDomainsWorked := false for i := start; i <= limit; i++ { domain := fmt.Sprintf("%d.download.real-debrid.com", i) - ips, err := net.LookupIP(domain) - if err == nil && len(ips) > 0 { - hasIPv6 := false - for _, ip := range ips { - if ip.To4() == nil { - hasIPv6 = true - } - } - // assume it always has ipv4 - r.ipv4 = append(r.ipv4, domain) - if hasIPv6 { - r.ipv6 = append(r.ipv6, domain) - } - if i == limit { - lastDomainWorked = true + // ips, err := net.LookupIP(domain) + // if err != nil || len(ips) == 0 { + // continue + // } + + latency, err := r.testDomainLatency(r.ipv4client, domain) + if err == nil { + r.ipv4latencyMap[domain] = latency + r.log.Debugf("Latency from ipv4 %s: %.5f seconds", domain, latency) + if i >= limit-2 { + lastDomainsWorked = true } } - domain2 := fmt.Sprintf("%d.download.real-debrid.cloud", i) - ips2, err := net.LookupIP(domain2) - if err == nil && len(ips2) > 0 { - hasIPv6 := false - for _, ip := range ips { - if ip.To4() == nil { - hasIPv6 = true - } + latency, err = r.testDomainLatency(r.ipv6client, domain) + if err == nil { + r.ipv6latencyMap[domain] = latency + r.log.Debugf("Latency from ipv6 %s: %.5f seconds", domain, latency) + if i >= limit-2 { + lastDomainsWorked = true } - r.ipv4 = append(r.ipv4, domain2) - if hasIPv6 { - r.ipv6 = append(r.ipv6, domain2) + } + + domain = fmt.Sprintf("%d.download.real-debrid.cloud", i) + // ips, err = net.LookupIP(domain) + // if err != nil || len(ips) == 0 { + // continue + // } + + latency, err = r.testDomainLatency(r.ipv4client, domain) + if err == nil { + r.ipv4latencyMap[domain] = latency + r.log.Debugf("Latency from ipv4 %s: %.5f seconds", domain, latency) + if i >= limit-2 { + lastDomainsWorked = true } - if i == limit { - lastDomainWorked = true + } + + latency, err = r.testDomainLatency(r.ipv6client, domain) + if err == nil { + r.ipv6latencyMap[domain] = latency + r.log.Debugf("Latency from ipv6 %s: %.5f seconds", domain, latency) + if i >= limit-2 { + lastDomainsWorked = true } } } - if lastDomainWorked { + if lastDomainsWorked { start = limit + 1 - limit += increment + limit += 10 } else { break } } - - r.log.Infof("Found %d IPv4 domains and %d IPv6 domains", len(r.ipv4), len(r.ipv6)) } -func (r *IPRepository) latencyTest(downloadClient *HTTPClient) { - const testFileSize = 1 +func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (float64, error) { + const testFileSize = 1 // byte const iterations = 3 + url := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64()) - for _, domain := range r.ipv4 { - url := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64()) + var totalDuration float64 + var retErr error + for i := 0; i < iterations; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() - var totalDuration float64 - hasError := false - for i := 0; i < iterations; i++ { - ctx, cancel := context.WithTimeout(context.Background(), iterations*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - r.log.Warnf("Failed to create request for %s: %v", domain, err) - hasError = true - break - } - - headers := make(http.Header) - headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1)) - req.Header = headers - - start := time.Now() - resp, err := downloadClient.Do(req) - if err != nil { - r.log.Warnf("Failed to download from %s: %v", domain, err) - hasError = true - break - } - - limitedReader := io.LimitReader(resp.Body, testFileSize) - - _, err = io.Copy(io.Discard, limitedReader) - resp.Body.Close() - - if err != nil && err != io.EOF { - r.log.Warnf("Failed to read from %s: %v", domain, err) - hasError = true - break - } - - duration := time.Since(start).Seconds() - totalDuration += duration - } - if hasError { - continue + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + r.log.Warnf("Failed to create request for %s: %v", domain, err) + retErr = err + break } - r.latencyMap[domain] = totalDuration / 3 + headers := make(http.Header) + headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1)) + req.Header = headers - r.log.Debugf("Latency from %s: %.5f seconds", domain, r.latencyMap[domain]) + start := time.Now() + resp, err := client.Do(req) + if err != nil { + r.log.Warnf("Failed to download from %s: %v", domain, err) + retErr = err + break + } + + limitedReader := io.LimitReader(resp.Body, testFileSize) + + _, err = io.Copy(io.Discard, limitedReader) + resp.Body.Close() + + if err != nil && err != io.EOF { + r.log.Warnf("Failed to read from %s: %v", domain, err) + retErr = err + break + } + + duration := time.Since(start).Seconds() + totalDuration += duration } + if retErr != nil { + return 0, retErr + } + + avgDuration := totalDuration / 3 + return avgDuration, nil } func (r *IPRepository) readLatencyFile(latencyFile string) *map[string]float64 { @@ -172,16 +182,16 @@ func (r *IPRepository) readLatencyFile(latencyFile string) *map[string]float64 { if err != nil { return nil } - var latencyMap map[string]float64 - if err := json.Unmarshal(jsonData, &latencyMap); err != nil { + var ipv4latencyMap map[string]float64 + if err := json.Unmarshal(jsonData, &ipv4latencyMap); err != nil { return nil } - return &latencyMap + return &ipv4latencyMap } return nil } -func (r *IPRepository) writeLatencyFile(latencyFile string) { +func (r *IPRepository) writeLatencyFile(latencyFile string, data interface{}) { file, err := os.Create(latencyFile) if err != nil { r.log.Warnf("Cannot create latency file %s: %v", latencyFile, err) @@ -189,7 +199,7 @@ func (r *IPRepository) writeLatencyFile(latencyFile string) { } defer file.Close() - jsonData, err := json.Marshal(r.latencyMap) + jsonData, err := json.Marshal(data) if err != nil { r.log.Warnf("Cannot marshal latency map: %v", err) return