From 9e3d12b008572354c399d29b2abc573555d349e1 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Thu, 11 Jan 2024 02:28:38 +0100 Subject: [PATCH] IPv6 hosts check --- internal/app.go | 12 ++- internal/commands.go | 4 +- pkg/hosts/hosts.go | 29 +++++++ pkg/http/client.go | 43 +++++++--- pkg/realdebrid/network.go | 160 ++++++++++++++++++++------------------ 5 files changed, 156 insertions(+), 92 deletions(-) create mode 100644 pkg/hosts/hosts.go diff --git a/internal/app.go b/internal/app.go index 0ad23e4..ef2e9c8 100644 --- a/internal/app.go +++ b/internal/app.go @@ -12,6 +12,7 @@ import ( "github.com/debridmediamanager/zurg/internal/handlers" "github.com/debridmediamanager/zurg/internal/torrent" "github.com/debridmediamanager/zurg/internal/universal" + "github.com/debridmediamanager/zurg/pkg/hosts" "github.com/debridmediamanager/zurg/pkg/http" "github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/premium" @@ -35,7 +36,7 @@ func MainApp(configPath string) { os.Exit(1) } - apiClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), config.GetRealDebridTimeout(), config, log.Named("httpclient")) + apiClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), config.GetRealDebridTimeout(), nil, config, log.Named("httpclient")) rd := realdebrid.NewRealDebrid(apiClient, log.Named("realdebrid")) @@ -51,7 +52,14 @@ func MainApp(configPath string) { utils.EnsureDirExists("data") // Ensure the data directory exists torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager")) - downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient")) + var ipv6List []string + if config.ShouldForceIPv6() { + ipv6List, err = hosts.FetchHosts(hosts.IPV6) + if err != nil { + panic(err) + } + } + downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, ipv6List, config, log.Named("dlclient")) downloader := universal.NewDownloader(downloadClient) router := chi.NewRouter() diff --git a/internal/commands.go b/internal/commands.go index fc22675..bfb0d99 100644 --- a/internal/commands.go +++ b/internal/commands.go @@ -16,8 +16,8 @@ func ShowVersion() { version.GetBuiltAt(), version.GetGitCommit(), version.GetVersion()) } -func NetworkTest(netTestType string) { - realdebrid.RunTest(netTestType) +func NetworkTest(testType string) { + realdebrid.RunTest(testType) } func ClearDownloads() { diff --git a/pkg/hosts/hosts.go b/pkg/hosts/hosts.go new file mode 100644 index 0000000..5b96b5e --- /dev/null +++ b/pkg/hosts/hosts.go @@ -0,0 +1,29 @@ +package hosts + +import ( + "bufio" + "net/http" +) + +const ( + IPV4 = "https://gist.githubusercontent.com/yowmamasita/d0c1c7353500d0928cb5242484e8ed06/raw/ipv4.txt" + IPV6 = "https://gist.githubusercontent.com/yowmamasita/d0c1c7353500d0928cb5242484e8ed06/raw/ipv6.txt" +) + +func FetchHosts(url string) ([]string, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var ips []string + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + ips = append(ips, scanner.Text()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return ips, nil +} diff --git a/pkg/http/client.go b/pkg/http/client.go index a2cf6d2..079b48c 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math" + "math/rand" "net" "net/http" "strings" @@ -22,14 +23,15 @@ const ( ) type HTTPClient struct { - client *http.Client - maxRetries int - backoff func(attempt int) time.Duration - getRetryIncr func(resp *http.Response, hasRangeHeader bool, err error) int - bearerToken string - cfg config.ConfigInterface - ipv6 cmap.ConcurrentMap[string, string] - log *logutil.Logger + client *http.Client + maxRetries int + backoff func(attempt int) time.Duration + getRetryIncr func(resp *http.Response, hasRangeHeader bool, err error) int + bearerToken string + restrictToHosts []string + cfg config.ConfigInterface + ipv6 cmap.ConcurrentMap[string, string] + log *logutil.Logger } // { @@ -46,7 +48,7 @@ func (e *ErrorResponse) Error() string { return fmt.Sprintf("api response error: %s (code: %d)", e.Message, e.Code) } -func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient { +func NewHTTPClient(token string, maxRetries int, timeoutSecs int, restrictToHosts []string, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient { client := HTTPClient{ bearerToken: token, client: &http.Client{ @@ -100,9 +102,10 @@ func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.Con } return RATE_LIMIT_FACTOR }, - cfg: cfg, - ipv6: cmap.New[string](), - log: log, + restrictToHosts: restrictToHosts, + cfg: cfg, + ipv6: cmap.New[string](), + log: log, } if cfg.ShouldForceIPv6() { @@ -115,6 +118,22 @@ func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.Con if err != nil { return nil, err } + if len(restrictToHosts) > 0 { + found := false + for _, h := range restrictToHosts { + if h == host { + found = true + break + } + } + if !found { + log.Warnf("Host %s is not an IPv6 host, replacing with a random host (ensure you have preferred_hosts properly set in your config.yml)", host) + // replace with a random ipv6 host + restrictToHostsLen := len(restrictToHosts) + randomHost := restrictToHosts[rand.Intn(restrictToHostsLen)] + host = randomHost + } + } ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, err diff --git a/pkg/realdebrid/network.go b/pkg/realdebrid/network.go index 2bf2fa5..88343cc 100644 --- a/pkg/realdebrid/network.go +++ b/pkg/realdebrid/network.go @@ -1,26 +1,33 @@ package realdebrid import ( - "bufio" "fmt" - "net/http" "os/exec" "sort" "strconv" "strings" "sync" "time" + + "github.com/debridmediamanager/zurg/pkg/hosts" ) -type IPInfo struct { +type HostInfo struct { Address string Hops int Latency time.Duration } -func traceroute(ip string) (int, time.Duration, error) { +func measureLatency(host, testType string) (int, time.Duration, error) { + traceroutePath := "traceroute" + pingPath := "ping" + if testType == "ipv6" { + traceroutePath = "traceroute6" + pingPath = "ping6" + } + // Try executing traceroute - cmd := exec.Command("traceroute", "-n", "-q", "1", "-w", "1", ip) + cmd := exec.Command(traceroutePath, "-n", "-q", "1", "-w", "1", host) out, err := cmd.CombinedOutput() if err == nil { @@ -45,7 +52,7 @@ func traceroute(ip string) (int, time.Duration, error) { } // Traceroute not successful, measure latency using ping - pingCmd := exec.Command("ping", "-c", "1", ip) + pingCmd := exec.Command(pingPath, "-c", "1", host) pingOut, pingErr := pingCmd.CombinedOutput() if pingErr != nil { @@ -75,110 +82,111 @@ func traceroute(ip string) (int, time.Duration, error) { return 0, 0, fmt.Errorf("failed to measure latency") } -func fetchIPs(url string) ([]string, error) { - resp, err := http.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var ips []string - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - ips = append(ips, scanner.Text()) - } - if err := scanner.Err(); err != nil { - return nil, err - } - return ips, nil -} - -func RunTest(netTestType string) { +func RunTest(testType string) { fmt.Print("Running network test...") - ipv4URL := "https://gist.githubusercontent.com/yowmamasita/d0c1c7353500d0928cb5242484e8ed06/raw/ipv4.txt" - ipv6URL := "https://gist.githubusercontent.com/yowmamasita/d0c1c7353500d0928cb5242484e8ed06/raw/ipv6.txt" + var ipv4Hosts, ipv6Hosts []string + var err error - var ips []string - - if netTestType == "ipv4" || netTestType == "both" { - ipv4IPs, err := fetchIPs(ipv4URL) + if testType == "ipv4" || testType == "both" { + ipv4Hosts, err = hosts.FetchHosts(hosts.IPV4) if err != nil { - fmt.Println("Error fetching IPv4 IPs:", err) + fmt.Println("Error fetching IPv4 hosts:", err) return } - ips = append(ips, ipv4IPs...) } - if netTestType == "ipv6" || netTestType == "both" { - ipv6IPs, err := fetchIPs(ipv6URL) + if testType == "ipv6" || testType == "both" { + ipv6Hosts, err = hosts.FetchHosts(hosts.IPV6) if err != nil { - fmt.Println("Error fetching IPv6 IPs:", err) + fmt.Println("Error fetching IPv6 hosts:", err) return } - ips = append(ips, ipv6IPs...) + } + + var totalHosts int + var hostInfos []HostInfo // Declare the slice to hold IPInfo objects + infoChan := make(chan HostInfo) + + if testType == "ipv4" || testType == "both" { + totalHosts += len(ipv4Hosts) + go runLatencyTests(ipv4Hosts, "ipv4", infoChan) + } + if testType == "ipv6" { + totalHosts += len(ipv6Hosts) + go runLatencyTests(ipv6Hosts, "ipv6", infoChan) + } + if testType == "both" { + totalHosts += len(ipv6Hosts) + go runLatencyTests(ipv6Hosts, "ipv4", infoChan) } var wg sync.WaitGroup - infoChan := make(chan IPInfo, len(ips)) - semaphore := make(chan struct{}, 10) - - for _, ip := range ips { - wg.Add(1) - semaphore <- struct{}{} - go func(ip string) { - defer wg.Done() - hops, latency, err := traceroute(ip) - if err != nil { - fmt.Printf("Error performing traceroute for %s:%s\n", ip, err) - } else { - infoChan <- IPInfo{Address: ip, Hops: hops, Latency: latency} - } - <-semaphore - }(ip) - } + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < totalHosts; i++ { + hostInfo := <-infoChan + hostInfos = append(hostInfos, hostInfo) + } + close(infoChan) + }() wg.Wait() - close(semaphore) - close(infoChan) - fmt.Printf("complete!\n\n") - var ipInfos []IPInfo - for info := range infoChan { - ipInfos = append(ipInfos, info) - } - - sort.Slice(ipInfos, func(i, j int) bool { - return ipInfos[i].Latency < ipInfos[j].Latency + sort.Slice(hostInfos, func(i, j int) bool { + return hostInfos[i].Latency < hostInfos[j].Latency }) const minResults = 10 const maxResults = 20 - var okIPs []IPInfo + var okHosts []HostInfo - if len(ipInfos) > 0 { + if len(hostInfos) > 0 { // Start by adding the best IPs based on hops and latency up to the minResults - for i := 0; i < min(len(ipInfos), minResults); i++ { - okIPs = append(okIPs, ipInfos[i]) + for i := 0; i < min(len(hostInfos), minResults); i++ { + okHosts = append(okHosts, hostInfos[i]) } - // Find the highest latency in the current okIPs list - highestLatency := okIPs[len(okIPs)-1].Latency + // Find the highest latency in the current okHosts list + highestLatency := okHosts[len(okHosts)-1].Latency - // Add any additional IPs that have latency within a reasonable range of the highest latency - for _, info := range ipInfos[minResults:] { - if len(okIPs) >= maxResults { - break // Stop adding IPs if maxResults is reached + // Add any additional hosts that have latency within a reasonable range of the highest latency + for _, info := range hostInfos[minResults:] { + if len(okHosts) >= maxResults { + break // Stop adding hosts if maxResults is reached } if info.Latency <= highestLatency+(highestLatency/3) { - okIPs = append(okIPs, info) + okHosts = append(okHosts, info) } } } fmt.Printf("Here are the results, you can copy-paste the following to your config.yml:\n\n") fmt.Println("preferred_hosts:") - for _, info := range okIPs { + for _, info := range okHosts { fmt.Printf(" - %s # hops: %d latency: %v\n", info.Address, info.Hops, info.Latency) } } + +func runLatencyTests(hosts []string, testType string, infoChan chan<- HostInfo) { + var wg sync.WaitGroup + semaphore := make(chan struct{}, 10) + + for _, host := range hosts { + wg.Add(1) + semaphore <- struct{}{} + go func(host string) { + defer wg.Done() + hops, latency, err := measureLatency(host, testType) + if err != nil { + fmt.Printf("Error measuring latency for %s: %s\n", host, err) + } else { + infoChan <- HostInfo{Address: host, Hops: hops, Latency: latency} + } + <-semaphore + }(host) + } + + wg.Wait() +}