package http import ( "context" "encoding/json" "fmt" "io" "math" "math/rand" "net" "net/http" "net/url" "strconv" "strings" "time" "github.com/debridmediamanager/zurg/internal/config" "github.com/debridmediamanager/zurg/pkg/hosts" "github.com/debridmediamanager/zurg/pkg/logutil" http_dialer "github.com/mwitkow/go-http-dialer" "golang.org/x/net/proxy" cmap "github.com/orcaman/concurrent-map/v2" ) type HTTPClient struct { client *http.Client maxRetries int timeoutSecs int backoff func(attempt int) time.Duration getRetryIncr func(resp *http.Response, reqHasRangeHeader bool, err error) int bearerToken string ensureIPv6Host bool cfg config.ConfigInterface ipv6 cmap.ConcurrentMap[string, string] ipv6Hosts []string log *logutil.Logger } // { // "error": "infringing_file", // "error_code": 35 // } type ApiErrorResponse struct { Message string `json:"error"` Code int `json:"error_code"` } func (e *ApiErrorResponse) Error() string { return fmt.Sprintf("api response error: %s (code: %d)", e.Message, e.Code) } func NewHTTPClient(token string, maxRetries int, timeoutSecs int, ensureIPv6Host bool, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient { client := HTTPClient{ bearerToken: token, client: &http.Client{}, maxRetries: maxRetries, timeoutSecs: timeoutSecs, backoff: func(attempt int) time.Duration { maxDuration := 60 backoff := int(math.Pow(2, float64(attempt))) if backoff > maxDuration { backoff = maxDuration } return time.Duration(backoff) * time.Second }, getRetryIncr: func(resp *http.Response, reqHasRangeHeader bool, err error) int { if err != nil && strings.HasPrefix(err.Error(), "api response error:") { if apiErr, ok := err.(*ApiErrorResponse); ok { switch apiErr.Code { case -1: // Internal error return 1 case 5: // Slow down (retry infinitely) time.Sleep(time.Duration(cfg.GetRateLimitSleepSeconds()) * time.Second) return -1 case 6: // Ressource unreachable return 1 case 17: // Hoster in maintenance return 1 case 19: // Hoster temporarily unavailable return 1 case 25: // Service unavailable return 1 case 34: // Too many requests (retry infinitely) time.Sleep(time.Duration(cfg.GetRateLimitSleepSeconds()) * time.Second) return -1 case 36: // Fair Usage Limit return 1 default: return 0 // don't retry } } } if resp != nil { if resp.StatusCode == 429 { time.Sleep(time.Duration(cfg.GetRateLimitSleepSeconds()) * time.Second) return -1 } if resp.Header.Get("Content-Range") == "" && reqHasRangeHeader { time.Sleep(10 * time.Millisecond) return -1 } return 0 // don't retry } return 1 }, ensureIPv6Host: ensureIPv6Host, cfg: cfg, ipv6: cmap.New[string](), log: log, } var dialer proxy.Dialer = &net.Dialer{ Timeout: time.Duration(timeoutSecs) * time.Second, // timeout to establish connection } if proxyURLString := cfg.GetProxy(); proxyURLString != "" { proxyURL, err := url.Parse(proxyURLString) if err != nil { log.Errorf("Failed to parse proxy URL: %v", err) return nil } dialer, err = proxyDialer(proxyURL) if err != nil { log.Errorf("Failed to create proxy dialer: %v", err) return nil } } maxConnections := cfg.GetNumOfWorkers() if maxConnections > 32 { maxConnections = 32 } if cfg.ShouldForceIPv6() { ipv6List, err := hosts.FetchHosts(hosts.IPV6) if err != nil { log.Warnf("Failed to fetch IPv6 hosts: %v", err) // Decide if you want to return nil here or continue without IPv6 } else { client.ipv6Hosts = ipv6List log.Debugf("Fetched %d IPv6 hosts", len(ipv6List)) } client.client.Transport = &http.Transport{ ResponseHeaderTimeout: time.Duration(timeoutSecs) * time.Second, MaxIdleConns: 0, MaxConnsPerHost: maxConnections, DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { if ipv6Address, ok := client.ipv6.Get(address); ok { return dialer.Dial(network, ipv6Address) } host, port, err := net.SplitHostPort(address) if err != nil { return nil, err } ips, err := net.DefaultResolver.LookupIPAddr(ctx, host) if err != nil { return nil, err } for _, ip := range ips { if ip.IP.To4() == nil { // IPv6 address found ipv6Address := net.JoinHostPort(ip.IP.String(), port) client.ipv6.Set(address, ipv6Address) return dialer.Dial(network, ipv6Address) } } return dialer.Dial(network, address) }, } } else { client.client.Transport = &http.Transport{ MaxIdleConns: maxConnections, MaxConnsPerHost: maxConnections, DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { return dialer.Dial(network, address) }, } } return &client } func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { if r.bearerToken != "" { req.Header.Set("Authorization", "Bearer "+r.bearerToken) } // check if Range header is set reqHasRangeHeader := req.Header.Get("Range") != "" && req.Header.Get("Range") != "bytes=0-" var resp *http.Response var err error attempt := 0 for { r.replaceHostIfNeeded(req) resp, err = r.client.Do(req) if resp != nil && (resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusPartialContent) { body, _ := io.ReadAll(resp.Body) if body != nil { var errResp ApiErrorResponse jsonErr := json.Unmarshal(body, &errResp) if jsonErr == nil { errResp.Message += fmt.Sprintf(" (status code: %d)", resp.StatusCode) err = &errResp } } } incr := r.getRetryIncr(resp, reqHasRangeHeader, err) if incr > 0 { attempt += incr if attempt > r.maxRetries { break } if incr > 0 { time.Sleep(r.backoff(attempt)) } } else if incr == 0 { // don't retry anymore break } // if incr < 0, retry infinitely } return resp, err } func (r *HTTPClient) replaceHostIfNeeded(req *http.Request) { if !r.ensureIPv6Host && !r.cfg.ShouldForceIPv6() || !strings.HasSuffix(req.URL.Host, "download.real-debrid.com") { return } // get subdomain of req.URL.Host subdomain := strings.Split(req.URL.Host, ".")[0] // check if subdomain is numeric _, err := strconv.Atoi(subdomain) if err == nil { // subdomain is numeric, replace it with .cloud req.URL.Host = strings.Replace(req.URL.Host, ".com", ".cloud", 1) } // check if host is in the list of IPv6 hosts found := false for _, h := range r.ipv6Hosts { if h == req.URL.Host { found = true break } } if !found { // random IPv6 host req.URL.Host = r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))] } fmt.Println(req.URL.Host) } func proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) { if proxyURL.Scheme == "http" || proxyURL.Scheme == "https" { // Create a new HTTP proxy dialer httpProxyDialer := http_dialer.New(proxyURL) return httpProxyDialer, nil } else if proxyURL.Scheme == "socks5" { // For SOCKS5 proxies, use the proxy package's FromURL return proxy.FromURL(proxyURL, proxy.Direct) } return nil, fmt.Errorf("unsupported proxy scheme: %s", proxyURL.Scheme) }