package http import ( "context" "math" "net" "net/http" "strings" "time" "github.com/debridmediamanager/zurg/internal/config" cmap "github.com/orcaman/concurrent-map/v2" "go.uber.org/zap" ) const ( RATE_LIMIT_FACTOR = 4 // should always be > 1 ) type HTTPClient struct { client *http.Client maxRetries int backoff func(attempt int) time.Duration getRetryIncr func(resp *http.Response, err error) int bearerToken string cfg config.ConfigInterface ipv6 cmap.ConcurrentMap[string, net.IP] log *zap.SugaredLogger } func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient { return &HTTPClient{ bearerToken: token, client: &http.Client{ Timeout: time.Duration(timeoutSecs) * time.Second, }, maxRetries: maxRetries * RATE_LIMIT_FACTOR, backoff: func(attempt int) time.Duration { maxDuration := 60 backoff := int(math.Pow(2, float64(attempt))) if backoff > maxDuration { backoff = maxDuration } return time.Duration(backoff) * time.Second }, getRetryIncr: func(resp *http.Response, err error) int { if resp != nil { if resp.StatusCode == 429 || resp.StatusCode == 400 || resp.StatusCode == 403 { return 1 // retry but don't increment attempt } return 0 // don't retry } else if err != nil { errStr := err.Error() if strings.Contains(errStr, "EOF") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "no such host") { return 1 // retry but don't increment attempt } else { return RATE_LIMIT_FACTOR } } return RATE_LIMIT_FACTOR // retry and increment attempt }, cfg: cfg, ipv6: cmap.New[net.IP](), log: log, } } func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { if r.cfg != nil && strings.Contains(req.Host, "download.real-debrid.") { prefHost := r.cfg.GetRandomPreferredHost() if prefHost != "" { req.Host = prefHost req.URL.Host = prefHost } } if r.bearerToken != "" { req.Header.Set("Authorization", "Bearer "+r.bearerToken) } if r.cfg.ShouldForceIPv6() { dialer := &net.Dialer{} transport := r.client.Transport.(*http.Transport).Clone() transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, err } if ipv6, ok := r.ipv6.Get(host); !ok { // Lookup IP address if not found in map ips, err := net.LookupIP(host) if err != nil { return nil, err } for _, ip := range ips { if ip.To4() == nil { // IPv6 ipv6 = ip r.ipv6.Set(host, ipv6) break } } if ipv6 == nil { // No IPv6 address found, fallback to default addr = net.JoinHostPort(host, port) } else { addr = net.JoinHostPort(ipv6.String(), port) } } else if ipv6 != nil && host == req.URL.Hostname() { addr = net.JoinHostPort(ipv6.String(), port) } return dialer.DialContext(ctx, network, addr) } r.client.Transport = transport } var resp *http.Response var err error attempt := 0 for { resp, err = r.client.Do(req) if incr := r.getRetryIncr(resp, err); incr > 0 { attempt += incr if attempt > r.maxRetries { break } if incr >= RATE_LIMIT_FACTOR { time.Sleep(r.backoff(attempt)) } else { time.Sleep(time.Duration(r.cfg.GetRateLimitSleepSeconds()) * time.Second) // extra delay } if resp != nil { resp.Body.Close() } } else { // if incr == 0, don't retry anymore break } } return resp, err }