Single worker pool, client adjustments
This commit is contained in:
Ben Sarmiento
@@ -13,38 +13,79 @@ import (
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
RATE_LIMIT_FACTOR = 4 // should always be > 1
|
||||
)
|
||||
|
||||
type HTTPClient struct {
|
||||
Client *http.Client
|
||||
MaxRetries int
|
||||
Backoff func(attempt int) time.Duration
|
||||
ShouldRetry func(resp *http.Response, err error) int
|
||||
BearerToken string
|
||||
log *zap.SugaredLogger
|
||||
config config.ConfigInterface
|
||||
IPv6 cmap.ConcurrentMap[string, net.IP]
|
||||
client *http.Client
|
||||
maxRetries int
|
||||
backoff func(attempt int) time.Duration
|
||||
getRetryIncr func(resp *http.Response, err error) int
|
||||
bearerToken string
|
||||
cfg config.ConfigInterface
|
||||
ipv6 cmap.ConcurrentMap[string, net.IP]
|
||||
log *zap.SugaredLogger
|
||||
}
|
||||
|
||||
func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient {
|
||||
return &HTTPClient{
|
||||
bearerToken: token,
|
||||
client: &http.Client{
|
||||
Timeout: time.Duration(timeoutSecs) * time.Second,
|
||||
},
|
||||
maxRetries: maxRetries * RATE_LIMIT_FACTOR,
|
||||
backoff: func(attempt int) time.Duration {
|
||||
maxDuration := 60
|
||||
backoff := int(math.Pow(2, float64(attempt)))
|
||||
if backoff > maxDuration {
|
||||
backoff = maxDuration
|
||||
}
|
||||
return time.Duration(backoff) * time.Second
|
||||
},
|
||||
getRetryIncr: func(resp *http.Response, err error) int {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 429 || resp.StatusCode == 400 || resp.StatusCode == 403 {
|
||||
return 1 // retry but don't increment attempt
|
||||
}
|
||||
return 0 // don't retry
|
||||
} else if err != nil {
|
||||
errStr := err.Error()
|
||||
if strings.Contains(errStr, "EOF") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "no such host") {
|
||||
return 1 // retry but don't increment attempt
|
||||
} else {
|
||||
return RATE_LIMIT_FACTOR
|
||||
}
|
||||
}
|
||||
return RATE_LIMIT_FACTOR // retry and increment attempt
|
||||
},
|
||||
cfg: cfg,
|
||||
ipv6: cmap.New[net.IP](),
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
|
||||
if r.config != nil && strings.Contains(req.Host, "download.real-debrid.") {
|
||||
prefHost := r.config.GetRandomPreferredHost()
|
||||
if r.cfg != nil && strings.Contains(req.Host, "download.real-debrid.") {
|
||||
prefHost := r.cfg.GetRandomPreferredHost()
|
||||
if prefHost != "" {
|
||||
req.Host = prefHost
|
||||
req.URL.Host = prefHost
|
||||
}
|
||||
}
|
||||
if r.BearerToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+r.BearerToken)
|
||||
if r.bearerToken != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+r.bearerToken)
|
||||
}
|
||||
|
||||
if r.config.ShouldForceIPv6() {
|
||||
if r.cfg.ShouldForceIPv6() {
|
||||
dialer := &net.Dialer{}
|
||||
transport := r.Client.Transport.(*http.Transport).Clone()
|
||||
transport := r.client.Transport.(*http.Transport).Clone()
|
||||
transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ipv6, ok := r.IPv6.Get(host); !ok {
|
||||
if ipv6, ok := r.ipv6.Get(host); !ok {
|
||||
// Lookup IP address if not found in map
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
@@ -53,7 +94,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
|
||||
for _, ip := range ips {
|
||||
if ip.To4() == nil { // IPv6
|
||||
ipv6 = ip
|
||||
r.IPv6.Set(host, ipv6)
|
||||
r.ipv6.Set(host, ipv6)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -67,66 +108,31 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
|
||||
}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
}
|
||||
r.Client.Transport = transport
|
||||
r.client.Transport = transport
|
||||
}
|
||||
|
||||
var resp *http.Response
|
||||
var err error
|
||||
attempt := 0
|
||||
for {
|
||||
resp, err = r.Client.Do(req)
|
||||
if val := r.ShouldRetry(resp, err); val == -1 {
|
||||
return resp, err
|
||||
} else {
|
||||
if val == 0 {
|
||||
time.Sleep(time.Duration(r.config.GetRateLimitSleepSeconds()) * time.Second) // extra delay
|
||||
resp, err = r.client.Do(req)
|
||||
if incr := r.getRetryIncr(resp, err); incr > 0 {
|
||||
attempt += incr
|
||||
if attempt > r.maxRetries {
|
||||
break
|
||||
}
|
||||
if incr >= RATE_LIMIT_FACTOR {
|
||||
time.Sleep(r.backoff(attempt))
|
||||
} else {
|
||||
attempt += val
|
||||
if attempt > r.MaxRetries {
|
||||
return resp, err
|
||||
}
|
||||
time.Sleep(r.Backoff(attempt))
|
||||
time.Sleep(time.Duration(r.cfg.GetRateLimitSleepSeconds()) * time.Second) // extra delay
|
||||
}
|
||||
if resp != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
} else {
|
||||
// if incr == 0, don't retry anymore
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient {
|
||||
return &HTTPClient{
|
||||
BearerToken: token,
|
||||
Client: &http.Client{
|
||||
Timeout: time.Duration(timeoutSecs) * time.Second,
|
||||
},
|
||||
MaxRetries: maxRetries,
|
||||
Backoff: func(attempt int) time.Duration {
|
||||
maxDuration := 60
|
||||
backoff := int(math.Pow(2, float64(attempt)))
|
||||
if backoff > maxDuration {
|
||||
backoff = maxDuration
|
||||
}
|
||||
return time.Duration(backoff) * time.Second
|
||||
},
|
||||
ShouldRetry: func(resp *http.Response, err error) int {
|
||||
if resp != nil {
|
||||
if resp.StatusCode == 429 || resp.StatusCode == 400 || resp.StatusCode == 403 {
|
||||
return 0 // retry but don't increment attempt
|
||||
}
|
||||
return -1 // don't retry
|
||||
} else if err != nil {
|
||||
errStr := err.Error()
|
||||
if strings.Contains(errStr, "EOF") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "no such host") {
|
||||
return 0 // retry but don't increment attempt
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 1 // retry and increment attempt
|
||||
},
|
||||
log: log,
|
||||
config: cfg,
|
||||
IPv6: cmap.New[net.IP](),
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user