Add rate limiter

pkg/http/client.go

@@ -30,6 +30,7 @@ type HTTPClient struct {
 	backoff            func(int, int) time.Duration
 	dnsCache           cmap.ConcurrentMap[string, string]
 	hosts              []string
+	rateLimiter        *RateLimiter
 	log                *logutil.Logger
 }
 
@@ -58,6 +59,7 @@ func NewHTTPClient(
 	forceIPv6 bool,
 	hosts []string,
 	proxyURL string,
+	rateLimiter *RateLimiter,
 	log *logutil.Logger,
 ) *HTTPClient {
 	client := HTTPClient{
@@ -69,6 +71,7 @@ func NewHTTPClient(
 		backoff:            backoffFunc,
 		dnsCache:           cmap.New[string](),
 		hosts:              hosts,
+		rateLimiter:        rateLimiter,
 		log:                log,
 	}
 
@@ -154,6 +157,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			r.ensureReachableHost(req)
 		}
 
+		r.rateLimiter.Wait()
+
 		resp, err = r.client.Do(req)
 
 		// http 4xx and 5xx errors

pkg/http/rate_limiter.go

@@ -0,0 +1,17 @@
+package http
+
+import "time"
+
+type RateLimiter struct {
+	ticker *time.Ticker
+}
+
+func NewRateLimiter(apiRateLimitPerSecond int) *RateLimiter {
+	return &RateLimiter{
+		ticker: time.NewTicker(time.Second / time.Duration(apiRateLimitPerSecond)),
+	}
+}
+
+func (r *RateLimiter) Wait() {
+	<-r.ticker.C
+}

pkg/realdebrid/api.go

@@ -17,19 +17,27 @@ import (
 )
 
 type RealDebrid struct {
-	UnrestrictMap    cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Download]]
-	TokenManager     *DownloadTokenManager
-	torrentsCache    []Torrent
-	verifiedLinks    cmap.ConcurrentMap[string, int64]
-	apiClient        *zurghttp.HTTPClient
-	unrestrictClient *zurghttp.HTTPClient
-	downloadClient   *zurghttp.HTTPClient
-	workerPool       *ants.Pool
-	cfg              config.ConfigInterface
-	log              *logutil.Logger
+	UnrestrictMap       cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Download]]
+	TokenManager        *DownloadTokenManager
+	torrentsCache       []Torrent
+	verifiedLinks       cmap.ConcurrentMap[string, int64]
+	apiClient           *zurghttp.HTTPClient
+	unrestrictClient    *zurghttp.HTTPClient
+	downloadClient      *zurghttp.HTTPClient
+	workerPool          *ants.Pool
+	torrentsRateLimiter *zurghttp.RateLimiter
+	cfg                 config.ConfigInterface
+	log                 *logutil.Logger
 }
 
-func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPClient, workerPool *ants.Pool, cfg config.ConfigInterface, log *logutil.Logger) *RealDebrid {
+func NewRealDebrid(apiClient,
+	unrestrictClient,
+	downloadClient *zurghttp.HTTPClient,
+	workerPool *ants.Pool,
+	torrentsRateLimiter *zurghttp.RateLimiter,
+	cfg config.ConfigInterface,
+	log *logutil.Logger,
+) *RealDebrid {
 	mainToken := cfg.GetToken()
 	downloadTokens := cfg.GetDownloadTokens()
 	if !strings.Contains(strings.Join(downloadTokens, ","), mainToken) {
@@ -37,16 +45,17 @@ func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPCli
 	}
 
 	rd := &RealDebrid{
-		UnrestrictMap:    cmap.New[cmap.ConcurrentMap[string, *Download]](),
-		TokenManager:     NewDownloadTokenManager(downloadTokens, log),
-		torrentsCache:    []Torrent{},
-		verifiedLinks:    cmap.New[int64](),
-		apiClient:        apiClient,
-		unrestrictClient: unrestrictClient,
-		downloadClient:   downloadClient,
-		workerPool:       workerPool,
-		cfg:              cfg,
-		log:              log,
+		UnrestrictMap:       cmap.New[cmap.ConcurrentMap[string, *Download]](),
+		TokenManager:        NewDownloadTokenManager(downloadTokens, log),
+		torrentsCache:       []Torrent{},
+		verifiedLinks:       cmap.New[int64](),
+		apiClient:           apiClient,
+		unrestrictClient:    unrestrictClient,
+		downloadClient:      downloadClient,
+		workerPool:          workerPool,
+		torrentsRateLimiter: torrentsRateLimiter,
+		cfg:                 cfg,
+		log:                 log,
 	}
 
 	for _, token := range downloadTokens {

pkg/realdebrid/torrents.go

@@ -29,7 +29,7 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) {
 
 	allTorrents := []Torrent{}
 	page := 1
-	pageSize := 250
+	pageSize := 500
 
 	maxPages := (totalElements + pageSize - 1) / pageSize
 	rd.log.Debugf("Torrents total count is %d", totalElements)
@@ -127,6 +127,8 @@ func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult {
 		}
 	}
 
+	rd.torrentsRateLimiter.Wait()
+
 	resp, err := rd.apiClient.Do(req)
 	if err != nil {
 		rd.log.Errorf("Error when executing the get torrents request: %v", err)