Add rate limiter

internal/app.go

@@ -57,8 +57,10 @@ func MainApp(configPath string) {
 		proxyURL = os.Getenv("PROXY")
 	}
 
-	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
+	dummyLimiter := http.NewRateLimiter(1000)
-	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
+
+	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
+	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
 	repo := http.NewIPRepository(repoClient4, repoClient6, "", log.Named("network_test"))
 
 	var hosts []string
@@ -79,6 +81,8 @@ func MainApp(configPath string) {
 		}
 	}
 
+	rateLimiter := http.NewRateLimiter(config.GetAPIRateLimitPerSecond())
+
 	apiClient := http.NewHTTPClient(
 		config.GetToken(),
 		config.GetRetriesUntilFailed(), // default retries = 2
@@ -86,6 +90,7 @@ func MainApp(configPath string) {
 		false,                          // no need for ipv6 support
 		[]string{},                     // no optimal hosts needed
 		proxyURL,
+		rateLimiter,
 		log.Named("api_client"),
 	)
 
@@ -96,6 +101,7 @@ func MainApp(configPath string) {
 		false,                           // no need for ipv6 support
 		[]string{},                      // no optimal hosts needed
 		proxyURL,
+		rateLimiter,
 		log.Named("unrestrict_client"),
 	)
 
@@ -106,6 +112,7 @@ func MainApp(configPath string) {
 		config.ShouldForceIPv6(),
 		hosts,
 		proxyURL,
+		rateLimiter,
 		log.Named("download_client"),
 	)
 
@@ -120,11 +127,14 @@ func MainApp(configPath string) {
 	}
 	defer workerPool.Release()
 
+	torrentsRateLimiter := http.NewRateLimiter(config.GetTorrentsRateLimitPerSecond())
+
 	rd := realdebrid.NewRealDebrid(
 		apiClient,
 		unrestrictClient,
 		downloadClient,
 		workerPool,
+		torrentsRateLimiter,
 		config,
 		log.Named("realdebrid"),
 	)

internal/commands.go

@@ -31,8 +31,9 @@ func NetworkTest(testURL string) {
 		log.Info("You can set a proxy by setting the PROXY environment variable")
 	}
 
-	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
+	dummyLimiter := http.NewRateLimiter(1000)
-	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
+	repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
+	repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
 	repo := http.NewIPRepository(repoClient4, repoClient6, testURL, log.Named("network_test"))
 	repo.NetworkTest(true, true)
 }

internal/config/types.go

@@ -10,6 +10,7 @@ type ConfigInterface interface {
 	EnableRepair() bool
 	EnableRetainFolderNameExtension() bool
 	EnableRetainRDTorrentName() bool
+	GetAPIRateLimitPerSecond() int
 	GetApiTimeoutSecs() int
 	GetConfig() ZurgConfig
 	GetDirectories() []string
@@ -30,6 +31,7 @@ type ConfigInterface interface {
 	GetRepairEveryMins() int
 	GetRetriesUntilFailed() int
 	GetToken() string
+	GetTorrentsRateLimitPerSecond() int
 	GetUsername() string
 	GetVersion() string
 	MeetsConditions(directory, torrentName string, torrentSize int64, torrentIDs, fileNames []string, fileSizes []int64, mediaInfos []*ffprobe.ProbeData) bool
@@ -46,6 +48,7 @@ type ZurgConfig struct {
 	Version string `yaml:"zurg" json:"-"`
 	Token   string `yaml:"token" json:"-"`
 
+	APIRateLimitPerSecond      int      `yaml:"api_rate_limit_per_second" json:"api_rate_limit_per_second"`
 	ApiTimeoutSecs             int      `yaml:"api_timeout_secs" json:"api_timeout_secs"`
 	AutoAnalyzeNewTorrents     bool     `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"`
 	CacheNetworkTestResults    bool     `yaml:"cache_network_test_results" json:"cache_network_test_results"`
@@ -73,6 +76,7 @@ type ZurgConfig struct {
 	RetainRDTorrentName        bool     `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
 	RetriesUntilFailed         int      `yaml:"retries_until_failed" json:"retries_until_failed"`
 	ServeFromRclone            bool     `yaml:"serve_from_rclone" json:"serve_from_rclone"`
+	TorrentsRateLimitPerSecond int      `yaml:"torrents_rate_limit_per_second" json:"torrents_rate_limit_per_second"`
 	Username                   string   `yaml:"username" json:"username"`
 }
 
@@ -238,3 +242,17 @@ func (z *ZurgConfig) GetDownloadTokens() []string {
 func (z *ZurgConfig) ShouldHideBrokenTorrents() bool {
 	return z.HideBrokenTorrents
 }
+
+func (z *ZurgConfig) GetAPIRateLimitPerSecond() int {
+	if z.APIRateLimitPerSecond == 0 {
+		return 250
+	}
+	return z.APIRateLimitPerSecond
+}
+
+func (z *ZurgConfig) GetTorrentsRateLimitPerSecond() int {
+	if z.TorrentsRateLimitPerSecond == 0 {
+		return 1
+	}
+	return z.TorrentsRateLimitPerSecond
+}

pkg/http/client.go

@@ -30,6 +30,7 @@ type HTTPClient struct {
 	backoff            func(int, int) time.Duration
 	dnsCache           cmap.ConcurrentMap[string, string]
 	hosts              []string
+	rateLimiter        *RateLimiter
 	log                *logutil.Logger
 }
 
@@ -58,6 +59,7 @@ func NewHTTPClient(
 	forceIPv6 bool,
 	hosts []string,
 	proxyURL string,
+	rateLimiter *RateLimiter,
 	log *logutil.Logger,
 ) *HTTPClient {
 	client := HTTPClient{
@@ -69,6 +71,7 @@ func NewHTTPClient(
 		backoff:            backoffFunc,
 		dnsCache:           cmap.New[string](),
 		hosts:              hosts,
+		rateLimiter:        rateLimiter,
 		log:                log,
 	}
 
@@ -154,6 +157,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			r.ensureReachableHost(req)
 		}
 
+		r.rateLimiter.Wait()
+
 		resp, err = r.client.Do(req)
 
 		// http 4xx and 5xx errors

pkg/http/rate_limiter.go

@@ -0,0 +1,17 @@
+package http
+
+import "time"
+
+type RateLimiter struct {
+	ticker *time.Ticker
+}
+
+func NewRateLimiter(apiRateLimitPerSecond int) *RateLimiter {
+	return &RateLimiter{
+		ticker: time.NewTicker(time.Second / time.Duration(apiRateLimitPerSecond)),
+	}
+}
+
+func (r *RateLimiter) Wait() {
+	<-r.ticker.C
+}

pkg/realdebrid/api.go

@@ -25,11 +25,19 @@ type RealDebrid struct {
 	unrestrictClient    *zurghttp.HTTPClient
 	downloadClient      *zurghttp.HTTPClient
 	workerPool          *ants.Pool
+	torrentsRateLimiter *zurghttp.RateLimiter
 	cfg                 config.ConfigInterface
 	log                 *logutil.Logger
 }
 
-func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPClient, workerPool *ants.Pool, cfg config.ConfigInterface, log *logutil.Logger) *RealDebrid {
+func NewRealDebrid(apiClient,
+	unrestrictClient,
+	downloadClient *zurghttp.HTTPClient,
+	workerPool *ants.Pool,
+	torrentsRateLimiter *zurghttp.RateLimiter,
+	cfg config.ConfigInterface,
+	log *logutil.Logger,
+) *RealDebrid {
 	mainToken := cfg.GetToken()
 	downloadTokens := cfg.GetDownloadTokens()
 	if !strings.Contains(strings.Join(downloadTokens, ","), mainToken) {
@@ -45,6 +53,7 @@ func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPCli
 		unrestrictClient:    unrestrictClient,
 		downloadClient:      downloadClient,
 		workerPool:          workerPool,
+		torrentsRateLimiter: torrentsRateLimiter,
 		cfg:                 cfg,
 		log:                 log,
 	}

pkg/realdebrid/torrents.go

@@ -29,7 +29,7 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) {
 
 	allTorrents := []Torrent{}
 	page := 1
-	pageSize := 250
+	pageSize := 500
 
 	maxPages := (totalElements + pageSize - 1) / pageSize
 	rd.log.Debugf("Torrents total count is %d", totalElements)
@@ -127,6 +127,8 @@ func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult {
 		}
 	}
 
+	rd.torrentsRateLimiter.Wait()
+
 	resp, err := rd.apiClient.Do(req)
 	if err != nil {
 		rd.log.Errorf("Error when executing the get torrents request: %v", err)