hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add rate limiter

Browse Source
This commit is contained in:
Ben Adrian Sarmiento
2024-07-12 14:00:10 +02:00
parent fded8ee8aa
commit fbc431b82b
7 changed files with 116 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/app.go
View File

@@ -57,8 +57,10 @@ func MainApp(configPath string) {
proxyURL = os.Getenv("PROXY")
}
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
dummyLimiter := http.NewRateLimiter(1000)
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
repo := http.NewIPRepository(repoClient4, repoClient6, "", log.Named("network_test"))
var hosts []string
@@ -79,6 +81,8 @@ func MainApp(configPath string) {
}
}
rateLimiter := http.NewRateLimiter(config.GetAPIRateLimitPerSecond())
apiClient := http.NewHTTPClient(
config.GetToken(),
config.GetRetriesUntilFailed(), // default retries = 2
@@ -86,6 +90,7 @@ func MainApp(configPath string) {
false, // no need for ipv6 support
[]string{}, // no optimal hosts needed
proxyURL,
rateLimiter,
log.Named("api_client"),
)
@@ -96,6 +101,7 @@ func MainApp(configPath string) {
false, // no need for ipv6 support
[]string{}, // no optimal hosts needed
proxyURL,
rateLimiter,
log.Named("unrestrict_client"),
)
@@ -106,6 +112,7 @@ func MainApp(configPath string) {
config.ShouldForceIPv6(),
hosts,
proxyURL,
rateLimiter,
log.Named("download_client"),
)
@@ -120,11 +127,14 @@ func MainApp(configPath string) {
}
defer workerPool.Release()
torrentsRateLimiter := http.NewRateLimiter(config.GetTorrentsRateLimitPerSecond())
rd := realdebrid.NewRealDebrid(
apiClient,
unrestrictClient,
downloadClient,
workerPool,
torrentsRateLimiter,
config,
log.Named("realdebrid"),
)

5
internal/commands.go
View File

@@ -31,8 +31,9 @@ func NetworkTest(testURL string) {
log.Info("You can set a proxy by setting the PROXY environment variable")
}
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
dummyLimiter := http.NewRateLimiter(1000)
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, dummyLimiter, log.Named("network_test"))
repo := http.NewIPRepository(repoClient4, repoClient6, testURL, log.Named("network_test"))
repo.NetworkTest(true, true)
}

74
internal/config/types.go
View File

@@ -10,6 +10,7 @@ type ConfigInterface interface {
EnableRepair() bool
EnableRetainFolderNameExtension() bool
EnableRetainRDTorrentName() bool
GetAPIRateLimitPerSecond() int
GetApiTimeoutSecs() int
GetConfig() ZurgConfig
GetDirectories() []string
@@ -30,6 +31,7 @@ type ConfigInterface interface {
GetRepairEveryMins() int
GetRetriesUntilFailed() int
GetToken() string
GetTorrentsRateLimitPerSecond() int
GetUsername() string
GetVersion() string
MeetsConditions(directory, torrentName string, torrentSize int64, torrentIDs, fileNames []string, fileSizes []int64, mediaInfos []*ffprobe.ProbeData) bool
@@ -46,34 +48,36 @@ type ZurgConfig struct {
Version string `yaml:"zurg" json:"-"`
Token string `yaml:"token" json:"-"`
ApiTimeoutSecs int `yaml:"api_timeout_secs" json:"api_timeout_secs"`
AutoAnalyzeNewTorrents bool `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"`
CacheNetworkTestResults bool `yaml:"cache_network_test_results" json:"cache_network_test_results"`
CanRepair bool `yaml:"enable_repair" json:"enable_repair"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
DownloadTimeoutSecs int `yaml:"download_timeout_secs" json:"download_timeout_secs"`
DownloadTokens []string `yaml:"download_tokens" json:"download_tokens"`
DumpTorrentsEveryMins int `yaml:"dump_torrents_every_mins" json:"dump_torrents_every_mins"`
ForceIPv6 bool `yaml:"force_ipv6" json:"force_ipv6"`
HideBrokenTorrents bool `yaml:"hide_broken_torrents" json:"hide_broken_torrents"`
Host string `yaml:"host" json:"host"`
IgnoreRenames bool `yaml:"ignore_renames" json:"ignore_renames"`
LogRequests bool `yaml:"log_requests" json:"log_requests"`
NumberOfHosts int `yaml:"number_of_hosts" json:"number_of_hosts"`
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
OnLibraryUpdate string `yaml:"on_library_update" json:"on_library_update"`
Password string `yaml:"password" json:"password"`
PlayableExtensions []string `yaml:"addl_playable_extensions" json:"addl_playable_extensions"`
Port string `yaml:"port" json:"port"`
Proxy string `yaml:"proxy" json:"proxy"`
RarAction string `yaml:"rar_action" json:"rar_action"`
RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"`
RetainFolderNameExtension bool `yaml:"retain_folder_name_extension" json:"retain_folder_name_extension"`
RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
RetriesUntilFailed int `yaml:"retries_until_failed" json:"retries_until_failed"`
ServeFromRclone bool `yaml:"serve_from_rclone" json:"serve_from_rclone"`
Username string `yaml:"username" json:"username"`
APIRateLimitPerSecond int `yaml:"api_rate_limit_per_second" json:"api_rate_limit_per_second"`
ApiTimeoutSecs int `yaml:"api_timeout_secs" json:"api_timeout_secs"`
AutoAnalyzeNewTorrents bool `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"`
CacheNetworkTestResults bool `yaml:"cache_network_test_results" json:"cache_network_test_results"`
CanRepair bool `yaml:"enable_repair" json:"enable_repair"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
DownloadTimeoutSecs int `yaml:"download_timeout_secs" json:"download_timeout_secs"`
DownloadTokens []string `yaml:"download_tokens" json:"download_tokens"`
DumpTorrentsEveryMins int `yaml:"dump_torrents_every_mins" json:"dump_torrents_every_mins"`
ForceIPv6 bool `yaml:"force_ipv6" json:"force_ipv6"`
HideBrokenTorrents bool `yaml:"hide_broken_torrents" json:"hide_broken_torrents"`
Host string `yaml:"host" json:"host"`
IgnoreRenames bool `yaml:"ignore_renames" json:"ignore_renames"`
LogRequests bool `yaml:"log_requests" json:"log_requests"`
NumberOfHosts int `yaml:"number_of_hosts" json:"number_of_hosts"`
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
OnLibraryUpdate string `yaml:"on_library_update" json:"on_library_update"`
Password string `yaml:"password" json:"password"`
PlayableExtensions []string `yaml:"addl_playable_extensions" json:"addl_playable_extensions"`
Port string `yaml:"port" json:"port"`
Proxy string `yaml:"proxy" json:"proxy"`
RarAction string `yaml:"rar_action" json:"rar_action"`
RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"`
RetainFolderNameExtension bool `yaml:"retain_folder_name_extension" json:"retain_folder_name_extension"`
RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
RetriesUntilFailed int `yaml:"retries_until_failed" json:"retries_until_failed"`
ServeFromRclone bool `yaml:"serve_from_rclone" json:"serve_from_rclone"`
TorrentsRateLimitPerSecond int `yaml:"torrents_rate_limit_per_second" json:"torrents_rate_limit_per_second"`
Username string `yaml:"username" json:"username"`
}
func (z *ZurgConfig) GetConfig() ZurgConfig {
@@ -238,3 +242,17 @@ func (z *ZurgConfig) GetDownloadTokens() []string {
func (z *ZurgConfig) ShouldHideBrokenTorrents() bool {
return z.HideBrokenTorrents
}
func (z *ZurgConfig) GetAPIRateLimitPerSecond() int {
if z.APIRateLimitPerSecond == 0 {
return 250
}
return z.APIRateLimitPerSecond
}
func (z *ZurgConfig) GetTorrentsRateLimitPerSecond() int {
if z.TorrentsRateLimitPerSecond == 0 {
return 1
}
return z.TorrentsRateLimitPerSecond
}