hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reachable hosts

Browse Source
This commit is contained in:
Ben Adrian Sarmiento
2024-06-24 01:21:09 +02:00
parent 449c0f71cf
commit 2e9a068780
5 changed files with 68 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/app.go
View File

@@ -64,7 +64,7 @@ func MainApp(configPath string) {
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
repo := http.NewIPRepository(repoClient4, repoClient6, "", log.Named("network_test"))
repo.NetworkTest(false)
repo.NetworkTest(false, config.ShouldCacheNetworkTestResults())
apiClient := http.NewHTTPClient(
config.GetToken(),
@@ -86,14 +86,11 @@ func MainApp(configPath string) {
log.Named("unrestrict_client"),
)
hosts := repo.GetOptimalHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6())
hosts := repo.GetHosts(config.GetNumberOfHosts(), config.ShouldForceIPv6())
if len(hosts) == 0 {
zurglog.Fatal("No optimal hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)")
zurglog.Fatal("No reachable hosts found. We cannot continue! (check if Real-Debrid is down or they have blocked your IP address)")
}
zurglog.Debugf("Optimal hosts (%d): %v", len(hosts), hosts)
// help message
zurglog.Debug("To reset optimal hosts, run 'zurg network-test' (Using docker compose? 'docker compose exec zurg ./zurg network-test')")
zurglog.Debug("To run network-test with a proxy, set the PROXY environment variable 'PROXY=http://xyz:123 zurg network-test'")
zurglog.Debugf("Reachable hosts (%d): %v", len(hosts), hosts)
downloadClient := http.NewHTTPClient(
"",

2
internal/commands.go
View File

@@ -38,7 +38,7 @@ func NetworkTest(testURL string) {
repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test"))
repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test"))
repo := http.NewIPRepository(repoClient4, repoClient6, testURL, log.Named("network_test"))
repo.NetworkTest(true)
repo.NetworkTest(true, true)
}
func ClearDownloads() {

9
internal/config/types.go
View File

@@ -34,6 +34,7 @@ type ConfigInterface interface {
GetVersion() string
MeetsConditions(directory, torrentName string, torrentSize int64, torrentIDs, fileNames []string, fileSizes []int64, mediaInfos []*ffprobe.ProbeData) bool
ShouldAutoAnalyzeNewTorrents() bool
ShouldCacheNetworkTestResults() bool
ShouldForceIPv6() bool
ShouldIgnoreRenames() bool
ShouldServeFromRclone() bool
@@ -45,6 +46,7 @@ type ZurgConfig struct {
ApiTimeoutSecs int `yaml:"api_timeout_secs" json:"api_timeout_secs"`
AutoAnalyzeNewTorrents bool `yaml:"auto_analyze_new_torrents" json:"auto_analyze_new_torrents"`
CacheNetworkTestResults bool `yaml:"cache_network_test_results" json:"cache_network_test_results"`
CanRepair bool `yaml:"enable_repair" json:"enable_repair"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
DownloadTimeoutSecs int `yaml:"download_timeout_secs" json:"download_timeout_secs"`
@@ -217,12 +219,13 @@ func (z *ZurgConfig) GetProxy() string {
}
func (z *ZurgConfig) GetNumberOfHosts() int {
if z.NumberOfHosts == 0 {
return 20
}
return z.NumberOfHosts
}
func (z *ZurgConfig) ShouldAutoAnalyzeNewTorrents() bool {
return z.AutoAnalyzeNewTorrents
}
func (z *ZurgConfig) ShouldCacheNetworkTestResults() bool {
return z.CacheNetworkTestResults
}

44
pkg/http/client.go
View File

@@ -29,7 +29,7 @@ type HTTPClient struct {
backoff func(attempt int) time.Duration
bearerToken string
dnsCache cmap.ConcurrentMap[string, string]
optimalHosts []string
hosts []string
log *logutil.Logger
}
@@ -56,7 +56,7 @@ func NewHTTPClient(
maxRetries int,
timeoutSecs int,
forceIPv6 bool,
optimalHosts []string,
hosts []string,
proxyURL string,
log *logutil.Logger,
) *HTTPClient {
@@ -68,7 +68,7 @@ func NewHTTPClient(
rateLimitSleepSecs: 4,
backoff: backoffFunc,
dnsCache: cmap.New[string](),
optimalHosts: optimalHosts,
hosts: hosts,
log: log,
}
@@ -152,8 +152,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
resp.Body.Close()
}
if len(r.optimalHosts) > 0 {
r.optimizeHost(req)
if len(r.hosts) > 0 {
r.ensureReachableHost(req)
}
resp, err = r.client.Do(req)
@@ -201,14 +201,44 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
return resp, err
}
func (r *HTTPClient) optimizeHost(req *http.Request) {
func (r *HTTPClient) ensureReachableHost(req *http.Request) {
if !strings.Contains(req.Host, ".download.real-debrid.") {
return
}
req.Host = r.optimalHosts[rand.Intn(len(r.optimalHosts))]
if req.Host[0] >= 'a' && req.Host[0] <= 'z' {
return
}
// check if req.Host is in r.hosts
if r.CheckIfHostIsReachable(req.Host) {
return
}
// replace prefix of req.Host from .com to .cloud or vice versa
var newHost string
if strings.HasSuffix(req.Host, ".com") {
newHost = strings.Replace(req.Host, ".com", ".cloud", 1)
} else if strings.HasSuffix(req.Host, ".cloud") {
newHost = strings.Replace(req.Host, ".cloud", ".com", 1)
}
if r.CheckIfHostIsReachable(newHost) {
req.Host = newHost
req.URL.Host = req.Host
return
}
req.Host = r.hosts[rand.Intn(len(r.hosts))]
req.URL.Host = req.Host
}
func (r *HTTPClient) CheckIfHostIsReachable(reqHost string) bool {
for _, host := range r.hosts {
if reqHost == host {
return true
}
}
return false
}
func (r *HTTPClient) proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) {
if proxyURL.Scheme == "http" || proxyURL.Scheme == "https" {
httpProxyDialer := http_dialer.New(proxyURL, http_dialer.WithConnectionTimeout(time.Duration(r.timeoutSecs)*time.Second))

50
pkg/http/ip.go
View File

@@ -38,18 +38,18 @@ func NewIPRepository(ipv4client *HTTPClient, ipv6client *HTTPClient, testURL str
return repo
}
func (r *IPRepository) NetworkTest(forceRun bool) {
ipv4latencyFile := "data/latency4.json"
ipv6latencyFile := "data/latency6.json"
func (r *IPRepository) NetworkTest(forceRun bool, persist bool) {
ipv4HostsFile := "data/ipv4-hosts.json"
ipv6HostsFile := "data/ipv6-hosts.json"
if !forceRun {
ipv4Loaded := false
ipv6Loaded := false
latencyData := r.readLatencyFile(ipv4latencyFile)
latencyData := r.readLatencyFile(ipv4HostsFile)
if latencyData != nil {
r.ipv4latencyMap = *latencyData
ipv4Loaded = true
}
latencyData = r.readLatencyFile(ipv6latencyFile)
latencyData = r.readLatencyFile(ipv6HostsFile)
if latencyData != nil {
r.ipv6latencyMap = *latencyData
ipv6Loaded = true
@@ -57,19 +57,23 @@ func (r *IPRepository) NetworkTest(forceRun bool) {
if ipv4Loaded && ipv6Loaded {
return
} else {
r.log.Warn("Network test files not found")
r.log.Warn("Network test files not found, running network test")
}
}
r.log.Info("Network test will start now. IGNORE THE WARNINGS!")
r.log.Info("zurg will check for all reachable download servers. You can set 'cache_network_test_results: true' in your config to skip this test in the future.")
r.log.Warn("IGNORE THE WARNINGS!")
r.runLatencyTest()
r.log.Info("Network test completed!")
r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4latencyFile, ipv6latencyFile)
r.writeLatencyFile(ipv4latencyFile, r.ipv4latencyMap)
r.writeLatencyFile(ipv6latencyFile, r.ipv6latencyMap)
if persist {
r.log.Infof("To rerun the network test, run 'zurg network-test', or delete the files %s and %s and run zurg again", ipv4HostsFile, ipv6HostsFile)
r.writeLatencyFile(ipv4HostsFile, r.ipv4latencyMap)
r.writeLatencyFile(ipv6HostsFile, r.ipv6latencyMap)
}
}
func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string {
func (r *IPRepository) GetHosts(numberOfHosts int, ipv6 bool) []string {
latencyMap := r.ipv4latencyMap
if ipv6 {
latencyMap = r.ipv6latencyMap
@@ -90,6 +94,9 @@ func (r *IPRepository) GetOptimalHosts(numberOfHosts int, ipv6 bool) []string {
})
var optimalHosts []string
if numberOfHosts == 0 {
numberOfHosts = len(kvList)
}
for i := 0; i < numberOfHosts && i < len(kvList); i++ {
optimalHosts = append(optimalHosts, kvList[i].Key)
}
@@ -161,7 +168,6 @@ func (r *IPRepository) runLatencyTest() {
}
func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (float64, error) {
const testFileSize = 1 // byte
const iterations = 3
testURL := fmt.Sprintf("https://%s/speedtest/test.rar/%f", domain, rand.Float64())
if r.testURL != "" {
@@ -177,17 +183,13 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, testURL, nil)
req, err := http.NewRequestWithContext(ctx, http.MethodHead, testURL, nil)
if err != nil {
r.log.Warnf("Failed to create request for %s: %v", domain, err)
retErr = err
break
}
headers := make(http.Header)
headers.Set("Range", fmt.Sprintf("bytes=0-%d", testFileSize-1))
req.Header = headers
start := time.Now()
resp, err := client.Do(req)
if err != nil {
@@ -195,23 +197,11 @@ func (r *IPRepository) testDomainLatency(client *HTTPClient, domain string) (flo
retErr = err
break
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
if resp.StatusCode != http.StatusOK {
r.log.Warnf("Failed to download from %s: %s", domain, resp.Status)
retErr = fmt.Errorf("status code: %s", resp.Status)
break
}
limitedReader := io.LimitReader(resp.Body, testFileSize)
_, err = io.Copy(io.Discard, limitedReader)
resp.Body.Close()
if err != nil && err != io.EOF {
r.log.Warnf("Failed to read from %s: %v", domain, err)
retErr = err
break
}
duration := time.Since(start).Seconds()
totalDuration += duration
}