From 3abf48514d437d46bba59b7a3d0916e83a31b937 Mon Sep 17 00:00:00 2001 From: Ben Adrian Sarmiento Date: Sun, 23 Jun 2024 22:08:54 +0200 Subject: [PATCH] Return an error for 503 --- internal/clear/downloads.go | 2 +- internal/clear/torrents.go | 2 +- internal/torrent/manager.go | 10 ++-- internal/torrent/repair.go | 8 +-- internal/universal/downloader.go | 4 +- pkg/http/client.go | 96 ++++++++++++++++++++------------ pkg/logutil/factory.go | 2 +- pkg/realdebrid/api.go | 24 ++++---- pkg/realdebrid/downloads.go | 2 +- pkg/realdebrid/torrents.go | 2 +- 10 files changed, 87 insertions(+), 65 deletions(-) diff --git a/internal/clear/downloads.go b/internal/clear/downloads.go index 60a32f6..2824dea 100644 --- a/internal/clear/downloads.go +++ b/internal/clear/downloads.go @@ -10,7 +10,7 @@ import ( func ClearDownloads(authCode string) { for { - req, err := http.NewRequest("GET", "https://real-debrid.com/downloads?del-all=1&p=1", nil) + req, err := http.NewRequest(http.MethodGet, "https://real-debrid.com/downloads?del-all=1&p=1", nil) if err != nil { fmt.Println("Error creating request:", err) return diff --git a/internal/clear/torrents.go b/internal/clear/torrents.go index f60c974..fc224a1 100644 --- a/internal/clear/torrents.go +++ b/internal/clear/torrents.go @@ -10,7 +10,7 @@ import ( func ClearTorrents(authCode string) { for { - req, err := http.NewRequest("GET", "https://real-debrid.com/torrents?del-all=1&p=1", nil) + req, err := http.NewRequest(http.MethodGet, "https://real-debrid.com/torrents?del-all=1&p=1", nil) if err != nil { fmt.Println("Error creating request:", err) return diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 9ce1765..2b02d56 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -138,7 +138,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w } // proxy function -func (t *TorrentManager) UnrestrictLinkUntilOk(link string, checkFirstByte bool) *realdebrid.Download { +func (t *TorrentManager) UnrestrictLink(link string, verifyURL bool) *realdebrid.Download { isRealDebrid := strings.HasPrefix(link, "https://real-debrid.com/d/") if isRealDebrid && t.UnrestrictMap.Has(link[0:39]) { ret, _ := t.UnrestrictMap.Get(link[0:39]) @@ -147,7 +147,7 @@ func (t *TorrentManager) UnrestrictLinkUntilOk(link string, checkFirstByte bool) ret, _ := t.UnrestrictMap.Get(link) return ret } - ret, err := t.api.UnrestrictLink(link, checkFirstByte) + ret, err := t.api.UnrestrictLink(link, verifyURL) if err != nil { t.log.Warnf("Cannot unrestrict link %s: %v", link, err) return nil @@ -160,11 +160,11 @@ func (t *TorrentManager) UnrestrictLinkUntilOk(link string, checkFirstByte bool) return ret } -func (t *TorrentManager) UnrestrictFileUntilOk(file *File, checkFirstByte bool) *realdebrid.Download { +func (t *TorrentManager) UnrestrictFile(file *File, checkFirstByte bool) *realdebrid.Download { if !file.State.Is("ok_file") { return nil } - return t.UnrestrictLinkUntilOk(file.Link, checkFirstByte) + return t.UnrestrictLink(file.Link, checkFirstByte) } func (t *TorrentManager) GetKey(torrent *Torrent) string { @@ -235,7 +235,7 @@ func (t *TorrentManager) applyMediaInfoDetails(torrent *Torrent) { if file.MediaInfo != nil || !file.State.Is("ok_file") || !isPlayable { return } - unrestrict := t.UnrestrictFileUntilOk(file, true) + unrestrict := t.UnrestrictFile(file, true) if unrestrict == nil { file.State.Event(context.Background(), "break_file") t.EnqueueForRepair(torrent) diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index 8de410d..83016e8 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -189,12 +189,12 @@ func (t *TorrentManager) repair(torrent *Torrent, wg *sync.WaitGroup) { return } - // check for other broken files + // check for other broken file torrent.SelectedFiles.IterCb(func(_ string, file *File) { if !file.State.Is("ok_file") { return } - if t.UnrestrictFileUntilOk(file, true) == nil { + if t.UnrestrictFile(file, true) == nil { file.State.Event(context.Background(), "break_file") } }) @@ -324,7 +324,7 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool { torrent.UnassignedLinks.Clone().Each(func(link string) bool { // unrestrict each unassigned link that was filled out during torrent init - unrestrict := t.UnrestrictLinkUntilOk(link, true) + unrestrict := t.UnrestrictLink(link, true) if unrestrict == nil { expiredCount++ return false // next unassigned link @@ -653,7 +653,7 @@ func (t *TorrentManager) isStillBroken(info *realdebrid.TorrentInfo, brokenFiles // check if the broken files can now be unrestricted and downloaded for _, oldFile := range brokenFiles { for idx, newFile := range selectedFiles { - if oldFile.ID == newFile.ID && t.UnrestrictFileUntilOk(selectedFiles[idx], true) == nil { + if oldFile.ID == newFile.ID && t.UnrestrictFile(selectedFiles[idx], true) == nil { return true } } diff --git a/internal/universal/downloader.go b/internal/universal/downloader.go index d8d7d17..58e0812 100644 --- a/internal/universal/downloader.go +++ b/internal/universal/downloader.go @@ -61,7 +61,7 @@ func (dl *Downloader) DownloadFile( return } - unrestrict := torMgr.UnrestrictFileUntilOk(file, cfg.ShouldServeFromRclone()) + unrestrict := torMgr.UnrestrictFile(file, cfg.ShouldServeFromRclone()) if unrestrict == nil { log.Warnf("File %s cannot be unrestricted (link=%s)", fileName, file.Link) if err := file.State.Event(context.Background(), "break_file"); err != nil { @@ -104,7 +104,7 @@ func (dl *Downloader) DownloadLink( log *logutil.Logger, ) { // log.Debugf("Opening file %s (%s)", fileName, link) - unrestrict := torMgr.UnrestrictLinkUntilOk(link, cfg.ShouldServeFromRclone()) + unrestrict := torMgr.UnrestrictLink(link, cfg.ShouldServeFromRclone()) if unrestrict == nil { log.Warnf("File %s cannot be unrestricted (link=%s)", fileName, link) http.Error(resp, "File is not available", http.StatusInternalServerError) diff --git a/pkg/http/client.go b/pkg/http/client.go index 3846227..23643ea 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -42,6 +42,15 @@ func (e *ApiErrorResponse) Error() string { return fmt.Sprintf("api response error: %s (code: %d)", e.Message, e.Code) } +type DownloadErrorResponse struct { + Message string + Code int +} + +func (e *DownloadErrorResponse) Error() string { + return fmt.Sprintf("download response error: %s (code: %d)", e.Message, e.Code) +} + func NewHTTPClient( token string, maxRetries int, @@ -152,13 +161,25 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { // http 4xx and 5xx errors if resp != nil && resp.StatusCode >= http.StatusBadRequest { body, _ := io.ReadAll(resp.Body) - if body != nil { - var errResp ApiErrorResponse - jsonErr := json.Unmarshal(body, &errResp) - if jsonErr == nil { - errResp.Message += fmt.Sprintf(" (status code: %d)", resp.StatusCode) + if req.Host == "api.real-debrid.com" { + if body != nil { + var errResp ApiErrorResponse + jsonErr := json.Unmarshal(body, &errResp) + if jsonErr == nil { + errResp.Message += fmt.Sprintf(" (status code: %d)", resp.StatusCode) + } else { + errResp.Message = string(body) + errResp.Code = resp.StatusCode + } err = &errResp } + } else { + // download servers + errResp := DownloadErrorResponse{ + Message: resp.Header.Get("X-Error"), + Code: resp.StatusCode, + } + err = &errResp } } @@ -177,10 +198,6 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { break } } - okWithTimeout := strings.HasSuffix(req.URL.Path, "unrestrict/link") && strings.HasSuffix(req.URL.Path, "torrents/addMagnet") - if err != nil && strings.Contains(err.Error(), "timeout") && req.Host == "api.real-debrid.com" && !okWithTimeout { - r.log.Warnf("Adjust your API timeout settings, request to %s timed out", req.URL.String()) - } return resp, err } @@ -206,30 +223,36 @@ func (r *HTTPClient) shouldRetry(req *http.Request, resp *http.Response, err err if strings.HasSuffix(req.URL.Path, "torrents/addMagnet") { return -1 // don't retry to prevent duplicate torrents } - if err != nil && strings.HasPrefix(err.Error(), "api response error:") { - if apiErr, ok := err.(*ApiErrorResponse); ok { - switch apiErr.Code { - case -1: // Internal error - return 1 - case 5: // Slow down (retry infinitely), default: 4 secs - time.Sleep(time.Duration(rateLimitSleep) * time.Second) - return 0 - case 6: // Ressource unreachable - return 1 - case 17: // Hoster in maintenance - return 1 - case 18: // Hoster limit reached - return 1 - case 25: // Service unavailable - return 1 - case 34: // Too many requests (retry infinitely), default: 4 secs - time.Sleep(time.Duration(rateLimitSleep) * time.Second) - return 0 - case 36: // Fair Usage Limit - return 1 - default: - return -1 // don't retry - } + if apiErr, ok := err.(*ApiErrorResponse); ok { + switch apiErr.Code { + case -1: // Internal error + return 1 + case 5: // Slow down (retry infinitely), default: 4 secs + time.Sleep(time.Duration(rateLimitSleep) * time.Second) + return 0 + case 6: // Ressource unreachable + return 1 + case 17: // Hoster in maintenance + return 1 + case 18: // Hoster limit reached + return 1 + case 25: // Service unavailable + return 1 + case 34: // Too many requests (retry infinitely), default: 4 secs + time.Sleep(time.Duration(rateLimitSleep) * time.Second) + return 0 + case 36: // Fair Usage Limit + time.Sleep(time.Duration(rateLimitSleep) * time.Second) + return 1 + default: + return -1 // don't retry + } + } else if downloadErr, ok := err.(*DownloadErrorResponse); ok { + switch downloadErr.Code { + case http.StatusServiceUnavailable: // Service unavailable + return -1 + default: + return 1 } } if err != nil && strings.Contains(err.Error(), "timeout") { @@ -270,12 +293,11 @@ func backoffFunc(attempt int) time.Duration { return time.Duration(backoff) * time.Second } -func (r *HTTPClient) CanFetchFirstByte(url string) bool { - req, err := http.NewRequest("GET", url, nil) +func (r *HTTPClient) VerifyURL(url string) bool { + req, err := http.NewRequest(http.MethodHead, url, nil) if err != nil { return false } - req.Header.Set("Range", "bytes=0-0") timeout := time.Duration(r.timeoutSecs) * time.Second ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -283,7 +305,7 @@ func (r *HTTPClient) CanFetchFirstByte(url string) bool { resp, _ := r.Do(req) if resp != nil { defer resp.Body.Close() - return resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusPartialContent + return resp.StatusCode == http.StatusOK } return false } diff --git a/pkg/logutil/factory.go b/pkg/logutil/factory.go index 2049d25..8252053 100644 --- a/pkg/logutil/factory.go +++ b/pkg/logutil/factory.go @@ -143,7 +143,7 @@ func (l *Logger) UploadLogFile() (string, error) { } // Create a request and add the proper headers. - req, err := http.NewRequest("POST", "https://0x0.st/", &b) + req, err := http.NewRequest(http.MethodPost, "https://0x0.st/", &b) if err != nil { return "", err } diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 834b381..b8212d0 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -43,7 +43,7 @@ func (rd *RealDebrid) UnrestrictCheck(link string) (*Download, error) { data.Set("link", link) requestBody := strings.NewReader(data.Encode()) - req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/check", requestBody) + req, err := http.NewRequest(http.MethodPost, "https://api.real-debrid.com/rest/1.0/unrestrict/check", requestBody) if err != nil { rd.log.Errorf("Error when creating a unrestrict check request: %v", err) return nil, err @@ -74,7 +74,7 @@ func (rd *RealDebrid) UnrestrictCheck(link string) (*Download, error) { return &response, nil } -func (rd *RealDebrid) UnrestrictLink(link string, checkFirstByte bool) (*Download, error) { +func (rd *RealDebrid) UnrestrictLink(link string, verifyDownloadURL bool) (*Download, error) { data := url.Values{} if strings.HasPrefix(link, "https://real-debrid.com/d/") { // set link to max 39 chars @@ -83,7 +83,7 @@ func (rd *RealDebrid) UnrestrictLink(link string, checkFirstByte bool) (*Downloa data.Set("link", link) requestBody := strings.NewReader(data.Encode()) - req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", requestBody) + req, err := http.NewRequest(http.MethodPost, "https://api.real-debrid.com/rest/1.0/unrestrict/link", requestBody) if err != nil { rd.log.Errorf("Error when creating a unrestrict link request: %v", err) return nil, err @@ -113,8 +113,8 @@ func (rd *RealDebrid) UnrestrictLink(link string, checkFirstByte bool) (*Downloa } // will only check for first byte if serving from rclone - if checkFirstByte && !rd.downloadClient.CanFetchFirstByte(response.Download) { - return nil, fmt.Errorf("can't fetch first byte") + if verifyDownloadURL && !rd.downloadClient.VerifyURL(response.Download) { + return nil, fmt.Errorf("download URL verification failed: %s", response.Download) } // rd.log.Debugf("Unrestricted link %s into %s", link, response.Download) @@ -124,7 +124,7 @@ func (rd *RealDebrid) UnrestrictLink(link string, checkFirstByte bool) (*Downloa func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) { url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { rd.log.Errorf("Error when creating a get info request: %v", err) return nil, err @@ -161,7 +161,7 @@ func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error { requestBody := strings.NewReader(data.Encode()) reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/selectFiles/%s", id) - req, err := http.NewRequest("POST", reqURL, requestBody) + req, err := http.NewRequest(http.MethodPost, reqURL, requestBody) if err != nil { rd.log.Errorf("Error when creating a select files request: %v", err) return err @@ -184,7 +184,7 @@ func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error { func (rd *RealDebrid) DeleteTorrent(id string) error { // Construct request URL reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/delete/%s", id) - req, err := http.NewRequest("DELETE", reqURL, nil) + req, err := http.NewRequest(http.MethodDelete, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a delete torrent request: %v", err) return err @@ -211,7 +211,7 @@ func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) { // Construct request URL reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet" - req, err := http.NewRequest("POST", reqURL, requestBody) + req, err := http.NewRequest(http.MethodPost, reqURL, requestBody) if err != nil { rd.log.Errorf("Error when creating an add magnet request: %v", err) return nil, err @@ -242,7 +242,7 @@ func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) { func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, error) { // Construct request URL reqURL := "https://api.real-debrid.com/rest/1.0/torrents/activeCount" - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequest(http.MethodGet, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a active torrents request: %v", err) return nil, err @@ -269,7 +269,7 @@ func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, erro func (rd *RealDebrid) GetUserInformation() (*User, error) { // Construct request URL reqURL := "https://api.real-debrid.com/rest/1.0/user" - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequest(http.MethodGet, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a user information request: %v", err) return nil, err @@ -302,7 +302,7 @@ func (rd *RealDebrid) AvailabilityCheck(hashes []string) (AvailabilityResponse, baseURL := "https://api.real-debrid.com/rest/1.0" url := fmt.Sprintf("%s/torrents/instantAvailability/%s", baseURL, strings.Join(hashes, "/")) - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err } diff --git a/pkg/realdebrid/downloads.go b/pkg/realdebrid/downloads.go index c8026b7..ef8163a 100644 --- a/pkg/realdebrid/downloads.go +++ b/pkg/realdebrid/downloads.go @@ -80,7 +80,7 @@ func (rd *RealDebrid) fetchPageOfDownloads(page, limit int) fetchDownloadsResult reqURL := baseURL + "?" + params.Encode() - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequest(http.MethodGet, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a get downloads request: %v", err) return fetchDownloadsResult{ diff --git a/pkg/realdebrid/torrents.go b/pkg/realdebrid/torrents.go index d10c63f..619a338 100644 --- a/pkg/realdebrid/torrents.go +++ b/pkg/realdebrid/torrents.go @@ -118,7 +118,7 @@ func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult { params.Set("limit", fmt.Sprintf("%d", limit)) reqURL := baseURL + "?" + params.Encode() - req, err := http.NewRequest("GET", reqURL, nil) + req, err := http.NewRequest(http.MethodGet, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a get torrents request: %v", err) return fetchTorrentsResult{