package realdebrid import ( "fmt" "net/http" "net/url" "strconv" ) type fetchDownloadsResult struct { downloads []Download page int total int err error } // GetDownloads returns all torrents, paginated func (rd *RealDebrid) GetDownloads() []Download { result := rd.fetchPageOfDownloads(1, 1) if result.err != nil { return nil } allDownloads := []Download{} page := 1 pageSize := 250 maxPages := (result.total + pageSize - 1) / pageSize rd.log.Debugf("Total downloads count is %d", result.total) maxParallelThreads := 4 if maxPages < maxParallelThreads { maxParallelThreads = maxPages } for { allResults := make(chan fetchDownloadsResult, maxParallelThreads) // Channel to collect results from goroutines for i := 0; i < maxParallelThreads; i++ { // Launch GET_PARALLEL concurrent fetches idx := i rd.workerPool.Submit(func() { if page+idx > maxPages { allResults <- fetchDownloadsResult{ downloads: nil, page: page + idx, total: result.total, err: nil, } return } result := rd.fetchPageOfDownloads(page+idx, pageSize) allResults <- result }) } // Collect results from all goroutines for i := 0; i < maxParallelThreads; i++ { result := <-allResults if result.err != nil { rd.log.Warnf("Encountered an error when fetching downloads pg %d: %v", result.page, result.err) continue } allDownloads = append(allDownloads, result.downloads...) } rd.log.Debugf("Got %d/%d downloads", len(allDownloads), result.total) if len(allDownloads) >= result.total || page >= maxPages { break } page += maxParallelThreads } return allDownloads } func (rd *RealDebrid) fetchPageOfDownloads(page, limit int) fetchDownloadsResult { baseURL := "https://api.real-debrid.com/rest/1.0/downloads" params := url.Values{} params.Set("page", fmt.Sprintf("%d", page)) params.Set("limit", fmt.Sprintf("%d", limit)) reqURL := baseURL + "?" + params.Encode() req, err := http.NewRequest("GET", reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a get downloads request: %v", err) return fetchDownloadsResult{ downloads: nil, page: page, total: 0, err: err, } } resp, err := rd.apiClient.Do(req) if err != nil { rd.log.Errorf("Error when executing the get downloads request: %v", err) return fetchDownloadsResult{ downloads: nil, page: page, total: 0, err: err, } } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { return fetchDownloadsResult{ downloads: []Download{}, page: page, total: 0, err: nil, } } if resp.StatusCode != http.StatusOK { err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) rd.log.Errorf("Error when executing the get downloads request: %v", err) return fetchDownloadsResult{ downloads: nil, page: page, total: 0, err: err, } } totalCountHeader := resp.Header.Get("x-total-count") totalCount, err := strconv.Atoi(totalCountHeader) if err != nil { totalCount = 0 } var downloads []Download decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&downloads) if err != nil { rd.log.Errorf("Error when decoding get downloads JSON: %v", err) return fetchDownloadsResult{ downloads: nil, page: page, total: totalCount, err: err, } } return fetchDownloadsResult{ downloads: downloads, page: page, total: totalCount, err: nil, } }