Optimize torrent fetching

pkg/realdebrid/downloads.go

@@ -0,0 +1,151 @@
+package realdebrid
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+type fetchDownloadsResult struct {
+	downloads []Download
+	page      int
+	total     int
+	err       error
+}
+
+// GetDownloads returns all torrents, paginated
+func (rd *RealDebrid) GetDownloads() []Download {
+	result := rd.fetchPageOfDownloads(1, 1)
+	if result.err != nil {
+		return nil
+	}
+
+	allDownloads := []Download{}
+	page := 1
+	pageSize := 250
+
+	maxPages := (result.total + pageSize - 1) / pageSize
+	rd.log.Debugf("Total downloads count is %d", result.total)
+	maxParallelThreads := 4
+	if maxPages < maxParallelThreads {
+		maxParallelThreads = maxPages
+	}
+	for {
+		allResults := make(chan fetchDownloadsResult, maxParallelThreads) // Channel to collect results from goroutines
+		for i := 0; i < maxParallelThreads; i++ {                         // Launch GET_PARALLEL concurrent fetches
+			idx := i
+			rd.workerPool.Submit(func() {
+				if page+idx > maxPages {
+					allResults <- fetchDownloadsResult{
+						downloads: nil,
+						page:      page + idx,
+						total:     result.total,
+						err:       nil,
+					}
+					return
+				}
+				result := rd.fetchPageOfDownloads(page+idx, pageSize)
+				allResults <- result
+			})
+		}
+		// Collect results from all goroutines
+		for i := 0; i < maxParallelThreads; i++ {
+			result := <-allResults
+			if result.err != nil {
+				rd.log.Warnf("Encountered an error when fetching downloads pg %d: %v", result.page, result.err)
+				continue
+			}
+			allDownloads = append(allDownloads, result.downloads...)
+		}
+
+		rd.log.Debugf("Got %d/%d downloads", len(allDownloads), result.total)
+
+		if len(allDownloads) >= result.total || page >= maxPages {
+			break
+		}
+
+		page += maxParallelThreads
+	}
+
+	return allDownloads
+}
+
+func (rd *RealDebrid) fetchPageOfDownloads(page, limit int) fetchDownloadsResult {
+	baseURL := "https://api.real-debrid.com/rest/1.0/downloads"
+
+	params := url.Values{}
+	params.Set("page", fmt.Sprintf("%d", page))
+	params.Set("limit", fmt.Sprintf("%d", limit))
+
+	reqURL := baseURL + "?" + params.Encode()
+
+	req, err := http.NewRequest("GET", reqURL, nil)
+	if err != nil {
+		rd.log.Errorf("Error when creating a get downloads request: %v", err)
+		return fetchDownloadsResult{
+			downloads: nil,
+			page:      page,
+			total:     0,
+			err:       err,
+		}
+	}
+
+	resp, err := rd.apiClient.Do(req)
+	if err != nil {
+		rd.log.Errorf("Error when executing the get downloads request: %v", err)
+		return fetchDownloadsResult{
+			downloads: nil,
+			page:      page,
+			total:     0,
+			err:       err,
+		}
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNoContent {
+		return fetchDownloadsResult{
+			downloads: []Download{},
+			page:      page,
+			total:     0,
+			err:       nil,
+		}
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		err := fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		rd.log.Errorf("Error when executing the get downloads request: %v", err)
+		return fetchDownloadsResult{
+			downloads: nil,
+			page:      page,
+			total:     0,
+			err:       err,
+		}
+	}
+
+	totalCountHeader := resp.Header.Get("x-total-count")
+	totalCount, err := strconv.Atoi(totalCountHeader)
+	if err != nil {
+		totalCount = 0
+	}
+
+	var downloads []Download
+	decoder := json.NewDecoder(resp.Body)
+	err = decoder.Decode(&downloads)
+	if err != nil {
+		rd.log.Errorf("Error when decoding get downloads JSON: %v", err)
+		return fetchDownloadsResult{
+			downloads: nil,
+			page:      page,
+			total:     totalCount,
+			err:       err,
+		}
+	}
+
+	return fetchDownloadsResult{
+		downloads: downloads,
+		page:      page,
+		total:     totalCount,
+		err:       nil,
+	}
+}