zurg/pkg/realdebrid/torrents.go

package realdebrid

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
)

type fetchTorrentsResult struct {
	torrents []Torrent
	page     int
	total    int
	err      error
}

// GetTorrents returns all torrents, paginated
func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) {
	result := rd.fetchPageOfTorrents(1, 1)
	if result.err != nil {
		return nil, 0, result.err
	}

	if onlyOne {
		return result.torrents, result.total, nil
	}

	allTorrents := []Torrent{}
	page := 1
	pageSize := 250

	maxPages := (result.total + pageSize - 1) / pageSize
	rd.log.Debugf("Torrents total count is %d", result.total)
	maxParallelThreads := 4
	if maxPages < maxParallelThreads {
		maxParallelThreads = maxPages
	}
	for {
		allResults := make(chan fetchTorrentsResult, maxParallelThreads) // Channel to collect results from goroutines
		for i := 0; i < maxParallelThreads; i++ {                        // Launch GET_PARALLEL concurrent fetches
			idx := i
			rd.workerPool.Submit(func() {
				if page+idx > maxPages {
					allResults <- fetchTorrentsResult{
						torrents: nil,
						page:     page + idx,
						total:    result.total,
						err:      nil,
					}
					return
				}
				allResults <- rd.fetchPageOfTorrents(page+idx, pageSize)
			})
		}
		batches := make([][]Torrent, maxParallelThreads)
		for i := 0; i < maxParallelThreads; i++ {
			result := <-allResults
			if result.err != nil {
				rd.log.Warnf("Ignoring error when fetching torrents pg %d: %v", result.page, result.err)
				continue
			}
			bIdx := (result.page - 1) % maxParallelThreads
			batches[bIdx] = []Torrent{}
			batches[bIdx] = append(batches[bIdx], result.torrents...)
		}
		for bIdx, batch := range batches { // 4 batches
			cachedCount := len(rd.torrentsCache)
			for cIdx, cached := range rd.torrentsCache { // N cached torrents
				cIdxEnd := cachedCount - 1 - cIdx
				for tIdx, torrent := range batch { // 250 torrents
					tIdxEnd := indexFromEnd(tIdx, page+bIdx, pageSize, result.total)
					if torrent.ID == cached.ID && tIdxEnd == cIdxEnd {
						allTorrents = append(allTorrents, batch[:tIdx]...)
						allTorrents = append(allTorrents, rd.torrentsCache[cIdx:]...)
						rd.log.Debugf("Got %d/%d torrents", len(allTorrents), result.total)
						rd.cacheTorrents(allTorrents)
						return allTorrents, len(allTorrents), nil
					}
				}
			}
			allTorrents = append(allTorrents, batch...)
		}

		rd.log.Debugf("Got %d/%d torrents", len(allTorrents), result.total)

		if len(allTorrents) >= result.total || page >= maxPages {
			break
		}

		page += maxParallelThreads
	}

	rd.cacheTorrents(allTorrents)
	return allTorrents, len(allTorrents), nil
}

func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult {
	baseURL := "https://api.real-debrid.com/rest/1.0/torrents"

	params := url.Values{}
	params.Set("page", fmt.Sprintf("%d", page))
	params.Set("limit", fmt.Sprintf("%d", limit))

	reqURL := baseURL + "?" + params.Encode()
	req, err := http.NewRequest("GET", reqURL, nil)
	if err != nil {
		rd.log.Errorf("Error when creating a get torrents request: %v", err)
		return fetchTorrentsResult{
			torrents: nil,
			page:     page,
			total:    0,
			err:      err,
		}
	}

	resp, err := rd.apiClient.Do(req)
	if err != nil {
		rd.log.Errorf("Error when executing the get torrents request: %v", err)
		return fetchTorrentsResult{
			torrents: nil,
			page:     page,
			total:    0,
			err:      err,
		}
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNoContent {
		return fetchTorrentsResult{
			torrents: []Torrent{},
			page:     page,
			total:    0,
			err:      nil,
		}
	}

	if resp.StatusCode != http.StatusOK {
		err := fmt.Errorf("unexpected status code: %d", resp.StatusCode)
		rd.log.Errorf("Error when executing the get torrents request: %v", err)
		return fetchTorrentsResult{
			torrents: nil,
			page:     page,
			total:    0,
			err:      err,
		}
	}

	totalCountHeader := resp.Header.Get("x-total-count")
	totalCount, err := strconv.Atoi(totalCountHeader)
	if err != nil {
		totalCount = 0
	}

	var torrents []Torrent
	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(&torrents)
	if err != nil {
		rd.log.Errorf("Error when decoding the body of get torrents response: %v", err)
		return fetchTorrentsResult{
			torrents: nil,
			page:     page,
			total:    0,
			err:      err,
		}
	}

	return fetchTorrentsResult{
		torrents: torrents,
		page:     page,
		total:    totalCount,
		err:      nil,
	}
}

func (rd *RealDebrid) cacheTorrents(torrents []Torrent) {
	filePath := "data/info/all.json"
	file, err := os.Create(filePath)
	if err != nil {
		rd.log.Warnf("Cannot create info file %s: %v", filePath, err)
		return
	}
	defer file.Close()

	jsonData, err := json.Marshal(torrents)
	if err != nil {
		rd.log.Warnf("Cannot marshal torrent info: %v", err)
		return
	}

	if _, err := file.Write(jsonData); err != nil {
		rd.log.Warnf("Cannot write to info file %s: %v", filePath, err)
		return
	}

	rd.torrentsCache = torrents
}

func (rd *RealDebrid) readCachedTorrents() {
	filePath := "data/info/all.json"
	file, err := os.Open(filePath)
	if err != nil {
		rd.log.Warnf("Cannot open info file %s: %v", filePath, err)
		return
	}
	defer file.Close()

	jsonData, err := io.ReadAll(file)
	if err != nil {
		rd.log.Warnf("Cannot read info file %s: %v", filePath, err)
		return
	}

	var torrents []Torrent
	err = json.Unmarshal(jsonData, &torrents)
	if err != nil {
		rd.log.Warnf("Cannot unmarshal torrent info: %v", err)
		return
	}

	rd.torrentsCache = torrents
}