package realdebrid import ( "fmt" "io" "net/http" "net/url" "os" "strconv" ) type fetchTorrentsResult struct { torrents []Torrent page int total int err error } // GetTorrents returns all torrents, paginated func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) { result := rd.fetchPageOfTorrents(1, 1) if result.err != nil { return nil, 0, result.err } totalElements := result.total if onlyOne { return result.torrents, totalElements, nil } allTorrents := []Torrent{} page := 1 pageSize := 250 maxPages := (totalElements + pageSize - 1) / pageSize rd.log.Debugf("Torrents total count is %d", totalElements) maxParallelThreads := 4 if maxPages < maxParallelThreads { maxParallelThreads = maxPages } found := -1 for { allResults := make(chan fetchTorrentsResult, maxParallelThreads) // Channel to collect results from goroutines for i := 0; i < maxParallelThreads; i++ { // Launch GET_PARALLEL concurrent fetches idx := i rd.workerPool.Submit(func() { if page+idx > maxPages { allResults <- fetchTorrentsResult{ torrents: nil, page: page + idx, total: totalElements, err: nil, } return } allResults <- rd.fetchPageOfTorrents(page+idx, pageSize) }) } batches := make([][]Torrent, maxParallelThreads) for i := 0; i < maxParallelThreads; i++ { result := <-allResults if result.err != nil { rd.log.Warnf("Ignoring error when fetching torrents pg %d: %v", result.page, result.err) return nil, 0, result.err } bIdx := (result.page - 1) % maxParallelThreads batches[bIdx] = []Torrent{} batches[bIdx] = append(batches[bIdx], result.torrents...) } for bIdx, batch := range batches { // 4 batches if found < 0 && len(batch) > 0 { cachedCount := len(rd.torrentsCache) for cIdx, cached := range rd.torrentsCache { // N cached torrents cIdxEnd := cachedCount - 1 - cIdx for tIdx, torrent := range batch { // 250 torrents in batch tIdxEnd := indexFromEnd(tIdx, page+bIdx, pageSize, totalElements) if torrent.ID == cached.ID && torrent.Progress == cached.Progress && tIdxEnd == cIdxEnd { found = ((page + bIdx - 1) * pageSize) + tIdx break } } if found >= 0 { break } } } allTorrents = append(allTorrents, batch...) } if found >= 0 { tIdx := found % pageSize pageNum := (found / pageSize) + 1 tIdxEnd := indexFromEnd(tIdx, pageNum, pageSize, totalElements) cIdx := len(rd.torrentsCache) - 1 - tIdxEnd last := len(allTorrents) - 1 cIdx += last - found + 1 allTorrents = append(allTorrents, rd.torrentsCache[cIdx:]...) } rd.log.Debugf("Got %d/%d torrents", len(allTorrents), totalElements) if len(allTorrents) >= totalElements || page >= maxPages { break } page += maxParallelThreads } rd.cacheTorrents(allTorrents) return allTorrents, len(allTorrents), nil } func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult { baseURL := "https://api.real-debrid.com/rest/1.0/torrents" params := url.Values{} params.Set("page", fmt.Sprintf("%d", page)) params.Set("limit", fmt.Sprintf("%d", limit)) reqURL := baseURL + "?" + params.Encode() req, err := http.NewRequest(http.MethodGet, reqURL, nil) if err != nil { rd.log.Errorf("Error when creating a get torrents request: %v", err) return fetchTorrentsResult{ torrents: nil, page: page, total: 0, err: err, } } resp, err := rd.apiClient.Do(req) if err != nil { rd.log.Errorf("Error when executing the get torrents request: %v", err) return fetchTorrentsResult{ torrents: nil, page: page, total: 0, err: err, } } defer resp.Body.Close() if resp.StatusCode == http.StatusNoContent { return fetchTorrentsResult{ torrents: []Torrent{}, page: page, total: 0, err: nil, } } if resp.StatusCode != http.StatusOK { err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) rd.log.Errorf("Error when executing the get torrents request: %v", err) return fetchTorrentsResult{ torrents: nil, page: page, total: 0, err: err, } } totalCountHeader := resp.Header.Get("x-total-count") totalCount, err := strconv.Atoi(totalCountHeader) if err != nil { totalCount = 0 } var torrents []Torrent decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&torrents) if err != nil { rd.log.Errorf("Error when decoding the body of get torrents response: %v", err) return fetchTorrentsResult{ torrents: nil, page: page, total: 0, err: err, } } return fetchTorrentsResult{ torrents: torrents, page: page, total: totalCount, err: nil, } } func (rd *RealDebrid) cacheTorrents(torrents []Torrent) { filePath := "data/info/all.json" file, err := os.Create(filePath) if err != nil { rd.log.Warnf("Cannot create info file %s: %v", filePath, err) return } defer file.Close() jsonData, err := json.Marshal(torrents) if err != nil { rd.log.Warnf("Cannot marshal torrent info: %v", err) return } if _, err := file.Write(jsonData); err != nil { rd.log.Warnf("Cannot write to info file %s: %v", filePath, err) return } rd.torrentsCache = torrents } func (rd *RealDebrid) readCachedTorrents() { filePath := "data/info/all.json" file, err := os.Open(filePath) if err != nil { rd.log.Warnf("Cannot open info file %s: %v", filePath, err) return } defer file.Close() jsonData, err := io.ReadAll(file) if err != nil { rd.log.Warnf("Cannot read info file %s: %v", filePath, err) return } var torrents []Torrent err = json.Unmarshal(jsonData, &torrents) if err != nil { rd.log.Warnf("Cannot unmarshal torrent info: %v", err) return } rd.torrentsCache = torrents }