diff --git a/internal/app.go b/internal/app.go index 558500b..22c1440 100644 --- a/internal/app.go +++ b/internal/app.go @@ -4,6 +4,7 @@ import ( "fmt" netHttp "net/http" "os" + "os/exec" "strings" "time" @@ -52,7 +53,13 @@ func MainApp(configPath string) { os.Exit(1) } - proxyURL := os.Getenv("PROXY") + var proxyURL string + if config.GetProxy() != "" { + proxyURL = config.GetProxy() + zurglog.Infof("Using proxy: %s", proxyURL) + } else { + proxyURL = os.Getenv("PROXY") + } repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test")) repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test")) @@ -135,6 +142,11 @@ func MainApp(configPath string) { log.Named("router"), ) + _, err = exec.LookPath("ffprobe") + if err != nil { + zurglog.Warn("ffprobe not found in PATH (do you have ffmpeg installed?), you won't be able to perform media analysis") + } + //// pprof // workerPool.Submit(func() { // if err := netHttp.ListenAndServe(":6060", nil); err != nil && err != netHttp.ErrServerClosed { diff --git a/internal/commands.go b/internal/commands.go index 9d3f67b..60a8775 100644 --- a/internal/commands.go +++ b/internal/commands.go @@ -29,6 +29,11 @@ func NetworkTest(testURL string) { log := logutil.NewLogger(logPath) proxyURL := os.Getenv("PROXY") + if proxyURL != "" { + log.Infof("Using proxy: %s", proxyURL) + } else { + log.Info("You can set a proxy by setting the PROXY environment variable") + } repoClient4 := http.NewHTTPClient("", 0, 1, false, []string{}, proxyURL, log.Named("network_test")) repoClient6 := http.NewHTTPClient("", 0, 1, true, []string{}, proxyURL, log.Named("network_test")) diff --git a/internal/config/types.go b/internal/config/types.go index 863c055..1f46879 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -34,6 +34,7 @@ type ConfigInterface interface { GetDownloadsEveryMins() int GetDumpTorrentsEveryMins() int GetPlayableExtensions() []string + GetProxy() string } type ZurgConfig struct { @@ -54,6 +55,7 @@ type ZurgConfig struct { Password string `yaml:"password" json:"password"` PlayableExtensions []string `yaml:"addl_playable_extensions" json:"addl_playable_extensions"` Port string `yaml:"port" json:"port"` + Proxy string `yaml:"proxy" json:"proxy"` RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"` RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"` RetainFolderNameExtension bool `yaml:"retain_folder_name_extension" json:"retain_folder_name_extension"` @@ -205,3 +207,7 @@ func (z *ZurgConfig) GetPlayableExtensions() []string { } return z.PlayableExtensions } + +func (z *ZurgConfig) GetProxy() string { + return z.Proxy +} diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 9298fdd..834b381 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -24,7 +24,7 @@ type RealDebrid struct { } func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPClient, workerPool *ants.Pool, cfg config.ConfigInterface, log *logutil.Logger) *RealDebrid { - return &RealDebrid{ + rd := &RealDebrid{ torrentsCache: []Torrent{}, apiClient: apiClient, unrestrictClient: unrestrictClient, @@ -33,6 +33,8 @@ func NewRealDebrid(apiClient, unrestrictClient, downloadClient *zurghttp.HTTPCli cfg: cfg, log: log, } + rd.readCachedTorrents() + return rd } // currently unused diff --git a/pkg/realdebrid/torrents.go b/pkg/realdebrid/torrents.go index d9cab66..d8d43fb 100644 --- a/pkg/realdebrid/torrents.go +++ b/pkg/realdebrid/torrents.go @@ -2,8 +2,10 @@ package realdebrid import ( "fmt" + "io" "net/http" "net/url" + "os" "strconv" ) @@ -52,29 +54,28 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) { allResults <- rd.fetchPageOfTorrents(page+idx, pageSize) }) } - // Collect results from all goroutines - buffer := make([][]Torrent, maxParallelThreads) + batches := make([][]Torrent, maxParallelThreads) for i := 0; i < maxParallelThreads; i++ { result := <-allResults - bufferIdx := (result.page - 1) % maxParallelThreads - buffer[bufferIdx] = []Torrent{} if result.err != nil { rd.log.Warnf("Ignoring error when fetching torrents pg %d: %v", result.page, result.err) continue } - buffer[bufferIdx] = append(buffer[bufferIdx], result.torrents...) + bIdx := (result.page - 1) % maxParallelThreads + batches[bIdx] = []Torrent{} + batches[bIdx] = append(batches[bIdx], result.torrents...) } - for bIdx, batch := range buffer { - for tIdx, torrent := range batch { - for cIdx, cached := range rd.torrentsCache { + for bIdx, batch := range batches { // 4 batches + cachedCount := len(rd.torrentsCache) + for cIdx, cached := range rd.torrentsCache { // N cached torrents + cIdxEnd := cachedCount - 1 - cIdx + for tIdx, torrent := range batch { // 250 torrents tIdxEnd := indexFromEnd(tIdx, page+bIdx, pageSize, result.total) - cIdxEnd := len(rd.torrentsCache) - 1 - cIdx if torrent.ID == cached.ID && tIdxEnd == cIdxEnd { allTorrents = append(allTorrents, batch[:tIdx]...) allTorrents = append(allTorrents, rd.torrentsCache[cIdx:]...) - rd.log.Debugf("Fresh %d, cached %d", len(batch[:tIdx]), len(rd.torrentsCache[cIdx:])) rd.log.Debugf("Got %d/%d torrents", len(allTorrents), result.total) - rd.torrentsCache = allTorrents + rd.cacheTorrents(allTorrents) return allTorrents, len(allTorrents), nil } } @@ -91,7 +92,7 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) { page += maxParallelThreads } - rd.torrentsCache = allTorrents + rd.cacheTorrents(allTorrents) return allTorrents, len(allTorrents), nil } @@ -172,3 +173,51 @@ func (rd *RealDebrid) fetchPageOfTorrents(page, limit int) fetchTorrentsResult { err: nil, } } + +func (rd *RealDebrid) cacheTorrents(torrents []Torrent) { + filePath := "data/info/all.json" + file, err := os.Create(filePath) + if err != nil { + rd.log.Warnf("Cannot create info file %s: %v", filePath, err) + return + } + defer file.Close() + + jsonData, err := json.Marshal(torrents) + if err != nil { + rd.log.Warnf("Cannot marshal torrent info: %v", err) + return + } + + if _, err := file.Write(jsonData); err != nil { + rd.log.Warnf("Cannot write to info file %s: %v", filePath, err) + return + } + + rd.torrentsCache = torrents +} + +func (rd *RealDebrid) readCachedTorrents() { + filePath := "data/info/all.json" + file, err := os.Open(filePath) + if err != nil { + rd.log.Warnf("Cannot open info file %s: %v", filePath, err) + return + } + defer file.Close() + + jsonData, err := io.ReadAll(file) + if err != nil { + rd.log.Warnf("Cannot read info file %s: %v", filePath, err) + return + } + + var torrents []Torrent + err = json.Unmarshal(jsonData, &torrents) + if err != nil { + rd.log.Warnf("Cannot unmarshal torrent info: %v", err) + return + } + + rd.torrentsCache = torrents +} diff --git a/pkg/realdebrid/util.go b/pkg/realdebrid/util.go index 93ff173..15bc4e0 100644 --- a/pkg/realdebrid/util.go +++ b/pkg/realdebrid/util.go @@ -1,14 +1,7 @@ package realdebrid func indexFromEnd(subIndex int, pageNumber int, pageSize int, totalElements int) int { - // Adjust pageNumber for 1-based index adjustedPageNumber := pageNumber - 1 - - // Calculate the overall index in the array overallIndex := (adjustedPageNumber * pageSize) + subIndex - - // Calculate the index from the end - indexFromEnd := totalElements - 1 - overallIndex - - return indexFromEnd + return totalElements - 1 - overallIndex }