Remove download mount config, it is now always enabled

internal/app.go

@@ -24,8 +24,11 @@ import (
 )
 
 func MainApp(configPath string) {
-	utils.EnsureDirExists("logs") // Ensure the logs directory exists
+	utils.EnsureDirExists("logs") // log files
-	logPath := fmt.Sprintf("logs/zurg-%s.log", time.Now().Format(time.DateOnly))
+	utils.EnsureDirExists("data") // cache files (info, bins, etc.)
+	utils.EnsureDirExists("dump") // "zurgtorrent" files
+
+	logPath := fmt.Sprintf("logs/zurg-%s-%s.log", time.Now().Format(time.DateOnly), time.Now().Format(time.TimeOnly))
 	log := logutil.NewLogger(logPath)
 
 	zurglog := log.Named("zurg") // logger for this main function
@@ -36,7 +39,7 @@ func MainApp(configPath string) {
 	zurglog.Infof("BuiltAt: %s", version.GetBuiltAt())
 
 	if log.Level() == zapcore.DebugLevel {
-		zurglog.Infof("Debug logging is enabled; if you are not debugging please set LOG_LEVEL=info in your environment")
+		zurglog.Infof("Debug logging is enabled; if you are not debugging please set LOG_LEVEL=info in your system environment")
 	}
 
 	config, configErr := config.LoadZurgConfig(configPath, log.Named("config"))
@@ -92,8 +95,6 @@ func MainApp(configPath string) {
 	}
 	defer workerPool.Release()
 
-	utils.EnsureDirExists("data") // Ensure the data directory exists
-	utils.EnsureDirExists("dump") // dump is a new directory for "torrent" files
 	torrentMgr := torrent.NewTorrentManager(
 		config,
 		api,

internal/config/types.go

@@ -27,7 +27,6 @@ type ConfigInterface interface {
 	GetApiTimeoutSecs() int
 	GetDownloadTimeoutSecs() int
 	GetRetriesUntilFailed() int
-	EnableDownloadMount() bool
 	GetRateLimitSleepSecs() int
 	ShouldDeleteRarFiles() bool
 	GetDownloadsEveryMins() int
@@ -42,7 +41,6 @@ type ZurgConfig struct {
 	ApiTimeoutSecs            int      `yaml:"api_timeout_secs" json:"api_timeout_secs"`
 	CanRepair                 bool     `yaml:"enable_repair" json:"enable_repair"`
 	DeleteRarFiles            bool     `yaml:"auto_delete_rar_torrents" json:"auto_delete_rar_torrents"`
-	DownloadMount             bool     `yaml:"enable_download_mount" json:"enable_download_mount"`
 	DownloadsEveryMins        int      `yaml:"downloads_every_mins" json:"downloads_every_mins"`
 	DownloadTimeoutSecs       int      `yaml:"download_timeout_secs" json:"download_timeout_secs"`
 	ForceIPv6                 bool     `yaml:"force_ipv6" json:"force_ipv6"`
@@ -176,10 +174,6 @@ func (z *ZurgConfig) GetRetriesUntilFailed() int {
 	return z.RetriesUntilFailed
 }
 
-func (z *ZurgConfig) EnableDownloadMount() bool {
-	return z.DownloadMount
-}
-
 func (z *ZurgConfig) GetApiTimeoutSecs() int {
 	if z.ApiTimeoutSecs == 0 {
 		return 60

internal/dav/infuse.go

@@ -26,9 +26,7 @@ func ServeRootDirectoryForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
 		buf.WriteString(dav.BaseDirectory(directory, ""))
 	}
 
-	if torMgr.Config.EnableDownloadMount() {
 	buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
-	}
 
 	_, size := version.GetFile()
 	buf.WriteString(dav.File(version.FILE, size, ""))
@@ -108,10 +106,6 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr
 
 func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) {
 	var buf bytes.Buffer
-	if !torMgr.Config.EnableDownloadMount() {
-		buf.WriteString("Enable download mount in config to use this feature")
-		return buf.Bytes(), nil
-	}
 
 	buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
 
@@ -120,7 +114,7 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
 
 	for _, filename := range filenames {
 		download, ok := torMgr.DownloadMap.Get(filename)
-		if !ok {
+		if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
 			continue
 		}
 		buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated))

internal/dav/listing.go

@@ -25,9 +25,7 @@ func ServeRootDirectory(torMgr *torrent.TorrentManager) ([]byte, error) {
 		}
 		buf.WriteString(dav.Directory(directory, ""))
 	}
-	if torMgr.Config.EnableDownloadMount() {
 	buf.WriteString(dav.Directory(config.DOWNLOADS, ""))
-	}
 	_, size := version.GetFile()
 	buf.WriteString(dav.File(version.FILE, size, ""))
 	buf.WriteString("</d:multistatus>")
@@ -121,17 +119,13 @@ func HandleSingleFile(directory, torrentName, fileName string, torMgr *torrent.T
 
 func ServeDownloadsList(torMgr *torrent.TorrentManager) ([]byte, error) {
 	var buf bytes.Buffer
-	if !torMgr.Config.EnableDownloadMount() {
-		buf.WriteString("Enable download mount in config to use this feature")
-		return buf.Bytes(), nil
-	}
 	buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
 	buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
 	filenames := torMgr.DownloadMap.Keys()
 	sort.Strings(filenames)
 	for _, filename := range filenames {
 		download, ok := torMgr.DownloadMap.Get(filename)
-		if !ok {
+		if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
 			continue
 		}
 		buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated))

internal/handlers/home.go

@@ -219,10 +219,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
 		<td>Download Timeout</td>
 		<td>%d secs</td>
 	</tr>
-	<tr>
-		<td>Use Download Mount</td>
-		<td>%t</td>
-	</tr>
 	<tr>
 		<td>Refresh Download Mount Every...</td>
 		<td>%d mins</td>
@@ -326,7 +322,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
 		response.Config.GetApiTimeoutSecs(),
 		response.Config.GetTorrentsCount(),
 		response.Config.GetDownloadTimeoutSecs(),
-		response.Config.EnableDownloadMount(),
 		response.Config.GetDownloadsEveryMins(),
 		response.Config.GetRateLimitSleepSecs(),
 		response.Config.GetRetriesUntilFailed(),

internal/http/listing.go

@@ -17,9 +17,7 @@ func ServeRootDirectory(torMgr *torrent.TorrentManager) ([]byte, error) {
 	var buf bytes.Buffer
 	buf.WriteString("<ol>")
 	directories := torMgr.DirectoryMap.Keys()
-	if torMgr.Config.EnableDownloadMount() {
 	directories = append(directories, config.DOWNLOADS)
-	}
 	sort.Strings(directories)
 	for _, directory := range directories {
 		if strings.HasPrefix(directory, "int__") {
@@ -95,16 +93,12 @@ func ServeFilesList(directory, torrentName string, torMgr *torrent.TorrentManage
 
 func ServeDownloadsList(torMgr *torrent.TorrentManager) ([]byte, error) {
 	var buf bytes.Buffer
-	if !torMgr.Config.EnableDownloadMount() {
-		buf.WriteString("Enable download mount in config to use this feature")
-		return buf.Bytes(), nil
-	}
 	buf.WriteString("<ol>")
 	filenames := torMgr.DownloadMap.Keys()
 	sort.Strings(filenames)
 	for _, filename := range filenames {
 		download, ok := torMgr.DownloadMap.Get(filename)
-		if !ok {
+		if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
 			continue
 		}
 		filePath := filepath.Join(config.DOWNLOADS, url.PathEscape(filename))

internal/torrent/manager.go

@@ -32,7 +32,6 @@ type TorrentManager struct {
 
 	DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
 	DownloadMap  cmap.ConcurrentMap[string, *realdebrid.Download]
-	DownloadCache cmap.ConcurrentMap[string, *realdebrid.Download]
 
 	RootNode *fs.FileNode
 
@@ -66,7 +65,6 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
 
 		DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
 		DownloadMap:  cmap.New[*realdebrid.Download](),
-		DownloadCache: cmap.New[*realdebrid.Download](),
 
 		RootNode: fs.NewFileNode("root", true),
 
@@ -96,19 +94,17 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
 
 // proxy function
 func (t *TorrentManager) UnrestrictLinkUntilOk(link string) *realdebrid.Download {
-	if strings.HasPrefix(link, "https://real-debrid.com/d/") && t.DownloadCache.Has(link[0:39]) {
+	if strings.HasPrefix(link, "https://real-debrid.com/d/") && t.DownloadMap.Has(link[0:39]) {
-		ret, _ := t.DownloadCache.Get(link[0:39])
+		ret, _ := t.DownloadMap.Get(link[0:39])
 		return ret
 	}
 	ret, err := t.api.UnrestrictLink(link, t.Config.ShouldServeFromRclone())
-	t.DownloadCache.Set(ret.Link[0:39], ret)
+	t.DownloadMap.Set(ret.Link[0:39], ret)
 	if err != nil {
 		t.log.Warnf("Cannot unrestrict link %s: %v", link, err)
 		return nil
 	}
-	if t.Config.EnableDownloadMount() {
 	t.DownloadMap.Set(ret.Filename, ret)
-	}
 	return ret
 }
 
@@ -273,22 +269,11 @@ func (t *TorrentManager) deleteInfoFile(torrentID string) {
 /// end info functions
 
 func (t *TorrentManager) mountDownloads() {
-	if !t.Config.EnableDownloadMount() {
-		return
-	}
 	t.DownloadMap.Clear()
-	t.DownloadCache.Clear()
 	_ = t.workerPool.Submit(func() {
-		downloads, totalDownloads, err := t.api.GetDownloads()
+		downloads := t.api.GetDownloads()
-		if err != nil {
-			t.log.Errorf("Cannot get downloads: %v", err)
-		}
-		t.log.Debugf("Got %d downloads", totalDownloads)
 		for i := range downloads {
 			idx := i
-			if strings.HasPrefix(downloads[idx].Link, "https://real-debrid.com/d/") {
-				t.DownloadCache.Set(downloads[idx].Link[0:39], &downloads[idx])
-			}
 			t.DownloadMap.Set(downloads[idx].Filename, &downloads[idx])
 		}
 	})

pkg/realdebrid/api.go

@@ -174,7 +174,7 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) {
 	page := 1
 	// compute ceiling of totalCount / limit
 	maxPages := (totalCount + rd.cfg.GetTorrentsCount() - 1) / rd.cfg.GetTorrentsCount()
-	rd.log.Debugf("Total count is %d, max pages is %d", totalCount, maxPages)
+	rd.log.Debugf("Torrents total count is %d, max pages is %d", totalCount, maxPages)
 	maxParallelThreads := 4
 	if maxPages < maxParallelThreads {
 		maxParallelThreads = maxPages
@@ -240,7 +240,7 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
 		return nil, err
 	}
 
-	rd.log.Debugf("Got info for torrent %s (progress=%d%%)", id, response.Progress)
+	// rd.log.Debugf("Got info for torrent %s (progress=%d%%)", id, response.Progress)
 	return &response, nil
 }
 
@@ -356,21 +356,22 @@ func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, erro
 }
 
 // GetDownloads returns all torrents, paginated
-func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
+func (rd *RealDebrid) GetDownloads() []Download {
-	_, totalCount, err := rd.fetchPageOfDownloads(1, 0)
+	_, totalCount, err := rd.fetchPageOfDownloads(1, 1)
 	if err != nil {
-		return nil, 0, err
+		return nil
 	}
 
+	const maxItems = 50000
+
 	// reset allDownloads
 	allDownloads := []Download{}
 	page := 1
-	offset := 0
 	limit := 100
 
 	// compute ceiling of totalCount / limit
 	maxPages := (totalCount + limit - 1) / limit
-	// rd.log.Debugf("Total count is %d, max pages is %d", totalCount, maxPages)
+	rd.log.Debugf("Total downloads count is %d, max pages is %d", totalCount, maxPages)
 	maxParallelThreads := 8
 	if maxPages < maxParallelThreads {
 		maxParallelThreads = maxPages
@@ -385,7 +386,7 @@ func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
 					errChan <- nil
 					return
 				}
-				result, _, err := rd.fetchPageOfDownloads(page+add, offset+add*limit)
+				result, _, err := rd.fetchPageOfDownloads(page+add, limit)
 				if err != nil {
 					allResults <- nil
 					errChan <- err
@@ -400,35 +401,34 @@ func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
 			res := <-allResults
 			err := <-errChan
 			if err != nil {
-				return nil, 0, err
+				return allDownloads
 			}
 			allDownloads = append(allDownloads, res...)
 		}
 
-		// rd.log.Debugf("Got %d/%d downloads", len(allDownloads), totalCount)
+		rd.log.Debugf("Got %d/%d downloads", len(allDownloads), totalCount)
 
-		if len(allDownloads) >= totalCount || page >= maxPages {
+		if len(allDownloads) >= totalCount || page >= maxPages || len(allDownloads) >= maxItems {
+			if len(allDownloads) > maxItems {
+				rd.log.Debugf("Capping it to %d downloads", len(allDownloads))
+				allDownloads = allDownloads[:maxItems]
+			}
 			break
 		}
 
 		page += maxParallelThreads
-		offset += maxParallelThreads * limit
 	}
 
-	rd.log.Debugf("Got %d downloads", len(allDownloads))
+	return allDownloads
-
-	return allDownloads, totalCount, nil
 }
 
-func (rd *RealDebrid) fetchPageOfDownloads(page, offset int) ([]Download, int, error) {
+func (rd *RealDebrid) fetchPageOfDownloads(page, limit int) ([]Download, int, error) {
 	baseURL := "https://api.real-debrid.com/rest/1.0/downloads"
 	var downloads []Download
-	limit := 500
 	totalCount := 0
 
 	params := url.Values{}
 	params.Set("page", fmt.Sprintf("%d", page))
-	params.Set("offset", fmt.Sprintf("%d", offset))
 	params.Set("limit", fmt.Sprintf("%d", limit))
 	// params.Set("filter", "active")