hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove download mount config, it is now always enabled

Browse Source
This commit is contained in:
Ben Sarmiento
2024-05-25 15:11:11 +02:00
parent 2f777f63e9
commit 264d62d0dc
8 changed files with 39 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/app.go
View File

@@ -24,8 +24,11 @@ import (
)
func MainApp(configPath string) {
utils.EnsureDirExists("logs") // Ensure the logs directory exists
logPath := fmt.Sprintf("logs/zurg-%s.log", time.Now().Format(time.DateOnly))
utils.EnsureDirExists("logs") // log files
utils.EnsureDirExists("data") // cache files (info, bins, etc.)
utils.EnsureDirExists("dump") // "zurgtorrent" files
logPath := fmt.Sprintf("logs/zurg-%s-%s.log", time.Now().Format(time.DateOnly), time.Now().Format(time.TimeOnly))
log := logutil.NewLogger(logPath)
zurglog := log.Named("zurg") // logger for this main function
@@ -36,7 +39,7 @@ func MainApp(configPath string) {
zurglog.Infof("BuiltAt: %s", version.GetBuiltAt())
if log.Level() == zapcore.DebugLevel {
zurglog.Infof("Debug logging is enabled; if you are not debugging please set LOG_LEVEL=info in your environment")
zurglog.Infof("Debug logging is enabled; if you are not debugging please set LOG_LEVEL=info in your system environment")
}
config, configErr := config.LoadZurgConfig(configPath, log.Named("config"))
@@ -92,8 +95,6 @@ func MainApp(configPath string) {
}
defer workerPool.Release()
utils.EnsureDirExists("data") // Ensure the data directory exists
utils.EnsureDirExists("dump") // dump is a new directory for "torrent" files
torrentMgr := torrent.NewTorrentManager(
config,
api,

6
internal/config/types.go
View File

@@ -27,7 +27,6 @@ type ConfigInterface interface {
GetApiTimeoutSecs() int
GetDownloadTimeoutSecs() int
GetRetriesUntilFailed() int
EnableDownloadMount() bool
GetRateLimitSleepSecs() int
ShouldDeleteRarFiles() bool
GetDownloadsEveryMins() int
@@ -42,7 +41,6 @@ type ZurgConfig struct {
ApiTimeoutSecs int `yaml:"api_timeout_secs" json:"api_timeout_secs"`
CanRepair bool `yaml:"enable_repair" json:"enable_repair"`
DeleteRarFiles bool `yaml:"auto_delete_rar_torrents" json:"auto_delete_rar_torrents"`
DownloadMount bool `yaml:"enable_download_mount" json:"enable_download_mount"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
DownloadTimeoutSecs int `yaml:"download_timeout_secs" json:"download_timeout_secs"`
ForceIPv6 bool `yaml:"force_ipv6" json:"force_ipv6"`
@@ -176,10 +174,6 @@ func (z *ZurgConfig) GetRetriesUntilFailed() int {
return z.RetriesUntilFailed
}
func (z *ZurgConfig) EnableDownloadMount() bool {
return z.DownloadMount
}
func (z *ZurgConfig) GetApiTimeoutSecs() int {
if z.ApiTimeoutSecs == 0 {
return 60

10
internal/dav/infuse.go
View File

@@ -26,9 +26,7 @@ func ServeRootDirectoryForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
buf.WriteString(dav.BaseDirectory(directory, ""))
}
if torMgr.Config.EnableDownloadMount() {
buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
}
buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
_, size := version.GetFile()
buf.WriteString(dav.File(version.FILE, size, ""))
@@ -108,10 +106,6 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr
func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) {
var buf bytes.Buffer
if !torMgr.Config.EnableDownloadMount() {
buf.WriteString("Enable download mount in config to use this feature")
return buf.Bytes(), nil
}
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
@@ -120,7 +114,7 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
for _, filename := range filenames {
download, ok := torMgr.DownloadMap.Get(filename)
if !ok {
if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
continue
}
buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated))

10
internal/dav/listing.go
View File

@@ -25,9 +25,7 @@ func ServeRootDirectory(torMgr *torrent.TorrentManager) ([]byte, error) {
}
buf.WriteString(dav.Directory(directory, ""))
}
if torMgr.Config.EnableDownloadMount() {
buf.WriteString(dav.Directory(config.DOWNLOADS, ""))
}
buf.WriteString(dav.Directory(config.DOWNLOADS, ""))
_, size := version.GetFile()
buf.WriteString(dav.File(version.FILE, size, ""))
buf.WriteString("</d:multistatus>")
@@ -121,17 +119,13 @@ func HandleSingleFile(directory, torrentName, fileName string, torMgr *torrent.T
func ServeDownloadsList(torMgr *torrent.TorrentManager) ([]byte, error) {
var buf bytes.Buffer
if !torMgr.Config.EnableDownloadMount() {
buf.WriteString("Enable download mount in config to use this feature")
return buf.Bytes(), nil
}
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
filenames := torMgr.DownloadMap.Keys()
sort.Strings(filenames)
for _, filename := range filenames {
download, ok := torMgr.DownloadMap.Get(filename)
if !ok {
if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
continue
}
buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated))

5
internal/handlers/home.go
View File

@@ -219,10 +219,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
<td>Download Timeout</td>
<td>%d secs</td>
</tr>
<tr>
<td>Use Download Mount</td>
<td>%t</td>
</tr>
<tr>
<td>Refresh Download Mount Every...</td>
<td>%d mins</td>
@@ -326,7 +322,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
response.Config.GetApiTimeoutSecs(),
response.Config.GetTorrentsCount(),
response.Config.GetDownloadTimeoutSecs(),
response.Config.EnableDownloadMount(),
response.Config.GetDownloadsEveryMins(),
response.Config.GetRateLimitSleepSecs(),
response.Config.GetRetriesUntilFailed(),

10
internal/http/listing.go
View File

@@ -17,9 +17,7 @@ func ServeRootDirectory(torMgr *torrent.TorrentManager) ([]byte, error) {
var buf bytes.Buffer
buf.WriteString("<ol>")
directories := torMgr.DirectoryMap.Keys()
if torMgr.Config.EnableDownloadMount() {
directories = append(directories, config.DOWNLOADS)
}
directories = append(directories, config.DOWNLOADS)
sort.Strings(directories)
for _, directory := range directories {
if strings.HasPrefix(directory, "int__") {
@@ -95,16 +93,12 @@ func ServeFilesList(directory, torrentName string, torMgr *torrent.TorrentManage
func ServeDownloadsList(torMgr *torrent.TorrentManager) ([]byte, error) {
var buf bytes.Buffer
if !torMgr.Config.EnableDownloadMount() {
buf.WriteString("Enable download mount in config to use this feature")
return buf.Bytes(), nil
}
buf.WriteString("<ol>")
filenames := torMgr.DownloadMap.Keys()
sort.Strings(filenames)
for _, filename := range filenames {
download, ok := torMgr.DownloadMap.Get(filename)
if !ok {
if !ok || strings.HasPrefix(download.Link, "https://real-debrid.com/d/") {
continue
}
filePath := filepath.Join(config.DOWNLOADS, url.PathEscape(filename))

33
internal/torrent/manager.go
View File

@@ -30,9 +30,8 @@ type TorrentManager struct {
log *logutil.Logger
repairLog *logutil.Logger
DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
DownloadMap cmap.ConcurrentMap[string, *realdebrid.Download]
DownloadCache cmap.ConcurrentMap[string, *realdebrid.Download]
DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
DownloadMap cmap.ConcurrentMap[string, *realdebrid.Download]
RootNode *fs.FileNode
@@ -64,9 +63,8 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
log: log,
repairLog: repairLog,
DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
DownloadMap: cmap.New[*realdebrid.Download](),
DownloadCache: cmap.New[*realdebrid.Download](),
DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
DownloadMap: cmap.New[*realdebrid.Download](),
RootNode: fs.NewFileNode("root", true),
@@ -96,19 +94,17 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
// proxy function
func (t *TorrentManager) UnrestrictLinkUntilOk(link string) *realdebrid.Download {
if strings.HasPrefix(link, "https://real-debrid.com/d/") && t.DownloadCache.Has(link[0:39]) {
ret, _ := t.DownloadCache.Get(link[0:39])
if strings.HasPrefix(link, "https://real-debrid.com/d/") && t.DownloadMap.Has(link[0:39]) {
ret, _ := t.DownloadMap.Get(link[0:39])
return ret
}
ret, err := t.api.UnrestrictLink(link, t.Config.ShouldServeFromRclone())
t.DownloadCache.Set(ret.Link[0:39], ret)
t.DownloadMap.Set(ret.Link[0:39], ret)
if err != nil {
t.log.Warnf("Cannot unrestrict link %s: %v", link, err)
return nil
}
if t.Config.EnableDownloadMount() {
t.DownloadMap.Set(ret.Filename, ret)
}
t.DownloadMap.Set(ret.Filename, ret)
return ret
}
@@ -273,22 +269,11 @@ func (t *TorrentManager) deleteInfoFile(torrentID string) {
/// end info functions
func (t *TorrentManager) mountDownloads() {
if !t.Config.EnableDownloadMount() {
return
}
t.DownloadMap.Clear()
t.DownloadCache.Clear()
_ = t.workerPool.Submit(func() {
downloads, totalDownloads, err := t.api.GetDownloads()
if err != nil {
t.log.Errorf("Cannot get downloads: %v", err)
}
t.log.Debugf("Got %d downloads", totalDownloads)
downloads := t.api.GetDownloads()
for i := range downloads {
idx := i
if strings.HasPrefix(downloads[idx].Link, "https://real-debrid.com/d/") {
t.DownloadCache.Set(downloads[idx].Link[0:39], &downloads[idx])
}
t.DownloadMap.Set(downloads[idx].Filename, &downloads[idx])
}
})

36
pkg/realdebrid/api.go
View File

@@ -174,7 +174,7 @@ func (rd *RealDebrid) GetTorrents(onlyOne bool) ([]Torrent, int, error) {
page := 1
// compute ceiling of totalCount / limit
maxPages := (totalCount + rd.cfg.GetTorrentsCount() - 1) / rd.cfg.GetTorrentsCount()
rd.log.Debugf("Total count is %d, max pages is %d", totalCount, maxPages)
rd.log.Debugf("Torrents total count is %d, max pages is %d", totalCount, maxPages)
maxParallelThreads := 4
if maxPages < maxParallelThreads {
maxParallelThreads = maxPages
@@ -240,7 +240,7 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
return nil, err
}
rd.log.Debugf("Got info for torrent %s (progress=%d%%)", id, response.Progress)
// rd.log.Debugf("Got info for torrent %s (progress=%d%%)", id, response.Progress)
return &response, nil
}
@@ -356,21 +356,22 @@ func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, erro
}
// GetDownloads returns all torrents, paginated
func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
_, totalCount, err := rd.fetchPageOfDownloads(1, 0)
func (rd *RealDebrid) GetDownloads() []Download {
_, totalCount, err := rd.fetchPageOfDownloads(1, 1)
if err != nil {
return nil, 0, err
return nil
}
const maxItems = 50000
// reset allDownloads
allDownloads := []Download{}
page := 1
offset := 0
limit := 100
// compute ceiling of totalCount / limit
maxPages := (totalCount + limit - 1) / limit
// rd.log.Debugf("Total count is %d, max pages is %d", totalCount, maxPages)
rd.log.Debugf("Total downloads count is %d, max pages is %d", totalCount, maxPages)
maxParallelThreads := 8
if maxPages < maxParallelThreads {
maxParallelThreads = maxPages
@@ -385,7 +386,7 @@ func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
errChan <- nil
return
}
result, _, err := rd.fetchPageOfDownloads(page+add, offset+add*limit)
result, _, err := rd.fetchPageOfDownloads(page+add, limit)
if err != nil {
allResults <- nil
errChan <- err
@@ -400,35 +401,34 @@ func (rd *RealDebrid) GetDownloads() ([]Download, int, error) {
res := <-allResults
err := <-errChan
if err != nil {
return nil, 0, err
return allDownloads
}
allDownloads = append(allDownloads, res...)
}
// rd.log.Debugf("Got %d/%d downloads", len(allDownloads), totalCount)
rd.log.Debugf("Got %d/%d downloads", len(allDownloads), totalCount)
if len(allDownloads) >= totalCount || page >= maxPages {
if len(allDownloads) >= totalCount || page >= maxPages || len(allDownloads) >= maxItems {
if len(allDownloads) > maxItems {
rd.log.Debugf("Capping it to %d downloads", len(allDownloads))
allDownloads = allDownloads[:maxItems]
}
break
}
page += maxParallelThreads
offset += maxParallelThreads * limit
}
rd.log.Debugf("Got %d downloads", len(allDownloads))
return allDownloads, totalCount, nil
return allDownloads
}
func (rd *RealDebrid) fetchPageOfDownloads(page, offset int) ([]Download, int, error) {
func (rd *RealDebrid) fetchPageOfDownloads(page, limit int) ([]Download, int, error) {
baseURL := "https://api.real-debrid.com/rest/1.0/downloads"
var downloads []Download
limit := 500
totalCount := 0
params := url.Values{}
params.Set("page", fmt.Sprintf("%d", page))
params.Set("offset", fmt.Sprintf("%d", offset))
params.Set("limit", fmt.Sprintf("%d", limit))
// params.Set("filter", "active")