hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix issue on downloads

Browse Source
This commit is contained in:
Ben Sarmiento
2024-01-31 22:39:42 +01:00
parent cdb04fcf52
commit 0810e8203f
5 changed files with 240 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/config/types.go
View File

@@ -29,20 +29,22 @@ type ConfigInterface interface {
EnableDownloadMount() bool EnableDownloadMount() bool
GetRateLimitSleepSecs() int GetRateLimitSleepSecs() int
ShouldDeleteRarFiles() bool ShouldDeleteRarFiles() bool
GetDownloadsEveryMins() int
} }
type ZurgConfig struct { type ZurgConfig struct {
Version string `yaml:"zurg" json:"-"` Version string `yaml:"zurg" json:"-"`
Token string `yaml:"token" json:"-"` Token string `yaml:"token" json:"-"`
Host string `yaml:"host" json:"host"` Host string `yaml:"host" json:"host"`
Port string `yaml:"port" json:"port"` Port string `yaml:"port" json:"port"`
Username string `yaml:"username" json:"username"` Username string `yaml:"username" json:"username"`
Password string `yaml:"password" json:"password"` Password string `yaml:"password" json:"password"`
Proxy string `yaml:"proxy" json:"proxy"` Proxy string `yaml:"proxy" json:"proxy"`
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"` NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"` RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"` RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
IgnoreRenames bool `yaml:"ignore_renames" json:"ignore_renames"` IgnoreRenames bool `yaml:"ignore_renames" json:"ignore_renames"`
RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"` RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
@@ -119,6 +121,13 @@ func (z *ZurgConfig) GetRepairEveryMins() int {
return z.RepairEveryMins return z.RepairEveryMins
} }
func (z *ZurgConfig) GetDownloadsEveryMins() int {
if z.DownloadsEveryMins == 0 {
return 60
}
return z.DownloadsEveryMins
}
func (z *ZurgConfig) EnableRepair() bool { func (z *ZurgConfig) EnableRepair() bool {
return z.CanRepair return z.CanRepair
} }

343
internal/handlers/home.go
View File

@@ -68,185 +68,204 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
} }
out := `<table border="1px"> out := `<table border="1px">
<tr> <tr>
<th colspan="3">zurg</th> <th colspan="3">zurg</th>
</tr> </tr>
<tr> <tr>
<td>Version</td> <td>Version</td>
<td colspan="2">%s</td> <td colspan="2">%s</td>
</tr> </tr>
<tr> <tr>
<td>Built At</td> <td>Built At</td>
<td colspan="2">%s</td> <td colspan="2">%s</td>
</tr> </tr>
<tr> <tr>
<td>Git Commit</td> <td>Git Commit</td>
<td colspan="2">%s</td> <td colspan="2">%s</td>
</tr> </tr>
<tr> <tr>
<td>HTML</td> <td>HTML</td>
<td colspan="2"><a href="%s">%s</a></td> <td colspan="2"><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>DAV</td> <td>DAV</td>
<td colspan="2"><a href="%s">%s</a></td> <td colspan="2"><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>Infuse</td> <td>Infuse</td>
<td colspan="2"><a href="%s">%s</a></td> <td colspan="2"><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>Logs</td> <td>Logs</td>
<td colspan="2"><a href="%s">%s</a></td> <td colspan="2"><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>Memory Allocation</td> <td>Memory Allocation</td>
<td colspan="2">%d MB</td> <td colspan="2">%d MB</td>
</tr> </tr>
<tr> <tr>
<td>Total Memory Allocated</td> <td>Total Memory Allocated</td>
<td colspan="2">%d MB</td> <td colspan="2">%d MB</td>
</tr> </tr>
<tr> <tr>
<td>System Memory</td> <td>System Memory</td>
<td colspan="2">%d MB</td> <td colspan="2">%d MB</td>
</tr> </tr>
<tr> <tr>
<td>Number of GC Cycles</td> <td>Number of GC Cycles</td>
<td colspan="2">%d</td> <td colspan="2">%d</td>
</tr> </tr>
<tr> <tr>
<td>Process ID</td> <td>Process ID</td>
<td colspan="2">%d</td> <td colspan="2">%d</td>
</tr> </tr>
<tr> <tr>
<td rowspan="3">Sponsor Zurg</td> <td rowspan="3">Sponsor Zurg</td>
<td>Patreon</td> <td>Patreon</td>
<td><a href="%s">%s</a></td> <td><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>Github</td> <td>Github</td>
<td><a href="%s">%s</a></td> <td><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td>Paypal</td> <td>Paypal</td>
<td><a href="%s">%s</a></td> <td><a href="%s">%s</a></td>
</tr> </tr>
<tr> <tr>
<td rowspan="6">User Info</td> <td rowspan="6">User Info</td>
<td>Username</td> <td>Username</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Points</td> <td>Points</td>
<td>%d</td> <td>%d</td>
</tr> </tr>
<tr> <tr>
<td>Locale</td> <td>Locale</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Type</td> <td>Type</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Premium</td> <td>Premium</td>
<td>%d days</td> <td>%d days</td>
</tr> </tr>
<tr> <tr>
<td>Expiration</td> <td>Expiration</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td rowspan="20">Config</td> <td rowspan="22">Config</td>
<td>Version</td> <td>Version</td>
<td>%s</td>
</tr>
<tr>
<td>Token</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Host</td> <td>Token</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Port</td> <td>Host</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Workers</td> <td>Port</td>
<td>%s</td>
</tr>
<tr>
<td>Workers</td>
<td>%d running / %d free / %d total</td> <td>%d running / %d free / %d total</td>
</tr> </tr>
<tr> <tr>
<td>Refresh Every...</td> <td>Refresh Every...</td>
<td>%d secs</td> <td>%d secs</td>
</tr> </tr>
<tr> <tr>
<td>Retain RD Torrent Name</td> <td>Retain RD Torrent Name</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Retain Folder Name Extension</td> <td>Retain Folder Name Extension</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Can Repair</td> <td>Can Repair</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Delete Rar Files</td> <td>Repair Every...</td>
<td>%d mins</td>
</tr>
<tr>
<td>Delete Rar Files</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>API Timeout</td> <td>API Timeout</td>
<td>%d secs</td> <td>%d secs</td>
</tr> </tr>
<tr> <tr>
<td>Download Timeout</td> <td>Download Timeout</td>
<td>%d secs</td> <td>%d secs</td>
</tr> </tr>
<tr> <tr>
<td>Use Download Mount</td> <td>Use Download Mount</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Rate Limit Sleep for...</td> <td>Refresh Download Mount Every...</td>
<td>%d mins</td>
</tr>
<tr>
<td>Rate Limit Sleep for...</td>
<td>%d secs</td> <td>%d secs</td>
</tr> </tr>
<tr> <tr>
<td>Retries Until Failed</td> <td>Retries Until Failed</td>
<td>%d</td> <td>%d</td>
</tr> </tr>
<tr> <tr>
<td>Network Buffer Size</td> <td>Network Buffer Size</td>
<td>%d bytes</td> <td>%d bytes</td>
</tr> </tr>
<tr> <tr>
<td>Serve From Rclone</td> <td>Serve From Rclone</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Verify Download Link</td> <td>Verify Download Link</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>Force IPv6</td> <td>Force IPv6</td>
<td>%t</td> <td>%t</td>
</tr> </tr>
<tr> <tr>
<td>On Library Update</td> <td>On Library Update</td>
<td>%s</td> <td>%s</td>
</tr> </tr>
<tr> <tr>
<td>Utilities</td> <td>Utilities</td>
<td colspan="2"> <td colspan="2">
<button type="button" onclick="window.open('/logs/upload')">Upload logs</button> <form method="get" action="/logs/upload">
<button type="button" onclick="window.open('/reboot/worker')">Reboot worker pool</button> <input type="submit" value="Upload logs" />
<button type="button" onclick="window.open('/reboot/refresh')">Reboot refresh worker</button> </form>
<button type="button" onclick="window.open('/reboot/repair')">Reboot repair worker</button> <form method="post" action="/reboot/worker">
</td> <input type="submit" value="Reboot worker pool" />
</tr> </form>
<form method="post" action="/reboot/refresh">
<input type="submit" value="Reboot refresh worker" />
</form>
<form method="post" action="/reboot/repair">
<input type="submit" value="Reboot repair worker" />
</form>
<form method="post" action="/remount/downloads">
<input type="submit" value="Remount downloads" />
</form>
</td>
</tr>
</table> </table>
` `
out = fmt.Sprintf(out, out = fmt.Sprintf(out,
@@ -289,10 +308,12 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
response.Config.EnableRetainRDTorrentName(), response.Config.EnableRetainRDTorrentName(),
response.Config.EnableRetainFolderNameExtension(), response.Config.EnableRetainFolderNameExtension(),
response.Config.EnableRepair(), response.Config.EnableRepair(),
response.Config.GetRepairEveryMins(),
response.Config.ShouldDeleteRarFiles(), response.Config.ShouldDeleteRarFiles(),
response.Config.GetApiTimeoutSecs(), response.Config.GetApiTimeoutSecs(),
response.Config.GetDownloadTimeoutSecs(), response.Config.GetDownloadTimeoutSecs(),
response.Config.EnableDownloadMount(), response.Config.EnableDownloadMount(),
response.Config.GetDownloadsEveryMins(),
response.Config.GetRateLimitSleepSecs(), response.Config.GetRateLimitSleepSecs(),
response.Config.GetRetriesUntilFailed(), response.Config.GetRetriesUntilFailed(),
response.Config.GetNetworkBufferSize(), response.Config.GetNetworkBufferSize(),
@@ -306,24 +327,34 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
} }
func (zr *Handlers) handleRebootWorkerPool(resp http.ResponseWriter, req *http.Request) { func (zr *Handlers) handleRebootWorkerPool(resp http.ResponseWriter, req *http.Request) {
// zr.workerPool.Release() resp.Header().Set("Refresh", "2; url=/")
zr.workerPool.Release()
zr.workerPool.Reboot() zr.workerPool.Reboot()
zr.log.Infof("Rebooted worker pool") zr.log.Infof("Rebooted worker pool")
fmt.Fprint(resp, "Rebooted worker pool, please close this window") fmt.Fprint(resp, "Rebooting worker pool...")
} }
func (zr *Handlers) handleRebootRefreshPool(resp http.ResponseWriter, req *http.Request) { func (zr *Handlers) handleRebootRefreshWorker(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RefreshKillSwitch <- struct{}{} zr.torMgr.RefreshKillSwitch <- struct{}{}
zr.torMgr.StartRefreshJob() zr.torMgr.StartRefreshJob()
zr.log.Infof("Rebooted refresh worker") zr.log.Infof("Rebooted refresh worker")
fmt.Fprint(resp, "Rebooted refresh worker, please close this window") fmt.Fprint(resp, "Rebooting refresh worker...")
} }
func (zr *Handlers) handleRebootRepairPool(resp http.ResponseWriter, req *http.Request) { func (zr *Handlers) handleRebootRepairWorker(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RepairKillSwitch <- struct{}{} zr.torMgr.RepairKillSwitch <- struct{}{}
zr.torMgr.StartRepairJob() zr.torMgr.StartRepairJob()
zr.log.Infof("Rebooted repair worker") zr.log.Infof("Rebooted repair worker")
fmt.Fprint(resp, "Rebooted repair worker, please close this window") fmt.Fprint(resp, "Rebooting repair worker...")
}
func (zr *Handlers) handleRemountDownloads(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RemountTrigger <- struct{}{}
zr.log.Infof("Triggered remount of downloads")
fmt.Fprint(resp, "Remounting downloads...")
} }
func bToMb(b uint64) uint64 { func bToMb(b uint64) uint64 {

21
internal/handlers/router.go
View File

@@ -20,14 +20,12 @@ import (
) )
type Handlers struct { type Handlers struct {
downloader *universal.Downloader downloader *universal.Downloader
torMgr *torrent.TorrentManager torMgr *torrent.TorrentManager
cfg config.ConfigInterface cfg config.ConfigInterface
api *realdebrid.RealDebrid api *realdebrid.RealDebrid
workerPool *ants.Pool workerPool *ants.Pool
refreshPool *ants.Pool log *logutil.Logger
repairPool *ants.Pool
log *logutil.Logger
} }
func init() { func init() {
@@ -52,9 +50,10 @@ func AttachHandlers(router *chi.Mux, downloader *universal.Downloader, torMgr *t
router.Use(hs.options) router.Use(hs.options)
router.Get("/", hs.handleHome) router.Get("/", hs.handleHome)
router.Get("/reboot/worker", hs.handleRebootWorkerPool) router.Post("/reboot/worker", hs.handleRebootWorkerPool)
router.Get("/reboot/refresh", hs.handleRebootRefreshPool) router.Post("/reboot/refresh", hs.handleRebootRefreshWorker)
router.Get("/reboot/repair", hs.handleRebootRepairPool) router.Post("/reboot/repair", hs.handleRebootRepairWorker)
router.Post("/remount/downloads", hs.handleRemountDownloads)
// version // version
router.Get(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleVersionFile) router.Get(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleVersionFile)
router.Head(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleCheckVersionFile) router.Head(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleCheckVersionFile)

30
internal/torrent/manager.go
View File

@@ -6,6 +6,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"sync" "sync"
"time"
"github.com/debridmediamanager/zurg/internal/config" "github.com/debridmediamanager/zurg/internal/config"
"github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/logutil"
@@ -33,6 +34,7 @@ type TorrentManager struct {
workerPool *ants.Pool workerPool *ants.Pool
RefreshKillSwitch chan struct{} RefreshKillSwitch chan struct{}
RepairKillSwitch chan struct{} RepairKillSwitch chan struct{}
RemountTrigger chan struct{}
repairTrigger chan *Torrent repairTrigger chan *Torrent
repairSet mapset.Set[*Torrent] repairSet mapset.Set[*Torrent]
repairRunning bool repairRunning bool
@@ -49,9 +51,9 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
Api: api, Api: api,
DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](), DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
DownloadCache: cmap.New[*realdebrid.Download](), DownloadCache: cmap.New[*realdebrid.Download](),
DownloadMap: cmap.New[*realdebrid.Download](),
RefreshKillSwitch: make(chan struct{}, 1), RefreshKillSwitch: make(chan struct{}, 1),
RepairKillSwitch: make(chan struct{}, 1), RepairKillSwitch: make(chan struct{}, 1),
RemountTrigger: make(chan struct{}, 1),
allAccessKeys: mapset.NewSet[string](), allAccessKeys: mapset.NewSet[string](),
latestState: &LibraryState{}, latestState: &LibraryState{},
requiredVersion: "0.9.3-hotfix.4", requiredVersion: "0.9.3-hotfix.4",
@@ -60,11 +62,12 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
} }
t.fixers = t.readFixersFromFile() t.fixers = t.readFixersFromFile()
t.initializeDirectories() t.initializeDirectories()
t.mountDownloads()
t.refreshTorrents() t.refreshTorrents()
t.setNewLatestState(t.getCurrentState()) t.setNewLatestState(t.getCurrentState())
t.StartRefreshJob() t.StartRefreshJob()
t.StartRepairJob() t.StartRepairJob()
t.mountDownloads()
t.StartDownloadsJob()
return t return t
} }
@@ -85,7 +88,9 @@ func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.Download {
} }
if ret != nil && ret.Link != "" && ret.Filename != "" { if ret != nil && ret.Link != "" && ret.Filename != "" {
t.DownloadCache.Set(ret.Link, ret) t.DownloadCache.Set(ret.Link, ret)
t.DownloadMap.Set(ret.Filename, ret) if t.Config.EnableDownloadMount() {
t.DownloadMap.Set(ret.Filename, ret)
}
} }
return ret return ret
} }
@@ -179,6 +184,7 @@ func (t *TorrentManager) mountDownloads() {
if !t.Config.EnableDownloadMount() { if !t.Config.EnableDownloadMount() {
return return
} }
t.DownloadMap = cmap.New[*realdebrid.Download]()
_ = t.workerPool.Submit(func() { _ = t.workerPool.Submit(func() {
page := 1 page := 1
offset := 0 offset := 0
@@ -196,7 +202,23 @@ func (t *TorrentManager) mountDownloads() {
break break
} }
} }
t.log.Infof("Compiled into %d downloads", t.DownloadCache.Count()) t.log.Infof("Compiled into %d downloads", t.DownloadMap.Count())
})
}
func (t *TorrentManager) StartDownloadsJob() {
_ = t.workerPool.Submit(func() {
remountTicker := time.NewTicker(time.Duration(t.Config.GetDownloadsEveryMins()) * time.Minute)
defer remountTicker.Stop()
for {
select {
case <-remountTicker.C:
t.mountDownloads()
case <-t.RemountTrigger:
t.mountDownloads()
}
}
}) })
} }

13
internal/universal/downloader.go
View File

@@ -104,19 +104,6 @@ func (dl *Downloader) DownloadLink(fileName, link string, resp http.ResponseWrit
http.Error(resp, "File is not available", http.StatusInternalServerError) http.Error(resp, "File is not available", http.StatusInternalServerError)
return return
} else { } else {
lFilename := strings.ToLower(fileName)
unrestrictFilename := strings.ToLower(strings.TrimPrefix(unrestrict.Filename, "/"))
if strings.Contains(lFilename, unrestrictFilename) {
// this is possible if there's only 1 streamable file in the torrent
// and then suddenly it's a rar file
actualExt := filepath.Ext(unrestrictFilename)
expectedExt := filepath.Ext(lFilename)
if actualExt != expectedExt && unrestrict.Streamable != 1 {
log.Warnf("File was changed and is not streamable: %s and %s (link=%s)", fileName, unrestrict.Filename, unrestrict.Link)
} else {
log.Warnf("Filename mismatch: %s and %s", fileName, unrestrict.Filename)
}
}
if cfg.ShouldServeFromRclone() { if cfg.ShouldServeFromRclone() {
if cfg.ShouldVerifyDownloadLink() { if cfg.ShouldVerifyDownloadLink() {
if !dl.client.CanFetchFirstByte(unrestrict.Download) { if !dl.client.CanFetchFirstByte(unrestrict.Download) {