hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix issue on downloads

Browse Source
This commit is contained in:
Ben Sarmiento
2024-01-31 22:39:42 +01:00
parent cdb04fcf52
commit 0810e8203f
5 changed files with 240 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/config/types.go
View File

@@ -29,20 +29,22 @@ type ConfigInterface interface {
EnableDownloadMount() bool
GetRateLimitSleepSecs() int
ShouldDeleteRarFiles() bool
GetDownloadsEveryMins() int
}
type ZurgConfig struct {
Version string `yaml:"zurg" json:"-"`
Token string `yaml:"token" json:"-"`
Host string `yaml:"host" json:"host"`
Port string `yaml:"port" json:"port"`
Username string `yaml:"username" json:"username"`
Password string `yaml:"password" json:"password"`
Proxy string `yaml:"proxy" json:"proxy"`
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"`
Host string `yaml:"host" json:"host"`
Port string `yaml:"port" json:"port"`
Username string `yaml:"username" json:"username"`
Password string `yaml:"password" json:"password"`
Proxy string `yaml:"proxy" json:"proxy"`
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
RefreshEverySecs int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
RepairEveryMins int `yaml:"repair_every_mins" json:"repair_every_mins"`
DownloadsEveryMins int `yaml:"downloads_every_mins" json:"downloads_every_mins"`
IgnoreRenames bool `yaml:"ignore_renames" json:"ignore_renames"`
RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
@@ -119,6 +121,13 @@ func (z *ZurgConfig) GetRepairEveryMins() int {
return z.RepairEveryMins
}
func (z *ZurgConfig) GetDownloadsEveryMins() int {
if z.DownloadsEveryMins == 0 {
return 60
}
return z.DownloadsEveryMins
}
func (z *ZurgConfig) EnableRepair() bool {
return z.CanRepair
}

343
internal/handlers/home.go
View File

@@ -68,185 +68,204 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
}
out := `<table border="1px">
<tr>
<th colspan="3">zurg</th>
</tr>
<tr>
<td>Version</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>Built At</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>Git Commit</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>HTML</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>DAV</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Infuse</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Logs</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Memory Allocation</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>Total Memory Allocated</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>System Memory</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>Number of GC Cycles</td>
<td colspan="2">%d</td>
</tr>
<tr>
<td>Process ID</td>
<td colspan="2">%d</td>
</tr>
<tr>
<td rowspan="3">Sponsor Zurg</td>
<td>Patreon</td>
<tr>
<th colspan="3">zurg</th>
</tr>
<tr>
<td>Version</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>Built At</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>Git Commit</td>
<td colspan="2">%s</td>
</tr>
<tr>
<td>HTML</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>DAV</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Infuse</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Logs</td>
<td colspan="2"><a href="%s">%s</a></td>
</tr>
<tr>
<td>Memory Allocation</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>Total Memory Allocated</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>System Memory</td>
<td colspan="2">%d MB</td>
</tr>
<tr>
<td>Number of GC Cycles</td>
<td colspan="2">%d</td>
</tr>
<tr>
<td>Process ID</td>
<td colspan="2">%d</td>
</tr>
<tr>
<td rowspan="3">Sponsor Zurg</td>
<td>Patreon</td>
<td><a href="%s">%s</a></td>
</tr>
<tr>
<td>Github</td>
</tr>
<tr>
<td>Github</td>
<td><a href="%s">%s</a></td>
</tr>
<tr>
<td>Paypal</td>
</tr>
<tr>
<td>Paypal</td>
<td><a href="%s">%s</a></td>
</tr>
<tr>
<td rowspan="6">User Info</td>
<td>Username</td>
</tr>
<tr>
<td rowspan="6">User Info</td>
<td>Username</td>
<td>%s</td>
</tr>
<tr>
<td>Points</td>
</tr>
<tr>
<td>Points</td>
<td>%d</td>
</tr>
<tr>
<td>Locale</td>
</tr>
<tr>
<td>Locale</td>
<td>%s</td>
</tr>
<tr>
<td>Type</td>
</tr>
<tr>
<td>Type</td>
<td>%s</td>
</tr>
<tr>
<td>Premium</td>
</tr>
<tr>
<td>Premium</td>
<td>%d days</td>
</tr>
<tr>
<td>Expiration</td>
</tr>
<tr>
<td>Expiration</td>
<td>%s</td>
</tr>
<tr>
<td rowspan="20">Config</td>
<td>Version</td>
<td>%s</td>
</tr>
<tr>
<td>Token</td>
</tr>
<tr>
<td rowspan="22">Config</td>
<td>Version</td>
<td>%s</td>
</tr>
<tr>
<td>Host</td>
</tr>
<tr>
<td>Token</td>
<td>%s</td>
</tr>
<tr>
<td>Port</td>
</tr>
<tr>
<td>Host</td>
<td>%s</td>
</tr>
<tr>
<td>Workers</td>
</tr>
<tr>
<td>Port</td>
<td>%s</td>
</tr>
<tr>
<td>Workers</td>
<td>%d running / %d free / %d total</td>
</tr>
<tr>
<td>Refresh Every...</td>
</tr>
<tr>
<td>Refresh Every...</td>
<td>%d secs</td>
</tr>
<tr>
<td>Retain RD Torrent Name</td>
</tr>
<tr>
<td>Retain RD Torrent Name</td>
<td>%t</td>
</tr>
<tr>
<td>Retain Folder Name Extension</td>
</tr>
<tr>
<td>Retain Folder Name Extension</td>
<td>%t</td>
</tr>
<tr>
<td>Can Repair</td>
</tr>
<tr>
<td>Can Repair</td>
<td>%t</td>
</tr>
<tr>
<td>Delete Rar Files</td>
</tr>
<tr>
<td>Repair Every...</td>
<td>%d mins</td>
</tr>
<tr>
<td>Delete Rar Files</td>
<td>%t</td>
</tr>
<tr>
<td>API Timeout</td>
</tr>
<tr>
<td>API Timeout</td>
<td>%d secs</td>
</tr>
<tr>
<td>Download Timeout</td>
</tr>
<tr>
<td>Download Timeout</td>
<td>%d secs</td>
</tr>
<tr>
<td>Use Download Mount</td>
</tr>
<tr>
<td>Use Download Mount</td>
<td>%t</td>
</tr>
<tr>
<td>Rate Limit Sleep for...</td>
</tr>
<tr>
<td>Refresh Download Mount Every...</td>
<td>%d mins</td>
</tr>
<tr>
<td>Rate Limit Sleep for...</td>
<td>%d secs</td>
</tr>
<tr>
<td>Retries Until Failed</td>
</tr>
<tr>
<td>Retries Until Failed</td>
<td>%d</td>
</tr>
<tr>
<td>Network Buffer Size</td>
</tr>
<tr>
<td>Network Buffer Size</td>
<td>%d bytes</td>
</tr>
<tr>
<td>Serve From Rclone</td>
</tr>
<tr>
<td>Serve From Rclone</td>
<td>%t</td>
</tr>
<tr>
<td>Verify Download Link</td>
</tr>
<tr>
<td>Verify Download Link</td>
<td>%t</td>
</tr>
<tr>
<td>Force IPv6</td>
</tr>
<tr>
<td>Force IPv6</td>
<td>%t</td>
</tr>
<tr>
<td>On Library Update</td>
</tr>
<tr>
<td>On Library Update</td>
<td>%s</td>
</tr>
<tr>
<td>Utilities</td>
</tr>
<tr>
<td>Utilities</td>
<td colspan="2">
<button type="button" onclick="window.open('/logs/upload')">Upload logs</button>
<button type="button" onclick="window.open('/reboot/worker')">Reboot worker pool</button>
<button type="button" onclick="window.open('/reboot/refresh')">Reboot refresh worker</button>
<button type="button" onclick="window.open('/reboot/repair')">Reboot repair worker</button>
</td>
</tr>
<form method="get" action="/logs/upload">
<input type="submit" value="Upload logs" />
</form>
<form method="post" action="/reboot/worker">
<input type="submit" value="Reboot worker pool" />
</form>
<form method="post" action="/reboot/refresh">
<input type="submit" value="Reboot refresh worker" />
</form>
<form method="post" action="/reboot/repair">
<input type="submit" value="Reboot repair worker" />
</form>
<form method="post" action="/remount/downloads">
<input type="submit" value="Remount downloads" />
</form>
</td>
</tr>
</table>
`
out = fmt.Sprintf(out,
@@ -289,10 +308,12 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
response.Config.EnableRetainRDTorrentName(),
response.Config.EnableRetainFolderNameExtension(),
response.Config.EnableRepair(),
response.Config.GetRepairEveryMins(),
response.Config.ShouldDeleteRarFiles(),
response.Config.GetApiTimeoutSecs(),
response.Config.GetDownloadTimeoutSecs(),
response.Config.EnableDownloadMount(),
response.Config.GetDownloadsEveryMins(),
response.Config.GetRateLimitSleepSecs(),
response.Config.GetRetriesUntilFailed(),
response.Config.GetNetworkBufferSize(),
@@ -306,24 +327,34 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
}
func (zr *Handlers) handleRebootWorkerPool(resp http.ResponseWriter, req *http.Request) {
// zr.workerPool.Release()
resp.Header().Set("Refresh", "2; url=/")
zr.workerPool.Release()
zr.workerPool.Reboot()
zr.log.Infof("Rebooted worker pool")
fmt.Fprint(resp, "Rebooted worker pool, please close this window")
fmt.Fprint(resp, "Rebooting worker pool...")
}
func (zr *Handlers) handleRebootRefreshPool(resp http.ResponseWriter, req *http.Request) {
func (zr *Handlers) handleRebootRefreshWorker(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RefreshKillSwitch <- struct{}{}
zr.torMgr.StartRefreshJob()
zr.log.Infof("Rebooted refresh worker")
fmt.Fprint(resp, "Rebooted refresh worker, please close this window")
fmt.Fprint(resp, "Rebooting refresh worker...")
}
func (zr *Handlers) handleRebootRepairPool(resp http.ResponseWriter, req *http.Request) {
func (zr *Handlers) handleRebootRepairWorker(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RepairKillSwitch <- struct{}{}
zr.torMgr.StartRepairJob()
zr.log.Infof("Rebooted repair worker")
fmt.Fprint(resp, "Rebooted repair worker, please close this window")
fmt.Fprint(resp, "Rebooting repair worker...")
}
func (zr *Handlers) handleRemountDownloads(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("Refresh", "2; url=/")
zr.torMgr.RemountTrigger <- struct{}{}
zr.log.Infof("Triggered remount of downloads")
fmt.Fprint(resp, "Remounting downloads...")
}
func bToMb(b uint64) uint64 {

21
internal/handlers/router.go
View File

@@ -20,14 +20,12 @@ import (
)
type Handlers struct {
downloader *universal.Downloader
torMgr *torrent.TorrentManager
cfg config.ConfigInterface
api *realdebrid.RealDebrid
workerPool *ants.Pool
refreshPool *ants.Pool
repairPool *ants.Pool
log *logutil.Logger
downloader *universal.Downloader
torMgr *torrent.TorrentManager
cfg config.ConfigInterface
api *realdebrid.RealDebrid
workerPool *ants.Pool
log *logutil.Logger
}
func init() {
@@ -52,9 +50,10 @@ func AttachHandlers(router *chi.Mux, downloader *universal.Downloader, torMgr *t
router.Use(hs.options)
router.Get("/", hs.handleHome)
router.Get("/reboot/worker", hs.handleRebootWorkerPool)
router.Get("/reboot/refresh", hs.handleRebootRefreshPool)
router.Get("/reboot/repair", hs.handleRebootRepairPool)
router.Post("/reboot/worker", hs.handleRebootWorkerPool)
router.Post("/reboot/refresh", hs.handleRebootRefreshWorker)
router.Post("/reboot/repair", hs.handleRebootRepairWorker)
router.Post("/remount/downloads", hs.handleRemountDownloads)
// version
router.Get(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleVersionFile)
router.Head(fmt.Sprintf("/{mountType}/%s", version.FILE), hs.handleCheckVersionFile)

30
internal/torrent/manager.go
View File

@@ -6,6 +6,7 @@ import (
"path/filepath"
"strings"
"sync"
"time"
"github.com/debridmediamanager/zurg/internal/config"
"github.com/debridmediamanager/zurg/pkg/logutil"
@@ -33,6 +34,7 @@ type TorrentManager struct {
workerPool *ants.Pool
RefreshKillSwitch chan struct{}
RepairKillSwitch chan struct{}
RemountTrigger chan struct{}
repairTrigger chan *Torrent
repairSet mapset.Set[*Torrent]
repairRunning bool
@@ -49,9 +51,9 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
Api: api,
DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
DownloadCache: cmap.New[*realdebrid.Download](),
DownloadMap: cmap.New[*realdebrid.Download](),
RefreshKillSwitch: make(chan struct{}, 1),
RepairKillSwitch: make(chan struct{}, 1),
RemountTrigger: make(chan struct{}, 1),
allAccessKeys: mapset.NewSet[string](),
latestState: &LibraryState{},
requiredVersion: "0.9.3-hotfix.4",
@@ -60,11 +62,12 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
}
t.fixers = t.readFixersFromFile()
t.initializeDirectories()
t.mountDownloads()
t.refreshTorrents()
t.setNewLatestState(t.getCurrentState())
t.StartRefreshJob()
t.StartRepairJob()
t.mountDownloads()
t.StartDownloadsJob()
return t
}
@@ -85,7 +88,9 @@ func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.Download {
}
if ret != nil && ret.Link != "" && ret.Filename != "" {
t.DownloadCache.Set(ret.Link, ret)
t.DownloadMap.Set(ret.Filename, ret)
if t.Config.EnableDownloadMount() {
t.DownloadMap.Set(ret.Filename, ret)
}
}
return ret
}
@@ -179,6 +184,7 @@ func (t *TorrentManager) mountDownloads() {
if !t.Config.EnableDownloadMount() {
return
}
t.DownloadMap = cmap.New[*realdebrid.Download]()
_ = t.workerPool.Submit(func() {
page := 1
offset := 0
@@ -196,7 +202,23 @@ func (t *TorrentManager) mountDownloads() {
break
}
}
t.log.Infof("Compiled into %d downloads", t.DownloadCache.Count())
t.log.Infof("Compiled into %d downloads", t.DownloadMap.Count())
})
}
func (t *TorrentManager) StartDownloadsJob() {
_ = t.workerPool.Submit(func() {
remountTicker := time.NewTicker(time.Duration(t.Config.GetDownloadsEveryMins()) * time.Minute)
defer remountTicker.Stop()
for {
select {
case <-remountTicker.C:
t.mountDownloads()
case <-t.RemountTrigger:
t.mountDownloads()
}
}
})
}

13
internal/universal/downloader.go
View File

@@ -104,19 +104,6 @@ func (dl *Downloader) DownloadLink(fileName, link string, resp http.ResponseWrit
http.Error(resp, "File is not available", http.StatusInternalServerError)
return
} else {
lFilename := strings.ToLower(fileName)
unrestrictFilename := strings.ToLower(strings.TrimPrefix(unrestrict.Filename, "/"))
if strings.Contains(lFilename, unrestrictFilename) {
// this is possible if there's only 1 streamable file in the torrent
// and then suddenly it's a rar file
actualExt := filepath.Ext(unrestrictFilename)
expectedExt := filepath.Ext(lFilename)
if actualExt != expectedExt && unrestrict.Streamable != 1 {
log.Warnf("File was changed and is not streamable: %s and %s (link=%s)", fileName, unrestrict.Filename, unrestrict.Link)
} else {
log.Warnf("Filename mismatch: %s and %s", fileName, unrestrict.Filename)
}
}
if cfg.ShouldServeFromRclone() {
if cfg.ShouldVerifyDownloadLink() {
if !dl.client.CanFetchFirstByte(unrestrict.Download) {