From 2aacff1125bdaa015127ca8fad6a10f84c544f9b Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Wed, 6 Dec 2023 19:18:04 +0100 Subject: [PATCH] Add logs route, add rar handler --- .gitignore | 2 + internal/app.go | 8 ++-- internal/config/load.go | 4 +- internal/config/types.go | 6 +++ internal/config/v1.go | 4 +- internal/config/v1types.go | 4 +- internal/dav/listing.go | 6 +-- internal/http/listing.go | 6 +-- internal/router/router.go | 19 +++++++-- internal/torrent/hooks.go | 4 +- internal/torrent/manager.go | 6 +-- internal/torrent/refresh.go | 22 ++++++++++- internal/torrent/repair.go | 2 +- internal/torrent/types.go | 1 + internal/universal/get.go | 8 ++-- internal/universal/head.go | 4 +- pkg/http/client.go | 6 +-- pkg/logutil/factory.go | 77 ++++++++++++++++++++++++++++++++++--- pkg/realdebrid/api.go | 6 +-- 19 files changed, 151 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index adfc22c..f26623e 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,5 @@ stressTestAddRemove.py *.zip pkg/anidb/ + +logs/ diff --git a/internal/app.go b/internal/app.go index 2c8000a..707e7f3 100644 --- a/internal/app.go +++ b/internal/app.go @@ -4,6 +4,7 @@ import ( "fmt" netHttp "net/http" "os" + "time" // _ "net/http/pprof" // Register pprof @@ -20,7 +21,9 @@ import ( ) func MainApp(configPath string) { - log := logutil.NewLogger() + utils.EnsureDirExists("logs") // Ensure the logs directory exists + logPath := fmt.Sprintf("logs/zurg-%s.log", time.Now().Format(time.DateOnly)) + log := logutil.NewLogger(logPath) zurglog := log.Named("zurg") zurglog.Debugf("PID: %d", os.Getpid()) @@ -42,8 +45,7 @@ func MainApp(configPath string) { } defer p.Release() - utils.EnsureDirExists("data") - + utils.EnsureDirExists("data") // Ensure the data directory exists torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager")) downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient")) diff --git a/internal/config/load.go b/internal/config/load.go index 6b01b20..f3bce8d 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -4,11 +4,11 @@ import ( "fmt" "os" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" "gopkg.in/yaml.v3" ) -func LoadZurgConfig(filename string, log *zap.SugaredLogger) (ConfigInterface, error) { +func LoadZurgConfig(filename string, log *logutil.Logger) (ConfigInterface, error) { log.Debug("Loading config file ", filename) content, err := os.ReadFile(filename) if err != nil { diff --git a/internal/config/types.go b/internal/config/types.go index a9b71cb..4a3890e 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -24,6 +24,7 @@ type ConfigInterface interface { GetRetriesUntilFailed() int EnableDownloadCache() bool GetRateLimitSleepSeconds() int + ShouldDeleteRarFiles() bool } type ZurgConfig struct { @@ -45,6 +46,7 @@ type ZurgConfig struct { RealDebridTimeout int `yaml:"realdebrid_timeout_secs" json:"realdebrid_timeout_secs"` RetriesUntilFailed int `yaml:"retries_until_failed" json:"retries_until_failed"` UseDownloadCache bool `yaml:"use_download_cache" json:"use_download_cache"` + DeleteRarFiles bool `yaml:"delete_rar_files" json:"delete_rar_files"` } func (z *ZurgConfig) GetConfig() ZurgConfig { @@ -146,3 +148,7 @@ func (z *ZurgConfig) GetRateLimitSleepSeconds() int { } return z.RateLimitSleepSeconds } + +func (z *ZurgConfig) ShouldDeleteRarFiles() bool { + return z.DeleteRarFiles +} diff --git a/internal/config/v1.go b/internal/config/v1.go index e15a6a5..89f7796 100644 --- a/internal/config/v1.go +++ b/internal/config/v1.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" + "github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/utils" - "go.uber.org/zap" "gopkg.in/yaml.v3" ) @@ -16,7 +16,7 @@ const ( ALL_TORRENTS = "__all__" ) -func loadV1Config(content []byte, log *zap.SugaredLogger) (*ZurgConfigV1, error) { +func loadV1Config(content []byte, log *logutil.Logger) (*ZurgConfigV1, error) { var configV1 ZurgConfigV1 if err := yaml.Unmarshal(content, &configV1); err != nil { return nil, err diff --git a/internal/config/v1types.go b/internal/config/v1types.go index 5e57a18..d1f8690 100644 --- a/internal/config/v1types.go +++ b/internal/config/v1types.go @@ -1,11 +1,11 @@ package config -import "go.uber.org/zap" +import "github.com/debridmediamanager/zurg/pkg/logutil" type ZurgConfigV1 struct { ZurgConfig `yaml:",inline"` Directories map[string]*DirectoryFilterConditionsV1 `yaml:"directories"` - log *zap.SugaredLogger + log *logutil.Logger } type DirectoryFilterConditionsV1 struct { GroupOrder int `yaml:"group_order"` diff --git a/internal/dav/listing.go b/internal/dav/listing.go index 792b5b4..cba0942 100644 --- a/internal/dav/listing.go +++ b/internal/dav/listing.go @@ -8,7 +8,7 @@ import ( "github.com/debridmediamanager/zurg/internal/torrent" "github.com/debridmediamanager/zurg/pkg/dav" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" ) func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) { @@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) { return &davDoc, nil } -func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) { +func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { return nil, fmt.Errorf("cannot find directory %s", directory) @@ -51,7 +51,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z return &davDoc, nil } -func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) { +func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { return nil, fmt.Errorf("cannot find directory %s", directory) diff --git a/internal/http/listing.go b/internal/http/listing.go index 0bb7b31..50adfb5 100644 --- a/internal/http/listing.go +++ b/internal/http/listing.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/debridmediamanager/zurg/internal/torrent" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" ) func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) { @@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) { return &htmlDoc, nil } -func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) { +func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { return nil, fmt.Errorf("cannot find directory %s", directory) @@ -49,7 +49,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z return &htmlDoc, nil } -func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) { +func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { return nil, fmt.Errorf("cannot find directory %s", directory) diff --git a/internal/router/router.go b/internal/router/router.go index fe153f2..5e5d3a1 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -12,9 +12,9 @@ import ( "github.com/debridmediamanager/zurg/internal/torrent" "github.com/debridmediamanager/zurg/internal/universal" "github.com/debridmediamanager/zurg/internal/version" + "github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/realdebrid" "github.com/julienschmidt/httprouter" - "go.uber.org/zap" jsoniter "github.com/json-iterator/go" ) @@ -26,10 +26,10 @@ type ZurgRouter struct { torMgr *torrent.TorrentManager cfg config.ConfigInterface api *realdebrid.RealDebrid - log *zap.SugaredLogger + log *logutil.Logger } -func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *zap.SugaredLogger) { +func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *logutil.Logger) { zr := &ZurgRouter{ getfile: getfile, torMgr: torMgr, @@ -64,6 +64,10 @@ func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torM // root route router.GET("/", zr.rootHandler) + + // logs route + router.GET("/logs", zr.logsHandler) + router.GET("/logs/", zr.logsHandler) } func (zr *ZurgRouter) httpTorrentDirectoryHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) { @@ -233,6 +237,15 @@ func (zr *ZurgRouter) rootHandler(resp http.ResponseWriter, req *http.Request, p } } +func (zr *ZurgRouter) logsHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) { + logs, err := zr.log.GetLogsFromFile() + if err != nil { + http.Error(resp, err.Error(), http.StatusInternalServerError) + return + } + fmt.Fprint(resp, logs) +} + func bToMb(b uint64) uint64 { return b / 1024 / 1024 } diff --git a/internal/torrent/hooks.go b/internal/torrent/hooks.go index e44afb0..69f5a37 100644 --- a/internal/torrent/hooks.go +++ b/internal/torrent/hooks.go @@ -6,7 +6,7 @@ import ( "os/exec" "github.com/debridmediamanager/zurg/internal/config" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" ) type ScriptExecutor struct { @@ -32,7 +32,7 @@ func (se *ScriptExecutor) Execute() (string, error) { return out.String(), nil } -func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *zap.SugaredLogger) { +func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *logutil.Logger) { executor := &ScriptExecutor{ Script: config.GetOnLibraryUpdate(), Args: paths, diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 6ee1185..80991fc 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -7,12 +7,12 @@ import ( "strings" "github.com/debridmediamanager/zurg/internal/config" + "github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/realdebrid" cmap "github.com/orcaman/concurrent-map/v2" "github.com/panjf2000/ants/v2" "github.com/scylladb/go-set" "github.com/scylladb/go-set/strset" - "go.uber.org/zap" ) const ( @@ -30,13 +30,13 @@ type TorrentManager struct { requiredVersion string workerPool *ants.Pool repairWorker *ants.Pool - log *zap.SugaredLogger + log *logutil.Logger } // NewTorrentManager creates a new torrent manager // it will fetch all torrents and their info in the background // and store them in-memory and cached in files -func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *zap.SugaredLogger) *TorrentManager { +func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *logutil.Logger) *TorrentManager { initialSate := EmptyState() t := &TorrentManager{ diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index c80e45d..5f21e51 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -149,8 +149,26 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { }) } if len(selectedFiles) > len(info.Links) && info.Progress == 100 { - t.log.Warnf("Torrent id=%s is partly expired, it has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links)) - torrent.ForRepair = true + if len(info.Links) == 1 { + // this might be a rar file so let's check + unrestrict := t.UnrestrictUntilOk(info.Links[0]) + if unrestrict != nil && strings.HasPrefix(strings.ToLower(unrestrict.Filename), ".rar") { + if t.Config.ShouldDeleteRarFiles() { + t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired. Deleting...", info.Name, info.ID) + t.Api.DeleteTorrent(info.ID) + return nil + } else { + t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired as it's a known Real-Debrid limitation. zurg recommends you delete this torrent or enable delete_rar_files in your config.yml", info.Name, info.ID) + torrent.Unfixable = true + } + } else { + t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only 1 link", info.ID, len(selectedFiles), len(info.Links)) + torrent.ForRepair = true + } + } else { + t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links)) + torrent.ForRepair = true + } } else if len(selectedFiles) == len(info.Links) { // all links are still intact! good! for i, file := range selectedFiles { diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index 3a1d989..e5321bc 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -22,7 +22,7 @@ func (t *TorrentManager) repairAll() { if torrent.AnyInProgress() && torrent.ForRepair { t.log.Warnf("Skipping %s for repairs because it is in progress", torrent.AccessKey) return - } else if torrent.ForRepair { + } else if torrent.ForRepair && !torrent.Unfixable { toRepair = append(toRepair, torrent) } }) diff --git a/internal/torrent/types.go b/internal/torrent/types.go index 0682471..7b3e5f6 100644 --- a/internal/torrent/types.go +++ b/internal/torrent/types.go @@ -18,6 +18,7 @@ type Torrent struct { SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` LatestAdded string `json:"LatestAdded"` ForRepair bool `json:"ForRepair"` + Unfixable bool `json:"Unfixable"` DownloadedIDs *strset.Set `json:"DownloadedIDs"` InProgressIDs *strset.Set `json:"InProgressIDs"` diff --git a/internal/universal/get.go b/internal/universal/get.go index dd4d264..3d46783 100644 --- a/internal/universal/get.go +++ b/internal/universal/get.go @@ -11,8 +11,8 @@ import ( "github.com/debridmediamanager/zurg/internal/config" intTor "github.com/debridmediamanager/zurg/internal/torrent" zurghttp "github.com/debridmediamanager/zurg/pkg/http" + "github.com/debridmediamanager/zurg/pkg/logutil" "github.com/debridmediamanager/zurg/pkg/realdebrid" - "go.uber.org/zap" ) type GetFile struct { @@ -24,7 +24,7 @@ func NewGetFile(client *zurghttp.HTTPClient) *GetFile { } // HandleGetRequest handles a GET request universally for both WebDAV and HTTP -func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) { +func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { log.Warnf("Cannot find directory %s", directory) @@ -96,7 +96,7 @@ func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, res } } -func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) error { +func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) error { // Create a new dlReq for the file download. dlReq, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -129,7 +129,7 @@ func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWrit return nil } -func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) { +func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) { // Create a new request for the file download. dlReq, err := http.NewRequest(http.MethodGet, unrestrict.Download, nil) if err != nil { diff --git a/internal/universal/head.go b/internal/universal/head.go index 8e645d8..f7680ed 100644 --- a/internal/universal/head.go +++ b/internal/universal/head.go @@ -7,10 +7,10 @@ import ( "strings" "github.com/debridmediamanager/zurg/internal/torrent" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" ) -func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) { +func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *logutil.Logger) { torrents, ok := torMgr.DirectoryMap.Get(directory) if !ok { diff --git a/pkg/http/client.go b/pkg/http/client.go index 6c14575..c9cb070 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -9,8 +9,8 @@ import ( "time" "github.com/debridmediamanager/zurg/internal/config" + "github.com/debridmediamanager/zurg/pkg/logutil" cmap "github.com/orcaman/concurrent-map/v2" - "go.uber.org/zap" ) const ( @@ -25,10 +25,10 @@ type HTTPClient struct { bearerToken string cfg config.ConfigInterface ipv6 cmap.ConcurrentMap[string, string] - log *zap.SugaredLogger + log *logutil.Logger } -func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient { +func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient { client := HTTPClient{ bearerToken: token, client: &http.Client{ diff --git a/pkg/logutil/factory.go b/pkg/logutil/factory.go index 417ffd9..219892d 100644 --- a/pkg/logutil/factory.go +++ b/pkg/logutil/factory.go @@ -1,13 +1,24 @@ package logutil import ( + "bytes" + "fmt" + "io" + "os" + "go.uber.org/zap" "go.uber.org/zap/zapcore" ) -func NewLogger() *zap.SugaredLogger { - zapConfig := zap.NewDevelopmentConfig() - zapConfig.EncoderConfig = zapcore.EncoderConfig{ +type Logger struct { + *zap.SugaredLogger + logPath string +} + +func NewLogger(logPath string) *Logger { + zapCfg := zap.NewDevelopmentConfig() + + consoleCfg := zapcore.EncoderConfig{ TimeKey: "time", LevelKey: "level", NameKey: "logger", @@ -19,9 +30,63 @@ func NewLogger() *zap.SugaredLogger { EncodeTime: zapcore.ISO8601TimeEncoder, EncodeDuration: zapcore.StringDurationEncoder, } - logger, _ := zapConfig.Build() - defer logger.Sync() + consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg) + + fileCfg := zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + MessageKey: "msg", + CallerKey: "", + StacktraceKey: "", + LineEnding: zapcore.DefaultLineEnding, + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeDuration: zapcore.StringDurationEncoder, + } + fileEncoder := zapcore.NewConsoleEncoder(fileCfg) + // Set up file logging with overwrite mode + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + panic("cannot open log file: " + err.Error()) + } + fmt.Println("Logging to", logPath) + + core := zapcore.NewTee( + zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), zapCfg.Level), + zapcore.NewCore(fileEncoder, zapcore.AddSync(logFile), zapCfg.Level), + ) + + logger := zap.New(core, zap.AddCaller(), zap.Development()) + defer logger.Sync() // flushes buffer, if any sugar := logger.Sugar() - return sugar + zLogger := &Logger{ + SugaredLogger: sugar, + logPath: logPath, + } + + return zLogger +} + +func (l *Logger) Named(name string) *Logger { + return &Logger{ + SugaredLogger: l.SugaredLogger.Named(name), + logPath: l.logPath, + } +} + +func (l *Logger) GetLogsFromFile() (string, error) { + file, err := os.Open(l.logPath) + if err != nil { + return "", err + } + defer file.Close() + + var buffer bytes.Buffer + _, err = io.Copy(&buffer, file) + if err != nil { + return "", err + } + + return buffer.String(), nil } diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 9ef2ccb..674c738 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -10,15 +10,15 @@ import ( "strings" zurghttp "github.com/debridmediamanager/zurg/pkg/http" - "go.uber.org/zap" + "github.com/debridmediamanager/zurg/pkg/logutil" ) type RealDebrid struct { - log *zap.SugaredLogger + log *logutil.Logger client *zurghttp.HTTPClient } -func NewRealDebrid(client *zurghttp.HTTPClient, log *zap.SugaredLogger) *RealDebrid { +func NewRealDebrid(client *zurghttp.HTTPClient, log *logutil.Logger) *RealDebrid { return &RealDebrid{ log: log, client: client,