Add logs route, add rar handler

.gitignore

@@ -39,3 +39,5 @@ stressTestAddRemove.py
 *.zip
 
 pkg/anidb/
+
+logs/

internal/app.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	netHttp "net/http"
 	"os"
+	"time"
 
 	// _ "net/http/pprof" // Register pprof
 
@@ -20,7 +21,9 @@ import (
 )
 
 func MainApp(configPath string) {
-	log := logutil.NewLogger()
+	utils.EnsureDirExists("logs") // Ensure the logs directory exists
+	logPath := fmt.Sprintf("logs/zurg-%s.log", time.Now().Format(time.DateOnly))
+	log := logutil.NewLogger(logPath)
 	zurglog := log.Named("zurg")
 
 	zurglog.Debugf("PID: %d", os.Getpid())
@@ -42,8 +45,7 @@ func MainApp(configPath string) {
 	}
 	defer p.Release()
 
-	utils.EnsureDirExists("data")
+	utils.EnsureDirExists("data") // Ensure the data directory exists
-
 	torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager"))
 
 	downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient"))

internal/config/load.go

@@ -4,11 +4,11 @@ import (
 	"fmt"
 	"os"
 
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	"gopkg.in/yaml.v3"
 )
 
-func LoadZurgConfig(filename string, log *zap.SugaredLogger) (ConfigInterface, error) {
+func LoadZurgConfig(filename string, log *logutil.Logger) (ConfigInterface, error) {
 	log.Debug("Loading config file ", filename)
 	content, err := os.ReadFile(filename)
 	if err != nil {

internal/config/types.go

@@ -24,6 +24,7 @@ type ConfigInterface interface {
 	GetRetriesUntilFailed() int
 	EnableDownloadCache() bool
 	GetRateLimitSleepSeconds() int
+	ShouldDeleteRarFiles() bool
 }
 
 type ZurgConfig struct {
@@ -45,6 +46,7 @@ type ZurgConfig struct {
 	RealDebridTimeout         int      `yaml:"realdebrid_timeout_secs" json:"realdebrid_timeout_secs"`
 	RetriesUntilFailed        int      `yaml:"retries_until_failed" json:"retries_until_failed"`
 	UseDownloadCache          bool     `yaml:"use_download_cache" json:"use_download_cache"`
+	DeleteRarFiles            bool     `yaml:"delete_rar_files" json:"delete_rar_files"`
 }
 
 func (z *ZurgConfig) GetConfig() ZurgConfig {
@@ -146,3 +148,7 @@ func (z *ZurgConfig) GetRateLimitSleepSeconds() int {
 	}
 	return z.RateLimitSleepSeconds
 }
+
+func (z *ZurgConfig) ShouldDeleteRarFiles() bool {
+	return z.DeleteRarFiles
+}

internal/config/v1.go

@@ -7,8 +7,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	"github.com/debridmediamanager/zurg/pkg/utils"
-	"go.uber.org/zap"
 	"gopkg.in/yaml.v3"
 )
 
@@ -16,7 +16,7 @@ const (
 	ALL_TORRENTS = "__all__"
 )
 
-func loadV1Config(content []byte, log *zap.SugaredLogger) (*ZurgConfigV1, error) {
+func loadV1Config(content []byte, log *logutil.Logger) (*ZurgConfigV1, error) {
 	var configV1 ZurgConfigV1
 	if err := yaml.Unmarshal(content, &configV1); err != nil {
 		return nil, err

internal/config/v1types.go

@@ -1,11 +1,11 @@
 package config
 
-import "go.uber.org/zap"
+import "github.com/debridmediamanager/zurg/pkg/logutil"
 
 type ZurgConfigV1 struct {
 	ZurgConfig  `yaml:",inline"`
 	Directories map[string]*DirectoryFilterConditionsV1 `yaml:"directories"`
-	log         *zap.SugaredLogger
+	log         *logutil.Logger
 }
 type DirectoryFilterConditionsV1 struct {
 	GroupOrder int                   `yaml:"group_order"`

internal/dav/listing.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/debridmediamanager/zurg/internal/torrent"
 	"github.com/debridmediamanager/zurg/pkg/dav"
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 )
 
 func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
@@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
 	return &davDoc, nil
 }
 
-func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
+func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {
 		return nil, fmt.Errorf("cannot find directory %s", directory)
@@ -51,7 +51,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z
 	return &davDoc, nil
 }
 
-func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
+func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {
 		return nil, fmt.Errorf("cannot find directory %s", directory)

internal/http/listing.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/debridmediamanager/zurg/internal/torrent"
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 )
 
 func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
@@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
 	return &htmlDoc, nil
 }
 
-func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
+func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {
 		return nil, fmt.Errorf("cannot find directory %s", directory)
@@ -49,7 +49,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z
 	return &htmlDoc, nil
 }
 
-func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
+func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {
 		return nil, fmt.Errorf("cannot find directory %s", directory)

internal/router/router.go

@@ -12,9 +12,9 @@ import (
 	"github.com/debridmediamanager/zurg/internal/torrent"
 	"github.com/debridmediamanager/zurg/internal/universal"
 	"github.com/debridmediamanager/zurg/internal/version"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	"github.com/debridmediamanager/zurg/pkg/realdebrid"
 	"github.com/julienschmidt/httprouter"
-	"go.uber.org/zap"
 
 	jsoniter "github.com/json-iterator/go"
 )
@@ -26,10 +26,10 @@ type ZurgRouter struct {
 	torMgr  *torrent.TorrentManager
 	cfg     config.ConfigInterface
 	api     *realdebrid.RealDebrid
-	log     *zap.SugaredLogger
+	log     *logutil.Logger
 }
 
-func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *zap.SugaredLogger) {
+func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *logutil.Logger) {
 	zr := &ZurgRouter{
 		getfile: getfile,
 		torMgr:  torMgr,
@@ -64,6 +64,10 @@ func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torM
 
 	// root route
 	router.GET("/", zr.rootHandler)
+
+	// logs route
+	router.GET("/logs", zr.logsHandler)
+	router.GET("/logs/", zr.logsHandler)
 }
 
 func (zr *ZurgRouter) httpTorrentDirectoryHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
@@ -233,6 +237,15 @@ func (zr *ZurgRouter) rootHandler(resp http.ResponseWriter, req *http.Request, p
 	}
 }
 
+func (zr *ZurgRouter) logsHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
+	logs, err := zr.log.GetLogsFromFile()
+	if err != nil {
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	fmt.Fprint(resp, logs)
+}
+
 func bToMb(b uint64) uint64 {
 	return b / 1024 / 1024
 }

internal/torrent/hooks.go

@@ -6,7 +6,7 @@ import (
 	"os/exec"
 
 	"github.com/debridmediamanager/zurg/internal/config"
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 )
 
 type ScriptExecutor struct {
@@ -32,7 +32,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
 	return out.String(), nil
 }
 
-func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *zap.SugaredLogger) {
+func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *logutil.Logger) {
 	executor := &ScriptExecutor{
 		Script: config.GetOnLibraryUpdate(),
 		Args:   paths,

internal/torrent/manager.go

@@ -7,12 +7,12 @@ import (
 	"strings"
 
 	"github.com/debridmediamanager/zurg/internal/config"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	"github.com/debridmediamanager/zurg/pkg/realdebrid"
 	cmap "github.com/orcaman/concurrent-map/v2"
 	"github.com/panjf2000/ants/v2"
 	"github.com/scylladb/go-set"
 	"github.com/scylladb/go-set/strset"
-	"go.uber.org/zap"
 )
 
 const (
@@ -30,13 +30,13 @@ type TorrentManager struct {
 	requiredVersion string
 	workerPool      *ants.Pool
 	repairWorker    *ants.Pool
-	log             *zap.SugaredLogger
+	log             *logutil.Logger
 }
 
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
 // and store them in-memory and cached in files
-func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *zap.SugaredLogger) *TorrentManager {
+func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *logutil.Logger) *TorrentManager {
 	initialSate := EmptyState()
 
 	t := &TorrentManager{

internal/torrent/refresh.go

@@ -149,8 +149,26 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		})
 	}
 	if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
-		t.log.Warnf("Torrent id=%s is partly expired, it has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links))
+		if len(info.Links) == 1 {
-		torrent.ForRepair = true
+			// this might be a rar file so let's check
+			unrestrict := t.UnrestrictUntilOk(info.Links[0])
+			if unrestrict != nil && strings.HasPrefix(strings.ToLower(unrestrict.Filename), ".rar") {
+				if t.Config.ShouldDeleteRarFiles() {
+					t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired. Deleting...", info.Name, info.ID)
+					t.Api.DeleteTorrent(info.ID)
+					return nil
+				} else {
+					t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired as it's a known Real-Debrid limitation. zurg recommends you delete this torrent or enable delete_rar_files in your config.yml", info.Name, info.ID)
+					torrent.Unfixable = true
+				}
+			} else {
+				t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only 1 link", info.ID, len(selectedFiles), len(info.Links))
+				torrent.ForRepair = true
+			}
+		} else {
+			t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links))
+			torrent.ForRepair = true
+		}
 	} else if len(selectedFiles) == len(info.Links) {
 		// all links are still intact! good!
 		for i, file := range selectedFiles {

internal/torrent/repair.go

@@ -22,7 +22,7 @@ func (t *TorrentManager) repairAll() {
 		if torrent.AnyInProgress() && torrent.ForRepair {
 			t.log.Warnf("Skipping %s for repairs because it is in progress", torrent.AccessKey)
 			return
-		} else if torrent.ForRepair {
+		} else if torrent.ForRepair && !torrent.Unfixable {
 			toRepair = append(toRepair, torrent)
 		}
 	})

internal/torrent/types.go

@@ -18,6 +18,7 @@ type Torrent struct {
 	SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"`
 	LatestAdded   string                            `json:"LatestAdded"`
 	ForRepair     bool                              `json:"ForRepair"`
+	Unfixable     bool                              `json:"Unfixable"`
 	DownloadedIDs *strset.Set                       `json:"DownloadedIDs"`
 	InProgressIDs *strset.Set                       `json:"InProgressIDs"`
 

internal/universal/get.go

@@ -11,8 +11,8 @@ import (
 	"github.com/debridmediamanager/zurg/internal/config"
 	intTor "github.com/debridmediamanager/zurg/internal/torrent"
 	zurghttp "github.com/debridmediamanager/zurg/pkg/http"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	"github.com/debridmediamanager/zurg/pkg/realdebrid"
-	"go.uber.org/zap"
 )
 
 type GetFile struct {
@@ -24,7 +24,7 @@ func NewGetFile(client *zurghttp.HTTPClient) *GetFile {
 }
 
 // HandleGetRequest handles a GET request universally for both WebDAV and HTTP
-func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) {
+func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) {
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {
 		log.Warnf("Cannot find directory %s", directory)
@@ -96,7 +96,7 @@ func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, res
 	}
 }
 
-func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) error {
+func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) error {
 	// Create a new dlReq for the file download.
 	dlReq, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
@@ -129,7 +129,7 @@ func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWrit
 	return nil
 }
 
-func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) {
+func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) {
 	// Create a new request for the file download.
 	dlReq, err := http.NewRequest(http.MethodGet, unrestrict.Download, nil)
 	if err != nil {

internal/universal/head.go

@@ -7,10 +7,10 @@ import (
 	"strings"
 
 	"github.com/debridmediamanager/zurg/internal/torrent"
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 )
 
-func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) {
+func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *logutil.Logger) {
 
 	torrents, ok := torMgr.DirectoryMap.Get(directory)
 	if !ok {

pkg/http/client.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/debridmediamanager/zurg/internal/config"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 	cmap "github.com/orcaman/concurrent-map/v2"
-	"go.uber.org/zap"
 )
 
 const (
@@ -25,10 +25,10 @@ type HTTPClient struct {
 	bearerToken  string
 	cfg          config.ConfigInterface
 	ipv6         cmap.ConcurrentMap[string, string]
-	log          *zap.SugaredLogger
+	log          *logutil.Logger
 }
 
-func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient {
+func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient {
 	client := HTTPClient{
 		bearerToken: token,
 		client: &http.Client{

pkg/logutil/factory.go

@@ -1,13 +1,24 @@
 package logutil
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 )
 
-func NewLogger() *zap.SugaredLogger {
+type Logger struct {
-	zapConfig := zap.NewDevelopmentConfig()
+	*zap.SugaredLogger
-	zapConfig.EncoderConfig = zapcore.EncoderConfig{
+	logPath string
+}
+
+func NewLogger(logPath string) *Logger {
+	zapCfg := zap.NewDevelopmentConfig()
+
+	consoleCfg := zapcore.EncoderConfig{
 		TimeKey:        "time",
 		LevelKey:       "level",
 		NameKey:        "logger",
@@ -19,9 +30,63 @@ func NewLogger() *zap.SugaredLogger {
 		EncodeTime:     zapcore.ISO8601TimeEncoder,
 		EncodeDuration: zapcore.StringDurationEncoder,
 	}
-	logger, _ := zapConfig.Build()
+	consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg)
-	defer logger.Sync()
+
+	fileCfg := zapcore.EncoderConfig{
+		TimeKey:        "time",
+		LevelKey:       "level",
+		NameKey:        "logger",
+		MessageKey:     "msg",
+		CallerKey:      "",
+		StacktraceKey:  "",
+		LineEnding:     zapcore.DefaultLineEnding,
+		EncodeTime:     zapcore.ISO8601TimeEncoder,
+		EncodeDuration: zapcore.StringDurationEncoder,
+	}
+	fileEncoder := zapcore.NewConsoleEncoder(fileCfg)
+	// Set up file logging with overwrite mode
+	logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		panic("cannot open log file: " + err.Error())
+	}
+	fmt.Println("Logging to", logPath)
+
+	core := zapcore.NewTee(
+		zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), zapCfg.Level),
+		zapcore.NewCore(fileEncoder, zapcore.AddSync(logFile), zapCfg.Level),
+	)
+
+	logger := zap.New(core, zap.AddCaller(), zap.Development())
+	defer logger.Sync() // flushes buffer, if any
 	sugar := logger.Sugar()
 
-	return sugar
+	zLogger := &Logger{
+		SugaredLogger: sugar,
+		logPath:       logPath,
+	}
+
+	return zLogger
+}
+
+func (l *Logger) Named(name string) *Logger {
+	return &Logger{
+		SugaredLogger: l.SugaredLogger.Named(name),
+		logPath:       l.logPath,
+	}
+}
+
+func (l *Logger) GetLogsFromFile() (string, error) {
+	file, err := os.Open(l.logPath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	var buffer bytes.Buffer
+	_, err = io.Copy(&buffer, file)
+	if err != nil {
+		return "", err
+	}
+
+	return buffer.String(), nil
 }

pkg/realdebrid/api.go

@@ -10,15 +10,15 @@ import (
 	"strings"
 
 	zurghttp "github.com/debridmediamanager/zurg/pkg/http"
-	"go.uber.org/zap"
+	"github.com/debridmediamanager/zurg/pkg/logutil"
 )
 
 type RealDebrid struct {
-	log    *zap.SugaredLogger
+	log    *logutil.Logger
 	client *zurghttp.HTTPClient
 }
 
-func NewRealDebrid(client *zurghttp.HTTPClient, log *zap.SugaredLogger) *RealDebrid {
+func NewRealDebrid(client *zurghttp.HTTPClient, log *logutil.Logger) *RealDebrid {
 	return &RealDebrid{
 		log:    log,
 		client: client,