hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add logs route, add rar handler

Browse Source
This commit is contained in:
Ben Sarmiento
2023-12-06 19:18:04 +01:00
parent 2a0b0fa9cd
commit 2aacff1125
19 changed files with 151 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -39,3 +39,5 @@ stressTestAddRemove.py
*.zip
pkg/anidb/
logs/

8
internal/app.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
netHttp "net/http"
"os"
"time"
// _ "net/http/pprof" // Register pprof
@@ -20,7 +21,9 @@ import (
)
func MainApp(configPath string) {
log := logutil.NewLogger()
utils.EnsureDirExists("logs") // Ensure the logs directory exists
logPath := fmt.Sprintf("logs/zurg-%s.log", time.Now().Format(time.DateOnly))
log := logutil.NewLogger(logPath)
zurglog := log.Named("zurg")
zurglog.Debugf("PID: %d", os.Getpid())
@@ -42,8 +45,7 @@ func MainApp(configPath string) {
}
defer p.Release()
utils.EnsureDirExists("data")
utils.EnsureDirExists("data") // Ensure the data directory exists
torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager"))
downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient"))

4
internal/config/load.go
View File

@@ -4,11 +4,11 @@ import (
"fmt"
"os"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
"gopkg.in/yaml.v3"
)
func LoadZurgConfig(filename string, log *zap.SugaredLogger) (ConfigInterface, error) {
func LoadZurgConfig(filename string, log *logutil.Logger) (ConfigInterface, error) {
log.Debug("Loading config file ", filename)
content, err := os.ReadFile(filename)
if err != nil {

6
internal/config/types.go
View File

@@ -24,6 +24,7 @@ type ConfigInterface interface {
GetRetriesUntilFailed() int
EnableDownloadCache() bool
GetRateLimitSleepSeconds() int
ShouldDeleteRarFiles() bool
}
type ZurgConfig struct {
@@ -45,6 +46,7 @@ type ZurgConfig struct {
RealDebridTimeout int `yaml:"realdebrid_timeout_secs" json:"realdebrid_timeout_secs"`
RetriesUntilFailed int `yaml:"retries_until_failed" json:"retries_until_failed"`
UseDownloadCache bool `yaml:"use_download_cache" json:"use_download_cache"`
DeleteRarFiles bool `yaml:"delete_rar_files" json:"delete_rar_files"`
}
func (z *ZurgConfig) GetConfig() ZurgConfig {
@@ -146,3 +148,7 @@ func (z *ZurgConfig) GetRateLimitSleepSeconds() int {
}
return z.RateLimitSleepSeconds
}
func (z *ZurgConfig) ShouldDeleteRarFiles() bool {
return z.DeleteRarFiles
}

4
internal/config/v1.go
View File

@@ -7,8 +7,8 @@ import (
"strconv"
"strings"
"github.com/debridmediamanager/zurg/pkg/logutil"
"github.com/debridmediamanager/zurg/pkg/utils"
"go.uber.org/zap"
"gopkg.in/yaml.v3"
)
@@ -16,7 +16,7 @@ const (
ALL_TORRENTS = "__all__"
)
func loadV1Config(content []byte, log *zap.SugaredLogger) (*ZurgConfigV1, error) {
func loadV1Config(content []byte, log *logutil.Logger) (*ZurgConfigV1, error) {
var configV1 ZurgConfigV1
if err := yaml.Unmarshal(content, &configV1); err != nil {
return nil, err

4
internal/config/v1types.go
View File

@@ -1,11 +1,11 @@
package config
import "go.uber.org/zap"
import "github.com/debridmediamanager/zurg/pkg/logutil"
type ZurgConfigV1 struct {
ZurgConfig `yaml:",inline"`
Directories map[string]*DirectoryFilterConditionsV1 `yaml:"directories"`
log *zap.SugaredLogger
log *logutil.Logger
}
type DirectoryFilterConditionsV1 struct {
GroupOrder int `yaml:"group_order"`

6
internal/dav/listing.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/debridmediamanager/zurg/internal/torrent"
"github.com/debridmediamanager/zurg/pkg/dav"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
)
func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
@@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
return &davDoc, nil
}
func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", directory)
@@ -51,7 +51,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z
return &davDoc, nil
}
func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", directory)

6
internal/http/listing.go
View File

@@ -8,7 +8,7 @@ import (
"strings"
"github.com/debridmediamanager/zurg/internal/torrent"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
)
func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
@@ -26,7 +26,7 @@ func HandleListDirectories(torMgr *torrent.TorrentManager) (*string, error) {
return &htmlDoc, nil
}
func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", directory)
@@ -49,7 +49,7 @@ func HandleListTorrents(directory string, torMgr *torrent.TorrentManager, log *z
return &htmlDoc, nil
}
func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
func HandleListFiles(directory, torrentName string, torMgr *torrent.TorrentManager, log *logutil.Logger) (*string, error) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", directory)

19
internal/router/router.go
View File

@@ -12,9 +12,9 @@ import (
"github.com/debridmediamanager/zurg/internal/torrent"
"github.com/debridmediamanager/zurg/internal/universal"
"github.com/debridmediamanager/zurg/internal/version"
"github.com/debridmediamanager/zurg/pkg/logutil"
"github.com/debridmediamanager/zurg/pkg/realdebrid"
"github.com/julienschmidt/httprouter"
"go.uber.org/zap"
jsoniter "github.com/json-iterator/go"
)
@@ -26,10 +26,10 @@ type ZurgRouter struct {
torMgr *torrent.TorrentManager
cfg config.ConfigInterface
api *realdebrid.RealDebrid
log *zap.SugaredLogger
log *logutil.Logger
}
func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *zap.SugaredLogger) {
func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torMgr *torrent.TorrentManager, cfg config.ConfigInterface, api *realdebrid.RealDebrid, log *logutil.Logger) {
zr := &ZurgRouter{
getfile: getfile,
torMgr: torMgr,
@@ -64,6 +64,10 @@ func ApplyRouteTable(router *httprouter.Router, getfile *universal.GetFile, torM
// root route
router.GET("/", zr.rootHandler)
// logs route
router.GET("/logs", zr.logsHandler)
router.GET("/logs/", zr.logsHandler)
}
func (zr *ZurgRouter) httpTorrentDirectoryHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
@@ -233,6 +237,15 @@ func (zr *ZurgRouter) rootHandler(resp http.ResponseWriter, req *http.Request, p
}
}
func (zr *ZurgRouter) logsHandler(resp http.ResponseWriter, req *http.Request, params httprouter.Params) {
logs, err := zr.log.GetLogsFromFile()
if err != nil {
http.Error(resp, err.Error(), http.StatusInternalServerError)
return
}
fmt.Fprint(resp, logs)
}
func bToMb(b uint64) uint64 {
return b / 1024 / 1024
}

4
internal/torrent/hooks.go
View File

@@ -6,7 +6,7 @@ import (
"os/exec"
"github.com/debridmediamanager/zurg/internal/config"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
)
type ScriptExecutor struct {
@@ -32,7 +32,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
return out.String(), nil
}
func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *zap.SugaredLogger) {
func OnLibraryUpdateHook(paths []string, config config.ConfigInterface, log *logutil.Logger) {
executor := &ScriptExecutor{
Script: config.GetOnLibraryUpdate(),
Args: paths,

6
internal/torrent/manager.go
View File

@@ -7,12 +7,12 @@ import (
"strings"
"github.com/debridmediamanager/zurg/internal/config"
"github.com/debridmediamanager/zurg/pkg/logutil"
"github.com/debridmediamanager/zurg/pkg/realdebrid"
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/panjf2000/ants/v2"
"github.com/scylladb/go-set"
"github.com/scylladb/go-set/strset"
"go.uber.org/zap"
)
const (
@@ -30,13 +30,13 @@ type TorrentManager struct {
requiredVersion string
workerPool *ants.Pool
repairWorker *ants.Pool
log *zap.SugaredLogger
log *logutil.Logger
}
// NewTorrentManager creates a new torrent manager
// it will fetch all torrents and their info in the background
// and store them in-memory and cached in files
func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *zap.SugaredLogger) *TorrentManager {
func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *logutil.Logger) *TorrentManager {
initialSate := EmptyState()
t := &TorrentManager{

22
internal/torrent/refresh.go
View File

@@ -149,8 +149,26 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
})
}
if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
t.log.Warnf("Torrent id=%s is partly expired, it has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links))
torrent.ForRepair = true
if len(info.Links) == 1 {
// this might be a rar file so let's check
unrestrict := t.UnrestrictUntilOk(info.Links[0])
if unrestrict != nil && strings.HasPrefix(strings.ToLower(unrestrict.Filename), ".rar") {
if t.Config.ShouldDeleteRarFiles() {
t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired. Deleting...", info.Name, info.ID)
t.Api.DeleteTorrent(info.ID)
return nil
} else {
t.log.Warnf("Torrent %s id=%s is a rar file, it cannot be repaired as it's a known Real-Debrid limitation. zurg recommends you delete this torrent or enable delete_rar_files in your config.yml", info.Name, info.ID)
torrent.Unfixable = true
}
} else {
t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only 1 link", info.ID, len(selectedFiles), len(info.Links))
torrent.ForRepair = true
}
} else {
t.log.Warnf("Torrent id=%s is partly expired. It has %d selected files but only %d links", info.ID, len(selectedFiles), len(info.Links))
torrent.ForRepair = true
}
} else if len(selectedFiles) == len(info.Links) {
// all links are still intact! good!
for i, file := range selectedFiles {

2
internal/torrent/repair.go
View File

@@ -22,7 +22,7 @@ func (t *TorrentManager) repairAll() {
if torrent.AnyInProgress() && torrent.ForRepair {
t.log.Warnf("Skipping %s for repairs because it is in progress", torrent.AccessKey)
return
} else if torrent.ForRepair {
} else if torrent.ForRepair && !torrent.Unfixable {
toRepair = append(toRepair, torrent)
}
})

1
internal/torrent/types.go
View File

@@ -18,6 +18,7 @@ type Torrent struct {
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"`
LatestAdded string `json:"LatestAdded"`
ForRepair bool `json:"ForRepair"`
Unfixable bool `json:"Unfixable"`
DownloadedIDs *strset.Set `json:"DownloadedIDs"`
InProgressIDs *strset.Set `json:"InProgressIDs"`

8
internal/universal/get.go
View File

@@ -11,8 +11,8 @@ import (
"github.com/debridmediamanager/zurg/internal/config"
intTor "github.com/debridmediamanager/zurg/internal/torrent"
zurghttp "github.com/debridmediamanager/zurg/pkg/http"
"github.com/debridmediamanager/zurg/pkg/logutil"
"github.com/debridmediamanager/zurg/pkg/realdebrid"
"go.uber.org/zap"
)
type GetFile struct {
@@ -24,7 +24,7 @@ func NewGetFile(client *zurghttp.HTTPClient) *GetFile {
}
// HandleGetRequest handles a GET request universally for both WebDAV and HTTP
func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) {
func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {
log.Warnf("Cannot find directory %s", directory)
@@ -96,7 +96,7 @@ func (gf *GetFile) HandleGetRequest(directory, torrentName, fileName string, res
}
}
func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) error {
func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) error {
// Create a new dlReq for the file download.
dlReq, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
@@ -129,7 +129,7 @@ func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWrit
return nil
}
func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) {
func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) {
// Create a new request for the file download.
dlReq, err := http.NewRequest(http.MethodGet, unrestrict.Download, nil)
if err != nil {

4
internal/universal/head.go
View File

@@ -7,10 +7,10 @@ import (
"strings"
"github.com/debridmediamanager/zurg/internal/torrent"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
)
func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *zap.SugaredLogger) {
func HandleHeadRequest(directory, torrentName, fileName string, w http.ResponseWriter, req *http.Request, torMgr *torrent.TorrentManager, log *logutil.Logger) {
torrents, ok := torMgr.DirectoryMap.Get(directory)
if !ok {

6
pkg/http/client.go
View File

@@ -9,8 +9,8 @@ import (
"time"
"github.com/debridmediamanager/zurg/internal/config"
"github.com/debridmediamanager/zurg/pkg/logutil"
cmap "github.com/orcaman/concurrent-map/v2"
"go.uber.org/zap"
)
const (
@@ -25,10 +25,10 @@ type HTTPClient struct {
bearerToken string
cfg config.ConfigInterface
ipv6 cmap.ConcurrentMap[string, string]
log *zap.SugaredLogger
log *logutil.Logger
}
func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient {
func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient {
client := HTTPClient{
bearerToken: token,
client: &http.Client{

77
pkg/logutil/factory.go
View File

@@ -1,13 +1,24 @@
package logutil
import (
"bytes"
"fmt"
"io"
"os"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func NewLogger() *zap.SugaredLogger {
zapConfig := zap.NewDevelopmentConfig()
zapConfig.EncoderConfig = zapcore.EncoderConfig{
type Logger struct {
*zap.SugaredLogger
logPath string
}
func NewLogger(logPath string) *Logger {
zapCfg := zap.NewDevelopmentConfig()
consoleCfg := zapcore.EncoderConfig{
TimeKey: "time",
LevelKey: "level",
NameKey: "logger",
@@ -19,9 +30,63 @@ func NewLogger() *zap.SugaredLogger {
EncodeTime: zapcore.ISO8601TimeEncoder,
EncodeDuration: zapcore.StringDurationEncoder,
}
logger, _ := zapConfig.Build()
defer logger.Sync()
consoleEncoder := zapcore.NewConsoleEncoder(consoleCfg)
fileCfg := zapcore.EncoderConfig{
TimeKey: "time",
LevelKey: "level",
NameKey: "logger",
MessageKey: "msg",
CallerKey: "",
StacktraceKey: "",
LineEnding: zapcore.DefaultLineEnding,
EncodeTime: zapcore.ISO8601TimeEncoder,
EncodeDuration: zapcore.StringDurationEncoder,
}
fileEncoder := zapcore.NewConsoleEncoder(fileCfg)
// Set up file logging with overwrite mode
logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
panic("cannot open log file: " + err.Error())
}
fmt.Println("Logging to", logPath)
core := zapcore.NewTee(
zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), zapCfg.Level),
zapcore.NewCore(fileEncoder, zapcore.AddSync(logFile), zapCfg.Level),
)
logger := zap.New(core, zap.AddCaller(), zap.Development())
defer logger.Sync() // flushes buffer, if any
sugar := logger.Sugar()
return sugar
zLogger := &Logger{
SugaredLogger: sugar,
logPath: logPath,
}
return zLogger
}
func (l *Logger) Named(name string) *Logger {
return &Logger{
SugaredLogger: l.SugaredLogger.Named(name),
logPath: l.logPath,
}
}
func (l *Logger) GetLogsFromFile() (string, error) {
file, err := os.Open(l.logPath)
if err != nil {
return "", err
}
defer file.Close()
var buffer bytes.Buffer
_, err = io.Copy(&buffer, file)
if err != nil {
return "", err
}
return buffer.String(), nil
}

6
pkg/realdebrid/api.go
View File

@@ -10,15 +10,15 @@ import (
"strings"
zurghttp "github.com/debridmediamanager/zurg/pkg/http"
"go.uber.org/zap"
"github.com/debridmediamanager/zurg/pkg/logutil"
)
type RealDebrid struct {
log *zap.SugaredLogger
log *logutil.Logger
client *zurghttp.HTTPClient
}
func NewRealDebrid(client *zurghttp.HTTPClient, log *zap.SugaredLogger) *RealDebrid {
func NewRealDebrid(client *zurghttp.HTTPClient, log *logutil.Logger) *RealDebrid {
return &RealDebrid{
log: log,
client: client,