From faba4e53ab776660b021304d366e07a51632dfca Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Thu, 19 Oct 2023 18:02:30 +0200 Subject: [PATCH] Finish config mapping --- cmd/zurg/main.go | 24 ++++--- config.yml | 48 ++++++++------ config.yml.example | 63 ++++++++++-------- go.mod | 19 +++++- go.sum | 36 +++++++++++ internal/config/v1.go | 39 +++++++++++- internal/config/v1types.go | 1 - internal/dav/propfind.go | 12 ++-- internal/dav/response.go | 7 ++ internal/dav/router.go | 124 +++++++++++++++++++++++++++++++++--- internal/dav/util.go | 2 +- internal/torrent/manager.go | 83 ++++++++++++++++++++++-- pkg/realdebrid/api.go | 24 ++++--- pkg/realdebrid/util.go | 4 +- 14 files changed, 397 insertions(+), 89 deletions(-) diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go index e956348..33b70c0 100644 --- a/cmd/zurg/main.go +++ b/cmd/zurg/main.go @@ -3,18 +3,26 @@ package main import ( "log" "net/http" + "os" + "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/dav" + "github.com/debridmediamanager.com/zurg/internal/torrent" ) func main() { - mux := http.NewServeMux() - - dav.Router(mux) - - log.Println("Listening on port 8123...") - err := http.ListenAndServe(":8123", mux) - if err != nil { - log.Println(err) + c, cErr := config.LoadZurgConfig("./config.yml") + if cErr != nil { + log.Panicf("Config failed to load: %v", cErr) } + + t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN"), c) + + // app := aero.New() + // dav.Setup(app, c, t) + // app.Run() + + mux := http.NewServeMux() + dav.Router(mux, c, t) + http.ListenAndServe(":8123", mux) } diff --git a/config.yml b/config.yml index 33515d4..91a80c3 100644 --- a/config.yml +++ b/config.yml @@ -1,38 +1,50 @@ +# Zurg configuration version zurg: v1 +# List of directory definitions and their filtering rules directories: - torrents: - group: asd - filters: - - not_contains: xxx - - not_contains_strict: trailer + # Configuration for TV shows shows: - group: sdf + group: media # directories on different groups have duplicates of the same torrent filters: - - regex: /season/i - - regex: /Season/i - - regex: /Saison/i + - regex: /season[\s\.]?\d/i # Capture torrent names with the term 'season' in any case + - regex: /Saison[\s\.]?\d/i # For non-English namings + - regex: /stage[\s\.]?\d/i + - regex: /s\d\d/i # Capture common season notations like S01, S02, etc. + # Configuration for movies + movies: + group: media # because movies and shows are in the same group, and shows come first before movies, all torrents that doesn't fall into shows will fall into movies + filters: + - regex: /.*/ # you cannot leave a directory without filters because it will not have any torrents in it + + # Configuration for remuxes remuxes: - group: gdfg + group: def filters: - - contains: remux + - contains: remux # Specifically target remuxed content + # Configuration for Dolby Vision content "dolby vision": - group: zxc + group: random filters: - and: - - regex: /\bdovi\b/i - - contains: 4k + - regex: /\bdovi\b/i # Matches abbreviations of 'dolby vision' + - contains: 4k # you can be quite greedy here, dolby vision + 4k! + + # Configuration for children's content kids: - group: wqe + group: kids filters: - - id: XFPQ5UCMUVAEG + - or: # you can also group conditions with 'or' which is useful especially inside 'and' conditions + - not_contains: xxx # Ensures adult content is excluded + - not_contains_strict: trailer # strict vs non-strict is just about case sensitivity; this ensures trailers aren't added + - id: XFPQ5UCMUVAEG # Specific inclusion by torrent ID - id: VDRPYNRPQHEXC - id: YELNX3XR5XJQM - default: - group: xcv + all: + group: default # because movies and shows are in the same group, and shows come first before movies, all torrents that doesn't fall into shows will fall into movies filters: - regex: /.*/ diff --git a/config.yml.example b/config.yml.example index 446db2f..263f513 100644 --- a/config.yml.example +++ b/config.yml.example @@ -1,35 +1,44 @@ +# Zurg configuration version zurg: v1 +# List of directory definitions and their filtering rules directories: - torrents: - or: - - regex: /.*/ # you can specify multiple conditions and it works as an OR - - contains: season # use 'contains' for string, case insensitive - - contains_strict: season # case sensitive - - and: - - not_contains: xxx - - not_contains_strict: trailer + + # Configuration for TV shows shows: - or: - - regex: /season/i # you can specify multiple regex conditions and it works as an OR - - regex: /Season/i - - regex: /Saison/i - "4k content": - regex: /\b4k\b/i + group: media # directories on different groups have duplicates of the same torrent + filters: + - regex: /season/i # Capture torrent names with the term 'season' in any case + - regex: /Saison/i # For non-English namings + - regex: /s\d\d/i # Capture common season notations like S01, S02, etc. + + # Configuration for movies + movies: + group: media # because movies and shows are in the same group, and shows come first before movies, all torrents that doesn't fall into shows will fall into movies + filters: + - regex: /.*/ # you cannot leave a directory without filters because it will not have any torrents in it + + # Configuration for remuxes + remuxes: + group: def + filters: + - contains: remux # Specifically target remuxed content + + # Configuration for Dolby Vision content "dolby vision": - and: - regex: /\bdovi\b/i - contains: 4k + group: random + filters: + - and: + - regex: /\bdovi\b/i # Matches abbreviations of 'dolby vision' + - contains: 4k # you can be quite greedy here, dolby vision + 4k! + + # Configuration for children's content kids: - or: - - id: XFPQ5UCMUVAEG # you can specify the torrent ID as well + group: kids + filters: + - or: # you can also group conditions with 'or' which is useful especially inside 'and' conditions + - not_contains: xxx # Ensures adult content is excluded + - not_contains_strict: trailer # strict vs non-strict is just about case sensitivity; this ensures trailers aren't added + - id: XFPQ5UCMUVAEG # Specific inclusion by torrent ID - id: VDRPYNRPQHEXC - id: YELNX3XR5XJQM - default: - regex: /.*/ # if duplicates=false, best practice to add a catch-all bucket - -duplicates: true -# if true, it means as long as a torrent satisfies the conditions set, then it will appear on that directory -# if false, it follows the order of directories and only appears once in one of them (or none so be careful) -# it processes each movie to see if it meets DIR1 conditions -# if not, it checks for DIR2 conditions, and so on... diff --git a/go.mod b/go.mod index 41a3792..11cd25f 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,21 @@ module github.com/debridmediamanager.com/zurg go 1.21.3 -require gopkg.in/yaml.v3 v3.0.1 // indirect +require ( + github.com/aerogo/aero v1.3.59 // indirect + github.com/aerogo/csp v0.1.10 // indirect + github.com/aerogo/http v1.1.3 // indirect + github.com/aerogo/session v0.1.9 // indirect + github.com/aerogo/session-store-memory v0.1.9 // indirect + github.com/akyoto/color v1.8.12 // indirect + github.com/akyoto/colorable v0.1.7 // indirect + github.com/akyoto/hash v0.5.0 // indirect + github.com/akyoto/stringutils v0.3.1 // indirect + github.com/akyoto/tty v0.1.4 // indirect + github.com/akyoto/uuid v1.1.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/zeebo/xxh3 v1.0.1 // indirect + golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 4bc0337..37e609f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,39 @@ +github.com/aerogo/aero v1.3.59 h1:5yu+kk/uIXAXADKSLCFKhxAzThCehvpbF6gst+G32Fw= +github.com/aerogo/aero v1.3.59/go.mod h1:ehwj+mb117xQRTvp11jlnrRNPgbcYL6s6aBk9wbIZ0o= +github.com/aerogo/csp v0.1.10 h1:2PJf9gkdRvCFYOA0baTUyp34vwPp5ZJJX8GZRCYc/nM= +github.com/aerogo/csp v0.1.10/go.mod h1:UrxbTXv+X9kJatyuLeu2yGFpOiWVPjbqA/DzqxSVhl8= +github.com/aerogo/http v1.1.3 h1:cvwOYL+zNEfNHvJcX6A6OgUwQ4KROlu8ypuQEQc1HtU= +github.com/aerogo/http v1.1.3/go.mod h1:h+m3WxevpaifyVpRAMV58qt8ScXSZhU1a5DdvBkRwwE= +github.com/aerogo/session v0.1.8/go.mod h1:Q9QqpT8nM6HTaklE14T+bzNSKrwW1M2wZ/NZV1HUTB0= +github.com/aerogo/session v0.1.9 h1:pgsFEtCteOQaZ/103q2/O+qrqZileiCZe+vboWKZMlU= +github.com/aerogo/session v0.1.9/go.mod h1:dgpdXvs9tZXcag5ay6tEoKuySPga226iSh748uIES/E= +github.com/aerogo/session-store-memory v0.1.9 h1:1OswTCtyqzffX5aGr6jI3H8gt/hkU3LKNiKpia7ntcs= +github.com/aerogo/session-store-memory v0.1.9/go.mod h1:z4ZxP+xLVdH69F/Cvgy93v8fWzeDmiJo+Mm+Th3un4c= +github.com/akyoto/assert v0.2.3/go.mod h1:g5e6ag+ksCEQENq/LnmU9z04wCAIFDr8KacBusVL0H8= +github.com/akyoto/assert v0.2.4/go.mod h1:SoqVayyOmM/YSBnwOxJHCt4BCocoIrgeceWtJV701C0= +github.com/akyoto/color v1.8.12 h1:7F/iF/POG6z+oppoGYWO6UOx8E2ZAypANO9rsfsBuHI= +github.com/akyoto/color v1.8.12/go.mod h1:rG1eiYoSE+arV6oLuGuuekPtgujUlIErWeqqM13pVoA= +github.com/akyoto/colorable v0.1.7 h1:ge91E25hiOiT/Zu47ij/rTO3cks7wMlTrcQspua1hFM= +github.com/akyoto/colorable v0.1.7/go.mod h1:zlc1+Es4DyoXzDdbKiSfvdM6R/DsWS8bFi4RHigkuu4= +github.com/akyoto/hash v0.5.0 h1:NAOZ8EySEOzlLpiURs4PLx26Hxsv8vkxpySElJ5U9FY= +github.com/akyoto/hash v0.5.0/go.mod h1:/ftTams8jMXYuc4NWDzdA6sEztxFslBS+VdqYZQZCNI= +github.com/akyoto/stringutils v0.3.1 h1:C+VGuXfud9SSo54QRfdQO+rgQiHmLS5f4nJ4yUOM+8I= +github.com/akyoto/stringutils v0.3.1/go.mod h1:I1F9f8FF7gnAQyYp4PVAl+GJ2WBnaN6kNoYjidCV5Qk= +github.com/akyoto/tty v0.1.3/go.mod h1:+VlbvviCaiwhS4oGpO+iBtC0lYG1ilIs3ZhUnT1Ppgo= +github.com/akyoto/tty v0.1.4 h1:TELbnAmrPTIrUJyuBLhrOSCcBnklC2fh0YeCTjksiDE= +github.com/akyoto/tty v0.1.4/go.mod h1:fkWwtA4F5Cq9kiQSlWdkPy5kAyySGYqalWyaRKn3zHo= +github.com/akyoto/uuid v1.1.3 h1:FEz14tNTfaUeY0Jrkz2F17rjKiks6hOALGcPmAmtn1s= +github.com/akyoto/uuid v1.1.3/go.mod h1:8dgzDQyrpuApBGIQHOX7JkvCZHusXZ0tGlQcxxv4bYg= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/zeebo/xxh3 v1.0.1 h1:FMSRIbkrLikb/0hZxmltpg84VkqDAT5M8ufXynuhXsI= +github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= +golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/v1.go b/internal/config/v1.go index 3d9e2dd..905b7c6 100644 --- a/internal/config/v1.go +++ b/internal/config/v1.go @@ -12,6 +12,7 @@ func loadV1Config(content []byte) (*ZurgConfigV1, error) { if err := yaml.Unmarshal(content, &configV1); err != nil { return nil, err } + return &configV1, nil } @@ -52,7 +53,7 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, filter *FilterCon return true } if filter.RegexStr != "" { - regex := regexp.MustCompile(filter.RegexStr) + regex := compilePattern(filter.RegexStr) if regex.MatchString(torrentName) { return true } @@ -88,3 +89,39 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, filter *FilterCon } return false } + +func compilePattern(pattern string) *regexp.Regexp { + flags := map[rune]string{ + 'i': "(?i)", + 'm': "(?m)", + 's': "(?s)", + 'x': "(?x)", + } + + lastSlash := strings.LastIndex(pattern, "/") + secondLastSlash := strings.LastIndex(pattern[:lastSlash], "/") + + // Extract the core pattern + corePattern := pattern[secondLastSlash+1 : lastSlash] + + // Extract and process flags + flagSection := pattern[lastSlash+1:] + flagString := "" + processedFlags := make(map[rune]bool) + for _, flag := range flagSection { + if replacement, ok := flags[flag]; ok && !processedFlags[flag] { + flagString += replacement + processedFlags[flag] = true + } + } + + // Combine the processed flags with the core pattern + finalPattern := flagString + corePattern + + // Validate pattern + if finalPattern == "" || finalPattern == flagString { + return nil + } + + return regexp.MustCompile(finalPattern) +} diff --git a/internal/config/v1types.go b/internal/config/v1types.go index 0d3e43b..0f2e769 100644 --- a/internal/config/v1types.go +++ b/internal/config/v1types.go @@ -3,7 +3,6 @@ package config type ZurgConfigV1 struct { ZurgConfig Directories map[string]*DirectoryFilterConditionsV1 `yaml:"directories"` - Duplicates bool `yaml:"duplicates"` } type DirectoryFilterConditionsV1 struct { Group string `yaml:"group"` diff --git a/internal/dav/propfind.go b/internal/dav/propfind.go index db50625..8381686 100644 --- a/internal/dav/propfind.go +++ b/internal/dav/propfind.go @@ -68,7 +68,7 @@ func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface return xml.MarshalIndent(rootResponse, "", " ") } -// handleListOfTorrents handles a PROPFIND request to the /torrents directory +// handleListOfTorrents handles a PROPFIND request to the base directory func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) ([]byte, error) { basePath := path.Base(requestPath) @@ -79,7 +79,7 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req } } if !found { - log.Println("Cannot find directory", requestPath) + log.Println("Cannot find directory when generating list", requestPath) http.Error(w, "Cannot find directory", http.StatusNotFound) return nil, nil } @@ -96,17 +96,17 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req // handleSingleTorrent handles a PROPFIND request to a single torrent directory func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) { - basePath := path.Dir(requestPath) + directory := strings.TrimPrefix(path.Dir(requestPath), "/") torrentName := path.Base(requestPath) - sameNameTorrents := findAllTorrentsWithName(t, basePath, torrentName) + sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName) if len(sameNameTorrents) == 0 { - log.Println("Cannot find directory", requestPath) + log.Println("Cannot find directory when generating single torrent", requestPath) http.Error(w, "Cannot find directory", http.StatusNotFound) return nil, nil } var resp *dav.MultiStatus - resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", basePath), sameNameTorrents, t) + resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", directory), sameNameTorrents, t) if err != nil { log.Printf("Cannot read directory (%s): %v\n", requestPath, err.Error()) http.Error(w, "Cannot read directory", http.StatusInternalServerError) diff --git a/internal/dav/response.go b/internal/dav/response.go index d026863..86b82f2 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -1,6 +1,7 @@ package dav import ( + "log" "path/filepath" "github.com/debridmediamanager.com/zurg/internal/torrent" @@ -47,6 +48,12 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent, t var torrentResponses []dav.Response for _, torrent := range torrents { for _, file := range torrent.SelectedFiles { + if file.Link == "" { + // TODO: trigger a re-add for the file + log.Println("File has no link, skipping", file.Path) + t.RefreshInfo(torrent.ID) + continue + } filename := filepath.Base(file.Path) fragment := davextra.GetLinkFragment(file.Link) filename = davextra.InsertLinkFragment(filename, fragment) diff --git a/internal/dav/router.go b/internal/dav/router.go index 79f8f22..b1ea876 100644 --- a/internal/dav/router.go +++ b/internal/dav/router.go @@ -1,23 +1,23 @@ package dav import ( + "encoding/xml" + "fmt" "log" "net/http" "os" + "strings" + "github.com/aerogo/aero" "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/torrent" + "github.com/debridmediamanager.com/zurg/pkg/dav" + "github.com/debridmediamanager.com/zurg/pkg/davextra" + "github.com/debridmediamanager.com/zurg/pkg/realdebrid" ) // Router creates a WebDAV router -func Router(mux *http.ServeMux) { - c, err := config.LoadZurgConfig("./config.yml") - if err != nil { - log.Panicf("Config failed to load: %v", err) - } - - t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN"), c) - +func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentManager) { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { switch r.Method { case "PROPFIND": @@ -25,7 +25,6 @@ func Router(mux *http.ServeMux) { case http.MethodGet: HandleGetRequest(w, r, t) - // default return case http.MethodOptions: w.WriteHeader(http.StatusOK) @@ -36,3 +35,110 @@ func Router(mux *http.ServeMux) { } }) } + +func Setup(app *aero.Application, c config.ConfigInterface, t *torrent.TorrentManager) { + // hack to make PROPFIND work + app.Rewrite(func(ctx aero.RewriteContext) { + newCtx := ctx.(aero.Context) + if newCtx.Request().Internal().Method == "PROPFIND" { + newCtx.Request().Internal().Method = http.MethodGet + } + }) + + app.Router().Add(http.MethodOptions, "/", func(ctx aero.Context) error { + ctx.SetStatus(http.StatusOK) + return nil + }) + + // hardcode the root directory + app.Get("/", func(ctx aero.Context) error { + var responses []dav.Response + responses = append(responses, dav.Directory("/")) + for _, directory := range c.GetDirectories() { + responses = append(responses, dav.Directory(fmt.Sprintf("/%s", directory))) + } + resp := dav.MultiStatus{ + XMLNS: "DAV:", + Response: responses, + } + return xmlResponse(ctx, resp) + }) + + for _, directoryPtr := range c.GetDirectories() { + directory := directoryPtr + + app.Get(fmt.Sprintf("/%s/", directory), func(ctx aero.Context) error { + torrentsInDirectory := t.GetByDirectory(directory) + resp, err := createMultiTorrentResponse(fmt.Sprintf("/%s", directory), torrentsInDirectory) + if err != nil { + log.Printf("Cannot read directory (%s): %v\n", directory, err.Error()) + return ctx.Error(http.StatusInternalServerError, "Cannot read directory") + } + return xmlResponse(ctx, *resp) + }) + + app.Get(fmt.Sprintf("/%s/:torrentName/", directory), func(ctx aero.Context) error { + torrentName := ctx.Get("torrentName") + + sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName) + resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", directory), sameNameTorrents, t) + if err != nil { + log.Printf("Cannot read directory (%s): %v\n", directory, err.Error()) + return ctx.Error(http.StatusInternalServerError, "Cannot read directory") + } + return xmlResponse(ctx, *resp) + }) + + app.Get(fmt.Sprintf("/%s/:torrentName/:filename", directory), func(ctx aero.Context) error { + torrentName := strings.TrimSpace(ctx.Get("torrentName")) + filename := strings.TrimSpace(ctx.Get("filename")) + + torrents := findAllTorrentsWithName(t, directory, torrentName) + if torrents == nil { + log.Println("Cannot find torrent", torrentName) + return ctx.Error(http.StatusNotFound, "Cannot find file") + } + + filenameV2, linkFragment := davextra.ExtractLinkFragment(filename) + link := getLink(torrents, filenameV2, linkFragment) + if link == "" { + log.Println("Link not found") + return ctx.Error(http.StatusNotFound, "Cannot find file") + } + + unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { + return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), link) + } + resp := realdebrid.RetryUntilOk(unrestrictFn) + if resp == nil { + // TODO: Readd the file + // when unrestricting fails, it means the file is not available anymore + // if it's the only file, tough luck + log.Println("Cannot unrestrict link") + return ctx.Error(http.StatusNotFound, "Cannot find file") + } + if resp.Filename != filenameV2 { + // TODO: Redo the logic to handle mismatch + // [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar + // Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv + // Action: schedule a "cleanup" job for the parent torrent + // do it in 2 batches with different selections + log.Println("Filename mismatch", resp.Filename, filenameV2) + } + + return ctx.Redirect(http.StatusFound, resp.Download) + }) + } +} + +func xmlResponse(ctx aero.Context, resp dav.MultiStatus) error { + output, err := xml.MarshalIndent(resp, "", " ") + if err != nil { + log.Printf("Cannot marshal xml: %v\n", err.Error()) + return ctx.Error(http.StatusInternalServerError, "Cannot read directory") + } + + ctx.SetStatus(http.StatusMultiStatus) + ctx.Response().SetHeader("Content-Type", "text/xml; charset=\"utf-8\"") + return ctx.String(fmt.Sprintf("\n%s\n", output)) +} diff --git a/internal/dav/util.go b/internal/dav/util.go index 8a64963..97ebba7 100644 --- a/internal/dav/util.go +++ b/internal/dav/util.go @@ -18,7 +18,7 @@ func convertDate(input string) string { return t.Format("Mon, 02 Jan 2006 15:04:05 GMT") } -// findAllTorrentsWithName finds all torrents with a given name +// findAllTorrentsWithName finds all torrents in a given directory with a given name func findAllTorrentsWithName(t *torrent.TorrentManager, directory, torrentName string) []torrent.Torrent { var matchingTorrents []torrent.Torrent diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index fc49f82..e60cbab 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -2,10 +2,12 @@ package torrent import ( "encoding/gob" + "fmt" "log" "os" "strings" "sync" + "time" "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" @@ -16,6 +18,29 @@ type TorrentManager struct { torrents []Torrent workerPool chan bool config config.ConfigInterface + checksum string +} + +func (handler *TorrentManager) refreshTorrents() { + log.Println("Starting periodic refresh") + for { + <-time.After(15 * time.Second) + checksum := handler.getChecksum() + if checksum == handler.checksum { + log.Println("No changes detected, skipping refresh") + continue + } + handler.checksum = checksum + handler.torrents = handler.getAll() + for _, torrent := range handler.torrents { + go func(id string) { + handler.workerPool <- true + handler.getInfo(id) + <-handler.workerPool + time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting + }(torrent.ID) + } + } } // NewTorrentManager creates a new torrent manager @@ -28,27 +53,46 @@ func NewTorrentManager(token string, config config.ConfigInterface) *TorrentMana config: config, } + // Initialize torrents for the first time handler.torrents = handler.getAll() for _, torrent := range handler.torrents { go func(id string) { handler.workerPool <- true handler.getInfo(id) - // sleep for 1 second to avoid rate limiting <-handler.workerPool + time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting }(torrent.ID) } + // Start the periodic refresh + go handler.refreshTorrents() + return handler } +func (t *TorrentManager) getChecksum() string { + torrents, totalCount, err := realdebrid.GetTorrents(t.token, 1) + if err != nil { + log.Printf("Cannot get torrents: %v\n", err.Error()) + return t.checksum + } + if len(torrents) == 0 { + return t.checksum + } + return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) +} + func (t *TorrentManager) getAll() []Torrent { log.Println("Getting all torrents") - torrents, err := realdebrid.GetTorrents(t.token) + + torrents, totalCount, err := realdebrid.GetTorrents(t.token, 0) if err != nil { log.Printf("Cannot get torrents: %v\n", err.Error()) return nil } + t.checksum = fmt.Sprintf("%d-%s", totalCount, torrents[0].ID) + var torrentsV2 []Torrent for _, torrent := range torrents { torrentV2 := Torrent{ @@ -64,7 +108,7 @@ func (t *TorrentManager) getAll() []Torrent { configV1 := t.config.(*config.ZurgConfigV1) groupMap := configV1.GetGroupMap() for group, directories := range groupMap { - log.Printf("Processing group %s\n", group) + log.Printf("Processing directory group: %s, %v\n", group, directories) for i := range torrents { for _, directory := range directories { if configV1.MeetsConditions(directory, torrentsV2[i].ID, torrentsV2[i].Name) { @@ -92,6 +136,28 @@ func (t *TorrentManager) GetByDirectory(directory string) []Torrent { return torrents } +func (t *TorrentManager) RefreshInfo(torrentID string) { + filePath := fmt.Sprintf("data/%s.bin", torrentID) + // Check the last modified time of the .bin file + fileInfo, err := os.Stat(filePath) + if err == nil { + modTime := fileInfo.ModTime() + // If the file was modified less than an hour ago, don't refresh + if time.Since(modTime) < time.Hour { + return + } + err = os.Remove(filePath) + if err != nil && !os.IsNotExist(err) { // File doesn't exist or other error + log.Printf("Cannot remove file: %v\n", err.Error()) + } + } else if !os.IsNotExist(err) { // Error other than file not existing + log.Printf("Error checking file info: %v\n", err.Error()) + return + } + info := t.getInfo(torrentID) + log.Println("Refreshed info for", info.Name) +} + func (t *TorrentManager) getInfo(torrentID string) *Torrent { torrentFromFile := t.readFromFile(torrentID) if torrentFromFile != nil { @@ -101,6 +167,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { } return torrent } + log.Println("Getting info for", torrentID) info, err := realdebrid.GetTorrentInfo(t.token, torrentID) if err != nil { log.Printf("Cannot get info: %v\n", err.Error()) @@ -117,6 +184,9 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { }) } if len(selectedFiles) != len(info.Links) { + // TODO: This means some files have expired + // we need to re-add the torrent + log.Println("Some links has expired for", info.Name) type Result struct { Filename string Link string @@ -136,7 +206,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { defer func() { <-sem }() // Release semaphore unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { - return realdebrid.UnrestrictLink(t.token, lnk) + return realdebrid.UnrestrictCheck(t.token, lnk) } resp := realdebrid.RetryUntilOk(unrestrictFn) if resp != nil { @@ -147,6 +217,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { go func() { wg.Wait() + close(sem) close(resultsChan) }() @@ -191,7 +262,7 @@ func (t *TorrentManager) getByID(torrentID string) *Torrent { } func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { - filePath := "data/" + torrentID + ".bin" + filePath := fmt.Sprintf("data/%s.bin", torrentID) file, err := os.Create(filePath) if err != nil { log.Fatalf("Failed creating file: %s", err) @@ -204,7 +275,7 @@ func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { } func (t *TorrentManager) readFromFile(torrentID string) *Torrent { - filePath := "data/" + torrentID + ".bin" + filePath := fmt.Sprintf("data/%s.bin", torrentID) file, err := os.Open(filePath) if err != nil { return nil diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 6aa3aa5..b7078b5 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -84,11 +84,17 @@ func UnrestrictLink(accessToken, link string) (*UnrestrictResponse, error) { return &response, nil } -func GetTorrents(accessToken string) ([]Torrent, error) { +// GetTorrents returns all torrents, paginated +// if customLimit is 0, the default limit of 2500 is used +func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) { baseURL := "https://api.real-debrid.com/rest/1.0/torrents" var allTorrents []Torrent page := 1 - limit := 100 + limit := customLimit + if limit == 0 { + limit = 2500 + } + totalCount := 0 for { params := url.Values{} @@ -99,7 +105,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { req, err := http.NewRequest("GET", reqURL, nil) if err != nil { - return nil, err + return nil, 0, err } req.Header.Set("Authorization", "Bearer "+accessToken) @@ -107,37 +113,37 @@ func GetTorrents(accessToken string) ([]Torrent, error) { client := &http.Client{} resp, err := client.Do(req) if err != nil { - return nil, err + return nil, 0, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP error: %s", resp.Status) + return nil, 0, fmt.Errorf("HTTP error: %s", resp.Status) } var torrents []Torrent decoder := json.NewDecoder(resp.Body) err = decoder.Decode(&torrents) if err != nil { - return nil, err + return nil, 0, err } allTorrents = append(allTorrents, torrents...) - totalCountHeader := "100" // resp.Header.Get("x-total-count") + totalCountHeader := resp.Header.Get("x-total-count") totalCount, err := strconv.Atoi(totalCountHeader) if err != nil { break } - if len(torrents) < limit || len(allTorrents) >= totalCount { + if len(allTorrents) >= totalCount || (customLimit != 0 && customLimit <= len(allTorrents) && customLimit <= totalCount) { break } page++ } - return allTorrents, nil + return allTorrents, totalCount, nil } func GetTorrentInfo(accessToken, id string) (*Torrent, error) { diff --git a/pkg/realdebrid/util.go b/pkg/realdebrid/util.go index 1f8507e..aaf4a54 100644 --- a/pkg/realdebrid/util.go +++ b/pkg/realdebrid/util.go @@ -7,11 +7,11 @@ import ( ) func RetryUntilOk[T any](fn func() (T, error)) T { - const initialDelay = 2 * time.Second + const initialDelay = 1 * time.Second const maxDelay = 128 * time.Second for i := 0; ; i++ { result, err := fn() - if err == nil || strings.Contains(err.Error(), "404") { + if err == nil || !strings.Contains(err.Error(), "429") { return result } delay := time.Duration(math.Min(float64(initialDelay*time.Duration(math.Pow(2, float64(i)))), float64(maxDelay)))