hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish config mapping

Browse Source
This commit is contained in:
Ben Sarmiento
2023-10-19 18:02:30 +02:00
parent ce0b2445b2
commit faba4e53ab
14 changed files with 397 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/config/v1.go
View File

@@ -12,6 +12,7 @@ func loadV1Config(content []byte) (*ZurgConfigV1, error) {
if err := yaml.Unmarshal(content, &configV1); err != nil {
return nil, err
}
return &configV1, nil
}
@@ -52,7 +53,7 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, filter *FilterCon
return true
}
if filter.RegexStr != "" {
regex := regexp.MustCompile(filter.RegexStr)
regex := compilePattern(filter.RegexStr)
if regex.MatchString(torrentName) {
return true
}
@@ -88,3 +89,39 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, filter *FilterCon
}
return false
}
func compilePattern(pattern string) *regexp.Regexp {
flags := map[rune]string{
'i': "(?i)",
'm': "(?m)",
's': "(?s)",
'x': "(?x)",
}
lastSlash := strings.LastIndex(pattern, "/")
secondLastSlash := strings.LastIndex(pattern[:lastSlash], "/")
// Extract the core pattern
corePattern := pattern[secondLastSlash+1 : lastSlash]
// Extract and process flags
flagSection := pattern[lastSlash+1:]
flagString := ""
processedFlags := make(map[rune]bool)
for _, flag := range flagSection {
if replacement, ok := flags[flag]; ok && !processedFlags[flag] {
flagString += replacement
processedFlags[flag] = true
}
}
// Combine the processed flags with the core pattern
finalPattern := flagString + corePattern
// Validate pattern
if finalPattern == "" || finalPattern == flagString {
return nil
}
return regexp.MustCompile(finalPattern)
}

1
internal/config/v1types.go
View File

@@ -3,7 +3,6 @@ package config
type ZurgConfigV1 struct {
ZurgConfig
Directories map[string]*DirectoryFilterConditionsV1 `yaml:"directories"`
Duplicates bool `yaml:"duplicates"`
}
type DirectoryFilterConditionsV1 struct {
Group string `yaml:"group"`

12
internal/dav/propfind.go
View File

@@ -68,7 +68,7 @@ func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface
return xml.MarshalIndent(rootResponse, "", " ")
}
// handleListOfTorrents handles a PROPFIND request to the /torrents directory
// handleListOfTorrents handles a PROPFIND request to the base directory
func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) ([]byte, error) {
basePath := path.Base(requestPath)
@@ -79,7 +79,7 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
}
}
if !found {
log.Println("Cannot find directory", requestPath)
log.Println("Cannot find directory when generating list", requestPath)
http.Error(w, "Cannot find directory", http.StatusNotFound)
return nil, nil
}
@@ -96,17 +96,17 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
// handleSingleTorrent handles a PROPFIND request to a single torrent directory
func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) {
basePath := path.Dir(requestPath)
directory := strings.TrimPrefix(path.Dir(requestPath), "/")
torrentName := path.Base(requestPath)
sameNameTorrents := findAllTorrentsWithName(t, basePath, torrentName)
sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName)
if len(sameNameTorrents) == 0 {
log.Println("Cannot find directory", requestPath)
log.Println("Cannot find directory when generating single torrent", requestPath)
http.Error(w, "Cannot find directory", http.StatusNotFound)
return nil, nil
}
var resp *dav.MultiStatus
resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", basePath), sameNameTorrents, t)
resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", directory), sameNameTorrents, t)
if err != nil {
log.Printf("Cannot read directory (%s): %v\n", requestPath, err.Error())
http.Error(w, "Cannot read directory", http.StatusInternalServerError)

7
internal/dav/response.go
View File

@@ -1,6 +1,7 @@
package dav
import (
"log"
"path/filepath"
"github.com/debridmediamanager.com/zurg/internal/torrent"
@@ -47,6 +48,12 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent, t
var torrentResponses []dav.Response
for _, torrent := range torrents {
for _, file := range torrent.SelectedFiles {
if file.Link == "" {
// TODO: trigger a re-add for the file
log.Println("File has no link, skipping", file.Path)
t.RefreshInfo(torrent.ID)
continue
}
filename := filepath.Base(file.Path)
fragment := davextra.GetLinkFragment(file.Link)
filename = davextra.InsertLinkFragment(filename, fragment)

124
internal/dav/router.go
View File

@@ -1,23 +1,23 @@
package dav
import (
"encoding/xml"
"fmt"
"log"
"net/http"
"os"
"strings"
"github.com/aerogo/aero"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/dav"
"github.com/debridmediamanager.com/zurg/pkg/davextra"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
)
// Router creates a WebDAV router
func Router(mux *http.ServeMux) {
c, err := config.LoadZurgConfig("./config.yml")
if err != nil {
log.Panicf("Config failed to load: %v", err)
}
t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN"), c)
func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentManager) {
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "PROPFIND":
@@ -25,7 +25,6 @@ func Router(mux *http.ServeMux) {
case http.MethodGet:
HandleGetRequest(w, r, t)
// default return
case http.MethodOptions:
w.WriteHeader(http.StatusOK)
@@ -36,3 +35,110 @@ func Router(mux *http.ServeMux) {
}
})
}
func Setup(app *aero.Application, c config.ConfigInterface, t *torrent.TorrentManager) {
// hack to make PROPFIND work
app.Rewrite(func(ctx aero.RewriteContext) {
newCtx := ctx.(aero.Context)
if newCtx.Request().Internal().Method == "PROPFIND" {
newCtx.Request().Internal().Method = http.MethodGet
}
})
app.Router().Add(http.MethodOptions, "/", func(ctx aero.Context) error {
ctx.SetStatus(http.StatusOK)
return nil
})
// hardcode the root directory
app.Get("/", func(ctx aero.Context) error {
var responses []dav.Response
responses = append(responses, dav.Directory("/"))
for _, directory := range c.GetDirectories() {
responses = append(responses, dav.Directory(fmt.Sprintf("/%s", directory)))
}
resp := dav.MultiStatus{
XMLNS: "DAV:",
Response: responses,
}
return xmlResponse(ctx, resp)
})
for _, directoryPtr := range c.GetDirectories() {
directory := directoryPtr
app.Get(fmt.Sprintf("/%s/", directory), func(ctx aero.Context) error {
torrentsInDirectory := t.GetByDirectory(directory)
resp, err := createMultiTorrentResponse(fmt.Sprintf("/%s", directory), torrentsInDirectory)
if err != nil {
log.Printf("Cannot read directory (%s): %v\n", directory, err.Error())
return ctx.Error(http.StatusInternalServerError, "Cannot read directory")
}
return xmlResponse(ctx, *resp)
})
app.Get(fmt.Sprintf("/%s/:torrentName/", directory), func(ctx aero.Context) error {
torrentName := ctx.Get("torrentName")
sameNameTorrents := findAllTorrentsWithName(t, directory, torrentName)
resp, err := createSingleTorrentResponse(fmt.Sprintf("/%s", directory), sameNameTorrents, t)
if err != nil {
log.Printf("Cannot read directory (%s): %v\n", directory, err.Error())
return ctx.Error(http.StatusInternalServerError, "Cannot read directory")
}
return xmlResponse(ctx, *resp)
})
app.Get(fmt.Sprintf("/%s/:torrentName/:filename", directory), func(ctx aero.Context) error {
torrentName := strings.TrimSpace(ctx.Get("torrentName"))
filename := strings.TrimSpace(ctx.Get("filename"))
torrents := findAllTorrentsWithName(t, directory, torrentName)
if torrents == nil {
log.Println("Cannot find torrent", torrentName)
return ctx.Error(http.StatusNotFound, "Cannot find file")
}
filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
link := getLink(torrents, filenameV2, linkFragment)
if link == "" {
log.Println("Link not found")
return ctx.Error(http.StatusNotFound, "Cannot find file")
}
unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), link)
}
resp := realdebrid.RetryUntilOk(unrestrictFn)
if resp == nil {
// TODO: Readd the file
// when unrestricting fails, it means the file is not available anymore
// if it's the only file, tough luck
log.Println("Cannot unrestrict link")
return ctx.Error(http.StatusNotFound, "Cannot find file")
}
if resp.Filename != filenameV2 {
// TODO: Redo the logic to handle mismatch
// [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar
// Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv
// Action: schedule a "cleanup" job for the parent torrent
// do it in 2 batches with different selections
log.Println("Filename mismatch", resp.Filename, filenameV2)
}
return ctx.Redirect(http.StatusFound, resp.Download)
})
}
}
func xmlResponse(ctx aero.Context, resp dav.MultiStatus) error {
output, err := xml.MarshalIndent(resp, "", " ")
if err != nil {
log.Printf("Cannot marshal xml: %v\n", err.Error())
return ctx.Error(http.StatusInternalServerError, "Cannot read directory")
}
ctx.SetStatus(http.StatusMultiStatus)
ctx.Response().SetHeader("Content-Type", "text/xml; charset=\"utf-8\"")
return ctx.String(fmt.Sprintf("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n%s\n", output))
}

2
internal/dav/util.go
View File

@@ -18,7 +18,7 @@ func convertDate(input string) string {
return t.Format("Mon, 02 Jan 2006 15:04:05 GMT")
}
// findAllTorrentsWithName finds all torrents with a given name
// findAllTorrentsWithName finds all torrents in a given directory with a given name
func findAllTorrentsWithName(t *torrent.TorrentManager, directory, torrentName string) []torrent.Torrent {
var matchingTorrents []torrent.Torrent

83
internal/torrent/manager.go
View File

@@ -2,10 +2,12 @@ package torrent
import (
"encoding/gob"
"fmt"
"log"
"os"
"strings"
"sync"
"time"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
@@ -16,6 +18,29 @@ type TorrentManager struct {
torrents []Torrent
workerPool chan bool
config config.ConfigInterface
checksum string
}
func (handler *TorrentManager) refreshTorrents() {
log.Println("Starting periodic refresh")
for {
<-time.After(15 * time.Second)
checksum := handler.getChecksum()
if checksum == handler.checksum {
log.Println("No changes detected, skipping refresh")
continue
}
handler.checksum = checksum
handler.torrents = handler.getAll()
for _, torrent := range handler.torrents {
go func(id string) {
handler.workerPool <- true
handler.getInfo(id)
<-handler.workerPool
time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting
}(torrent.ID)
}
}
}
// NewTorrentManager creates a new torrent manager
@@ -28,27 +53,46 @@ func NewTorrentManager(token string, config config.ConfigInterface) *TorrentMana
config: config,
}
// Initialize torrents for the first time
handler.torrents = handler.getAll()
for _, torrent := range handler.torrents {
go func(id string) {
handler.workerPool <- true
handler.getInfo(id)
// sleep for 1 second to avoid rate limiting
<-handler.workerPool
time.Sleep(1 * time.Second) // sleep for 1 second to avoid rate limiting
}(torrent.ID)
}
// Start the periodic refresh
go handler.refreshTorrents()
return handler
}
func (t *TorrentManager) getChecksum() string {
torrents, totalCount, err := realdebrid.GetTorrents(t.token, 1)
if err != nil {
log.Printf("Cannot get torrents: %v\n", err.Error())
return t.checksum
}
if len(torrents) == 0 {
return t.checksum
}
return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID)
}
func (t *TorrentManager) getAll() []Torrent {
log.Println("Getting all torrents")
torrents, err := realdebrid.GetTorrents(t.token)
torrents, totalCount, err := realdebrid.GetTorrents(t.token, 0)
if err != nil {
log.Printf("Cannot get torrents: %v\n", err.Error())
return nil
}
t.checksum = fmt.Sprintf("%d-%s", totalCount, torrents[0].ID)
var torrentsV2 []Torrent
for _, torrent := range torrents {
torrentV2 := Torrent{
@@ -64,7 +108,7 @@ func (t *TorrentManager) getAll() []Torrent {
configV1 := t.config.(*config.ZurgConfigV1)
groupMap := configV1.GetGroupMap()
for group, directories := range groupMap {
log.Printf("Processing group %s\n", group)
log.Printf("Processing directory group: %s, %v\n", group, directories)
for i := range torrents {
for _, directory := range directories {
if configV1.MeetsConditions(directory, torrentsV2[i].ID, torrentsV2[i].Name) {
@@ -92,6 +136,28 @@ func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
return torrents
}
func (t *TorrentManager) RefreshInfo(torrentID string) {
filePath := fmt.Sprintf("data/%s.bin", torrentID)
// Check the last modified time of the .bin file
fileInfo, err := os.Stat(filePath)
if err == nil {
modTime := fileInfo.ModTime()
// If the file was modified less than an hour ago, don't refresh
if time.Since(modTime) < time.Hour {
return
}
err = os.Remove(filePath)
if err != nil && !os.IsNotExist(err) { // File doesn't exist or other error
log.Printf("Cannot remove file: %v\n", err.Error())
}
} else if !os.IsNotExist(err) { // Error other than file not existing
log.Printf("Error checking file info: %v\n", err.Error())
return
}
info := t.getInfo(torrentID)
log.Println("Refreshed info for", info.Name)
}
func (t *TorrentManager) getInfo(torrentID string) *Torrent {
torrentFromFile := t.readFromFile(torrentID)
if torrentFromFile != nil {
@@ -101,6 +167,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
}
return torrent
}
log.Println("Getting info for", torrentID)
info, err := realdebrid.GetTorrentInfo(t.token, torrentID)
if err != nil {
log.Printf("Cannot get info: %v\n", err.Error())
@@ -117,6 +184,9 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
})
}
if len(selectedFiles) != len(info.Links) {
// TODO: This means some files have expired
// we need to re-add the torrent
log.Println("Some links has expired for", info.Name)
type Result struct {
Filename string
Link string
@@ -136,7 +206,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
defer func() { <-sem }() // Release semaphore
unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
return realdebrid.UnrestrictLink(t.token, lnk)
return realdebrid.UnrestrictCheck(t.token, lnk)
}
resp := realdebrid.RetryUntilOk(unrestrictFn)
if resp != nil {
@@ -147,6 +217,7 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
go func() {
wg.Wait()
close(sem)
close(resultsChan)
}()
@@ -191,7 +262,7 @@ func (t *TorrentManager) getByID(torrentID string) *Torrent {
}
func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) {
filePath := "data/" + torrentID + ".bin"
filePath := fmt.Sprintf("data/%s.bin", torrentID)
file, err := os.Create(filePath)
if err != nil {
log.Fatalf("Failed creating file: %s", err)
@@ -204,7 +275,7 @@ func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) {
}
func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
filePath := "data/" + torrentID + ".bin"
filePath := fmt.Sprintf("data/%s.bin", torrentID)
file, err := os.Open(filePath)
if err != nil {
return nil