From 0886c93250da9e508bab6ae2cd70b5c768005c06 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Wed, 18 Oct 2023 00:17:07 +0200 Subject: [PATCH] A lot of rewrite here --- cmd/zurg/main.go | 8 +- go.mod | 21 ++-- go.sum | 29 +++-- internal/dav/getfile.go | 91 ++++++++++++++++ internal/dav/propfind.go | 82 ++++++++++++++ internal/dav/response.go | 123 +++++++++++---------- internal/dav/router.go | 114 ++----------------- internal/dav/util.go | 22 +++- internal/torrent/cache.go | 1 + internal/torrent/manager.go | 93 ++++++++++++++++ pkg/realdebrid/api.go | 73 +------------ pkg/realdebrid/types.go | 2 +- pkg/repo/mysql.go | 211 ------------------------------------ pkg/repo/types.go | 11 -- 14 files changed, 399 insertions(+), 482 deletions(-) create mode 100644 internal/dav/getfile.go create mode 100644 internal/dav/propfind.go create mode 100644 internal/torrent/cache.go create mode 100644 internal/torrent/manager.go delete mode 100644 pkg/repo/mysql.go delete mode 100644 pkg/repo/types.go diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go index c5d6c51..e956348 100644 --- a/cmd/zurg/main.go +++ b/cmd/zurg/main.go @@ -3,20 +3,14 @@ package main import ( "log" "net/http" - "os" "github.com/debridmediamanager.com/zurg/internal/dav" - "github.com/debridmediamanager.com/zurg/pkg/repo" ) func main() { mux := http.NewServeMux() - db, dbErr := repo.NewDatabase(os.Getenv("DB_DSN")) - if dbErr != nil { - log.Println(dbErr) - } - dav.Router(mux, db) + dav.Router(mux) log.Println("Listening on port 8123...") err := http.ListenAndServe(":8123", mux) diff --git a/go.mod b/go.mod index 7148f83..1609985 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,17 @@ module github.com/debridmediamanager.com/zurg go 1.21.3 require ( - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/qianbin/directcache v0.9.7 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect - google.golang.org/protobuf v1.26.0 // indirect + github.com/go-sql-driver/mysql v1.7.1 + github.com/qianbin/directcache v0.9.7 + github.com/zeebo/xxh3 v1.0.2 +) + +require ( + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect ) diff --git a/go.sum b/go.sum index a16b321..d578d4e 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,35 @@ +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/qianbin/directcache v0.9.7 h1:DH6MdmU0fVjcKry57ju7U6akTFDBnLhHd0xOHZDq948= github.com/qianbin/directcache v0.9.7/go.mod h1:gZBpa9NqO1Qz7wZKO7t7atBA76bT8X0eM01PdveW4qc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/dav/getfile.go b/internal/dav/getfile.go new file mode 100644 index 0000000..18bc093 --- /dev/null +++ b/internal/dav/getfile.go @@ -0,0 +1,91 @@ +package dav + +import ( + "fmt" + "log" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/debridmediamanager.com/zurg/internal/torrent" + "github.com/debridmediamanager.com/zurg/pkg/realdebrid" +) + +// HandleGetRequest handles a GET request to a file +func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) { + requestPath := path.Clean(r.URL.Path) + + segments := strings.Split(requestPath, "/") + // If there are less than 3 segments, return an error or adjust as needed + if len(segments) < 3 { + log.Println("Invalid url", requestPath) + http.Error(w, "Cannot find file", http.StatusNotFound) + } + + // Get the last two segments + torrentName := segments[len(segments)-2] + torrents := findAllTorrentsWithName(t, torrentName) + if torrents == nil { + log.Println("Cannot find directory", requestPath) + http.Error(w, "Cannot find file", http.StatusNotFound) + return + } + + filename := segments[len(segments)-1] + + filenameV2, linkFragment := extractIDFromFilename(filename) + link := findLinkByFragment(torrents, linkFragment) + + unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { + return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), link) + } + resp := realdebrid.RetryUntilOk(unrestrictFn) + if resp == nil { + // TODO: Delete the link from the database + log.Println("Cannot unrestrict link") + http.Error(w, "Cannot find file", http.StatusNotFound) + return + } + if resp.Filename != filenameV2 { + // TODO: Redo the logic to handle mismatch + log.Println("Filename mismatch", resp.Filename, filenameV2) + } + http.Redirect(w, r, resp.Download, http.StatusFound) +} + +// extractIDFromFilename extracts the link ID from a filename +func extractIDFromFilename(filename string) (string, string) { + filenameV2, err := url.PathUnescape(filename) + if err != nil { + filenameV2 = filename + } + ext := filepath.Ext(filenameV2) + name := strings.TrimSuffix(filenameV2, ext) + + r := regexp.MustCompile(`\sDMM(\w+)`) + matches := r.FindStringSubmatch(name) + if len(matches) < 2 { + // No ID found + return filenameV2, "" + } + + // Remove ID from filename + originalName := strings.Replace(name, matches[0], "", 1) + return originalName + ext, matches[1] +} + +// findLinkByFragment finds a link by a fragment, it might be wrong +func findLinkByFragment(torrents []realdebrid.Torrent, fragment string) string { + for _, torrent := range torrents { + for _, link := range torrent.Links { + if strings.HasPrefix(link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) { + return link + } + } + } + return "" +} diff --git a/internal/dav/propfind.go b/internal/dav/propfind.go new file mode 100644 index 0000000..d2b9d80 --- /dev/null +++ b/internal/dav/propfind.go @@ -0,0 +1,82 @@ +package dav + +import ( + "encoding/xml" + "fmt" + "log" + "net/http" + "path" + + "github.com/debridmediamanager.com/zurg/internal/torrent" + "github.com/debridmediamanager.com/zurg/pkg/dav" +) + +// HandlePropfindRequest handles a PROPFIND request +func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) { + var output []byte + var err error + + requestPath := path.Clean(r.URL.Path) + if requestPath == "/" { + output, err = handleRoot(w, r) + } else if requestPath == "/torrents" { + output, err = handleListOfTorrents(w, r, t) + } else { + output, err = handleSingleTorrent(w, r, t) + } + if err != nil { + log.Printf("Cannot marshal xml: %v\n", err.Error()) + http.Error(w, fmt.Sprintf("Cannot marshal xml: %v", err.Error()), http.StatusInternalServerError) + return + } + if output != nil { + w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"") + w.WriteHeader(http.StatusMultiStatus) + fmt.Fprintf(w, "\n%s\n", output) + return + } +} + +// handleRoot handles a PROPFIND request to the root directory +func handleRoot(w http.ResponseWriter, r *http.Request) ([]byte, error) { + rootResponse := dav.MultiStatus{ + XMLNS: "DAV:", + Response: []dav.Response{ + dav.Directory("/"), + dav.Directory("/torrents"), + }, + } + return xml.MarshalIndent(rootResponse, "", " ") +} + +// handleListOfTorrents handles a PROPFIND request to the /torrents directory +func handleListOfTorrents(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) { + allTorrents := t.GetAll() + allTorrentsResponse, err := createMultiTorrentResponse(allTorrents) + if err != nil { + log.Printf("Cannot read directory: %v\n", err.Error()) + http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) + return nil, nil + } + return xml.MarshalIndent(allTorrentsResponse, "", " ") +} + +// handleSingleTorrent handles a PROPFIND request to a single torrent directory +func handleSingleTorrent(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) { + requestPath := path.Clean(r.URL.Path) + torrentName := path.Base(requestPath) + foundTorrents := findAllTorrentsWithName(t, torrentName) + if len(foundTorrents) == 0 { + log.Println("Cannot find directory") + http.Error(w, "Cannot find directory", http.StatusNotFound) + return nil, nil + } + var torrentResponse *dav.MultiStatus + torrentResponse, err := createCombinedTorrentResponse(foundTorrents, t) + if err != nil { + log.Printf("Cannot read directory: %v\n", err.Error()) + http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) + return nil, nil + } + return xml.MarshalIndent(torrentResponse, "", " ") +} diff --git a/internal/dav/response.go b/internal/dav/response.go index e35510e..d7072be 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -1,26 +1,29 @@ package dav import ( - "os" + "fmt" "path/filepath" + "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/dav" - "github.com/debridmediamanager.com/zurg/pkg/davextra" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" - "github.com/debridmediamanager.com/zurg/pkg/repo" ) +// createMultiTorrentResponse creates a WebDAV response for a list of torrents func createMultiTorrentResponse(torrents []realdebrid.Torrent) (*dav.MultiStatus, error) { var responses []dav.Response - - // initial response is the directory itself responses = append(responses, dav.Directory("/torrents")) - // add all files and directories in the directory + seen := make(map[string]bool) + for _, item := range torrents { if item.Progress != 100 { continue } + if _, exists := seen[item.Filename]; exists { + continue + } + seen[item.Filename] = true path := filepath.Join("/torrents", item.Filename) responses = append(responses, dav.Directory(path)) @@ -32,66 +35,76 @@ func createMultiTorrentResponse(torrents []realdebrid.Torrent) (*dav.MultiStatus }, nil } -func createSingleTorrentResponse(torrent realdebrid.Torrent, db *repo.Database) (*dav.MultiStatus, error) { +// createTorrentResponse creates a WebDAV response for torrents with the same name +func createCombinedTorrentResponse(torrents []realdebrid.Torrent, t *torrent.TorrentManager) (*dav.MultiStatus, error) { var responses []dav.Response - // initial response is the directory itself - currentPath := filepath.Join("/torrents", torrent.Filename) + currentPath := filepath.Join("/torrents", torrents[0].Filename) responses = append(responses, dav.Directory(currentPath)) - davFiles, err := db.GetMultiple(torrent.Hash) - if err != nil { - return nil, err - } + seen := make(map[string]bool) + idx := 0 - // Create a map for O(1) lookups of the cached links - cachedLinksMap := make(map[string]*repo.DavFile) - for _, cached := range davFiles.Files { - cachedLinksMap[cached.Link] = cached - } - for _, link := range torrent.Links { - if unrestrict, exists := cachedLinksMap[link]; exists { - if unrestrict.Filesize == 0 { - // This link is cached but the filesize is 0 - // This means that the link is dead + var torrentResponses []dav.Response + for _, torrent := range torrents { + info := t.GetInfo(torrent.ID) + if info == nil { + continue + } + + var selectedFiles []realdebrid.File + for _, file := range info.Files { + if file.Selected == 0 { continue } - filenameV2 := davextra.InsertLinkFragment(unrestrict.Filename, davextra.GetLinkFragment(unrestrict.Link)) - path := filepath.Join(currentPath, filenameV2) - response := dav.File( - path, - unrestrict.Filesize, - convertDate(torrent.Added), - link, - ) - responses = append(responses, response) + selectedFiles = append(selectedFiles, file) + } + + if len(selectedFiles) != len(info.Links) { + fmt.Println("Links and files do not match", info.Filename) + // TODO: Add auto-healing for this + // for _, link := range info.Links { + // unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { + // return realdebrid.UnrestrictCheck(os.Getenv("RD_TOKEN"), link) + // } + // resp := realdebrid.RetryUntilOk(unrestrictFn) + // if resp == nil { + // continue + // } else { + // if _, exists := seen[resp.Filename]; exists { + // continue + // } + // seen[resp.Filename] = true + // filePath := filepath.Join(currentPath, resp.Filename) + // torrentResponses = append(torrentResponses, + // dav.File( + // filePath, + // resp.Filesize, + // info.Added, + // resp.Link, + // ), + // ) + // } + // } } else { - // This link is not cached yet - unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { - return realdebrid.UnrestrictCheck(os.Getenv("RD_TOKEN"), link) + for _, file := range selectedFiles { + filename := filepath.Base(file.Path) + if _, exists := seen[filename]; exists { + continue + } + seen[filename] = true + filePath := filepath.Join(currentPath, filename) + torrentResponses = append(torrentResponses, dav.File( + filePath, + file.Bytes, + convertDate(info.Added), + info.Links[idx], + )) + idx++ } - unrestrict := realdebrid.RetryUntilOk(unrestrictFn) - if unrestrict == nil { - db.Insert(torrent.Hash, torrent.Filename, realdebrid.UnrestrictResponse{ - Filename: "", - Filesize: 0, - Link: link, - Host: "", - }) - continue - } - db.Insert(torrent.Hash, torrent.Filename, *unrestrict) - filenameV2 := davextra.InsertLinkFragment(unrestrict.Filename, davextra.GetLinkFragment(unrestrict.Link)) - path := filepath.Join(currentPath, filenameV2) - response := dav.File( - path, - unrestrict.Filesize, - convertDate(torrent.Added), - link, - ) - responses = append(responses, response) } } + responses = append(responses, torrentResponses...) return &dav.MultiStatus{ XMLNS: "DAV:", diff --git a/internal/dav/router.go b/internal/dav/router.go index a259e3e..6b47ed2 100644 --- a/internal/dav/router.go +++ b/internal/dav/router.go @@ -1,42 +1,17 @@ package dav import ( - "encoding/xml" - "fmt" "log" "net/http" "os" "path" - "strings" - "github.com/debridmediamanager.com/zurg/pkg/dav" - "github.com/debridmediamanager.com/zurg/pkg/realdebrid" - "github.com/debridmediamanager.com/zurg/pkg/repo" + "github.com/debridmediamanager.com/zurg/internal/torrent" ) -func findTorrentByName(torrents []realdebrid.Torrent, filename string) *realdebrid.Torrent { - for _, torrent := range torrents { - if torrent.Filename == filename { - return &torrent - } - } - return nil -} - -func Router(mux *http.ServeMux, db *repo.Database) { - torrents, err := realdebrid.GetTorrents(os.Getenv("RD_TOKEN")) - if err != nil { - log.Printf("Cannot get torrents: %v", err.Error()) - return - } - - rootResponse := dav.MultiStatus{ - XMLNS: "DAV:", - Response: []dav.Response{ - dav.Directory("/"), - dav.Directory("/torrents"), - }, - } +// Router creates a WebDAV router +func Router(mux *http.ServeMux) { + t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN")) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { requestPath := path.Clean(r.URL.Path) @@ -45,89 +20,14 @@ func Router(mux *http.ServeMux, db *repo.Database) { switch r.Method { case "PROPFIND": - var output []byte - var err error + HandlePropfindRequest(w, r, t) - if requestPath == "/" { - output, err = xml.MarshalIndent(rootResponse, "", " ") - } else if requestPath == "/torrents" { - var allTorrentsResponse *dav.MultiStatus - allTorrentsResponse, err = createMultiTorrentResponse(torrents) - if err != nil { - log.Printf("Cannot read directory: %v", err.Error()) - http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) - return - } - output, err = xml.MarshalIndent(allTorrentsResponse, "", " ") - } else { - torrentName := path.Base(requestPath) - torrent := findTorrentByName(torrents, torrentName) - if torrent == nil { - log.Println("Cannot find directory") - http.Error(w, "Cannot find directory", http.StatusNotFound) - return - } - - var torrentResponse *dav.MultiStatus - torrentResponse, err = createSingleTorrentResponse(*torrent, db) - if err != nil { - log.Printf("Cannot read directory: %v", err.Error()) - http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) - return - } - output, err = xml.MarshalIndent(torrentResponse, "", " ") - } - - if err != nil { - log.Printf("Cannot marshal xml: %v", err.Error()) - http.Error(w, fmt.Sprintf("Cannot marshal xml: %v", err.Error()), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"") - w.WriteHeader(http.StatusMultiStatus) - fmt.Fprintf(w, "\n%s\n", output) + case http.MethodGet: + HandleGetRequest(w, r, t) case http.MethodOptions: w.WriteHeader(http.StatusOK) - case http.MethodGet: - segments := strings.Split(requestPath, "/") - // If there are less than 3 segments, return an error or adjust as needed - if len(segments) < 3 { - log.Println("Cannot find file") - http.Error(w, "Cannot find file", http.StatusNotFound) - } - - // Get the last two segments - torrentName := segments[len(segments)-2] - torrent := findTorrentByName(torrents, torrentName) - if torrent == nil { - log.Println("Cannot find directory") - http.Error(w, "Cannot find directory", http.StatusNotFound) - return - } - - filename := segments[len(segments)-1] - unrestrict, dbErr := db.Get(torrent.Hash, filename) - if dbErr != nil { - log.Printf("Cannot find file in db: %v", dbErr.Error()) - http.Error(w, fmt.Sprintf("Cannot find file in db: %v", dbErr.Error()), http.StatusInternalServerError) - return - } - - unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { - return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), unrestrict.Link) - } - resp := realdebrid.RetryUntilOk(unrestrictFn) - if resp == nil { - // TODO: Delete the link from the database - log.Printf("Cannot unrestrict link: %v", err.Error()) - http.Error(w, fmt.Sprintf("Cannot unrestrict link: %v", err.Error()), http.StatusNotFound) - return - } - http.Redirect(w, r, resp.Download, http.StatusFound) - default: log.Println("Method not implemented") http.Error(w, "Method not implemented", http.StatusMethodNotAllowed) diff --git a/internal/dav/util.go b/internal/dav/util.go index ac35cee..d0dfe0b 100644 --- a/internal/dav/util.go +++ b/internal/dav/util.go @@ -1,15 +1,33 @@ package dav import ( - "fmt" + "log" "time" + + "github.com/debridmediamanager.com/zurg/internal/torrent" + "github.com/debridmediamanager.com/zurg/pkg/realdebrid" ) +// convertDate converts a date from RFC3339 to RFC1123 func convertDate(input string) string { t, err := time.Parse(time.RFC3339, input) if err != nil { - fmt.Println("Error:", err) + log.Println("Error:", err) return "" } return t.Format("Mon, 02 Jan 2006 15:04:05 GMT") } + +// findAllTorrentsWithName finds all torrents with a given name +func findAllTorrentsWithName(t *torrent.TorrentManager, filename string) []realdebrid.Torrent { + var matchingTorrents []realdebrid.Torrent + + torrents := t.GetAll() + for _, torrent := range torrents { + if torrent.Filename == filename { + matchingTorrents = append(matchingTorrents, torrent) + } + } + + return matchingTorrents +} diff --git a/internal/torrent/cache.go b/internal/torrent/cache.go new file mode 100644 index 0000000..10cbafc --- /dev/null +++ b/internal/torrent/cache.go @@ -0,0 +1 @@ +package torrent diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go new file mode 100644 index 0000000..ab5386b --- /dev/null +++ b/internal/torrent/manager.go @@ -0,0 +1,93 @@ +package torrent + +import ( + "log" + + "github.com/debridmediamanager.com/zurg/pkg/realdebrid" + "github.com/dgraph-io/ristretto" +) + +type TorrentManager struct { + token string + cache *ristretto.Cache + workerPool chan bool +} + +// NewTorrentManager creates a new torrent manager +// it will fetch all torrents and their info in the background +// and cache them +func NewTorrentManager(token string) *TorrentManager { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e7, // number of keys to track frequency of (10M). + MaxCost: 1 << 30, // maximum cost of cache (1GB). + BufferItems: 64, // number of keys per Get buffer. + }) + if err != nil { + panic(err) + } + + handler := &TorrentManager{ + token: token, + cache: cache, + workerPool: make(chan bool, 10), + } + + torrents := handler.getAll() + + for _, torrent := range torrents { + go func(id string) { + handler.workerPool <- true + handler.getInfo(id) + // sleep for 1 second to avoid rate limiting + <-handler.workerPool + }(torrent.ID) + } + + return handler +} + +func (t *TorrentManager) getAll() []realdebrid.Torrent { + cacheKey := "t:all" + torrents, err := realdebrid.GetTorrents(t.token) + if err != nil { + log.Printf("Cannot get torrents: %v\n", err.Error()) + return nil + } + t.cache.Set(cacheKey, torrents, 0) + return torrents +} + +func (t *TorrentManager) GetAll() []realdebrid.Torrent { + cacheKey := "t:all" + if data, found := t.cache.Get(cacheKey); found { + if cachedTorrents, ok := data.([]realdebrid.Torrent); ok { + return cachedTorrents + } else { + t.cache.Del(cacheKey) + } + } + return t.getAll() +} + +func (t *TorrentManager) getInfo(torrentID string) *realdebrid.Torrent { + cacheKey := "t:" + torrentID + info, err := realdebrid.GetTorrentInfo(t.token, torrentID) + if err != nil { + log.Printf("Cannot get info: %v\n", err.Error()) + return nil + } + t.cache.Set(cacheKey, info, 0) + return info +} + +func (t *TorrentManager) GetInfo(torrentID string) *realdebrid.Torrent { + cacheKey := "t:" + torrentID + if data, found := t.cache.Get(cacheKey); found { + if torrent, ok := data.(*realdebrid.Torrent); ok { + return torrent + } else { + t.cache.Del(cacheKey) + } + } + return t.getInfo(torrentID) +} diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 736a746..304269e 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -8,7 +8,6 @@ import ( "net/http" "net/url" "strconv" - "time" ) func UnrestrictCheck(accessToken, link string) (*UnrestrictResponse, error) { @@ -89,7 +88,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { baseURL := "https://api.real-debrid.com/rest/1.0/torrents" var allTorrents []Torrent page := 1 - limit := 100 + limit := 2500 for { params := url.Values{} @@ -125,7 +124,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { allTorrents = append(allTorrents, torrents...) - totalCountHeader := "100" // resp.Header.Get("x-total-count") + totalCountHeader := resp.Header.Get("x-total-count") totalCount, err := strconv.Atoi(totalCountHeader) if err != nil { break @@ -138,7 +137,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { page++ } - return deduplicateTorrents(allTorrents), nil + return allTorrents, nil } func GetTorrentInfo(accessToken, id string) (*Torrent, error) { @@ -175,69 +174,3 @@ func GetTorrentInfo(accessToken, id string) (*Torrent, error) { return &response, nil } - -func deduplicateTorrents(torrents []Torrent) []Torrent { - mappedTorrents := make(map[string]Torrent) - - for _, t := range torrents { - torrentName := t.Filename - if existing, ok := mappedTorrents[torrentName]; ok { - if existing.Hash == t.Hash { - // If hash is the same, combine the links - existing.ID += "," + t.ID - // existing.Links = append(existing.Links, t.Links...) - for _, link := range t.Links { - existing.Links = appendIfNotExists(existing.Links, link) - } - existing.Bytes += t.Bytes - existing.Added = moreRecent(existing.Added, t.Added) - mappedTorrents[torrentName] = existing - } else { - // If hash is different, delete old entry and create two new entries - delete(mappedTorrents, torrentName) - newKey1 := fmt.Sprintf("%s - %s", torrentName, t.Hash[:4]) - mappedTorrents[newKey1] = t - newKey2 := fmt.Sprintf("%s - %s", existing.Filename, existing.Hash[:4]) - mappedTorrents[newKey2] = existing - } - } else { - mappedTorrents[torrentName] = t - } - } - - // Convert the map back to a slice - deduplicated := make([]Torrent, 0, len(mappedTorrents)) - for _, value := range mappedTorrents { - deduplicated = append(deduplicated, value) - } - - return deduplicated -} - -func contains(slice []string, str string) bool { - for _, v := range slice { - if v == str { - return true - } - } - return false -} - -func appendIfNotExists(slice []string, str string) []string { - if !contains(slice, str) { - slice = append(slice, str) - } - return slice -} - -func moreRecent(time1, time2 string) string { - tTime1, err1 := time.Parse(time.RFC3339, time1) - tTime2, err2 := time.Parse(time.RFC3339, time2) - if err1 != nil || err2 != nil { - return time1 - } - if tTime2.After(tTime1) { - time1 = time2 - } - return time1 -} diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go index 7f6c0fd..eef4dff 100644 --- a/pkg/realdebrid/types.go +++ b/pkg/realdebrid/types.go @@ -27,6 +27,6 @@ type Torrent struct { type File struct { ID int `json:"id"` Path string `json:"path"` - Bytes int `json:"bytes"` + Bytes int64 `json:"bytes"` Selected int `json:"selected"` } diff --git a/pkg/repo/mysql.go b/pkg/repo/mysql.go deleted file mode 100644 index 4f3c030..0000000 --- a/pkg/repo/mysql.go +++ /dev/null @@ -1,211 +0,0 @@ -package repo - -import ( - "bytes" - "database/sql" - "encoding/gob" - "fmt" - "log" - "net/url" - "path" - "path/filepath" - "regexp" - "strings" - - "github.com/debridmediamanager.com/zurg/pkg/davextra" - "github.com/debridmediamanager.com/zurg/pkg/realdebrid" - _ "github.com/go-sql-driver/mysql" - "github.com/qianbin/directcache" - "github.com/zeebo/xxh3" -) - -type Database struct { - Connection *sql.DB - Cache *directcache.Cache -} - -func GenerateID(segment1, segment2, segment3 string) string { - fullPath := path.Join(segment1, segment2, segment3) - hash := xxh3.HashString(fullPath) - return fmt.Sprintf("%016x", hash) -} - -func NewDatabase(dsn string) (*Database, error) { - db, err := sql.Open("mysql", dsn) - if err != nil { - return nil, err - } - - cache := directcache.New(10 << 20) // This initializes a cache with 10 MB - - return &Database{Connection: db, Cache: cache}, nil -} - -func (db *Database) Insert(parentHash, torrentName string, resp realdebrid.UnrestrictResponse) { - // Generate the ID for the link - var id string - if resp.Filename == "" { - // alternative ID for 404 links - id = GenerateID(parentHash, resp.Link, "") - } else { - id = GenerateID(parentHash, resp.Filename, davextra.GetLinkFragment(resp.Link)) - } - // Check if the link already exists in the database - var exists int - err := db.Connection.QueryRow("SELECT COUNT(*) FROM Links WHERE ID = ?", id).Scan(&exists) - if err != nil { - log.Printf("failed to check existence: %v", err) - } - - // If link does not exist in the database, insert the new record - if exists == 0 { - _, err = db.Connection.Exec(` - INSERT INTO Links (ID, ParentHash, Directory, Filename, Filesize, Link, Host) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - id, - parentHash, - torrentName, - resp.Filename, - resp.Filesize, - resp.Link, - resp.Host, - ) - if err != nil { - log.Printf("failed to insert record: %v", err) - } - - // Clear cache for parentHash - db.Cache.Del([]byte(parentHash)) - } -} - -func (db *Database) Get(parentHash, filename string) (*DavFile, error) { - filenameV2, linkFragment := extractIDFromFilename(filename) - id := GenerateID(parentHash, filenameV2, linkFragment) - data, ok := db.Cache.Get([]byte(id)) - if !ok { - resp, err := fetchFromDatabaseByID(db.Connection, id, linkFragment) - if err != nil { - return nil, err - } - - buffer := &bytes.Buffer{} - encoder := gob.NewEncoder(buffer) - if err := encoder.Encode(resp); err != nil { - return nil, err - } - - db.Cache.Set([]byte(id), buffer.Bytes()) - return resp, nil - } - - buffer := bytes.NewBuffer(data) - decoder := gob.NewDecoder(buffer) - var resp DavFile - if err := decoder.Decode(&resp); err != nil { - return nil, err - } - return &resp, nil -} - -func extractIDFromFilename(filename string) (string, string) { - filenameV2, err := url.PathUnescape(filename) - if err != nil { - filenameV2 = filename - } - ext := filepath.Ext(filenameV2) - name := strings.TrimSuffix(filenameV2, ext) - - r := regexp.MustCompile(`\sDMM(\w+)`) - matches := r.FindStringSubmatch(name) - if len(matches) < 2 { - // No ID found - return filenameV2, "" - } - - // Remove ID from filename - originalName := strings.Replace(name, matches[0], "", 1) - return originalName + ext, matches[1] -} - -func (db *Database) GetMultiple(parentHash string) (*DavFiles, error) { - key := []byte(parentHash) - data, ok := db.Cache.Get(key) - if !ok { - resps, err := fetchMultipleFromDatabase(db.Connection, parentHash) - if err != nil { - return nil, err - } - - buffer := &bytes.Buffer{} - encoder := gob.NewEncoder(buffer) - if err := encoder.Encode(resps); err != nil { - return nil, err - } - - db.Cache.Set(key, buffer.Bytes()) - return resps, nil - } - - buffer := bytes.NewBuffer(data) - decoder := gob.NewDecoder(buffer) - var resps DavFiles - if err := decoder.Decode(&resps); err != nil { - return nil, err - } - - return &resps, nil -} - -func fetchFromDatabaseByID(conn *sql.DB, id, linkFragment string) (*DavFile, error) { - log.Printf("fetching from database: %s", id) - var resp DavFile - - query := ` - SELECT Filename, Filesize, Link - FROM Links WHERE ID = ? AND Link LIKE ?` - row := conn.QueryRow(query, id, "https://real-debrid.com/d/"+linkFragment+"%") - - err := row.Scan(&resp.Filename, &resp.Filesize, &resp.Link) - if err != nil { - if err == sql.ErrNoRows { - return nil, nil - } - log.Printf("failed to fetch record: %v", err) - return nil, err - } - - return &resp, nil -} - -func fetchMultipleFromDatabase(conn *sql.DB, parentHash string) (*DavFiles, error) { - log.Printf("fetching multiple from database: %s", parentHash) - rows, err := conn.Query(` - SELECT Filename, Filesize, Link - FROM Links WHERE ParentHash = ?`, - parentHash, - ) - if err != nil { - return nil, fmt.Errorf("failed to fetch records: %v", err) - } - defer rows.Close() - - var responses []*DavFile - - for rows.Next() { - resp := &DavFile{} - if err := rows.Scan(&resp.Filename, &resp.Filesize, &resp.Link); err != nil { - return nil, fmt.Errorf("failed to scan row: %v", err) - } - responses = append(responses, resp) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error while iterating over rows: %v", err) - } - - result := &DavFiles{ - Files: responses, - } - - return result, nil -} diff --git a/pkg/repo/types.go b/pkg/repo/types.go deleted file mode 100644 index 01d36c4..0000000 --- a/pkg/repo/types.go +++ /dev/null @@ -1,11 +0,0 @@ -package repo - -type DavFile struct { - Filename string - Filesize int64 - Link string -} - -type DavFiles struct { - Files []*DavFile -}