From acee02377ef75d35ee019855d7faa27185b114fc Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Wed, 18 Oct 2023 12:36:34 +0200 Subject: [PATCH] Proper caching --- .gitignore | 3 ++ internal/dav/getfile.go | 56 ++++++++++++++------------------- internal/dav/propfind.go | 12 +++---- internal/dav/response.go | 18 +++++------ internal/dav/router.go | 7 ++--- internal/dav/util.go | 3 +- internal/torrent/manager.go | 62 +++++++++++++++++++++++++++++++------ pkg/davextra/util.go | 26 ++++++++++++++++ pkg/realdebrid/api.go | 4 +-- 9 files changed, 126 insertions(+), 65 deletions(-) diff --git a/.gitignore b/.gitignore index 3b735ec..cb3100c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ # Go workspace file go.work + +data/ +rclone diff --git a/internal/dav/getfile.go b/internal/dav/getfile.go index a69095d..d3a3497 100644 --- a/internal/dav/getfile.go +++ b/internal/dav/getfile.go @@ -4,14 +4,13 @@ import ( "fmt" "log" "net/http" - "net/url" "os" "path" "path/filepath" - "regexp" "strings" "github.com/debridmediamanager.com/zurg/internal/torrent" + "github.com/debridmediamanager.com/zurg/pkg/davextra" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" ) @@ -24,21 +23,27 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent if len(segments) < 3 { log.Println("Invalid url", requestPath) http.Error(w, "Cannot find file", http.StatusNotFound) + return } // Get the last two segments torrentName := segments[len(segments)-2] + filename := segments[len(segments)-1] + torrents := findAllTorrentsWithName(t, torrentName) if torrents == nil { - log.Println("Cannot find directory", requestPath) + log.Println("Cannot find torrent", torrentName) http.Error(w, "Cannot find file", http.StatusNotFound) return } - filename := segments[len(segments)-1] - - filenameV2, linkFragment := extractIDFromFilename(filename) - link := findLinkByFragment(torrents, linkFragment) + filenameV2, linkFragment := davextra.ExtractLinkFragment(filename) + link := findLinkByFragment(torrents, filenameV2, linkFragment) + if link == "" { + log.Println("Link not found") + http.Error(w, "Cannot find file", http.StatusNotFound) + return + } unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), link) @@ -52,38 +57,25 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent } if resp.Filename != filenameV2 { // TODO: Redo the logic to handle mismatch + // Filename mismatch + // [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar + // Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv + // Filename mismatch + // Adventure.Time.S06E10.Something.Big.1080p.HMAX.WEBRip.DD.2.0.H.265.-EDGE2020.mkv + // Adventure.Time.S06E03.James.II.1080p.HMAX.WEBRip.DD.2.0.H.265.-EDGE2020.mkv + // Action: schedule a "cleanup" job for the parent torrent log.Println("Filename mismatch", resp.Filename, filenameV2) } http.Redirect(w, r, resp.Download, http.StatusFound) } -// extractIDFromFilename extracts the link ID from a filename -func extractIDFromFilename(filename string) (string, string) { - filenameV2, err := url.PathUnescape(filename) - if err != nil { - filenameV2 = filename - } - ext := filepath.Ext(filenameV2) - name := strings.TrimSuffix(filenameV2, ext) - - r := regexp.MustCompile(`\sDMM(\w+)`) - matches := r.FindStringSubmatch(name) - if len(matches) < 2 { - // No ID found - return filenameV2, "" - } - - // Remove ID from filename - originalName := strings.Replace(name, matches[0], "", 1) - return originalName + ext, matches[1] -} - // findLinkByFragment finds a link by a fragment, it might be wrong -func findLinkByFragment(torrents []torrent.Torrent, fragment string) string { +func findLinkByFragment(torrents []torrent.Torrent, filename, fragment string) string { for _, torrent := range torrents { - for _, link := range torrent.Links { - if strings.HasPrefix(link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) { - return link + for _, file := range torrent.SelectedFiles { + fname := filepath.Base(file.Path) + if filename == fname && strings.HasPrefix(file.Link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) { + return file.Link } } } diff --git a/internal/dav/propfind.go b/internal/dav/propfind.go index d2b9d80..fff0d82 100644 --- a/internal/dav/propfind.go +++ b/internal/dav/propfind.go @@ -26,7 +26,7 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To } if err != nil { log.Printf("Cannot marshal xml: %v\n", err.Error()) - http.Error(w, fmt.Sprintf("Cannot marshal xml: %v", err.Error()), http.StatusInternalServerError) + http.Error(w, "Cannot read directory", http.StatusInternalServerError) return } if output != nil { @@ -54,8 +54,8 @@ func handleListOfTorrents(w http.ResponseWriter, r *http.Request, t *torrent.Tor allTorrents := t.GetAll() allTorrentsResponse, err := createMultiTorrentResponse(allTorrents) if err != nil { - log.Printf("Cannot read directory: %v\n", err.Error()) - http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) + log.Printf("Cannot read directory (/torrents): %v\n", err.Error()) + http.Error(w, "Cannot read directory", http.StatusInternalServerError) return nil, nil } return xml.MarshalIndent(allTorrentsResponse, "", " ") @@ -67,15 +67,15 @@ func handleSingleTorrent(w http.ResponseWriter, r *http.Request, t *torrent.Torr torrentName := path.Base(requestPath) foundTorrents := findAllTorrentsWithName(t, torrentName) if len(foundTorrents) == 0 { - log.Println("Cannot find directory") + log.Println("Cannot find directory", requestPath) http.Error(w, "Cannot find directory", http.StatusNotFound) return nil, nil } var torrentResponse *dav.MultiStatus torrentResponse, err := createCombinedTorrentResponse(foundTorrents, t) if err != nil { - log.Printf("Cannot read directory: %v\n", err.Error()) - http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError) + log.Printf("Cannot read directory (%s): %v\n", requestPath, err.Error()) + http.Error(w, "Cannot read directory", http.StatusInternalServerError) return nil, nil } return xml.MarshalIndent(torrentResponse, "", " ") diff --git a/internal/dav/response.go b/internal/dav/response.go index bef863a..65254ff 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -5,6 +5,7 @@ import ( "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/dav" + "github.com/debridmediamanager.com/zurg/pkg/davextra" ) // createMultiTorrentResponse creates a WebDAV response for a list of torrents @@ -33,7 +34,8 @@ func createMultiTorrentResponse(torrents []torrent.Torrent) (*dav.MultiStatus, e }, nil } -// createTorrentResponse creates a WebDAV response for torrents with the same name +// createTorrentResponse creates a WebDAV response for a single torrent +// but it also handles the case where there are many torrents with the same name func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.TorrentManager) (*dav.MultiStatus, error) { var responses []dav.Response // initial response is the directory itself @@ -41,16 +43,13 @@ func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.Torren responses = append(responses, dav.Directory(currentPath)) seen := make(map[string]bool) - idx := 0 var torrentResponses []dav.Response for _, torrent := range torrents { - info := t.GetInfo(torrent.ID) - if info == nil { - continue - } - for _, file := range info.SelectedFiles { + for _, file := range torrent.SelectedFiles { filename := filepath.Base(file.Path) + fragment := davextra.GetLinkFragment(file.Link) + filename = davextra.InsertLinkFragment(filename, fragment) if _, exists := seen[filename]; exists { continue } @@ -59,10 +58,9 @@ func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.Torren torrentResponses = append(torrentResponses, dav.File( filePath, file.Bytes, - convertDate(info.Added), - info.Links[idx], + convertDate(torrent.Added), + file.Link, )) - idx++ } } responses = append(responses, torrentResponses...) diff --git a/internal/dav/router.go b/internal/dav/router.go index 6b47ed2..c569c7a 100644 --- a/internal/dav/router.go +++ b/internal/dav/router.go @@ -4,7 +4,6 @@ import ( "log" "net/http" "os" - "path" "github.com/debridmediamanager.com/zurg/internal/torrent" ) @@ -14,9 +13,8 @@ func Router(mux *http.ServeMux) { t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN")) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - requestPath := path.Clean(r.URL.Path) - - log.Println(r.Method, requestPath) + // requestPath := path.Clean(r.URL.Path) + // log.Println(r.Method, requestPath) switch r.Method { case "PROPFIND": @@ -24,6 +22,7 @@ func Router(mux *http.ServeMux) { case http.MethodGet: HandleGetRequest(w, r, t) + // default return case http.MethodOptions: w.WriteHeader(http.StatusOK) diff --git a/internal/dav/util.go b/internal/dav/util.go index bd5f7ed..2d645ff 100644 --- a/internal/dav/util.go +++ b/internal/dav/util.go @@ -2,6 +2,7 @@ package dav import ( "log" + "strings" "time" "github.com/debridmediamanager.com/zurg/internal/torrent" @@ -23,7 +24,7 @@ func findAllTorrentsWithName(t *torrent.TorrentManager, filename string) []torre torrents := t.GetAll() for _, torrent := range torrents { - if torrent.Filename == filename { + if torrent.Filename == filename || strings.HasPrefix(torrent.Filename, filename) { matchingTorrents = append(matchingTorrents, torrent) } } diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 8caffb7..b17b6cb 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -1,7 +1,9 @@ package torrent import ( + "encoding/gob" "log" + "os" "strings" "sync" @@ -58,6 +60,14 @@ func (t *TorrentManager) GetAll() []Torrent { } func (t *TorrentManager) getInfo(torrentID string) *Torrent { + torrentFromFile := t.readFromFile(torrentID) + if torrentFromFile != nil { + torrent := t.getByID(torrentID) + if torrent != nil { + torrent.SelectedFiles = torrentFromFile.SelectedFiles + } + return torrent + } info, err := realdebrid.GetTorrentInfo(t.token, torrentID) if err != nil { log.Printf("Cannot get info: %v\n", err.Error()) @@ -123,26 +133,58 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent { if torrent != nil { torrent.SelectedFiles = selectedFiles } - log.Println("Fetched info for", info.Filename) + if len(torrent.SelectedFiles) > 0 { + t.writeToFile(torrentID, torrent) + } return torrent } func (t *TorrentManager) GetInfo(torrentID string) *Torrent { for _, torrent := range t.torrents { if torrent.ID == torrentID { - if torrent.SelectedFiles != nil { - return t.getInfo(torrentID) - } + return &torrent + } + } + return t.getInfo(torrentID) +} + +func (t *TorrentManager) getByID(torrentID string) *Torrent { + for i, torrent := range t.torrents { + if torrent.ID == torrentID { + return &t.torrents[i] } } return nil } -func (t *TorrentManager) getByID(torrentID string) *Torrent { - for _, torrent := range t.torrents { - if torrent.ID == torrentID { - return &torrent - } +func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) { + filePath := "data/" + torrentID + ".bin" + file, err := os.Create(filePath) + if err != nil { + log.Fatalf("Failed creating file: %s", err) + return } - return nil + defer file.Close() + + dataEncoder := gob.NewEncoder(file) + dataEncoder.Encode(torrent) +} + +func (t *TorrentManager) readFromFile(torrentID string) *Torrent { + filePath := "data/" + torrentID + ".bin" + file, err := os.Open(filePath) + if err != nil { + return nil + } + defer file.Close() + + var torrent Torrent + dataDecoder := gob.NewDecoder(file) + err = dataDecoder.Decode(&torrent) + if err != nil { + log.Fatalf("Failed decoding file: %s", err) + return nil + } + + return &torrent } diff --git a/pkg/davextra/util.go b/pkg/davextra/util.go index 5cc8d60..fdda349 100644 --- a/pkg/davextra/util.go +++ b/pkg/davextra/util.go @@ -2,11 +2,13 @@ package davextra import ( "fmt" + "net/url" "path/filepath" "regexp" "strings" ) +// GetLinkFragment returns the longest alphanumeric string from a link func GetLinkFragment(link string) string { re := regexp.MustCompile(`\w+`) matches := re.FindAllString(link, -1) // Returns all matches @@ -19,8 +21,32 @@ func GetLinkFragment(link string) string { return longestMatch[:4] } +// InsertLinkFragment inserts the link ID into a filename +// returns the new filename func InsertLinkFragment(filename, dupeID string) string { ext := filepath.Ext(filename) name := strings.TrimSuffix(filename, ext) return fmt.Sprintf("%s DMM%s%s", name, dupeID, ext) } + +// ExtractLinkFragment extracts the link ID from a filename +// returns the original filename and the link ID +func ExtractLinkFragment(filename string) (string, string) { + filenameV2, err := url.PathUnescape(filename) + if err != nil { + filenameV2 = filename + } + ext := filepath.Ext(filenameV2) + name := strings.TrimSuffix(filenameV2, ext) + + r := regexp.MustCompile(`\sDMM(\w+)`) + matches := r.FindStringSubmatch(name) + if len(matches) < 2 { + // No ID found + return filenameV2, "" + } + + // Remove ID from filename + originalName := strings.Replace(name, matches[0], "", 1) + return originalName + ext, matches[1] +} diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 304269e..6aa3aa5 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -88,7 +88,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { baseURL := "https://api.real-debrid.com/rest/1.0/torrents" var allTorrents []Torrent page := 1 - limit := 2500 + limit := 100 for { params := url.Values{} @@ -124,7 +124,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) { allTorrents = append(allTorrents, torrents...) - totalCountHeader := resp.Header.Get("x-total-count") + totalCountHeader := "100" // resp.Header.Get("x-total-count") totalCount, err := strconv.Atoi(totalCountHeader) if err != nil { break