From a2d6d501b3cdbd7d6292d34329e933d487854104 Mon Sep 17 00:00:00 2001 From: Laptop Date: Wed, 16 Apr 2025 22:28:05 +0300 Subject: [PATCH] inject metadata on the fly when downloading, refactoring a bit --- docs/USER_GUIDE.md | 6 +- lib/preferences/init.go | 2 +- lib/proxy_images/init.go | 2 +- lib/restream/init.go | 341 +++++---------------------------------- lib/restream/reader.go | 195 ++++++++++++++++++++++ lib/sc/track.go | 24 --- 6 files changed, 246 insertions(+), 324 deletions(-) create mode 100644 lib/restream/reader.go diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index f3d22d8..18136eb 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -24,4 +24,8 @@ Scroll down to the end of the preferences page. There you can see a management t soundcloak tries to keep the URL schemes same to SoundCloud's, so you can just replace `soundcloud.com` with your instance URL. For short links: `https://on.soundcloud.com/boiKDP46fayYDoVK9` -> `/on/boiKDP46fayYDoVK9` -To automatically redirect, you can use [LibRedirect](https://libredirect.github.io/) extension. Soundcloak is supported \ No newline at end of file +To automatically redirect, you can use [LibRedirect](https://libredirect.github.io/) extension. Soundcloak is supported + +# Extra notes + +If you find music that you like, make sure to download it! Stuff that's on there may be deleted or changed at any moment, without any warning or ability to experience it again, unless you download it for yourself. Download button is available if `Restream` is enabled in backend config. You can configure audio preset for downloading in preferences page. For easily and quickly downloading entire users or playlists, you can use my tool [scrip](https://git.maid.zone/laptop/scrip) \ No newline at end of file diff --git a/lib/preferences/init.go b/lib/preferences/init.go index 1aa2cb4..d80d63a 100644 --- a/lib/preferences/init.go +++ b/lib/preferences/init.go @@ -117,7 +117,7 @@ func Load(r *fiber.App) { return err } - c.Set("Content-Type", "text/html") + c.Request().Header.SetContentType("text/html") return templates.Base("preferences", templates.Preferences(p), nil).Render(context.Background(), c) }) diff --git a/lib/proxy_images/init.go b/lib/proxy_images/init.go index 687244e..77979ae 100644 --- a/lib/proxy_images/init.go +++ b/lib/proxy_images/init.go @@ -63,7 +63,7 @@ func Load(r *fiber.App) { return err } - c.Set("Content-Type", "image/jpeg") + c.Request().Header.SetContentType("image/jpeg") c.Set("Cache-Control", cfg.ImageCacheControl) //return c.Send(resp.Body()) pr := misc.AcquireProxyReader() diff --git a/lib/restream/init.go b/lib/restream/init.go index 337bb13..1a6373b 100644 --- a/lib/restream/init.go +++ b/lib/restream/init.go @@ -2,11 +2,8 @@ package restream import ( "bytes" - "encoding/binary" "image" - "io" "strings" - "sync" _ "image/jpeg" _ "image/png" @@ -19,191 +16,8 @@ import ( "github.com/gcottom/mp4meta" "github.com/gcottom/oggmeta" "github.com/gofiber/fiber/v3" - "github.com/valyala/fasthttp" ) -const defaultPartsCapacity = 24 - -type reader struct { - parts [][]byte - leftover []byte - index int - duration *uint32 - - req *fasthttp.Request - resp *fasthttp.Response - client *fasthttp.HostClient -} - -var readerpool = sync.Pool{ - New: func() any { - return &reader{} - }, -} - -func acquireReader() *reader { - return readerpool.Get().(*reader) -} - -func clone(buf []byte) []byte { - out := make([]byte, len(buf)) - copy(out, buf) - return out -} - -var mvhd = []byte("mvhd") - -func fixDuration(data []byte, duration *uint32) { - i := bytes.Index(data, mvhd) - if i != -1 { - i += 20 - - bt := make([]byte, 4) - binary.BigEndian.PutUint32(bt, *duration) - copy(data[i:], bt) - // timescale is already 1000 in the files - } -} - -func (r *reader) Setup(url string, aac bool, duration *uint32) error { - if r.req == nil { - r.req = fasthttp.AcquireRequest() - } - - if r.resp == nil { - r.resp = fasthttp.AcquireResponse() - } - - r.req.SetRequestURI(url) - r.req.Header.SetUserAgent(cfg.UserAgent) - r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") - - if aac { - r.client = misc.HlsAacClient - r.duration = duration - } else { - r.client = misc.HlsClient - } - - err := sc.DoWithRetry(r.client, r.req, r.resp) - if err != nil { - return err - } - - data, err := r.resp.BodyUncompressed() - if err != nil { - data = r.resp.Body() - } - - if r.parts == nil { - misc.Log("make() r.parts") - r.parts = make([][]byte, 0, defaultPartsCapacity) - } else { - misc.Log(cap(r.parts), len(r.parts)) - } - if aac { - // clone needed to mitigate memory skill issues here - for _, s := range bytes.Split(data, []byte{'\n'}) { - if len(s) == 0 { - continue - } - if s[0] == '#' { - if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) { - r.parts = append(r.parts, clone(s[16:len(s)-1])) - } - - continue - } - - r.parts = append(r.parts, clone(s)) - } - } else { - for _, s := range bytes.Split(data, []byte{'\n'}) { - if len(s) == 0 || s[0] == '#' { - continue - } - - r.parts = append(r.parts, s) - } - } - - return nil -} - -func (r *reader) Close() error { - misc.Log("closed :D") - r.req.Reset() - r.resp.Reset() - - r.leftover = nil - r.index = 0 - r.parts = r.parts[:0] - - readerpool.Put(r) - return nil -} - -// you could prob make this a bit faster by concurrency (make a bunch of workers => make them download the parts => temporarily add them to a map => fully assemble the result => make reader.Read() read out the result as the parts are coming in) but whatever, fine for now -func (r *reader) Read(buf []byte) (n int, err error) { - misc.Log("we read") - if len(r.leftover) != 0 { - h := len(buf) - if h > len(r.leftover) { - h = len(r.leftover) - } - - n = copy(buf, r.leftover[:h]) - - if n > len(r.leftover) { - r.leftover = r.leftover[:0] - } else { - r.leftover = r.leftover[n:] - } - - if n < len(buf) && r.index == len(r.parts) { - err = io.EOF - } - - return - } - - if r.index == len(r.parts) { - err = io.EOF - return - } - - r.req.SetRequestURIBytes(r.parts[r.index]) - - err = sc.DoWithRetry(r.client, r.req, r.resp) - if err != nil { - return - } - - data, err := r.resp.BodyUncompressed() - if err != nil { - data = r.resp.Body() - } - - if r.index == 0 && r.duration != nil { - fixDuration(data, r.duration) // I'm guessing that mvhd will always be in first part - } - - if len(data) > len(buf) { - n = copy(buf, data[:len(buf)]) - } else { - n = copy(buf, data) - } - - r.leftover = data[n:] - r.index++ - - if n < len(buf) && r.index == len(r.parts) { - err = io.EOF - } - - return -} - type collector struct { data []byte } @@ -250,9 +64,10 @@ func Load(r *fiber.App) { return err } - c.Set("Content-Type", tr.Format.MimeType) + c.Request().Header.SetContentType(tr.Format.MimeType) c.Set("Cache-Control", cfg.RestreamCacheControl) + r := acquireReader() if isDownload { if t.Artwork != "" { t.Artwork = strings.Replace(t.Artwork, "t500x500", "original", 1) @@ -260,8 +75,8 @@ func Load(r *fiber.App) { switch audio { case cfg.AudioMP3: - r := acquireReader() - if err := r.Setup(u, false, nil); err != nil { + err := r.Setup(u, false, nil) + if err != nil { return err } @@ -275,70 +90,37 @@ func Load(r *fiber.App) { tag.SetTitle(t.Title) if t.Artwork != "" { - data, mime, err := t.DownloadImage() + r.req.SetRequestURI(t.Artwork) + r.req.Header.Del("Accept-Encoding") + + err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp) if err != nil { return err } - tag.AddAttachedPicture(id3v2.PictureFrame{MimeType: mime, Picture: data, PictureType: id3v2.PTFrontCover, Encoding: id3v2.EncodingUTF8}) + tag.AddAttachedPicture(id3v2.PictureFrame{MimeType: cfg.B2s(r.req.Header.ContentType()), Picture: r.req.Body(), PictureType: id3v2.PTFrontCover, Encoding: id3v2.EncodingUTF8}) + r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") } var col collector tag.WriteTo(&col) r.leftover = col.data - // id3 is quite flexible and the files streamed by soundcloud don't have it so its easy to restream the stuff like this return c.SendStream(r) - case cfg.AudioOpus: - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - req.SetRequestURI(u) - req.Header.SetUserAgent(cfg.UserAgent) - req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") - - resp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(resp) - - err = sc.DoWithRetry(misc.HlsClient, req, resp) + err := r.Setup(u, false, nil) if err != nil { return err } - data, err := resp.BodyUncompressed() + r.req.SetRequestURIBytes(r.parts[0]) + err = sc.DoWithRetry(r.client, r.req, r.resp) if err != nil { - data = resp.Body() + return err } - parts := make([][]byte, 0, defaultPartsCapacity) - for _, s := range bytes.Split(data, []byte{'\n'}) { - if len(s) == 0 || s[0] == '#' { - continue - } - - parts = append(parts, s) - } - - result := []byte{} - - for _, part := range parts { - req.SetRequestURIBytes(part) - - err = sc.DoWithRetry(misc.HlsClient, req, resp) - if err != nil { - return err - } - - data, err = resp.BodyUncompressed() - if err != nil { - data = resp.Body() - } - - result = append(result, data...) - } - - tag, err := oggmeta.ReadOGG(bytes.NewReader(result)) + r.index++ + tag, err := oggmeta.ReadOGG(bytes.NewReader(r.resp.Body())) if err != nil { return err } @@ -351,82 +133,43 @@ func Load(r *fiber.App) { tag.SetTitle(t.Title) if t.Artwork != "" { - req.SetRequestURI(t.Artwork) - req.Header.Del("Accept-Encoding") + r.req.SetRequestURI(t.Artwork) + r.req.Header.Del("Accept-Encoding") - err := sc.DoWithRetry(misc.ImageClient, req, resp) + err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp) if err != nil { return err } - defer resp.CloseBodyStream() - parsed, _, err := image.Decode(resp.BodyStream()) + parsed, _, err := image.Decode(r.resp.BodyStream()) + r.resp.CloseBodyStream() if err != nil { return err } tag.SetCoverArt(&parsed) + r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") } - return tag.Save(c) + var col collector + tag.Save(&col) + r.leftover = col.data + + return c.SendStream(r) case cfg.AudioAAC: - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - req.SetRequestURI(u) - req.Header.SetUserAgent(cfg.UserAgent) - req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") - - resp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(resp) - - err = sc.DoWithRetry(misc.HlsAacClient, req, resp) + err := r.Setup(u, true, &t.Duration) if err != nil { return err } - data, err := resp.BodyUncompressed() + r.req.SetRequestURIBytes(r.parts[0]) + err = sc.DoWithRetry(r.client, r.req, r.resp) if err != nil { - data = resp.Body() + return err } - parts := make([][]byte, 0, defaultPartsCapacity) - // clone needed to mitigate memory skill issues here - for _, s := range bytes.Split(data, []byte{'\n'}) { - if len(s) == 0 { - continue - } - if s[0] == '#' { - if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) { - parts = append(parts, clone(s[16:len(s)-1])) - } - - continue - } - - parts = append(parts, clone(s)) - } - - result := []byte{} - for _, part := range parts { - req.SetRequestURIBytes(part) - - err = sc.DoWithRetry(misc.HlsAacClient, req, resp) - if err != nil { - return err - } - - data, err = resp.BodyUncompressed() - if err != nil { - data = resp.Body() - } - - result = append(result, data...) - } - - fixDuration(result, &t.Duration) - - tag, err := mp4meta.ReadMP4(bytes.NewReader(result)) + r.index++ + tag, err := mp4meta.ReadMP4(bytes.NewReader(r.resp.Body())) if err != nil { return err } @@ -439,28 +182,32 @@ func Load(r *fiber.App) { tag.SetTitle(t.Title) if t.Artwork != "" { - req.SetRequestURI(t.Artwork) - req.Header.Del("Accept-Encoding") + r.req.SetRequestURI(t.Artwork) + r.req.Header.Del("Accept-Encoding") - err := sc.DoWithRetry(misc.ImageClient, req, resp) + err := sc.DoWithRetry(misc.ImageClient, r.req, r.resp) if err != nil { return err } - defer resp.CloseBodyStream() - parsed, _, err := image.Decode(resp.BodyStream()) + parsed, _, err := image.Decode(r.resp.BodyStream()) + r.resp.CloseBodyStream() if err != nil { return err } tag.SetCoverArt(&parsed) + r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") } - return tag.Save(c) + var col collector + tag.Save(&col) + r.leftover = col.data + + return c.SendStream(r) } } - r := acquireReader() if audio == cfg.AudioAAC { err = r.Setup(u, true, &t.Duration) } else { diff --git a/lib/restream/reader.go b/lib/restream/reader.go new file mode 100644 index 0000000..51171ef --- /dev/null +++ b/lib/restream/reader.go @@ -0,0 +1,195 @@ +package restream + +import ( + "bytes" + "encoding/binary" + "io" + "sync" + + "git.maid.zone/stuff/soundcloak/lib/cfg" + "git.maid.zone/stuff/soundcloak/lib/misc" + "git.maid.zone/stuff/soundcloak/lib/sc" + "github.com/valyala/fasthttp" +) + +const defaultPartsCapacity = 24 + +type reader struct { + parts [][]byte + leftover []byte + index int + duration *uint32 + + req *fasthttp.Request + resp *fasthttp.Response + client *fasthttp.HostClient +} + +var readerpool = sync.Pool{ + New: func() any { + return &reader{} + }, +} + +func acquireReader() *reader { + return readerpool.Get().(*reader) +} + +func clone(buf []byte) []byte { + out := make([]byte, len(buf)) + copy(out, buf) + return out +} + +var mvhd = []byte("mvhd") + +func fixDuration(data []byte, duration *uint32) { + i := bytes.Index(data, mvhd) + if i != -1 { + i += 20 + + bt := make([]byte, 4) + binary.BigEndian.PutUint32(bt, *duration) + copy(data[i:], bt) + // timescale is already 1000 in the files + } +} + +func (r *reader) Setup(url string, aac bool, duration *uint32) error { + if r.req == nil { + r.req = fasthttp.AcquireRequest() + } + + if r.resp == nil { + r.resp = fasthttp.AcquireResponse() + } + + r.req.SetRequestURI(url) + r.req.Header.SetUserAgent(cfg.UserAgent) + r.req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + + if aac { + r.client = misc.HlsAacClient + r.duration = duration + } else { + r.client = misc.HlsClient + } + + err := sc.DoWithRetry(r.client, r.req, r.resp) + if err != nil { + return err + } + + data, err := r.resp.BodyUncompressed() + if err != nil { + data = r.resp.Body() + } + + if r.parts == nil { + misc.Log("make() r.parts") + r.parts = make([][]byte, 0, defaultPartsCapacity) + } else { + misc.Log(cap(r.parts), len(r.parts)) + } + if aac { + // clone needed to mitigate memory skill issues here + for _, s := range bytes.Split(data, []byte{'\n'}) { + if len(s) == 0 { + continue + } + if s[0] == '#' { + if bytes.HasPrefix(s, []byte(`#EXT-X-MAP:URI="`)) { + r.parts = append(r.parts, clone(s[16:len(s)-1])) + } + + continue + } + + r.parts = append(r.parts, clone(s)) + } + } else { + for _, s := range bytes.Split(data, []byte{'\n'}) { + if len(s) == 0 || s[0] == '#' { + continue + } + + r.parts = append(r.parts, s) + } + } + + return nil +} + +func (r *reader) Close() error { + misc.Log("closed :D") + r.req.Reset() + r.resp.Reset() + + r.leftover = nil + r.index = 0 + r.parts = r.parts[:0] + + readerpool.Put(r) + return nil +} + +// you could prob make this a bit faster by concurrency (make a bunch of workers => make them download the parts => temporarily add them to a map => fully assemble the result => make reader.Read() read out the result as the parts are coming in) but whatever, fine for now +func (r *reader) Read(buf []byte) (n int, err error) { + misc.Log("we read") + if len(r.leftover) != 0 { + h := len(buf) + if h > len(r.leftover) { + h = len(r.leftover) + } + + n = copy(buf, r.leftover[:h]) + + if n > len(r.leftover) { + r.leftover = r.leftover[:0] + } else { + r.leftover = r.leftover[n:] + } + + if n < len(buf) && r.index == len(r.parts) { + err = io.EOF + } + + return + } + + if r.index == len(r.parts) { + err = io.EOF + return + } + + r.req.SetRequestURIBytes(r.parts[r.index]) + + err = sc.DoWithRetry(r.client, r.req, r.resp) + if err != nil { + return + } + + data, err := r.resp.BodyUncompressed() + if err != nil { + data = r.resp.Body() + } + + if r.index == 0 && r.duration != nil { + fixDuration(data, r.duration) // I'm guessing that mvhd will always be in first part + } + + if len(data) > len(buf) { + n = copy(buf, data[:len(buf)]) + } else { + n = copy(buf, data) + } + + r.leftover = data[n:] + r.index++ + + if n < len(buf) && r.index == len(r.parts) { + err = io.EOF + } + + return +} diff --git a/lib/sc/track.go b/lib/sc/track.go index 5a204cc..771c772 100644 --- a/lib/sc/track.go +++ b/lib/sc/track.go @@ -448,30 +448,6 @@ func GetTrackByID(cid string, id string) (Track, error) { return t, nil } -func (t Track) DownloadImage() ([]byte, string, error) { - req := fasthttp.AcquireRequest() - defer fasthttp.ReleaseRequest(req) - - req.SetRequestURI(t.Artwork) - req.Header.SetUserAgent(cfg.UserAgent) - //req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") images not big enough to be compressed - - resp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(resp) - - err := DoWithRetry(misc.ImageClient, req, resp) - if err != nil { - return nil, "", err - } - - data, err := resp.BodyUncompressed() - if err != nil { - data = resp.Body() - } - - return data, string(resp.Header.Peek("Content-Type")), nil -} - func (t Track) Href() string { return "/" + t.Author.Permalink + "/" + t.Permalink }