package sc import ( "bytes" "errors" "fmt" "net/url" "os" "sync" "syscall" "time" "git.maid.zone/stuff/soundcloak/lib/cfg" "git.maid.zone/stuff/soundcloak/lib/misc" "github.com/dlclark/regexp2" "github.com/goccy/go-json" "github.com/valyala/fasthttp" ) type clientIdCache struct { NextCheck time.Time ClientID string Version string } var ClientIDCache clientIdCache const api = "api-v2.soundcloud.com" const H = len("https://" + api) var newline = []byte("\n") var sc_version = []byte(`$`, 2) // var scriptsRegex = regexp2.MustCompile(`^$`, 2) // var scriptRegex = regexp2.MustCompile(`^$`, 2) //go:generate regexp2cg -package sc -o regexp2_codegen.go var clientIdRegex = regexp2.MustCompile(`client_id:"([A-Za-z0-9]{32})"`, 0) //regexp2.MustCompile(`\("client_id=([A-Za-z0-9]{32})"\)`, 0) var ErrVersionNotFound = errors.New("version not found") var ErrScriptNotFound = errors.New("script not found") var ErrIDNotFound = errors.New("clientid not found") var ErrKindNotCorrect = errors.New("entity of incorrect kind") type cached[T any] struct { Value T Expires time.Time } // don't be spooked by misc.Log, it will be removed during compilation if cfg.Debug == false func processFile(wg *sync.WaitGroup, ch chan string, uri []byte, isDone *bool) { misc.Log(string(uri)) req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURIBytes(uri) req.Header.SetUserAgent(cfg.UserAgent) req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) defer wg.Done() if *isDone { misc.Log("early 1") return } err := DoWithRetryAll(genericClient, req, resp) if err != nil { misc.Log(err) return } if *isDone { misc.Log("early 2") return } data, err := resp.BodyUncompressed() if err != nil { data = resp.Body() } if *isDone { misc.Log("early 3") return } m2, _ := clientIdRegex.FindStringMatch(string(data)) if m2 != nil { g := m2.GroupByNumber(1) if g != nil { if *isDone { misc.Log("early 4") return } ch <- g.String() misc.Log("found in", string(uri)) return } } misc.Log("not found in", string(uri)) } // Experimental method, which asserts that the clientId is inside the file that starts with "0-" const experimental_GetClientID = true // inspired by github.com/imputnet/cobalt func GetClientID() (string, error) { if cfg.ClientID != "" { return cfg.ClientID, nil } if ClientIDCache.NextCheck.After(time.Now()) { misc.Log("clientidcache hit @ 1") return ClientIDCache.ClientID, nil } req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURI("https://soundcloud.com") req.Header.SetUserAgent(cfg.UserAgent) req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) err := DoWithRetryAll(genericClient, req, resp) if err != nil { return "", err } data, err := resp.BodyUncompressed() if err != nil { data = resp.Body() } if experimental_GetClientID { var ver string var scriptUrl []byte for l := range bytes.SplitSeq(data, newline) { // version usually comes earlier, but retest this sometimes !!! if ver == "" && bytes.HasPrefix(l, sc_version) { ver = cfg.B2s(l[len(sc_version) : len(l)-len(`"`)]) misc.Log("found ver:", ver) if ClientIDCache.Version != "" && ver == ClientIDCache.Version { goto verCacheHit } } else if bytes.HasPrefix(l, script0) { scriptUrl = l[len(``)] misc.Log("found scriptUrl:", string(scriptUrl)) break } } if ver == "" { return "", ErrVersionNotFound } if scriptUrl == nil { return "", ErrScriptNotFound } req.SetRequestURIBytes(scriptUrl) err = DoWithRetryAll(genericClient, req, resp) if err != nil { return "", err } data, err = resp.BodyUncompressed() if err != nil { data = resp.Body() } m, _ := clientIdRegex.FindStringMatch(cfg.B2s(data)) if m != nil { g := m.GroupByNumber(1) if g != nil { ClientIDCache.ClientID = g.String() ClientIDCache.Version = ver ClientIDCache.NextCheck = time.Now().Add(cfg.ClientIDTTL) misc.Log(ClientIDCache) return ClientIDCache.ClientID, nil } } } else { ch := make(chan string, 1) wg := &sync.WaitGroup{} done := false var ver string var scriptUrls = make([][]byte, 0, 10) // Usually only 8 chunks, so I went with jsut a bit more to be safe for l := range bytes.SplitSeq(data, newline) { if ver == "" && bytes.HasPrefix(l, sc_version) { ver = cfg.B2s(l[len(sc_version) : len(l)-len(`"`)]) if ver == ClientIDCache.Version { goto verCacheHit } } else if bytes.HasPrefix(l, script) { scriptUrls = append(scriptUrls, l[len(``)]) } } if ver == "" { return "", ErrVersionNotFound } if len(scriptUrls) == 0 { return "", ErrScriptNotFound } for _, s := range scriptUrls { wg.Add(1) go processFile(wg, ch, s, &done) } go func() { defer func() { err := recover() misc.Log("-- GetClientID recovered:", err) }() wg.Wait() //time.Sleep(time.Millisecond) // maybe race? if !done { ch <- "" } }() res := <-ch done = true close(ch) if res == "" { err = ErrIDNotFound } else { ClientIDCache.ClientID = res ClientIDCache.Version = ver ClientIDCache.NextCheck = time.Now().Add(cfg.ClientIDTTL) misc.Log(ClientIDCache) } return res, err } return "", ErrIDNotFound verCacheHit: misc.Log("clientidcache hit @ ver") ClientIDCache.NextCheck = time.Now().Add(cfg.ClientIDTTL) return ClientIDCache.ClientID, nil } // Just retry any kind of errors, why not func DoWithRetryAll(httpc *fasthttp.Client, req *fasthttp.Request, resp *fasthttp.Response) (err error) { for range 10 { err = httpc.Do(req, resp) if err == nil { return nil } } return } // Since the http client is setup to always keep connections idle (great for speed, no need to open a new one everytime), those connections may be closed by soundcloud after some time of inactivity, this ensures that we retry those requests that fail due to the connection closing/timing out func DoWithRetry(httpc *fasthttp.HostClient, req *fasthttp.Request, resp *fasthttp.Response) (err error) { for range 10 { err = httpc.Do(req, resp) if err == nil { return nil } if err != fasthttp.ErrTimeout && err != fasthttp.ErrDialTimeout && err != fasthttp.ErrTLSHandshakeTimeout && err != fasthttp.ErrConnectionClosed && !os.IsTimeout(err) && !errors.Is(err, syscall.EPIPE) && // EPIPE is "broken pipe" error err.Error() != "timeout" { return } misc.Log("we failed haha", err) } return } func Resolve(cid string, path string, out any) error { var err error if cid == "" { cid, err = GetClientID() if err != nil { return err } } req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) req.SetRequestURI("https://" + api + "/resolve?url=https%3A%2F%2Fsoundcloud.com%2F" + url.QueryEscape(path) + "&client_id=" + cid) req.Header.SetUserAgent(cfg.UserAgent) req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) err = DoWithRetry(httpc, req, resp) if err != nil { return err } if resp.StatusCode() != 200 { return fmt.Errorf("resolve: got status code %d", resp.StatusCode()) } data, err := resp.BodyUncompressed() if err != nil { data = resp.Body() } return json.Unmarshal(data, out) } type Paginated[T any] struct { Next string `json:"next_href"` Collection []T `json:"collection"` Total int64 `json:"total_results"` } func (p *Paginated[T]) Proceed(cid string, shouldUnfold bool) error { var err error if cid == "" { cid, err = GetClientID() if err != nil { return err } } req := fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) oldNext := p.Next req.SetRequestURI(p.Next + "&client_id=" + cid) req.Header.SetUserAgent(cfg.UserAgent) req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) err = DoWithRetry(httpc, req, resp) if err != nil { return err } if resp.StatusCode() != 200 { return fmt.Errorf("paginated.proceed: got status code %d", resp.StatusCode()) } data, err := resp.BodyUncompressed() if err != nil { data = resp.Body() } err = json.Unmarshal(data, p) if err != nil { return err } if p.Next == oldNext { // prevent loops of nothingness p.Next = "" } // in soundcloud api, pagination may not immediately return you something! // loading users who haven't released anything recently may require you to do a bunch of requests for nothing :/ // maybe there could be a way to cache the last useless layer of pagination so soundcloak can start loading from there? might be a bit complicated, but would be great // another note: in featured tracks it seems to just be forever stuck after 2-3~ pages so i added a way to disable this behaviour if shouldUnfold && len(p.Collection) == 0 && p.Next != "" { // this will make sure that we actually proceed to something useful and not emptiness return p.Proceed(cid, true) } return nil } func TagListParser(taglist string) (res []string) { inString := false cur := []rune{} for i, c := range taglist { if c == '"' { if i == len(taglist)-1 { res = append(res, string(cur)) return } inString = !inString continue } if !inString && c == ' ' { res = append(res, string(cur)) cur = []rune{} continue } cur = append(cur, c) } if len(cur) != 0 { res = append(res, string(cur)) } return } type SearchSuggestion struct { Query string `json:"query"` } func GetSearchSuggestions(cid string, query string) ([]string, error) { p := Paginated[SearchSuggestion]{Next: "https://" + api + "/search/queries?limit=10&q=" + url.QueryEscape(query)} err := p.Proceed(cid, false) if err != nil { return nil, err } l := make([]string, len(p.Collection)) // just so the response is a bit smaller, why not for i, r := range p.Collection { l[i] = r.Query } return l, nil } // could probably make a generic function, whatever func init() { go func() { ticker := time.NewTicker(cfg.UserCacheCleanDelay) for range ticker.C { usersCacheLock.Lock() now := time.Now() for key, val := range UsersCache { if val.Expires.Before(now) { delete(UsersCache, key) } } usersCacheLock.Unlock() } }() go func() { ticker := time.NewTicker(cfg.TrackCacheCleanDelay) for range ticker.C { tracksCacheLock.Lock() now := time.Now() for key, val := range TracksCache { if val.Expires.Before(now) { delete(TracksCache, key) } } tracksCacheLock.Unlock() } }() go func() { ticker := time.NewTicker(cfg.PlaylistCacheCleanDelay) for range ticker.C { playlistsCacheLock.Lock() now := time.Now() for key, val := range PlaylistsCache { if val.Expires.Before(now) { delete(PlaylistsCache, key) } } playlistsCacheLock.Unlock() } }() }