Improvements towards supporting pagination

Not there yet, as Apollo stations always return autoplay
recommendations even if you set autoplay to false. Along the way
as an effort to bring the protocol up to spec:

- And support for and use different Apollo station scopes depending
  on whether we are using autoplay or not. For autoplay, get a
  "stations" scope and follow the "tracks" pages from there. Otherwise
  use "tracks" immediately for the active scope (playlist, album).

- For the above point we only need the fields from `PageContext`
  so use that instead of a `StationContext`.

- Add some documentation from API reverse engineering: things seen
  in the wild, some of them to do, others documented for posterity's
  sake.

- Update the Spirc device state based on what the latest desktop
  client puts out. Unfortunately none of it seems to change the
  behavior necessary to support external episodes, shows, but
  at least we're doing the right thing.

- Add a salt to HTTPS queries to defeat any caching.

- Add country metrics to HTTPS queries.

- Fix `get_radio_for_track` to use the right Spotify ID format.

- Fix a bug from the previous commit, where the playback position
  might not advance when hitting next and the autoplay context
  is loaded initially.

Author: roderickvd

connect/src/spirc.rs

@@ -16,7 +16,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream;
 
 use crate::{
     config::ConnectConfig,
-    context::{PageContext, StationContext},
+    context::PageContext,
     core::{
         authentication::Credentials, mercury::MercurySender, session::UserAttributes,
         util::SeqGenerator, version, Error, Session, SpotifyId,
@@ -100,7 +100,7 @@ struct SpircTask {
     session: Session,
     resolve_context: Option<String>,
     autoplay_context: bool,
-    context: Option<StationContext>,
+    context: Option<PageContext>,
 
     spirc_id: usize,
 }
@@ -124,7 +124,6 @@ pub enum SpircCommand {
     SetVolume(u16),
 }
 
-const CONTEXT_TRACKS_COUNT: usize = 50;
 const CONTEXT_TRACKS_HISTORY: usize = 10;
 const CONTEXT_FETCH_THRESHOLD: u32 = 5;
 
@@ -184,6 +183,7 @@ fn initial_device_state(config: ConnectConfig) -> DeviceState {
             };
             {
                 let msg = repeated.push_default();
+                // TODO: implement logout
                 msg.set_typ(protocol::spirc::CapabilityType::kSupportsLogout);
                 {
                     let repeated = msg.mut_intValue();
@@ -224,17 +224,51 @@ fn initial_device_state(config: ConnectConfig) -> DeviceState {
             };
             {
                 let msg = repeated.push_default();
-                msg.set_typ(protocol::spirc::CapabilityType::kSupportedContexts);
+                msg.set_typ(protocol::spirc::CapabilityType::kSupportsExternalEpisodes);
                 {
-                    let repeated = msg.mut_stringValue();
-                    repeated.push(::std::convert::Into::into("album"));
-                    repeated.push(::std::convert::Into::into("playlist"));
-                    repeated.push(::std::convert::Into::into("search"));
-                    repeated.push(::std::convert::Into::into("inbox"));
-                    repeated.push(::std::convert::Into::into("toplist"));
-                    repeated.push(::std::convert::Into::into("starred"));
-                    repeated.push(::std::convert::Into::into("publishedstarred"));
-                    repeated.push(::std::convert::Into::into("track"))
+                    let repeated = msg.mut_intValue();
+                    repeated.push(1)
+                };
+                msg
+            };
+            {
+                let msg = repeated.push_default();
+                // TODO: how would such a rename command be triggered? Handle it.
+                msg.set_typ(protocol::spirc::CapabilityType::kSupportsRename);
+                {
+                    let repeated = msg.mut_intValue();
+                    repeated.push(1)
+                };
+                msg
+            };
+            {
+                let msg = repeated.push_default();
+                msg.set_typ(protocol::spirc::CapabilityType::kCommandAcks);
+                {
+                    let repeated = msg.mut_intValue();
+                    repeated.push(0)
+                };
+                msg
+            };
+            {
+                let msg = repeated.push_default();
+                // TODO: does this mean local files or the local network?
+                // LAN may be an interesting privacy toggle.
+                msg.set_typ(protocol::spirc::CapabilityType::kRestrictToLocal);
+                {
+                    let repeated = msg.mut_intValue();
+                    repeated.push(0)
+                };
+                msg
+            };
+            {
+                let msg = repeated.push_default();
+                // TODO: what does this hide, or who do we hide from?
+                // May be an interesting privacy toggle.
+                msg.set_typ(protocol::spirc::CapabilityType::kHidden);
+                {
+                    let repeated = msg.mut_intValue();
+                    repeated.push(0)
                 };
                 msg
             };
@@ -243,9 +277,15 @@ fn initial_device_state(config: ConnectConfig) -> DeviceState {
                 msg.set_typ(protocol::spirc::CapabilityType::kSupportedTypes);
                 {
                     let repeated = msg.mut_stringValue();
-                    repeated.push(::std::convert::Into::into("audio/track"));
-                    repeated.push(::std::convert::Into::into("audio/episode"));
-                    repeated.push(::std::convert::Into::into("track"))
+                    repeated.push("audio/episode".to_string());
+                    repeated.push("audio/episode+track".to_string());
+                    repeated.push("audio/track".to_string());
+                    // other known types:
+                    // - "audio/ad"
+                    // - "audio/interruption"
+                    // - "audio/local"
+                    // - "video/ad"
+                    // - "video/episode"
                 };
                 msg
             };
@@ -498,35 +538,30 @@ impl SpircTask {
                     break;
                 },
                 context_uri = async { self.resolve_context.take() }, if self.resolve_context.is_some() => {
-                    let context_uri = context_uri.unwrap();
-                    let is_next_page = context_uri.starts_with("hm://");
+                    let context_uri = context_uri.unwrap(); // guaranteed above
+                    if context_uri.contains("spotify:show:") || context_uri.contains("spotify:episode:") {
+                        continue; // not supported by apollo stations
+                    }
 
-                    let context = if is_next_page {
+                    let context = if context_uri.starts_with("hm://") {
                         self.session.spclient().get_next_page(&context_uri).await
                     } else {
-                        let previous_tracks = self.state.get_track().iter().filter_map(|t| SpotifyId::try_from(t).ok()).collect();
-                        self.session.spclient().get_apollo_station(&context_uri, CONTEXT_TRACKS_COUNT, previous_tracks, self.autoplay_context).await
+                        // only send previous tracks that were before the current playback position
+                        let current_position = self.state.get_playing_track_index() as usize;
+                        let previous_tracks = self.state.get_track()[..current_position].iter().filter_map(|t| SpotifyId::try_from(t).ok()).collect();
+
+                        let scope = if self.autoplay_context {
+                            "stations" // this returns a `StationContext` but we deserialize it into a `PageContext`
+                        } else {
+                            "tracks" // this returns a `PageContext`
+                        };
+
+                        self.session.spclient().get_apollo_station(scope, &context_uri, None, previous_tracks, self.autoplay_context).await
                     };
 
                     match context {
                         Ok(value) => {
-                            let r_context = if is_next_page {
-                                match serde_json::from_slice::<PageContext>(&value) {
-                                    Ok(page_context) => {
-                                        // page contexts don't have the stations full metadata, so decorate it
-                                        let mut station_context = self.context.clone().unwrap_or_default();
-                                        station_context.tracks = page_context.tracks;
-                                        station_context.next_page_url = page_context.next_page_url;
-                                        station_context.correlation_id = page_context.correlation_id;
-                                        Ok(station_context)
-                                    },
-                                    Err(e) => Err(e),
-                                }
-                            } else {
-                                serde_json::from_slice::<StationContext>(&value)
-                            };
-
-                            self.context = match r_context {
+                            self.context = match serde_json::from_slice::<PageContext>(&value) {
                                 Ok(context) => {
                                     info!(
                                         "Resolved {:?} tracks from <{:?}>",
@@ -829,7 +864,7 @@ impl SpircTask {
 
         for entry in update.get_device_state().get_metadata().iter() {
             match entry.get_field_type() {
-                "client-id" => self.session.set_client_id(entry.get_metadata()),
+                "client_id" => self.session.set_client_id(entry.get_metadata()),
                 "brand_display_name" => self.session.set_client_brand_name(entry.get_metadata()),
                 "model_display_name" => self.session.set_client_model_name(entry.get_metadata()),
                 _ => (),
@@ -1207,17 +1242,23 @@ impl SpircTask {
 
         // When in autoplay, keep topping up the playlist when it nears the end
         if update_tracks {
-            self.update_tracks_from_context();
-            new_index = self.state.get_playing_track_index();
-            tracks_len = self.state.get_track().len() as u32;
+            if let Some(ref context) = self.context {
+                self.resolve_context = Some(context.next_page_url.to_owned());
+                self.update_tracks_from_context();
+                tracks_len = self.state.get_track().len() as u32;
+            }
         }
 
         // When not in autoplay, either start autoplay or loop back to the start
         if new_index >= tracks_len {
-            if self.session.autoplay() {
+            // for some contexts there is no autoplay, such as shows and episodes
+            // in such cases there is no context in librespot.
+            if self.context.is_some() && self.session.autoplay() {
                 // Extend the playlist
                 debug!("Starting autoplay for <{}>", context_uri);
+                // force reloading the current context with an autoplay context
                 self.autoplay_context = true;
+                self.resolve_context = Some(self.state.get_context_uri().to_owned());
                 self.update_tracks_from_context();
                 self.player.set_auto_normalise_as_album(false);
             } else {
@@ -1306,17 +1347,10 @@ impl SpircTask {
 
     fn update_tracks_from_context(&mut self) {
         if let Some(ref context) = self.context {
-            self.resolve_context =
-                if !self.autoplay_context || context.next_page_url.contains("autoplay=true") {
-                    Some(context.next_page_url.to_owned())
-                } else {
-                    // this arm means: we need to resolve for autoplay,
-                    // and were previously resolving for the original context
-                    Some(context.uri.to_owned())
-                };
-
             let new_tracks = &context.tracks;
+
             debug!("Adding {:?} tracks from context to frame", new_tracks.len());
+
             let mut track_vec = self.state.take_track().into_vec();
             if let Some(head) = track_vec.len().checked_sub(CONTEXT_TRACKS_HISTORY) {
                 track_vec.drain(0..head);
@@ -1342,22 +1376,22 @@ impl SpircTask {
         trace!("State: {:#?}", frame.get_state());
 
         let index = frame.get_state().get_playing_track_index();
-        let context_uri = frame.get_state().get_context_uri().to_owned();
+        let context_uri = frame.get_state().get_context_uri();
         let tracks = frame.get_state().get_track();
 
         trace!("Frame has {:?} tracks", tracks.len());
 
         // First the tracks from the requested context, without autoplay.
         // We will transition into autoplay after the latest track of this context.
         self.autoplay_context = false;
-        self.resolve_context = Some(context_uri.clone());
+        self.resolve_context = Some(context_uri.to_owned());
 
         self.player
             .set_auto_normalise_as_album(context_uri.starts_with("spotify:album:"));
 
         self.state.set_playing_track_index(index);
         self.state.set_track(tracks.iter().cloned().collect());
-        self.state.set_context_uri(context_uri);
+        self.state.set_context_uri(context_uri.to_owned());
         // has_shuffle/repeat seem to always be true in these replace msgs,
         // but to replicate the behaviour of the Android client we have to
         // ignore false values.
@@ -1517,8 +1551,11 @@ struct CommandSender<'a> {
 impl<'a> CommandSender<'a> {
     fn new(spirc: &'a mut SpircTask, cmd: MessageType) -> CommandSender<'_> {
         let mut frame = protocol::spirc::Frame::new();
+        // frame version
         frame.set_version(1);
-        frame.set_protocol_version(::std::convert::Into::into("2.0.0"));
+        // Latest known Spirc version is 3.2.6, but we need another interface to announce support for Spirc V3.
+        // Setting anything higher than 2.0.0 here just seems to limit it to 2.0.0.
+        frame.set_protocol_version("2.0.0".to_string());
         frame.set_ident(spirc.ident.clone());
         frame.set_seq_nr(spirc.sequence.get());
         frame.set_typ(cmd);

core/src/spclient.rs

@@ -15,6 +15,7 @@ use hyper::{
     Body, HeaderMap, Method, Request,
 };
 use protobuf::{Message, ProtobufEnum};
+use rand::RngCore;
 use sha1::{Digest, Sha1};
 use sysinfo::{System, SystemExt};
 use thiserror::Error;
@@ -435,13 +436,26 @@ impl SpClient {
             let mut url = self.base_url().await?;
             url.push_str(endpoint);
 
-            // Add metrics. There is also an optional `partner` key with a value like
-            // `vodafone-uk` but we've yet to discover how we can find that value.
             let separator = match url.find('?') {
                 Some(_) => "&",
                 None => "?",
             };
-            let _ = write!(url, "{}product=0", separator);
+
+            // Add metrics. There is also an optional `partner` key with a value like
+            // `vodafone-uk` but we've yet to discover how we can find that value.
+            // For the sake of documentation you could also do "product=free" but
+            // we only support premium anyway.
+            let _ = write!(
+                url,
+                "{}product=0&country={}",
+                separator,
+                self.session().country()
+            );
+
+            // Defeat caches. Spotify-generated URLs already contain this.
+            if !url.contains("salt=") {
+                let _ = write!(url, "&salt={}", rand::thread_rng().next_u32());
+            }
 
             let mut request = Request::builder()
                 .method(method)
@@ -616,29 +630,49 @@ impl SpClient {
     pub async fn get_radio_for_track(&self, track_id: &SpotifyId) -> SpClientResult {
         let endpoint = format!(
             "/inspiredby-mix/v2/seed_to_playlist/{}?response-format=json",
-            track_id.to_base62()?
+            track_id.to_uri()?
         );
 
         self.request_as_json(&Method::GET, &endpoint, None, None)
             .await
     }
 
+    // Known working scopes: stations, tracks
+    // For others see: https://gist.github.com/roderickvd/62df5b74d2179a12de6817a37bb474f9
+    //
+    // Seen-in-the-wild but unimplemented query parameters:
+    // - image_style=gradient_overlay
+    // - excludeClusters=true
+    // - language=en
+    // - count_tracks=0
+    // - market=from_token
     pub async fn get_apollo_station(
         &self,
+        scope: &str,
         context_uri: &str,
-        count: usize,
+        count: Option<usize>,
         previous_tracks: Vec<SpotifyId>,
         autoplay: bool,
     ) -> SpClientResult {
+        let mut endpoint = format!(
+            "/radio-apollo/v3/{}/{}?autoplay={}",
+            scope, context_uri, autoplay,
+        );
+
+        // Spotify has a default of 50
+        if let Some(count) = count {
+            let _ = write!(endpoint, "&count={}", count);
+        }
+
         let previous_track_str = previous_tracks
             .iter()
             .map(|track| track.to_base62())
             .collect::<Result<Vec<_>, _>>()?
             .join(",");
-        let endpoint = format!(
-            "/radio-apollo/v3/stations/{}?count={}&prev_tracks={}&autoplay={}",
-            context_uri, count, previous_track_str, autoplay,
-        );
+        // better than checking `previous_tracks.len() > 0` because the `filter_map` could still return 0 items
+        if !previous_track_str.is_empty() {
+            let _ = write!(endpoint, "&prev_tracks={}", previous_track_str);
+        }
 
         self.request_as_json(&Method::GET, &endpoint, None, None)
             .await
@@ -650,6 +684,9 @@ impl SpClient {
             .await
     }
 
+    // TODO: Seen-in-the-wild but unimplemented endpoints
+    // - /presence-view/v1/buddylist
+
     // TODO: Find endpoint for newer canvas.proto and upgrade to that.
     pub async fn get_canvases(&self, request: EntityCanvazRequest) -> SpClientResult {
         let endpoint = "/canvaz-cache/v0/canvases";