This post is part of a series of posts to analyse the digital me.

Collecting Spotify data with R

Spotify has made its data very accessible to endusers to play with. Their API provides you with a plethora of options to get to their data, which is a ton of fun to analyse.

To get started, you wil need to R packages to access the data and process it afterwards.

library('httr')
library('jsonlite')
library('dplyr')
library('tidyr')
library('zoo')
library('purrr')
library('RCurl')

You will also need access to the Spotify API, which is granted through a Client ID and Client Secret. You can get these by using your Spotify account and go to the Developers Page at Spotify.

Once you have your client data, you are ready to get access to the data with R. You will need to get a token the first time, after that, you can store it in a local file to ensure you don’t have to go back to the popup screen each time to get another one.

if(!file.exists(".spotify")){
    print("no file")

    #to get token FIRST TIME
    browseURL(paste0('https://accounts.spotify.com/authorize?client_id=',client_id,'&response_type=code&redirect_uri=',website_uri,'/&scope=user-read-recently-played'),browser = getOption("browser"), encodeIfNeeded = FALSE)
    
    #add new token
    user_code <- user_code_value
    
    #construct body of POST request FIRST TIME
    request_body <- list(grant_type='authorization_code',
                         code=user_code,
                         redirect_uri=website_uri, #input your domain name
                         client_id = sp_client_id, #input your Spotify Client ID
                         client_secret = sp_client_secret) #input your Spotify Client Secret
    
    #get user tokens FIRST TIME
    user_token <- httr::content(httr::POST('https://accounts.spotify.com/api/token',
                                           body=request_body,
                                           encode='form'))
    
    user_token$access_token -> token
    auth_header <- httr::add_headers('Authorization'= paste('Bearer',token))
    write(user_token$refresh_token, ".spotify")

}

if(file.exists(".spotify")) {
    print("we have file")
  
    #REFRESH
    scan(file = ".spotify", what= list(id="")) -> red
    as.character(red) -> refresh_code
    request_body_refresh <- list(grant_type='refresh_token',
                            refresh_token=refresh_code,
                            redirect_uri=website_uri,
                            client_id = sp_client_id,
                            client_secret = sp_client_secret)
    
    #get user tokens REFRESH
    user_token_refresh <- httr::content(httr::POST('https://accounts.spotify.com/api/token',
                                           body=request_body_refresh,
                                           encode='form'))
    user_token_refresh$access_token -> token
}
## [1] "we have file"
#THIS RUNS EVERYTIME
auth_header <- httr::add_headers('Authorization'= paste('Bearer',token))
recently_played <- httr::content(httr::GET('https://api.spotify.com/v1/me/player/recently-played',
                        query=list(limit=50,time_range='long_range'),auth_header))

The script returns a recently played list of trackes of maximum of 50. Unfortunately, Spotify doesn’t allow you to get all of your historical data. It is easy to run this script on a server hourly or daily to collect fresh data and update your dataset.

Get Spotify track data

Now that we have collected the 50 most recently played tracks, we will create a clean dataframe that we can store and analyse.

# generate the proper data.frame
toJSON(recently_played$items) -> df1
fromJSON(df1) %>% as.data.frame -> df2
df2$track -> df_track
df_track$name -> track_name
df_track$duration_ms -> track_duration_ms
df_track$id -> track_id
df_track$popularity -> track_popularity
df_track$album$name -> track_album
df_track$explicit -> track_explicit
as.list(df_track$external_urls) -> track_href
placeholder <- data.frame(list(height=1:3),list(url=1:3),list(width=1:3))
df_track$album$images[!lengths(df_track$album$images)] <- list(placeholder)

lapply(df_track$album$images, "[[", 2) -> album_images
lapply(album_images, "[[", 1) -> album_image_big
lapply(album_images, "[[", 3) -> album_image_small
df_track$album$id -> track_album_id
df2$played_at -> track_played_at

df_track$artists -> track_artists
lapply(track_artists, "[[", 4) -> track_artist_name
lapply(track_artist_name, `[[`, 1) -> track_artist_name
lapply(track_artists, "[[", 3) -> track_artist_id
lapply(track_artist_id, `[[`, 1) -> track_artist_id

do.call(rbind.data.frame, Map('c', track_id, track_name, track_album_id, track_album, track_artist_id, track_popularity, track_duration_ms, track_explicit, album_image_big, album_image_small, track_played_at)) -> track_details
colnames(track_details) <- c("track_id", "track_name", "track_album_id", "track_album", "artist_id", "track_popularity", "track_duration_ms", "track_explicit", "album_image_big", 'album_image_small', "track_played_at" )

This generates a clean dataframe with all the track data I need to analyse and visualise my Spotify data.

head(track_details,5)
##                 track_id                              track_name
## 1 6HjJLMT0VgAr2DDKAtSRsY A Little Respect - David Thoener Mix #1
## 2 4rXZ7zmDDk739IWHrLcVt1                          Old Dan Tucker
## 3 3ftHrCjsTUPLgI48m67byk                        Plush - Acoustic
## 4 7bCKBqdYJI8fvFH48sT2IN                              Dune Buggy
## 5 4uGIJG1jYFonGc4LGp5uQL                        Somebody to Love
##           track_album_id
## 1 2WHNZHbZ9TshfsZWRbTmq2
## 2 2O3avtEIE8xX9F0zcIjIZI
## 3 1fyLNx6wxgDA59wFInnyup
## 4 5xxeAo8AVneH1OKO5vR604
## 5 6lPb7Eoon6QPbscWbMsk6a
##                                                                                        track_album
## 1                                                                                 A Little Respect
## 2                                   We Shall Overcome: The Seeger Sessions (American Land Edition)
## 3                                                                                        Thank You
## 4 The Presidents of The United States of America: Ten Year Super Bonus Special Anniversary Edition
## 5                                                                              Surrealistic Pillow
##                artist_id track_popularity track_duration_ms track_explicit
## 1 4mYFgEjpQT4IKOrjOUKyXu               33            203866          FALSE
## 2 3eqjTLE0HfPfh78zjh6TqT               39            151906          FALSE
## 3 2UazAtjfzqBF0Nho2awK4z               62            230293          FALSE
## 4 1lZvg4fNAqHoj6I9N8naBM               48            164573          FALSE
## 5 2qFr8w5sWUITRlzZ9kZotF               68            174840          FALSE
##                                                    album_image_big
## 1 https://i.scdn.co/image/97739f8e8d3e584c7e28b63a832ec33e31aaf957
## 2 https://i.scdn.co/image/623b680211596bf1a06190d7faa0ac8118aacd58
## 3 https://i.scdn.co/image/2e3a77fd23cdd87181e21fb3e9a70d6074329422
## 4 https://i.scdn.co/image/224dc65b4e81e3d395c66f36cf2a141edf50de66
## 5 https://i.scdn.co/image/a93ab07d6c3718f1d8f01185b53d336d1b2a986d
##                                                  album_image_small
## 1 https://i.scdn.co/image/4f5757de78b72f72f5173a16f9d9a9b51cf898b2
## 2 https://i.scdn.co/image/0717466f95895b5c6fba39d62ed4e6e9d6a9f4fb
## 3 https://i.scdn.co/image/ee928502fe2546e5215692fa3357bcba58963abf
## 4 https://i.scdn.co/image/3332efb89725df4626a05bc73cac165b938618c2
## 5 https://i.scdn.co/image/f50c6c8f5e065280bfe9039d0ab462c741b1e39d
##            track_played_at
## 1 2018-10-02T15:27:46.764Z
## 2 2018-10-02T15:24:23.972Z
## 3 2018-10-02T15:21:54.916Z
## 4 2018-10-02T15:18:01.809Z
## 5 2018-10-02T15:15:18.712Z

Get Spotify artist data

I also want to add data about the artist to my dataset. So from each of the tracks in my dataset, I will collect the artist ID and add them to my following query to collect the corresponding artist data

#prepare to get data about artists
response = POST(
  'https://accounts.spotify.com/api/token',
  accept_json(),
  authenticate(sp_client_id, sp_client_secret),
  body = list(grant_type = 'client_credentials'),
  encode = 'form',
  verbose()
)
token1 <- content(response)$access_token

#get list of artist ID's from recently played list
paste(as.character(track_artist_id),collapse=",",sep="") -> artist_comma

#query open api data
HeaderValue = paste0('Bearer ', token1)
URI = paste0('https://api.spotify.com/v1/artists?ids=', artist_comma)
response2 = GET(url = URI, add_headers(Authorization = HeaderValue))
artist_details = content(response2)

toJSON(artist_details) -> df4
fromJSON(df4) %>% as.data.frame -> df5

df6 <- data.frame(df5$artists.followers)
as.numeric(df6$total) -> artist_followers
df5$artists.popularity -> artist_popularity
df5$artists.id -> artist_id
df5$artists.genres -> artist_genres
df5$artists.name -> artist_names

lapply(df5$artists.images, `[`,1,2) -> track_artist_image

df6_6 <- data.frame(df5$artists.external_urls)
df6_6$spotify -> artist_url

do.call(rbind.data.frame, Map('c', artist_id, artist_popularity, artist_followers, artist_names, artist_url, track_artist_image)) -> artist_details
colnames(artist_details) <- c("artist_id", "artist_popularity", "artist_followers", "artist_name" ,"artist_url", "track_artist_image")
sapply(artist_genres, paste0 , collapse = "|") -> artist_details$artist_genres

head(artist_details,5)
##                  artist_id artist_popularity artist_followers
## 2   4mYFgEjpQT4IKOrjOUKyXu                60            59838
## 210 3eqjTLE0HfPfh78zjh6TqT                76          1716348
## 3   2UazAtjfzqBF0Nho2awK4z                69          1277165
## 4   1lZvg4fNAqHoj6I9N8naBM                61           192727
## 5   2qFr8w5sWUITRlzZ9kZotF                63           580023
##                                        artist_name
## 2                                          Wheatus
## 210                              Bruce Springsteen
## 3                              Stone Temple Pilots
## 4   The Presidents Of The United States Of America
## 5                               Jefferson Airplane
##                                                 artist_url
## 2   https://open.spotify.com/artist/4mYFgEjpQT4IKOrjOUKyXu
## 210 https://open.spotify.com/artist/3eqjTLE0HfPfh78zjh6TqT
## 3   https://open.spotify.com/artist/2UazAtjfzqBF0Nho2awK4z
## 4   https://open.spotify.com/artist/1lZvg4fNAqHoj6I9N8naBM
## 5   https://open.spotify.com/artist/2qFr8w5sWUITRlzZ9kZotF
##                                                   track_artist_image
## 2   https://i.scdn.co/image/709fbfbdc988b8e3c6011a64fa0d3117d41b1bb8
## 210 https://i.scdn.co/image/bfebac3159d5a6a5be5c05d6eb133fbceafe08c6
## 3   https://i.scdn.co/image/d242ac9d8c7a93c021e8d61863e98d77386cb0f5
## 4   https://i.scdn.co/image/fea0ca15171adee5a512bc9f417767d81c18cf35
## 5   https://i.scdn.co/image/a9ae31801b767795dd3060e5529478e534b02c16
##                                                                                          artist_genres
## 2                                                                                             pop rock
## 210                                   heartland rock|mellow gold|permanent wave|rock|singer-songwriter
## 3   alternative metal|alternative rock|grunge|hard rock|modern rock|nu metal|pop rock|post-grunge|rock
## 4             alternative metal|alternative rock|comic|funk rock|garage rock|pop rock|post-grunge|rock
## 5               album rock|art rock|blues-rock|classic rock|folk rock|psychedelic rock|rock|roots rock

One of the few caveats I have found in the Spotify data is that tracks are not assigned genres, but artists are. This means that there is no way to define the genre of a song, unless, you connect it to the performing artist. This is not always accurate for eclectic artists performing multiple genres.

Get Spotify track audio details

Spotify provides very detailed data for each individual track. The Spotify Insights blog has some cool posts on this topic. There are more technical details on audio analysis in the Spotify API documentation.

To get the Audio details for the tracks, we will use the track_id variable from our track dataframe and get more detailed data.

#get list of track ID's from recently played list
paste(as.character(track_id),collapse=",",sep="") -> track_comma

URI2 = paste0('https://api.spotify.com/v1/audio-features/?ids=', track_comma)
response3 = GET(url = URI2, add_headers(Authorization = HeaderValue))
track_special_details = content(response3)

toJSON(track_special_details) -> df9
fromJSON(df9) %>% as.data.frame -> df10
unnest(df10) ->df10
df10$audio_features.id -> track_special_id
#https://developer.spotify.com/web-api/get-audio-features/
as.numeric(df10$audio_features.key) -> track_special_key
as.numeric(df10$audio_features.mode) -> track_special_mode
as.numeric(df10$audio_features.acousticness) -> track_special_acousticness
as.numeric(df10$audio_features.danceability) -> track_special_danceability
as.numeric(df10$audio_features.energy) -> track_special_energy
as.numeric(df10$audio_features.tempo) -> track_special_tempo
as.numeric(df10$audio_features.speechiness) -> track_special_speechiness
as.numeric(df10$audio_features.instrumentalness) -> track_special_instrumentalness
as.numeric(df10$audio_features.liveness) -> track_special_liveliness
as.numeric(df10$audio_features.valence) -> track_special_valence
df10$audio_features.uri -> track_features_uri
df10$audio_features.track_href -> track_features_href

do.call(rbind.data.frame, Map('c', track_special_id, track_special_key, track_special_mode, track_special_acousticness, track_special_danceability, track_special_energy, track_special_speechiness, track_special_tempo, track_special_instrumentalness, track_special_liveliness, track_special_valence)) -> track_special_details
colnames(track_special_details) <- c("track_id", "track_key", "track_special_mode", "track_auccoustiness", "track_danceability", "track_energy", "track_speechiness", "track_tempo", "track_special_instrumentalness", "track_special_liveliness", "track_special_valence")
head(track_special_details,5)
##                 track_id track_key track_special_mode track_auccoustiness
## 1 6HjJLMT0VgAr2DDKAtSRsY         0                  1              0.0496
## 2 4rXZ7zmDDk739IWHrLcVt1         3                  1              0.0754
## 3 3ftHrCjsTUPLgI48m67byk         5                  0               0.712
## 4 7bCKBqdYJI8fvFH48sT2IN         1                  1              0.0092
## 5 4uGIJG1jYFonGc4LGp5uQL         4                  1               0.408
##   track_danceability track_energy track_speechiness track_tempo
## 1              0.426        0.785            0.0478     113.478
## 2              0.592        0.677            0.0305      96.645
## 3              0.644        0.363            0.0397     133.396
## 4               0.69        0.828            0.0992      92.973
## 5              0.504        0.578             0.031     132.411
##   track_special_instrumentalness track_special_liveliness
## 1                          1e-04                    0.105
## 2                              0                    0.317
## 3                              0                    0.659
## 4                         0.0026                    0.065
## 5                              0                    0.321
##   track_special_valence
## 1                 0.429
## 2                 0.945
## 3                 0.303
## 4                 0.822
## 5                 0.654

Merging it to one flat dataframe

To make it easier to use, I will use our three dataframes and merge them into one big dataframe. I will also clean up some of the data for future use.

# merge 3 data.frames to have one big one for all tracks
merged1 <- track_details %>% 
  mutate(track_played_at = as.POSIXct(track_played_at, 
                                tz = "CET", 
                                format = "%Y-%m-%dT%H:%M:%S"))  %>%
  left_join(artist_details, by="artist_id")

merged <- merged1 %>%  left_join(track_special_details, by="track_id")
#kill duplicates
subset(merged,!duplicated(merged$track_played_at)) -> merged

as.numeric(as.character(merged$track_speechiness)) -> merged$track_speechiness
as.numeric(as.character(merged$track_danceability)) -> merged$track_danceability
as.numeric(as.character(merged$track_popularity)) -> merged$track_popularity
as.numeric(as.character(merged$track_duration_ms)) -> merged$track_duration_ms
as.numeric(as.character(merged$artist_popularity)) -> merged$artist_popularity
as.numeric(as.character(merged$artist_followers)) -> merged$artist_followers
as.numeric(as.character(merged$track_key)) -> merged$track_key
as.numeric(as.character(merged$track_special_mode)) -> merged$track_special_mode
as.numeric(as.character(merged$track_auccoustiness)) -> merged$track_auccoustiness
as.numeric(as.character(merged$track_energy)) -> merged$track_energy
as.numeric(as.character(merged$track_tempo)) -> merged$track_tempo
as.numeric(as.character(merged$track_special_instrumentalness)) -> merged$track_special_instrumentalness
as.numeric(as.character(merged$track_special_liveliness)) -> merged$track_special_liveliness
as.numeric(as.character(merged$track_special_valence)) -> merged$track_special_valence