data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os, json, logging\n",
    "from requests import get, post\n",
    "from time import time, sleep\n",
    "from random import randint\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# initialize log settings for error\n",
    "logging.basicConfig(filename=\"error.log\", level=logging.INFO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain token and prepare headers to access Spotify's APIs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# post Client Credentials to Spotify's Token API to obtain Token\n",
    "with open(\"./token/token.json\", \"r\") as token_file:\n",
    "    token_file = json.load(token_file)\n",
    "res = post('https://accounts.spotify.com/api/token', headers = {'Authorization': '{}'.format(token_file[\"token\"])}, data= {'grant_type': 'client_credentials'})\n",
    "token = 'Bearer {}'.format(res.json()['access_token'])\n",
    "\n",
    "# define headers to include Token\n",
    "headers = {'Authorization': token, \"Accept\": 'application/json', 'Content-Type': \"application/json\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function that tracks progress of specified task."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def track_progress(task, step, num, i, start_time):\n",
    "    \"\"\"\n",
    "    Function that helps track progress of specified task.\n",
    "    Args:\n",
    "        task: the task to track\n",
    "        step: the increment between each for loop\n",
    "        num: number of songs to track\n",
    "        i: current iteration of the loop\n",
    "        start_time: time to start tracking from\n",
    "    \"\"\"\n",
    "    try:\n",
    "        print(\"{}: [{}{}] {}/{} songs completed. Approx time left: {} minutes.\".format(task, \"#\"*round(((i+step)/(num/20))), \"-\"*round((num/(num/20)-((i+step)/(num/20)))), i+step, num, round(((time()-start_time)/i)*(num-i)/60, 2)), end=\"\\r\", flush=True)\n",
    "    except:\n",
    "        print(\"{}: [{}{}] {}/{} songs completed. Approx time left: {}\".format(task, \"#\"*round(((i+step)/(num/20))), \"-\"*round((num/(num/20)-((i+step)/(num/20)))), i+step, num, \"-\"), end=\"\\r\", flush=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function that extracts basic info of songs from specified year (name, id, popularity, release date) and stores them in dataframe songs_info.\n",
    "\n",
    "approx. 5secs per 50 songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_songs(year, num):\n",
    "    \"\"\"\n",
    "    Function to extract specified number of songs from a specified year.\n",
    "    Args:\n",
    "        year: year to extract songs from\n",
    "        num: number of songs to extract\n",
    "    \"\"\"\n",
    "    # for songs released in specified year, create data frame for songs' basic info\n",
    "    songs_info = pd.DataFrame(columns=[\"name\", \"id\", \"popularity\", \"release_date\"])\n",
    "\n",
    "    # set start_time and songs_extracted to track progress\n",
    "    start_time = time()\n",
    "    songs_extracted = 0\n",
    "\n",
    "    # from Spotify API, get songs and basic info released in specified year, in batches of 50, with rate limit of 1-3 secs between batches\n",
    "    # maximum songs per year is 10k\n",
    "    for i in range(0, num, 50):\n",
    "        url=\"https://api.spotify.com/v1/search?q=%20year:{}&limit=50&offset={}&type=track\".format(year, i)\n",
    "        r=get(url, headers=headers)\n",
    "        \n",
    "        # to ensure response contains track details, which is not the case sometimes\n",
    "        while \"tracks\" not in r.json():\n",
    "            r=get(url, headers=headers)\n",
    "\n",
    "        for item in r.json()[\"tracks\"][\"items\"]:\n",
    "            songs_info = songs_info.append({\"name\": item[\"name\"], \"id\": item[\"id\"], \"popularity\": item[\"popularity\"],\n",
    "                                           \"release_date\": item[\"album\"][\"release_date\"]}, ignore_index = True)\n",
    "        \n",
    "        # track progress\n",
    "        track_progress(\"Extracting songs\", 50, num, i, start_time)\n",
    "        \n",
    "        sleep(randint(1,3))\n",
    "        songs_extracted = i + 50\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"EXTRACTION COMPLETE. Songs extracted: {}; Elapsed Time: {} minutes.\".format(songs_extracted, round((time()-start_time)/60, 2)))\n",
    "    return songs_info\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function that labels data with hit/miss and balance datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_songs(songs_info):\n",
    "    \"\"\"\n",
    "    Function to label songs with hit/miss.\n",
    "    Args:\n",
    "        songs_info: dataframe containing songs' basic info\n",
    "    \"\"\"\n",
    "    # number of songs to label\n",
    "    num = len(songs_info[\"popularity\"])\n",
    "    \n",
    "    # using cutoff_popularity, classify songs as hit or miss using hit_miss list, then add hit_miss list to DataFrame\n",
    "    hit_miss = []\n",
    "    hit_count = 0\n",
    "    \n",
    "    # set start_time and songs_labeled to track progress\n",
    "    start_time = time()\n",
    "    songs_labeled = 0\n",
    "\n",
    "    for i in range(num):\n",
    "\n",
    "        if songs_info[\"popularity\"][i] >= 70:\n",
    "            hit_count += 1\n",
    "            hit_miss.append(\"hit\")\n",
    "\n",
    "        else:\n",
    "            hit_miss.append(\"miss\")\n",
    "            \n",
    "        # track progress\n",
    "        track_progress(\"Labeling songs\", 1, num, i, start_time)\n",
    "            \n",
    "        songs_labeled = i + 1\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"LABELING COMPLETE. Songs labeled: {}; Elapsed Time: {} minutes.\".format(songs_labeled, round((time()-start_time)/60, 2)))\n",
    "    \n",
    "    songs_info[\"hit_miss\"] = hit_miss\n",
    "            \n",
    "    if hit_count < len(songs_info[\"popularity\"])/2:\n",
    "        drop_count = len(songs_info[\"popularity\"]) - 2*hit_count\n",
    "        songs_info = songs_info.drop(songs_info[songs_info[\"hit_miss\"]==\"miss\"].sample(n=drop_count).index)\n",
    "        songs_info = songs_info.reset_index(drop=True)\n",
    "        \n",
    "    # save temp song list in case analysis later fails\n",
    "    songs_info.to_csv(\"./temp/{}_{}_song_list.csv\".format(year, num), index=False)\n",
    "\n",
    "    return songs_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## On songs extracted, conduct audio analysis (e.g. duration, loudness, tempo etc.), stored in dataframe songs_analysis.\n",
    "\n",
    "approx. 3mins per 10 songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def audio_analysis(songs_info, half_analyzed):\n",
    "    \"\"\"\n",
    "    Function to conduct audio analysis on extracted songs.\n",
    "    Args:\n",
    "        songs_info: dataframe containing songs' basic info\n",
    "        half_analyzed: determines number of pre-analyzed songs if applicable\n",
    "    \"\"\"\n",
    "    # number of songs for audio analysis\n",
    "    num = len(songs_info[\"hit_miss\"])\n",
    "    \n",
    "    # create data frame for songs' audio analysis\n",
    "    songs_analysis = pd.DataFrame(columns=[\"duration\", \"loudness\", \"tempo\", \"tempo_confidence\", \"time_signature\", \n",
    "                                                \"time_signature_confidence\", \"key\", \"key_confidence\", \"mode\", \"mode_confidence\"])\n",
    "\n",
    "    # set start_time and songs_analyzed to track progress\n",
    "    start_time = time()\n",
    "    songs_analyzed = 0\n",
    "    \n",
    "    # write headings for new file\n",
    "    if half_analyzed == 0:\n",
    "        with open(\"./temp/{}_temp_audio_analysis.csv\".format(year), \"w+\") as file:\n",
    "            file.write(\",duration,loudness,tempo,tempo_confidence,time_signature,time_signature_confidence,key,key_confidence,mode,mode_confidence\\n\")\n",
    "\n",
    "    # from Spotify API, get songs' audio analysis with rate limit of 1-2 secs between songs\n",
    "    # maximum songs per year is 10k\n",
    "    for i in range(num):\n",
    "        url=\"https://api.spotify.com/v1/audio-analysis/{}\".format(songs_info[\"id\"][i])\n",
    "        r=get(url, headers=headers)\n",
    "\n",
    "        # to ensure response contains track details, which is not the case sometimes\n",
    "        while \"track\" not in r.json():\n",
    "            r=get(url, headers=headers)\n",
    "\n",
    "        r=r.json()[\"track\"]\n",
    "\n",
    "        # track progress\n",
    "        track_progress(\"Conducting audio analysis\", 1, num, i, start_time)\n",
    "\n",
    "        # write analyzed songs to temp file in case analysis fails halfway\n",
    "        with open(\"./temp/{}_temp_audio_analysis.csv\".format(year), \"a\") as file:\n",
    "            file.write(\"{},{},{},{},{},{},{},{},{},{},{}\\n\".format(half_analyzed+i, r[\"duration\"], r[\"loudness\"], r[\"tempo\"], r[\"tempo_confidence\"], r[\"time_signature\"], \n",
    "                                                             r[\"time_signature_confidence\"], r[\"key\"],r[\"key_confidence\"], r[\"mode\"], r[\"mode_confidence\"]))\n",
    "\n",
    "        sleep(randint(1,2))\n",
    "        songs_analyzed = i + 1\n",
    "        \n",
    "    songs_analysis = pd.read_csv(\"./temp/{}_temp_audio_analysis.csv\".format(year), index_col=0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"AUDIO ANALYSIS COMPLETE. Songs analyzed: {}; Elapsed Time: {} minutes.\".format(songs_analyzed, round((time()-start_time)/60, 2)))\n",
    "    return songs_analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## On songs extracted, conduct audio feature analysis (e.g. acousticness, danceability, energy etc.), stored in dataframe songs_features.\n",
    "\n",
    "approx. 15secs per 10 songs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features_analysis(songs_info, half_analyzed):\n",
    "    \"\"\"\n",
    "    Function to conduct features analysis on extracted songs.\n",
    "    Args:\n",
    "        songs_info: dataframe containing songs' basic info\n",
    "        half_analyzed: determines number of pre-analyzed songs if applicable\n",
    "    \"\"\"\n",
    "    # number of songs for features analysis\n",
    "    num = len(songs_info[\"hit_miss\"])\n",
    "    \n",
    "    # create data frame for songs' audio analysis\n",
    "    songs_features = pd.DataFrame(columns=[\"acousticness\", \"danceability\", \"energy\", \"instrumentalness\", \"liveness\", \n",
    "                                                \"speechiness\", \"valence\"])\n",
    "\n",
    "    # set start_time and songs_analyzed to track progress\n",
    "    start_time = time()\n",
    "    songs_analyzed = 0\n",
    "    \n",
    "    # write headings for new file\n",
    "    if half_analyzed == 0:\n",
    "        with open(\"./temp/{}_temp_features_analysis.csv\".format(year), \"w+\") as file:\n",
    "            file.write(\",acousticness,danceability,energy,instrumentalness,liveness,speechiness,valence\\n\")\n",
    "\n",
    "    # get songs' audio features with rate limit of 1-2 secs between songs\n",
    "    # maximum songs per year is 10k\n",
    "    for i in range(num):\n",
    "        url=\"https://api.spotify.com/v1/audio-features/{}\".format(songs_info[\"id\"][i])\n",
    "        r=get(url, headers=headers)\n",
    "        \n",
    "        # to ensure response contains features details, which is not the case sometimes\n",
    "        while \"acousticness\" not in r.json():\n",
    "            r=get(url, headers=headers)\n",
    "\n",
    "        r=r.json()\n",
    "\n",
    "        # track progress\n",
    "        track_progress(\"Conducting features analysis\", 1, num, i, start_time)\n",
    "        \n",
    "        # write analyzed songs to temp file in case analysis fails halfway\n",
    "        with open(\"./temp/{}_temp_features_analysis.csv\".format(year), \"a\") as file:\n",
    "            file.write(\"{},{},{},{},{},{},{},{}\\n\".format(half_analyzed+i, r[\"acousticness\"], r[\"danceability\"], r[\"energy\"], r[\"instrumentalness\"], r[\"liveness\"], \n",
    "                                                             r[\"speechiness\"], r[\"valence\"]))\n",
    "        \n",
    "        sleep(randint(1,2))\n",
    "        songs_analyzed = i + 1\n",
    "        \n",
    "    songs_features = pd.read_csv(\"./temp/{}_temp_features_analysis.csv\".format(year), index_col=0)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"FEATURES ANALYSIS COMPLETE. Songs analyzed: {}; Elapsed Time: {} minutes.\".format(songs_analyzed, round((time()-start_time)/60, 2)))\n",
    "    return songs_features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Link up above processes through a single function for easier management."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(year, num, resume=[]):\n",
    "    \"\"\"\n",
    "    Function that initiates entire process of extracting specified number of song data from specified year.\n",
    "    Args:\n",
    "        year: year to extract songs from\n",
    "        num: number of songs to extract\n",
    "        resume: list of year(s) to resume audio analysis from, defaults to an empty list\n",
    "    \"\"\"\n",
    "    try:\n",
    "        # check whether to execute a fresh start or resume for current year\n",
    "        if year not in resume:\n",
    "            print(\"Obtaining data from {}...\".format(year))\n",
    "            # obtain data\n",
    "            songs_info = extract_songs(year, num)\n",
    "            # label songs\n",
    "            songs_info = label_songs(songs_info)\n",
    "            # audio analyze songs\n",
    "            songs_analysis = audio_analysis(songs_info, half_analyzed=0)\n",
    "            # features analyze songs\n",
    "            songs_features = features_analysis(songs_info, half_analyzed=0)\n",
    "        else:\n",
    "            print(\"Resuming analysis from {}...\".format(year))\n",
    "            # read in song list\n",
    "            song_list = pd.read_csv(\"./temp/{}_{}_song_list.csv\".format(year, num))\n",
    "            # read in half-analyzed audio data\n",
    "            half_audio_analyzed_songs = pd.read_csv(\"./temp/{}_temp_audio_analysis.csv\".format(year))\n",
    "            songs_info = song_list.iloc[len(half_audio_analyzed_songs):]\n",
    "            songs_info = songs_info.reset_index(drop=True)\n",
    "            # audio analyze songs\n",
    "            songs_analysis = audio_analysis(songs_info, half_analyzed=len(half_audio_analyzed_songs))\n",
    "            if os.path.isfile(\"./temp/{}_temp_features_analysis.csv\".format(year)):\n",
    "                # read in half-analyzed features data\n",
    "                half_features_analyzed_songs = pd.read_csv(\"./temp/{}_temp_features_analysis.csv\".format(year))\n",
    "                songs_info = song_list.iloc[len(half_features_analyzed_songs):]\n",
    "                songs_info = songs_info.reset_index(drop=True)\n",
    "                # features analyze songs\n",
    "                songs_features = features_analysis(songs_info, half_analyzed=len(half_features_analyzed_songs))\n",
    "            else:\n",
    "                songs_info = song_list\n",
    "                songs_features = features_analysis(songs_info, half_analyzed=0)\n",
    "            songs_info = pd.read_csv(\"./temp/{}_{}_song_list.csv\".format(year, num))\n",
    "        # combine all 3 data frames to obtain songs, a dataframe which contains metadata for songs released in specified year.\n",
    "        songs = pd.concat([songs_info, songs_analysis, songs_features], axis = 1)\n",
    "        # save results to csv file\n",
    "        songs.to_csv(\"./song_features/{}_{}_song_features.csv\".format(year, num), index=False)\n",
    "        # remove temp files on success\n",
    "        if len(songs) != 0:\n",
    "            os.remove(\"./temp/{}_temp_audio_analysis.csv\".format(year))\n",
    "            os.remove(\"./temp/{}_temp_features_analysis.csv\".format(year))\n",
    "            os.remove(\"./temp/{}_{}_song_list.csv\".format(year, num))\n",
    "    except Exception as ex:\n",
    "        print(\"\")\n",
    "        print(\"Error for scraping from year {} with num {} and exception: {}\".format(year, num, ex))\n",
    "        logging.exception(str(ex) + \" (Year: {} - Num: {})\".format(year, num))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Where it all begins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# edit the range values in the for loop to input the range of years to scrape from\n",
    "# edit the second parameter in the get_data function to input the number of songs to scrape from each year\n",
    "# edit the resume list to add or remove years which was half analyzed previously\n",
    "for year in range(2008, 2019):\n",
    "    get_data(year, 50, resume=[])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}