Wave Direction 2
# Howard Wang 05/14/19
# CSE 145: Embedded systems and design
# Main Database: https://surf.smartfin.org/
# Analyzing data from Buoy Calibration experiment to get wave direction.
# First, parse the data from the .CSV file containing ocean and wave motion data.
# This data comes from a controlled experiment (CE3), so we are assuming that
# all of the vertical accelerations are contained in IMUA2.
# MATPLOTLIB
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
#from mpl_toolkits.basemap import Basemap
# DATAFRAMES
import pandas as pd
import numpy as np
# SCIPY
from scipy import stats
from scipy import constants
from scipy import signal #added
from scipy.interpolate import CubicSpline
from scipy.interpolate import interp1d
from scipy.integrate import simps
from scipy.integrate import cumtrapz
import pylab as pylab
# SYSTEM and CONVERSION TOOLS
import math
import abc
import sys
import csv
import io
import os
import datetime
import pytz
import re
# MODELING AND GRAPHS
import peakutils
import statsmodels.api as sm
# URL REQUESTS
import requests
# VECTORS AND GRAPHICS
import mpld3
import folium
# import cmocean
import skinematics as skin
from skinematics import quat, vector, misc, rotmat, imus, view
import pygame
# PLOTTING TOOLS
from plotly import tools
import plotly.offline
import plotly.graph_objs as go
%matplotlib notebook
%matplotlib inline
print("Done!")Done!
ride_ids = ['16083']
# This is the recent (May 15th buoy float session ID)
# A smartfin was placed next to a real buoy and data was collected
# Now data must be scrutinized against buoy float data
print("Done!")Done!
#%% Fin ID scraper
# Input fin ID, get all ride IDs
# base URL to which we'll append given fin IDs
fin_url_base = 'http://surf.smartfin.org/fin/'
# Look for the following text in the HTML contents in fcn below
str_id_ride = 'rideId = \'' # backslash allows us to look for single quote
str_id_date = 'var date = \'' # backslash allows us to look for single quote
#%% Ride ID scraper
# Input ride ID, get ocean and motion CSVs
# Base URL to which we'll append given ride IDs
ride_url_base = 'https://surf.smartfin.org/ride/'
# Look for the following text in the HTML contents in fcn below
str_id_csv = 'img id="temperatureChart" class="chart" src="'
def get_csv_from_ride_id(rid):
# Build URL for each individual ride
ride_url = ride_url_base+str(rid)
print(ride_url)
# Get contents of ride_url
html_contents = requests.get(ride_url).text
# Find CSV identifier
loc_csv_id = html_contents.find(str_id_csv)
# Different based on whether user logged in with FB or Google
offset_googleOAuth = [46, 114]
offset_facebkOAuth = [46, 112]
if html_contents[loc_csv_id+59] == 'f': # Facebook login
off0 = offset_facebkOAuth[0]
off1 = offset_facebkOAuth[1]
else: # Google login
off0 = offset_googleOAuth[0]
off1 = offset_googleOAuth[1]
csv_id_longstr = html_contents[loc_csv_id+off0:loc_csv_id+off1]
# Stitch together full URL for CSV
if ("media" in csv_id_longstr) & ("Calibration" not in html_contents): # other junk URLs can exist and break everything
ocean_csv_url = 'https://surf.smartfin.org/'+csv_id_longstr+'Ocean.CSV'
motion_csv_url = 'https://surf.smartfin.org/'+csv_id_longstr+'Motion.CSV'
print(ocean_csv_url)
# Go to ocean_csv_url and grab contents (theoretically, a CSV)
oceanCSV = requests.get(ocean_csv_url).content
ocean_df_small = pd.read_csv(io.StringIO(oceanCSV.decode('utf-8')))
# Grab CSV from motion url
motionCSV = requests.get(motion_csv_url).content
motion_df_small = pd.read_csv(io.StringIO(motionCSV.decode('utf-8')))
return ocean_df_small, motion_df_small
else:
ocean_df_small_resample = pd.DataFrame() # empty DF just so something is returned
motion_df_small_resample = pd.DataFrame()
return ocean_df_small_resample, motion_df_small_resample
print("Done!")Done!
appended_ocean_list = [] # list of DataFrames from original CSVs
appended_motion_list = []
appended_multiIndex = [] # fin_id & ride_id used to identify each DataFrame
## Nested loops (for each fin ID, find all ride IDs, then build a DataFrame from all ride CSVs)
## (Here, ride IDS are either ocean or motion dataframes)
count_good_fins = 0
# Loop over ride_ids and find CSVs
for rid in ride_ids:
try:
new_ocean_df, new_motion_df = get_csv_from_ride_id(rid) # get given ride's CSV from its ride ID using function above
#print(len(new_ocean_df))
#print(len(new_motion_df))
if not new_ocean_df.empty: # Calibration rides, for example
# Append only if DF isn't empty. There may be a better way to control empty DFs which are created above
appended_multiIndex.append(str(rid)) # build list to be multiIndex of future DataFrame
appended_ocean_list.append(new_ocean_df)
appended_motion_list.append(new_motion_df)
print("Ride data has been uploaded.")
#print("Ride: ", rid, "data has been uploaded.")
count_good_fins += 1
except:
print("Ride threw an exception!")
#print("Ride ", rid, "threw an exception!")
#%% Build the "Master" DataFrame
# appended_ocean_df.summary()
df_keys = tuple(appended_multiIndex) # keys gotta be a tuple, a list which data in it cannot be changed
ocean_df = pd.concat(appended_ocean_list, keys = df_keys, names=['ride_id'])
motion_df = pd.concat(appended_motion_list, keys = df_keys, names = ['ride_id'])
# Print motion and Ocean dataframes
print(motion_df)
print(ocean_df)https://surf.smartfin.org/ride/16083
https://surf.smartfin.org/media/201905/google_105349665704999793400_0006667E229D_190515161151_Ocean.CSV
Ride data has been uploaded.
UTC Time IMU A1 IMU A2 \
ride_id
16083 0 2019-05-15T16:11:51.2410+00:00 762168371 -59.0 505.0
1 2019-05-15T16:11:51.4940+00:00 762168623 -56.0 493.0
2 2019-05-15T16:11:51.7460+00:00 762168873 -60.0 524.0
3 2019-05-15T16:11:51.9990+00:00 762169125 -58.0 551.0
4 2019-05-15T16:11:52.2510+00:00 762169375 -57.0 519.0
5 2019-05-15T16:11:52.5030+00:00 762169625 -56.0 450.0
6 2019-05-15T16:11:52.7540+00:00 762169875 -57.0 439.0
7 2019-05-15T16:11:53.0070+00:00 762170126 -56.0 497.0
8 2019-05-15T16:11:53.2580+00:00 762170376 -59.0 506.0
9 2019-05-15T16:11:53.5110+00:00 762170627 -57.0 501.0
10 2019-05-15T16:11:53.7630+00:00 762170877 -59.0 499.0
11 2019-05-15T16:11:54.0140+00:00 762171127 -60.0 502.0
12 2019-05-15T16:11:54.2680+00:00 762171379 -59.0 446.0
13 2019-05-15T16:11:54.5190+00:00 762171629 -56.0 478.0
14 2019-05-15T16:11:54.7710+00:00 762171879 -59.0 535.0
15 2019-05-15T16:11:55.0240+00:00 762172130 -59.0 553.0
16 2019-05-15T16:11:55.2670+00:00 762172372 -60.0 520.0
17 2019-05-15T16:11:55.5200+00:00 762172623 -61.0 567.0
18 2019-05-15T16:11:55.7710+00:00 762172873 -63.0 589.0
19 2019-05-15T16:11:56.0250+00:00 762173125 -64.0 601.0
20 2019-05-15T16:11:56.2770+00:00 762173375 -62.0 510.0
21 2019-05-15T16:11:56.5290+00:00 762173626 -59.0 470.0
22 2019-05-15T16:11:56.7810+00:00 762173876 -60.0 537.0
23 2019-05-15T16:11:57.0330+00:00 762174127 -56.0 457.0
24 2019-05-15T16:11:57.2850+00:00 762174377 -56.0 421.0
25 2019-05-15T16:11:57.5370+00:00 762174627 -58.0 456.0
26 2019-05-15T16:11:57.7880+00:00 762174877 -63.0 527.0
27 2019-05-15T16:11:58.0410+00:00 762175128 -59.0 492.0
28 2019-05-15T16:11:58.2940+00:00 762175379 -57.0 480.0
29 2019-05-15T16:11:58.5460+00:00 762175630 -56.0 440.0
... ... ... ... ...
6440 2019-05-15T16:35:41.0010+00:00 763589006 465.0 42.0
6441 2019-05-15T16:35:41.2040+00:00 763589207 495.0 48.0
6442 2019-05-15T16:35:41.4050+00:00 763589407 482.0 72.0
6443 2019-05-15T16:35:41.6070+00:00 763589608 511.0 47.0
6444 2019-05-15T16:35:41.8090+00:00 763589808 540.0 22.0
6445 2019-05-15T16:35:42.0110+00:00 763590009 546.0 -31.0
6446 2019-05-15T16:35:42.2120+00:00 763590209 547.0 -69.0
6447 2019-05-15T16:35:42.4130+00:00 763590409 486.0 -21.0
6448 2019-05-15T16:35:42.6160+00:00 763590610 453.0 -31.0
6449 2019-05-15T16:35:42.8160+00:00 763590809 445.0 -22.0
6450 2019-05-15T16:35:43.0170+00:00 763591009 431.0 -12.0
6451 2019-05-15T16:35:43.2190+00:00 763591209 440.0 -29.0
6452 2019-05-15T16:35:43.4200+00:00 763591409 477.0 5.0
6453 2019-05-15T16:35:43.6220+00:00 763591610 465.0 -35.0
6454 2019-05-15T16:35:43.8240+00:00 763591811 465.0 -38.0
6455 2019-05-15T16:35:44.0260+00:00 763592011 474.0 -77.0
6456 2019-05-15T16:35:44.2270+00:00 763592211 475.0 -15.0
6457 2019-05-15T16:35:44.4290+00:00 763592412 509.0 -13.0
6458 2019-05-15T16:35:44.6310+00:00 763592612 520.0 -3.0
6459 2019-05-15T16:35:44.8320+00:00 763592812 500.0 -308.0
6460 2019-05-15T16:35:45.0000+00:00 763592979 NaN NaN
6461 2019-05-15T16:35:45.0340+00:00 763593013 533.0 -5.0
6462 2019-05-15T16:35:45.2370+00:00 763593214 544.0 -18.0
6463 2019-05-15T16:35:45.4290+00:00 763593405 529.0 -21.0
6464 2019-05-15T16:35:45.6300+00:00 763593605 542.0 -20.0
6465 2019-05-15T16:35:45.8310+00:00 763593805 549.0 -93.0
6466 2019-05-15T16:35:46.0330+00:00 763594005 521.0 -29.0
6467 2019-05-15T16:35:46.2350+00:00 763594206 522.0 -22.0
6468 2019-05-15T16:35:46.4370+00:00 763594407 499.0 -17.0
6469 2019-05-15T16:35:46.6390+00:00 763594608 421.0 371.0
IMU A3 IMU G1 IMU G2 IMU G3 IMU M1 IMU M2 IMU M3 \
ride_id
16083 0 63.0 3.0 15.0 56.0 29.0 -145.0 197.0
1 62.0 -68.0 -14.0 65.0 12.0 -148.0 194.0
2 87.0 -27.0 -12.0 23.0 16.0 -150.0 188.0
3 97.0 66.0 22.0 -84.0 15.0 -161.0 177.0
4 91.0 6.0 30.0 -103.0 29.0 -139.0 187.0
5 82.0 -36.0 34.0 -31.0 37.0 -141.0 185.0
6 78.0 28.0 36.0 74.0 36.0 -148.0 192.0
7 63.0 27.0 25.0 107.0 28.0 -138.0 200.0
8 76.0 -103.0 3.0 0.0 23.0 -155.0 195.0
9 89.0 -1.0 26.0 -36.0 26.0 -154.0 184.0
10 88.0 77.0 48.0 -38.0 28.0 -144.0 184.0
11 77.0 -4.0 36.0 -31.0 26.0 -142.0 194.0
12 63.0 -59.0 11.0 32.0 37.0 -145.0 187.0
13 75.0 -42.0 -5.0 104.0 23.0 -155.0 181.0
14 87.0 29.0 2.0 90.0 18.0 -158.0 178.0
15 99.0 14.0 22.0 -77.0 18.0 -156.0 180.0
16 87.0 3.0 22.0 -32.0 17.0 -153.0 191.0
17 93.0 2.0 28.0 -30.0 29.0 -151.0 179.0
18 97.0 9.0 30.0 16.0 26.0 -150.0 186.0
19 81.0 52.0 35.0 13.0 26.0 -148.0 188.0
20 71.0 -3.0 12.0 22.0 26.0 -146.0 190.0
21 81.0 -6.0 0.0 46.0 19.0 -145.0 195.0
22 91.0 36.0 18.0 28.0 26.0 -152.0 190.0
23 70.0 -18.0 22.0 -22.0 18.0 -144.0 200.0
24 67.0 -55.0 13.0 -25.0 24.0 -150.0 194.0
25 72.0 -8.0 3.0 27.0 25.0 -145.0 187.0
26 90.0 -12.0 6.0 24.0 23.0 -147.0 175.0
27 81.0 -19.0 19.0 -23.0 23.0 -147.0 181.0
28 77.0 -33.0 22.0 -63.0 33.0 -153.0 181.0
29 73.0 -24.0 21.0 -57.0 33.0 -141.0 177.0
... ... ... ... ... ... ... ...
6440 230.0 38.0 24.0 -28.0 -219.0 43.0 -133.0
6441 213.0 -3.0 18.0 0.0 -221.0 39.0 -123.0
6442 209.0 -12.0 10.0 75.0 -220.0 44.0 -120.0
6443 194.0 -56.0 -12.0 151.0 -225.0 49.0 -123.0
6444 167.0 -55.0 -26.0 163.0 -230.0 70.0 -126.0
6445 147.0 -34.0 -23.0 88.0 -234.0 80.0 -124.0
6446 137.0 49.0 25.0 3.0 -238.0 94.0 -112.0
6447 127.0 -106.0 -8.0 -62.0 -241.0 89.0 -123.0
6448 118.0 -117.0 -4.0 -61.0 -250.0 88.0 -118.0
6449 127.0 30.0 -15.0 -10.0 -245.0 93.0 -111.0
6450 127.0 -78.0 -20.0 7.0 -246.0 92.0 -110.0
6451 125.0 21.0 -27.0 65.0 -255.0 91.0 -105.0
6452 139.0 -59.0 3.0 49.0 -256.0 100.0 -96.0
6453 142.0 -157.0 5.0 -2.0 -250.0 112.0 -100.0
6454 123.0 -291.0 26.0 -31.0 -250.0 130.0 -100.0
6455 130.0 -31.0 10.0 -61.0 -255.0 131.0 -103.0
6456 142.0 11.0 23.0 -57.0 -255.0 125.0 -105.0
6457 136.0 25.0 17.0 -16.0 -259.0 125.0 -97.0
6458 137.0 -32.0 0.0 24.0 -249.0 113.0 -93.0
6459 156.0 39.0 4.0 26.0 -259.0 121.0 -89.0
6460 NaN NaN NaN NaN NaN NaN NaN
6461 122.0 -19.0 -8.0 57.0 -262.0 120.0 -92.0
6462 101.0 -26.0 4.0 36.0 -261.0 129.0 -93.0
6463 124.0 2.0 21.0 -3.0 -261.0 129.0 -93.0
6464 107.0 -3.0 26.0 -24.0 -267.0 125.0 -99.0
6465 82.0 -29.0 30.0 -44.0 -257.0 119.0 -93.0
6466 97.0 21.0 29.0 -38.0 -269.0 119.0 -99.0
6467 108.0 0.0 12.0 -22.0 -266.0 120.0 -100.0
6468 97.0 -17.0 0.0 5.0 -255.0 121.0 -93.0
6469 45.0 88.0 4.0 46.0 -272.0 112.0 -90.0
Latitude Longitude
ride_id
16083 0 NaN NaN
1 NaN NaN
2 NaN NaN
3 NaN NaN
4 NaN NaN
5 NaN NaN
6 NaN NaN
7 NaN NaN
8 NaN NaN
9 NaN NaN
10 NaN NaN
11 NaN NaN
12 NaN NaN
13 NaN NaN
14 NaN NaN
15 NaN NaN
16 NaN NaN
17 NaN NaN
18 NaN NaN
19 NaN NaN
20 NaN NaN
21 NaN NaN
22 NaN NaN
23 NaN NaN
24 NaN NaN
25 NaN NaN
26 NaN NaN
27 NaN NaN
28 NaN NaN
29 NaN NaN
... ... ...
6440 NaN NaN
6441 NaN NaN
6442 NaN NaN
6443 NaN NaN
6444 NaN NaN
6445 NaN NaN
6446 NaN NaN
6447 NaN NaN
6448 NaN NaN
6449 NaN NaN
6450 NaN NaN
6451 NaN NaN
6452 NaN NaN
6453 NaN NaN
6454 NaN NaN
6455 NaN NaN
6456 NaN NaN
6457 NaN NaN
6458 NaN NaN
6459 NaN NaN
6460 3286983.0 -11726667.0
6461 NaN NaN
6462 NaN NaN
6463 NaN NaN
6464 NaN NaN
6465 NaN NaN
6466 NaN NaN
6467 NaN NaN
6468 NaN NaN
6469 NaN NaN
[6470 rows x 13 columns]
UTC Time Temperature 1 \
ride_id
16083 0 2019-05-15T16:11:50.7700+00:00 762167903 303
1 2019-05-15T16:11:56.8280+00:00 762173923 303
2 2019-05-15T16:12:02.8860+00:00 762179942 303
3 2019-05-15T16:12:08.9450+00:00 762185962 303
4 2019-05-15T16:12:15.0030+00:00 762191982 303
5 2019-05-15T16:12:21.0620+00:00 762198002 303
6 2019-05-15T16:12:27.1200+00:00 762204021 303
7 2019-05-15T16:12:33.1780+00:00 762210041 303
8 2019-05-15T16:12:39.2360+00:00 762216060 303
9 2019-05-15T16:12:45.2950+00:00 762222080 303
10 2019-05-15T16:12:51.3540+00:00 762228101 303
11 2019-05-15T16:12:57.4130+00:00 762234121 303
12 2019-05-15T16:13:03.4720+00:00 762240141 303
13 2019-05-15T16:13:09.5310+00:00 762246162 303
14 2019-05-15T16:13:15.5910+00:00 762252183 303
15 2019-05-15T16:13:21.6500+00:00 762258203 303
16 2019-05-15T16:13:27.7080+00:00 762264223 303
17 2019-05-15T16:13:33.7660+00:00 762270242 303
18 2019-05-15T16:13:39.8260+00:00 762276263 303
19 2019-05-15T16:13:45.8830+00:00 762282282 303
20 2019-05-15T16:13:51.9420+00:00 762288302 303
21 2019-05-15T16:13:58.0010+00:00 762294322 302
22 2019-05-15T16:14:04.0590+00:00 762300342 302
23 2019-05-15T16:14:10.1170+00:00 762306361 303
24 2019-05-15T16:14:16.1770+00:00 762312382 303
25 2019-05-15T16:14:22.2350+00:00 762318402 303
26 2019-05-15T16:14:28.2940+00:00 762324422 303
27 2019-05-15T16:14:34.3530+00:00 762330442 302
28 2019-05-15T16:14:40.4100+00:00 762336461 303
29 2019-05-15T16:14:46.4680+00:00 762342480 303
... ... ... ...
208 2019-05-15T16:32:50.9970+00:00 763420087 303
209 2019-05-15T16:32:57.0570+00:00 763426108 303
210 2019-05-15T16:33:03.1170+00:00 763432129 304
211 2019-05-15T16:33:09.1750+00:00 763438149 304
212 2019-05-15T16:33:15.2340+00:00 763444169 305
213 2019-05-15T16:33:21.2920+00:00 763450188 306
214 2019-05-15T16:33:27.3500+00:00 763456208 306
215 2019-05-15T16:33:33.4080+00:00 763462227 306
216 2019-05-15T16:33:39.4680+00:00 763468248 307
217 2019-05-15T16:33:45.5270+00:00 763474269 307
218 2019-05-15T16:33:51.5860+00:00 763480289 308
219 2019-05-15T16:33:57.6440+00:00 763486308 308
220 2019-05-15T16:34:03.7020+00:00 763492328 309
221 2019-05-15T16:34:09.7610+00:00 763498348 309
222 2019-05-15T16:34:15.8210+00:00 763504369 309
223 2019-05-15T16:34:21.8790+00:00 763510389 310
224 2019-05-15T16:34:27.9380+00:00 763516409 310
225 2019-05-15T16:34:33.9990+00:00 763522431 310
226 2019-05-15T16:34:40.0570+00:00 763528451 311
227 2019-05-15T16:34:46.1160+00:00 763534471 311
228 2019-05-15T16:34:52.1740+00:00 763540490 311
229 2019-05-15T16:34:58.2320+00:00 763546510 311
230 2019-05-15T16:35:04.2910+00:00 763552530 312
231 2019-05-15T16:35:10.3510+00:00 763558551 312
232 2019-05-15T16:35:16.4100+00:00 763564571 312
233 2019-05-15T16:35:22.4680+00:00 763570591 312
234 2019-05-15T16:35:28.5270+00:00 763576611 312
235 2019-05-15T16:35:34.5870+00:00 763582632 312
236 2019-05-15T16:35:40.6450+00:00 763588652 312
237 2019-05-15T16:35:46.7050+00:00 763594673 313
Calibrated Temperature 1 Temperature 1 Stable Temperature 2 \
ride_id
16083 0 18.938 False 4734
1 18.938 False 4744
2 18.938 False 4743
3 18.938 False 4745
4 18.938 False 4745
5 18.938 True 4751
6 18.938 True 4747
7 18.938 True 4748
8 18.938 True 4741
9 18.938 True 4746
10 18.938 True 4748
11 18.938 True 4749
12 18.938 True 4743
13 18.938 True 4747
14 18.938 True 4750
15 18.938 True 4747
16 18.938 True 4756
17 18.938 True 4744
18 18.938 True 4744
19 18.938 True 4743
20 18.938 True 4745
21 18.875 True 4739
22 18.875 True 4743
23 18.938 True 4740
24 18.938 False 4744
25 18.938 False 4741
26 18.938 True 4746
27 18.875 True 4746
28 18.938 True 4748
29 18.938 True 4748
... ... ... ...
208 18.938 False 4703
209 18.938 False 4711
210 19.000 False 4714
211 19.000 False 4719
212 19.062 True 4717
213 19.125 True 4725
214 19.125 True 4731
215 19.125 True 4732
216 19.188 True 4731
217 19.188 False 4730
218 19.250 False 4736
219 19.250 False 4734
220 19.312 False 4742
221 19.312 False 4746
222 19.312 False 4746
223 19.375 False 4755
224 19.375 False 4753
225 19.375 False 4756
226 19.438 False 4773
227 19.438 False 4765
228 19.438 False 4788
229 19.438 False 4783
230 19.500 False 4780
231 19.500 False 4776
232 19.500 False 4771
233 19.500 False 4770
234 19.500 False 4766
235 19.500 False 4761
236 19.500 False 4763
237 19.562 False 4771
Calibrated Temperature 2 Temperature 2 Stable salinity \
ride_id
16083 0 18.657 False NaN
1 18.696 False NaN
2 18.692 False NaN
3 18.700 False NaN
4 18.700 False NaN
5 18.724 False NaN
6 18.708 True NaN
7 18.712 True NaN
8 18.684 True NaN
9 18.704 True NaN
10 18.712 True NaN
11 18.716 True NaN
12 18.692 True NaN
13 18.708 True NaN
14 18.720 True NaN
15 18.708 True NaN
16 18.743 True NaN
17 18.696 True NaN
18 18.696 True NaN
19 18.692 True NaN
20 18.700 True NaN
21 18.676 True NaN
22 18.692 True NaN
23 18.680 True NaN
24 18.696 False NaN
25 18.684 False NaN
26 18.704 True NaN
27 18.704 True NaN
28 18.712 True NaN
29 18.712 True NaN
... ... ... ...
208 18.535 False NaN
209 18.567 False NaN
210 18.578 False NaN
211 18.598 False NaN
212 18.590 True NaN
213 18.622 True NaN
214 18.645 False NaN
215 18.649 False NaN
216 18.645 False NaN
217 18.641 False NaN
218 18.665 False NaN
219 18.657 False NaN
220 18.688 False NaN
221 18.704 False NaN
222 18.704 False NaN
223 18.739 False NaN
224 18.731 False NaN
225 18.743 False NaN
226 18.810 False NaN
227 18.778 False NaN
228 18.869 False NaN
229 18.849 False NaN
230 18.837 False NaN
231 18.822 False NaN
232 18.802 False NaN
233 18.798 False NaN
234 18.782 False NaN
235 18.763 False NaN
236 18.771 False NaN
237 18.802 False NaN
Calibrated Salinity Salinity Stable pH Calibrated pH \
ride_id
16083 0 NaN NaN NaN NaN
1 NaN NaN NaN NaN
2 NaN NaN NaN NaN
3 NaN NaN NaN NaN
4 NaN NaN NaN NaN
5 NaN NaN NaN NaN
6 NaN NaN NaN NaN
7 NaN NaN NaN NaN
8 NaN NaN NaN NaN
9 NaN NaN NaN NaN
10 NaN NaN NaN NaN
11 NaN NaN NaN NaN
12 NaN NaN NaN NaN
13 NaN NaN NaN NaN
14 NaN NaN NaN NaN
15 NaN NaN NaN NaN
16 NaN NaN NaN NaN
17 NaN NaN NaN NaN
18 NaN NaN NaN NaN
19 NaN NaN NaN NaN
20 NaN NaN NaN NaN
21 NaN NaN NaN NaN
22 NaN NaN NaN NaN
23 NaN NaN NaN NaN
24 NaN NaN NaN NaN
25 NaN NaN NaN NaN
26 NaN NaN NaN NaN
27 NaN NaN NaN NaN
28 NaN NaN NaN NaN
29 NaN NaN NaN NaN
... ... ... .. ...
208 NaN NaN NaN NaN
209 NaN NaN NaN NaN
210 NaN NaN NaN NaN
211 NaN NaN NaN NaN
212 NaN NaN NaN NaN
213 NaN NaN NaN NaN
214 NaN NaN NaN NaN
215 NaN NaN NaN NaN
216 NaN NaN NaN NaN
217 NaN NaN NaN NaN
218 NaN NaN NaN NaN
219 NaN NaN NaN NaN
220 NaN NaN NaN NaN
221 NaN NaN NaN NaN
222 NaN NaN NaN NaN
223 NaN NaN NaN NaN
224 NaN NaN NaN NaN
225 NaN NaN NaN NaN
226 NaN NaN NaN NaN
227 NaN NaN NaN NaN
228 NaN NaN NaN NaN
229 NaN NaN NaN NaN
230 NaN NaN NaN NaN
231 NaN NaN NaN NaN
232 NaN NaN NaN NaN
233 NaN NaN NaN NaN
234 NaN NaN NaN NaN
235 NaN NaN NaN NaN
236 NaN NaN NaN NaN
237 NaN NaN NaN NaN
pH Stable
ride_id
16083 0 NaN
1 NaN
2 NaN
3 NaN
4 NaN
5 NaN
6 NaN
7 NaN
8 NaN
9 NaN
10 NaN
11 NaN
12 NaN
13 NaN
14 NaN
15 NaN
16 NaN
17 NaN
18 NaN
19 NaN
20 NaN
21 NaN
22 NaN
23 NaN
24 NaN
25 NaN
26 NaN
27 NaN
28 NaN
29 NaN
... ...
208 NaN
209 NaN
210 NaN
211 NaN
212 NaN
213 NaN
214 NaN
215 NaN
216 NaN
217 NaN
218 NaN
219 NaN
220 NaN
221 NaN
222 NaN
223 NaN
224 NaN
225 NaN
226 NaN
227 NaN
228 NaN
229 NaN
230 NaN
231 NaN
232 NaN
233 NaN
234 NaN
235 NaN
236 NaN
237 NaN
[238 rows x 14 columns]
#print(motion_df)
saved_copy_motion_df = motion_df.copy(deep=True) #make a copy of the dataframe with raw data
print(saved_copy_motion_df)
#Drop the "nan" values from the columns that we care about.
dropped_motion_df = motion_df.dropna(subset=['Time', 'IMU A1', 'IMU A2', 'IMU A3', 'IMU G1', 'IMU G2', 'IMU G3', 'IMU M1',
'IMU M2', 'IMU M3'])
print(dropped_motion_df) UTC Time IMU A1 IMU A2 \
ride_id
16083 0 2019-05-15T16:11:51.2410+00:00 762168371 -59.0 505.0
1 2019-05-15T16:11:51.4940+00:00 762168623 -56.0 493.0
2 2019-05-15T16:11:51.7460+00:00 762168873 -60.0 524.0
3 2019-05-15T16:11:51.9990+00:00 762169125 -58.0 551.0
4 2019-05-15T16:11:52.2510+00:00 762169375 -57.0 519.0
5 2019-05-15T16:11:52.5030+00:00 762169625 -56.0 450.0
6 2019-05-15T16:11:52.7540+00:00 762169875 -57.0 439.0
7 2019-05-15T16:11:53.0070+00:00 762170126 -56.0 497.0
8 2019-05-15T16:11:53.2580+00:00 762170376 -59.0 506.0
9 2019-05-15T16:11:53.5110+00:00 762170627 -57.0 501.0
10 2019-05-15T16:11:53.7630+00:00 762170877 -59.0 499.0
11 2019-05-15T16:11:54.0140+00:00 762171127 -60.0 502.0
12 2019-05-15T16:11:54.2680+00:00 762171379 -59.0 446.0
13 2019-05-15T16:11:54.5190+00:00 762171629 -56.0 478.0
14 2019-05-15T16:11:54.7710+00:00 762171879 -59.0 535.0
15 2019-05-15T16:11:55.0240+00:00 762172130 -59.0 553.0
16 2019-05-15T16:11:55.2670+00:00 762172372 -60.0 520.0
17 2019-05-15T16:11:55.5200+00:00 762172623 -61.0 567.0
18 2019-05-15T16:11:55.7710+00:00 762172873 -63.0 589.0
19 2019-05-15T16:11:56.0250+00:00 762173125 -64.0 601.0
20 2019-05-15T16:11:56.2770+00:00 762173375 -62.0 510.0
21 2019-05-15T16:11:56.5290+00:00 762173626 -59.0 470.0
22 2019-05-15T16:11:56.7810+00:00 762173876 -60.0 537.0
23 2019-05-15T16:11:57.0330+00:00 762174127 -56.0 457.0
24 2019-05-15T16:11:57.2850+00:00 762174377 -56.0 421.0
25 2019-05-15T16:11:57.5370+00:00 762174627 -58.0 456.0
26 2019-05-15T16:11:57.7880+00:00 762174877 -63.0 527.0
27 2019-05-15T16:11:58.0410+00:00 762175128 -59.0 492.0
28 2019-05-15T16:11:58.2940+00:00 762175379 -57.0 480.0
29 2019-05-15T16:11:58.5460+00:00 762175630 -56.0 440.0
... ... ... ... ...
6440 2019-05-15T16:35:41.0010+00:00 763589006 465.0 42.0
6441 2019-05-15T16:35:41.2040+00:00 763589207 495.0 48.0
6442 2019-05-15T16:35:41.4050+00:00 763589407 482.0 72.0
6443 2019-05-15T16:35:41.6070+00:00 763589608 511.0 47.0
6444 2019-05-15T16:35:41.8090+00:00 763589808 540.0 22.0
6445 2019-05-15T16:35:42.0110+00:00 763590009 546.0 -31.0
6446 2019-05-15T16:35:42.2120+00:00 763590209 547.0 -69.0
6447 2019-05-15T16:35:42.4130+00:00 763590409 486.0 -21.0
6448 2019-05-15T16:35:42.6160+00:00 763590610 453.0 -31.0
6449 2019-05-15T16:35:42.8160+00:00 763590809 445.0 -22.0
6450 2019-05-15T16:35:43.0170+00:00 763591009 431.0 -12.0
6451 2019-05-15T16:35:43.2190+00:00 763591209 440.0 -29.0
6452 2019-05-15T16:35:43.4200+00:00 763591409 477.0 5.0
6453 2019-05-15T16:35:43.6220+00:00 763591610 465.0 -35.0
6454 2019-05-15T16:35:43.8240+00:00 763591811 465.0 -38.0
6455 2019-05-15T16:35:44.0260+00:00 763592011 474.0 -77.0
6456 2019-05-15T16:35:44.2270+00:00 763592211 475.0 -15.0
6457 2019-05-15T16:35:44.4290+00:00 763592412 509.0 -13.0
6458 2019-05-15T16:35:44.6310+00:00 763592612 520.0 -3.0
6459 2019-05-15T16:35:44.8320+00:00 763592812 500.0 -308.0
6460 2019-05-15T16:35:45.0000+00:00 763592979 NaN NaN
6461 2019-05-15T16:35:45.0340+00:00 763593013 533.0 -5.0
6462 2019-05-15T16:35:45.2370+00:00 763593214 544.0 -18.0
6463 2019-05-15T16:35:45.4290+00:00 763593405 529.0 -21.0
6464 2019-05-15T16:35:45.6300+00:00 763593605 542.0 -20.0
6465 2019-05-15T16:35:45.8310+00:00 763593805 549.0 -93.0
6466 2019-05-15T16:35:46.0330+00:00 763594005 521.0 -29.0
6467 2019-05-15T16:35:46.2350+00:00 763594206 522.0 -22.0
6468 2019-05-15T16:35:46.4370+00:00 763594407 499.0 -17.0
6469 2019-05-15T16:35:46.6390+00:00 763594608 421.0 371.0
IMU A3 IMU G1 IMU G2 IMU G3 IMU M1 IMU M2 IMU M3 \
ride_id
16083 0 63.0 3.0 15.0 56.0 29.0 -145.0 197.0
1 62.0 -68.0 -14.0 65.0 12.0 -148.0 194.0
2 87.0 -27.0 -12.0 23.0 16.0 -150.0 188.0
3 97.0 66.0 22.0 -84.0 15.0 -161.0 177.0
4 91.0 6.0 30.0 -103.0 29.0 -139.0 187.0
5 82.0 -36.0 34.0 -31.0 37.0 -141.0 185.0
6 78.0 28.0 36.0 74.0 36.0 -148.0 192.0
7 63.0 27.0 25.0 107.0 28.0 -138.0 200.0
8 76.0 -103.0 3.0 0.0 23.0 -155.0 195.0
9 89.0 -1.0 26.0 -36.0 26.0 -154.0 184.0
10 88.0 77.0 48.0 -38.0 28.0 -144.0 184.0
11 77.0 -4.0 36.0 -31.0 26.0 -142.0 194.0
12 63.0 -59.0 11.0 32.0 37.0 -145.0 187.0
13 75.0 -42.0 -5.0 104.0 23.0 -155.0 181.0
14 87.0 29.0 2.0 90.0 18.0 -158.0 178.0
15 99.0 14.0 22.0 -77.0 18.0 -156.0 180.0
16 87.0 3.0 22.0 -32.0 17.0 -153.0 191.0
17 93.0 2.0 28.0 -30.0 29.0 -151.0 179.0
18 97.0 9.0 30.0 16.0 26.0 -150.0 186.0
19 81.0 52.0 35.0 13.0 26.0 -148.0 188.0
20 71.0 -3.0 12.0 22.0 26.0 -146.0 190.0
21 81.0 -6.0 0.0 46.0 19.0 -145.0 195.0
22 91.0 36.0 18.0 28.0 26.0 -152.0 190.0
23 70.0 -18.0 22.0 -22.0 18.0 -144.0 200.0
24 67.0 -55.0 13.0 -25.0 24.0 -150.0 194.0
25 72.0 -8.0 3.0 27.0 25.0 -145.0 187.0
26 90.0 -12.0 6.0 24.0 23.0 -147.0 175.0
27 81.0 -19.0 19.0 -23.0 23.0 -147.0 181.0
28 77.0 -33.0 22.0 -63.0 33.0 -153.0 181.0
29 73.0 -24.0 21.0 -57.0 33.0 -141.0 177.0
... ... ... ... ... ... ... ...
6440 230.0 38.0 24.0 -28.0 -219.0 43.0 -133.0
6441 213.0 -3.0 18.0 0.0 -221.0 39.0 -123.0
6442 209.0 -12.0 10.0 75.0 -220.0 44.0 -120.0
6443 194.0 -56.0 -12.0 151.0 -225.0 49.0 -123.0
6444 167.0 -55.0 -26.0 163.0 -230.0 70.0 -126.0
6445 147.0 -34.0 -23.0 88.0 -234.0 80.0 -124.0
6446 137.0 49.0 25.0 3.0 -238.0 94.0 -112.0
6447 127.0 -106.0 -8.0 -62.0 -241.0 89.0 -123.0
6448 118.0 -117.0 -4.0 -61.0 -250.0 88.0 -118.0
6449 127.0 30.0 -15.0 -10.0 -245.0 93.0 -111.0
6450 127.0 -78.0 -20.0 7.0 -246.0 92.0 -110.0
6451 125.0 21.0 -27.0 65.0 -255.0 91.0 -105.0
6452 139.0 -59.0 3.0 49.0 -256.0 100.0 -96.0
6453 142.0 -157.0 5.0 -2.0 -250.0 112.0 -100.0
6454 123.0 -291.0 26.0 -31.0 -250.0 130.0 -100.0
6455 130.0 -31.0 10.0 -61.0 -255.0 131.0 -103.0
6456 142.0 11.0 23.0 -57.0 -255.0 125.0 -105.0
6457 136.0 25.0 17.0 -16.0 -259.0 125.0 -97.0
6458 137.0 -32.0 0.0 24.0 -249.0 113.0 -93.0
6459 156.0 39.0 4.0 26.0 -259.0 121.0 -89.0
6460 NaN NaN NaN NaN NaN NaN NaN
6461 122.0 -19.0 -8.0 57.0 -262.0 120.0 -92.0
6462 101.0 -26.0 4.0 36.0 -261.0 129.0 -93.0
6463 124.0 2.0 21.0 -3.0 -261.0 129.0 -93.0
6464 107.0 -3.0 26.0 -24.0 -267.0 125.0 -99.0
6465 82.0 -29.0 30.0 -44.0 -257.0 119.0 -93.0
6466 97.0 21.0 29.0 -38.0 -269.0 119.0 -99.0
6467 108.0 0.0 12.0 -22.0 -266.0 120.0 -100.0
6468 97.0 -17.0 0.0 5.0 -255.0 121.0 -93.0
6469 45.0 88.0 4.0 46.0 -272.0 112.0 -90.0
Latitude Longitude
ride_id
16083 0 NaN NaN
1 NaN NaN
2 NaN NaN
3 NaN NaN
4 NaN NaN
5 NaN NaN
6 NaN NaN
7 NaN NaN
8 NaN NaN
9 NaN NaN
10 NaN NaN
11 NaN NaN
12 NaN NaN
13 NaN NaN
14 NaN NaN
15 NaN NaN
16 NaN NaN
17 NaN NaN
18 NaN NaN
19 NaN NaN
20 NaN NaN
21 NaN NaN
22 NaN NaN
23 NaN NaN
24 NaN NaN
25 NaN NaN
26 NaN NaN
27 NaN NaN
28 NaN NaN
29 NaN NaN
... ... ...
6440 NaN NaN
6441 NaN NaN
6442 NaN NaN
6443 NaN NaN
6444 NaN NaN
6445 NaN NaN
6446 NaN NaN
6447 NaN NaN
6448 NaN NaN
6449 NaN NaN
6450 NaN NaN
6451 NaN NaN
6452 NaN NaN
6453 NaN NaN
6454 NaN NaN
6455 NaN NaN
6456 NaN NaN
6457 NaN NaN
6458 NaN NaN
6459 NaN NaN
6460 3286983.0 -11726667.0
6461 NaN NaN
6462 NaN NaN
6463 NaN NaN
6464 NaN NaN
6465 NaN NaN
6466 NaN NaN
6467 NaN NaN
6468 NaN NaN
6469 NaN NaN
[6470 rows x 13 columns]
UTC Time IMU A1 IMU A2 \
ride_id
16083 0 2019-05-15T16:11:51.2410+00:00 762168371 -59.0 505.0
1 2019-05-15T16:11:51.4940+00:00 762168623 -56.0 493.0
2 2019-05-15T16:11:51.7460+00:00 762168873 -60.0 524.0
3 2019-05-15T16:11:51.9990+00:00 762169125 -58.0 551.0
4 2019-05-15T16:11:52.2510+00:00 762169375 -57.0 519.0
5 2019-05-15T16:11:52.5030+00:00 762169625 -56.0 450.0
6 2019-05-15T16:11:52.7540+00:00 762169875 -57.0 439.0
7 2019-05-15T16:11:53.0070+00:00 762170126 -56.0 497.0
8 2019-05-15T16:11:53.2580+00:00 762170376 -59.0 506.0
9 2019-05-15T16:11:53.5110+00:00 762170627 -57.0 501.0
10 2019-05-15T16:11:53.7630+00:00 762170877 -59.0 499.0
11 2019-05-15T16:11:54.0140+00:00 762171127 -60.0 502.0
12 2019-05-15T16:11:54.2680+00:00 762171379 -59.0 446.0
13 2019-05-15T16:11:54.5190+00:00 762171629 -56.0 478.0
14 2019-05-15T16:11:54.7710+00:00 762171879 -59.0 535.0
15 2019-05-15T16:11:55.0240+00:00 762172130 -59.0 553.0
16 2019-05-15T16:11:55.2670+00:00 762172372 -60.0 520.0
17 2019-05-15T16:11:55.5200+00:00 762172623 -61.0 567.0
18 2019-05-15T16:11:55.7710+00:00 762172873 -63.0 589.0
19 2019-05-15T16:11:56.0250+00:00 762173125 -64.0 601.0
20 2019-05-15T16:11:56.2770+00:00 762173375 -62.0 510.0
21 2019-05-15T16:11:56.5290+00:00 762173626 -59.0 470.0
22 2019-05-15T16:11:56.7810+00:00 762173876 -60.0 537.0
23 2019-05-15T16:11:57.0330+00:00 762174127 -56.0 457.0
24 2019-05-15T16:11:57.2850+00:00 762174377 -56.0 421.0
25 2019-05-15T16:11:57.5370+00:00 762174627 -58.0 456.0
26 2019-05-15T16:11:57.7880+00:00 762174877 -63.0 527.0
27 2019-05-15T16:11:58.0410+00:00 762175128 -59.0 492.0
28 2019-05-15T16:11:58.2940+00:00 762175379 -57.0 480.0
29 2019-05-15T16:11:58.5460+00:00 762175630 -56.0 440.0
... ... ... ... ...
6439 2019-05-15T16:35:40.8000+00:00 763588806 449.0 31.0
6440 2019-05-15T16:35:41.0010+00:00 763589006 465.0 42.0
6441 2019-05-15T16:35:41.2040+00:00 763589207 495.0 48.0
6442 2019-05-15T16:35:41.4050+00:00 763589407 482.0 72.0
6443 2019-05-15T16:35:41.6070+00:00 763589608 511.0 47.0
6444 2019-05-15T16:35:41.8090+00:00 763589808 540.0 22.0
6445 2019-05-15T16:35:42.0110+00:00 763590009 546.0 -31.0
6446 2019-05-15T16:35:42.2120+00:00 763590209 547.0 -69.0
6447 2019-05-15T16:35:42.4130+00:00 763590409 486.0 -21.0
6448 2019-05-15T16:35:42.6160+00:00 763590610 453.0 -31.0
6449 2019-05-15T16:35:42.8160+00:00 763590809 445.0 -22.0
6450 2019-05-15T16:35:43.0170+00:00 763591009 431.0 -12.0
6451 2019-05-15T16:35:43.2190+00:00 763591209 440.0 -29.0
6452 2019-05-15T16:35:43.4200+00:00 763591409 477.0 5.0
6453 2019-05-15T16:35:43.6220+00:00 763591610 465.0 -35.0
6454 2019-05-15T16:35:43.8240+00:00 763591811 465.0 -38.0
6455 2019-05-15T16:35:44.0260+00:00 763592011 474.0 -77.0
6456 2019-05-15T16:35:44.2270+00:00 763592211 475.0 -15.0
6457 2019-05-15T16:35:44.4290+00:00 763592412 509.0 -13.0
6458 2019-05-15T16:35:44.6310+00:00 763592612 520.0 -3.0
6459 2019-05-15T16:35:44.8320+00:00 763592812 500.0 -308.0
6461 2019-05-15T16:35:45.0340+00:00 763593013 533.0 -5.0
6462 2019-05-15T16:35:45.2370+00:00 763593214 544.0 -18.0
6463 2019-05-15T16:35:45.4290+00:00 763593405 529.0 -21.0
6464 2019-05-15T16:35:45.6300+00:00 763593605 542.0 -20.0
6465 2019-05-15T16:35:45.8310+00:00 763593805 549.0 -93.0
6466 2019-05-15T16:35:46.0330+00:00 763594005 521.0 -29.0
6467 2019-05-15T16:35:46.2350+00:00 763594206 522.0 -22.0
6468 2019-05-15T16:35:46.4370+00:00 763594407 499.0 -17.0
6469 2019-05-15T16:35:46.6390+00:00 763594608 421.0 371.0
IMU A3 IMU G1 IMU G2 IMU G3 IMU M1 IMU M2 IMU M3 \
ride_id
16083 0 63.0 3.0 15.0 56.0 29.0 -145.0 197.0
1 62.0 -68.0 -14.0 65.0 12.0 -148.0 194.0
2 87.0 -27.0 -12.0 23.0 16.0 -150.0 188.0
3 97.0 66.0 22.0 -84.0 15.0 -161.0 177.0
4 91.0 6.0 30.0 -103.0 29.0 -139.0 187.0
5 82.0 -36.0 34.0 -31.0 37.0 -141.0 185.0
6 78.0 28.0 36.0 74.0 36.0 -148.0 192.0
7 63.0 27.0 25.0 107.0 28.0 -138.0 200.0
8 76.0 -103.0 3.0 0.0 23.0 -155.0 195.0
9 89.0 -1.0 26.0 -36.0 26.0 -154.0 184.0
10 88.0 77.0 48.0 -38.0 28.0 -144.0 184.0
11 77.0 -4.0 36.0 -31.0 26.0 -142.0 194.0
12 63.0 -59.0 11.0 32.0 37.0 -145.0 187.0
13 75.0 -42.0 -5.0 104.0 23.0 -155.0 181.0
14 87.0 29.0 2.0 90.0 18.0 -158.0 178.0
15 99.0 14.0 22.0 -77.0 18.0 -156.0 180.0
16 87.0 3.0 22.0 -32.0 17.0 -153.0 191.0
17 93.0 2.0 28.0 -30.0 29.0 -151.0 179.0
18 97.0 9.0 30.0 16.0 26.0 -150.0 186.0
19 81.0 52.0 35.0 13.0 26.0 -148.0 188.0
20 71.0 -3.0 12.0 22.0 26.0 -146.0 190.0
21 81.0 -6.0 0.0 46.0 19.0 -145.0 195.0
22 91.0 36.0 18.0 28.0 26.0 -152.0 190.0
23 70.0 -18.0 22.0 -22.0 18.0 -144.0 200.0
24 67.0 -55.0 13.0 -25.0 24.0 -150.0 194.0
25 72.0 -8.0 3.0 27.0 25.0 -145.0 187.0
26 90.0 -12.0 6.0 24.0 23.0 -147.0 175.0
27 81.0 -19.0 19.0 -23.0 23.0 -147.0 181.0
28 77.0 -33.0 22.0 -63.0 33.0 -153.0 181.0
29 73.0 -24.0 21.0 -57.0 33.0 -141.0 177.0
... ... ... ... ... ... ... ...
6439 226.0 52.0 24.0 -26.0 -225.0 51.0 -131.0
6440 230.0 38.0 24.0 -28.0 -219.0 43.0 -133.0
6441 213.0 -3.0 18.0 0.0 -221.0 39.0 -123.0
6442 209.0 -12.0 10.0 75.0 -220.0 44.0 -120.0
6443 194.0 -56.0 -12.0 151.0 -225.0 49.0 -123.0
6444 167.0 -55.0 -26.0 163.0 -230.0 70.0 -126.0
6445 147.0 -34.0 -23.0 88.0 -234.0 80.0 -124.0
6446 137.0 49.0 25.0 3.0 -238.0 94.0 -112.0
6447 127.0 -106.0 -8.0 -62.0 -241.0 89.0 -123.0
6448 118.0 -117.0 -4.0 -61.0 -250.0 88.0 -118.0
6449 127.0 30.0 -15.0 -10.0 -245.0 93.0 -111.0
6450 127.0 -78.0 -20.0 7.0 -246.0 92.0 -110.0
6451 125.0 21.0 -27.0 65.0 -255.0 91.0 -105.0
6452 139.0 -59.0 3.0 49.0 -256.0 100.0 -96.0
6453 142.0 -157.0 5.0 -2.0 -250.0 112.0 -100.0
6454 123.0 -291.0 26.0 -31.0 -250.0 130.0 -100.0
6455 130.0 -31.0 10.0 -61.0 -255.0 131.0 -103.0
6456 142.0 11.0 23.0 -57.0 -255.0 125.0 -105.0
6457 136.0 25.0 17.0 -16.0 -259.0 125.0 -97.0
6458 137.0 -32.0 0.0 24.0 -249.0 113.0 -93.0
6459 156.0 39.0 4.0 26.0 -259.0 121.0 -89.0
6461 122.0 -19.0 -8.0 57.0 -262.0 120.0 -92.0
6462 101.0 -26.0 4.0 36.0 -261.0 129.0 -93.0
6463 124.0 2.0 21.0 -3.0 -261.0 129.0 -93.0
6464 107.0 -3.0 26.0 -24.0 -267.0 125.0 -99.0
6465 82.0 -29.0 30.0 -44.0 -257.0 119.0 -93.0
6466 97.0 21.0 29.0 -38.0 -269.0 119.0 -99.0
6467 108.0 0.0 12.0 -22.0 -266.0 120.0 -100.0
6468 97.0 -17.0 0.0 5.0 -255.0 121.0 -93.0
6469 45.0 88.0 4.0 46.0 -272.0 112.0 -90.0
Latitude Longitude
ride_id
16083 0 NaN NaN
1 NaN NaN
2 NaN NaN
3 NaN NaN
4 NaN NaN
5 NaN NaN
6 NaN NaN
7 NaN NaN
8 NaN NaN
9 NaN NaN
10 NaN NaN
11 NaN NaN
12 NaN NaN
13 NaN NaN
14 NaN NaN
15 NaN NaN
16 NaN NaN
17 NaN NaN
18 NaN NaN
19 NaN NaN
20 NaN NaN
21 NaN NaN
22 NaN NaN
23 NaN NaN
24 NaN NaN
25 NaN NaN
26 NaN NaN
27 NaN NaN
28 NaN NaN
29 NaN NaN
... ... ...
6439 NaN NaN
6440 NaN NaN
6441 NaN NaN
6442 NaN NaN
6443 NaN NaN
6444 NaN NaN
6445 NaN NaN
6446 NaN NaN
6447 NaN NaN
6448 NaN NaN
6449 NaN NaN
6450 NaN NaN
6451 NaN NaN
6452 NaN NaN
6453 NaN NaN
6454 NaN NaN
6455 NaN NaN
6456 NaN NaN
6457 NaN NaN
6458 NaN NaN
6459 NaN NaN
6461 NaN NaN
6462 NaN NaN
6463 NaN NaN
6464 NaN NaN
6465 NaN NaN
6466 NaN NaN
6467 NaN NaN
6468 NaN NaN
6469 NaN NaN
[6203 rows x 13 columns]
# To store time elapsed between each measurement, and time offset from 0s
time_e_list = []
time_o_list = []
#Remove all nan instances in time:
time_array_nans = np.array(dropped_motion_df.loc[:,"Time"], dtype=float)
time_array = []
imuA1_array_nans = np.array(dropped_motion_df.loc[:,"IMU A1"], dtype=float)
imu_array_A1 = []
imuA2_array_nans = np.array(dropped_motion_df.loc[:,"IMU A2"], dtype=float)
imu_array_A2 = []
imuA3_array_nans = np.array(dropped_motion_df.loc[:,"IMU A3"], dtype=float)
imu_array_A3 = []
imuG1_array_nans = np.array(dropped_motion_df.loc[:,"IMU G1"], dtype=float)
imu_array_G1 = []
imuG2_array_nans = np.array(dropped_motion_df.loc[:,"IMU G2"], dtype=float)
imu_array_G2 = []
imuG3_array_nans = np.array(dropped_motion_df.loc[:,"IMU G3"], dtype=float)
imu_array_G3 = []
imuM1_array_nans = np.array(dropped_motion_df.loc[:,"IMU M1"], dtype=float)
imu_array_M1 = []
imuM2_array_nans = np.array(dropped_motion_df.loc[:,"IMU M2"], dtype=float)
imu_array_M2 = []
imuM3_array_nans = np.array(dropped_motion_df.loc[:,"IMU M3"], dtype=float)
imu_array_M3 = []
#Get all the times and imus where time, imu a1, imu a2, and imu a3 are NOT nan values:
for t,x,y,z,a,b,c,d,e,f in zip(time_array_nans, imuA1_array_nans, imuA2_array_nans, imuA3_array_nans, imuG1_array_nans,
imuG2_array_nans, imuG3_array_nans, imuM1_array_nans, imuM2_array_nans, imuM3_array_nans):
if (np.isnan(t)==0 and np.isnan(x)==0 and np.isnan(y)==0 and np.isnan(z)==0):
time_array.append(t)
imu_array_A1.append(x)
imu_array_A2.append(y)
imu_array_A3.append(z)
imu_array_G1.append(a)
imu_array_G2.append(b)
imu_array_G3.append(c)
imu_array_M1.append(d)
imu_array_M2.append(e)
imu_array_M3.append(f)
#for x in time_array:
# print(x)
start_time = time_array[0]
time_len = len(time_array)
i = 0
while (i < time_len - 1):
prev = time_array[i]
after = time_array[i+1]
offset = after - prev
#if (np.isnan(offset)==0):
time_o_list.append(offset)
elapsed = time_array[i] - start_time
#if (np.isnan(elapsed)==0):
time_e_list.append(elapsed)
i = i + 1
##Check to make sure there are no "nan" values:
i = 0
while (i < len(time_o_list)):
if (np.isnan(time_o_list[i])):
print("Error! Value at index: ", i, " is nan")
i = i + 1
#Drop the last value from each of the imu lists to make it match the time list.
del(imu_array_A1[-1])
del(imu_array_A2[-1])
del(imu_array_A3[-1])
del(imu_array_G1[-1])
del(imu_array_G2[-1])
del(imu_array_G3[-1])
del(imu_array_M1[-1])
del(imu_array_M2[-1])
del(imu_array_M3[-1])
print(len(time_e_list))
print(len(time_o_list))
print(len(imu_array_A1))
print(len(imu_array_A2))
print(len(imu_array_A3))
print(len(imu_array_G1))
print(len(imu_array_G2))
print(len(imu_array_G3))
print(len(imu_array_M1))
print(len(imu_array_M2))
print(len(imu_array_M3))
CheckLength = len(time_e_list) + len(time_o_list) + len(imu_array_A1) + len(imu_array_A2) + len(imu_array_A3) + len(imu_array_G1) + len(imu_array_G2) + len(imu_array_G3) + len(imu_array_M1)+ len(imu_array_M2) + len(imu_array_M3)
if CheckLength//11 == len(time_e_list):
print("All columns are matching!")6202
6202
6202
6202
6202
6202
6202
6202
6202
6202
6202
All columns are matching!
#Raw acceleration constant 512 = 1g (accelerometer's measured force due to gravity)
g_const = 512
#Approximate measurement for gravity:
gravity = 9.80665
# Correct the IMU Acceleration columns into units of meters
def convert_acc_units(acc_array):
ret_array = []
for a in acc_array:
#Acceleration is now in m/s^2, need to subtract gravity from vertical axis. (??)
new_a = a / g_const * gravity
ret_array.append(new_a)
return ret_array
imu1_array = convert_acc_units(imu_array_A1) #new units in m/s^2
imu2_array = convert_acc_units(imu_array_A2) #new units in m/s^2
imu3_array = convert_acc_units(imu_array_A3) #new units in m/s^2
# To check:
#for x,y in zip(imu2_array, imu_array_A2):
# print(x,y)
def convert_time_units(time_array):
ret_array = []
for t in time_array:
new_t = t * (10**(-3)) #converting units in milliseconds to seconds
ret_array.append(new_t)
return ret_array
time_o_array = convert_time_units(time_o_list) #new units in seconds
time_e_array = convert_time_units(time_e_list) #new units in seconds
# To check:
# for t in time_e_array:
# print(t)
print("Done!")Done!
print("Graph of X Acceleration vs. Time")
plt.plot(time_e_array, imu1_array)
plt.xlabel("Time (s)")
plt.ylabel("Acceleration-X (m/s^2)")
plt.show()Graph of X Acceleration vs. Time
# Offset variables help in recentering the magnetic data in order to define direction and use trig functions
M1_offset_var = 219.786
M2_offset_var = 180
M3_offset_var = 280
def calibrate_magn_data(magn_array, offset_value):
ret_array = []
for m in magn_array:
new_m = m - offset_value
ret_array.append(new_m)
return ret_array
imuM1_array = calibrate_magn_data(imu_array_M1, M1_offset_var)
imuM2_array = calibrate_magn_data(imu_array_M2, M2_offset_var)
imuM3_array = calibrate_magn_data(imu_array_M3, M3_offset_var)
# Check
# print(imuM1_array)
print("Done.")Done.
# Create N x 3 arrays for functions that need them later on, such as Scikit Kinematics
magn_height = len(imuM1_array)
acc_height = len(imu1_array)
acc_array = np.zeros(shape=(acc_height,3))
magn_array = np.zeros(shape=(magn_height,3))
print("For Accelerometer: ")
for x in range(len(acc_array)):
acc_array[x,0] = imu1_array[x]
acc_array[x,1] = imu2_array[x]
acc_array[x,2] = imu3_array[x]
print(acc_array)
print("\nFor Magnetometer: ")
for x in range(len(magn_array)):
magn_array[x,0] = imuM1_array[x]
magn_array[x,1] = imuM2_array[x]
magn_array[x,2] = imuM3_array[x]
print(magn_array)
print("Done.")For Accelerometer:
[[-1.13006318 9.67257471 1.20667764]
[-1.07260234 9.44273135 1.18752402]
[-1.1492168 10.03649336 1.66636436]
...
[ 9.97903252 -0.55545479 1.85790049]
[ 9.99818613 -0.42137949 2.06859023]
[ 9.55765303 -0.32561143 1.85790049]]
For Magnetometer:
[[-190.786 -325. -83. ]
[-207.786 -328. -86. ]
[-203.786 -330. -92. ]
...
[-488.786 -61. -379. ]
[-485.786 -60. -380. ]
[-474.786 -59. -373. ]]
Done.
Orientation from here onwards will be from the board/surfers reference frame (yaw left = turning left)
x = -IMU1, y = -IMU3, z = -IMU2
# The new array for board reference frame will have the IMUs in columns according to X,Y,Z directions
print("For Accelerometer:")
board_acc = acc_array.copy() # Reassign to the correct axes as stated above
temp_x_acc = board_acc[:,0] * (-1)
temp_y_acc = board_acc[:,1] * (-1)
temp_z_acc = board_acc[:,2] * (-1)
board_acc[:,0] = temp_x_acc # X acceleration
board_acc[:,1] = temp_y_acc # Y acceleration
board_acc[:,2] = temp_z_acc # Z acceleration
print(board_acc)
print("\nFor Magnetometer:")
board_magn = magn_array.copy()
temp_x_magn = board_magn[:,0] * (-1)
temp_y_magn = board_magn[:,1] * (-1)
temp_z_magn = board_magn[:,2] * (-1)
board_magn[:,0] = temp_x_magn
board_magn[:,1] = temp_y_magn
board_magn[:,2] = temp_z_magn
print(board_magn)
print("Done.")For Accelerometer:
[[ 1.13006318 -9.67257471 -1.20667764]
[ 1.07260234 -9.44273135 -1.18752402]
[ 1.1492168 -10.03649336 -1.66636436]
...
[ -9.97903252 0.55545479 -1.85790049]
[ -9.99818613 0.42137949 -2.06859023]
[ -9.55765303 0.32561143 -1.85790049]]
For Magnetometer:
[[190.786 325. 83. ]
[207.786 328. 86. ]
[203.786 330. 92. ]
...
[488.786 61. 379. ]
[485.786 60. 380. ]
[474.786 59. 373. ]]
Done.
# Azimuth and Altitude LEGEND:
# Altitude is the angle between the ground and the vector
# Azimuth is the angle going clockwise from 0 deg North:
# N - 0/360deg, E - 90deg, S - 180deg, W - 270deg
# This will get complicated (ie make cases or lots of if statements) when rotations about the heading become more prevalent
def azimuth(x,y,z):
real_y = y * (-1) # This is to account for y
return (180/math.pi * math.atan2(real_y,x)) % 360
def altitude(x,y,z):
h = math.hypot(y, x)
return 180/math.pi * math.atan2(z,h)
def printAltAzi(alt, azi):
print ("Alt:", alt, "\n", "Azi:",azi,"\n")
# These values are uncorrected values: still need to add or subtract 'declination'
# (for AziMuth) and 'inclination' (for Altitude) correction values for geographical location
heading_altitude = board_magn[:,0].copy()
heading_azimuth = board_magn[:,0].copy()
i = 0 #iterator
#for i in range(len(M1_no_out)):
while i < len(heading_altitude):
factor = 0;
# Use acceleration values to calibrate magnetometer
if (board_acc[i, 0] >= 0):
factor = 1
else:
factor = -1
heading_altitude[i] = altitude(board_magn[i,0], board_magn[i,1], board_magn[i,2])
heading_azimuth[i] = azimuth(board_magn[i,0], board_magn[i,1], board_magn[i,2])
heading_azimuth[i] = heading_azimuth[i] * factor
#printAltAzi(heading_altitude[i],heading_azimuth[i])
i += 1
#for t in range(len(time_e_array)):
#printAltAzi(heading_altitude[t], heading_azimuth[t])# Fixing random state for reproducibility
heading_alt_plot = plt.figure(figsize=(10,5))
alt_plot = heading_alt_plot.add_subplot(111)
alt_plot.plot(time_e_array, heading_altitude)
alt_plot.set_title("Altitude vs. Time")
alt_plot.set_xlabel("Time Elapsed [sec]")
alt_plot.set_ylabel("Altitude [deg]")
plt.show()
np.random.seed(19680801)
# Compute areas and colors
r = [i for i in range(0, len(board_magn))]
theta = heading_azimuth/360 * 2 * np.pi
area = 1
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='polar')
c = ax.scatter(theta, r, c='red', s=area, cmap='hsv', alpha=1)
# Compute areas and colors
alt_r = [i for i in range(0, len(board_magn))]
alt_theta = heading_altitude/360 * 2 * np.pi
alt_area = 1
alt_colors = theta
fig2 = plt.figure(figsize=(10,10))
ax2 = fig2.add_subplot(111, projection='polar')
c2 = ax2.scatter(alt_theta, alt_r, c='blue', s=alt_area, cmap='hsv', alpha=1)
# Plot first 10 directional changes
magnfig = plt.figure(figsize=(20,20))
magnaxi = magnfig.add_subplot(111, projection='3d')
magnaxi.scatter(board_magn[0,0], board_magn[0,1], board_magn[0,2], c='black', s=300, marker = "<")
magnaxi.scatter(board_magn[1:,0], board_magn[1:,1], board_magn[1:,2], c='black', s=10, marker = "o")
magnaxi.plot(board_magn[:,0], board_magn[:,1], board_magn[:,2], color='green')
magnaxi.set_xlabel("X Axis ->", fontsize=30)
magnaxi.set_ylabel("Y Axis ->", fontsize=30)
magnaxi.set_zlabel("Z Axis ->", fontsize=30)
plt.show()
# Plot Smartfin data
heading_azi_plot = plt.figure(figsize=(10,5))
azi_plot = heading_azi_plot.add_subplot(111)
azi_plot.plot(time_e_array, heading_azimuth)
azi_plot.set_title("Azimuth vs. Time")
azi_plot.set_xlabel("Time Elapsed[sec]")
azi_plot.set_ylabel("Azimuth [deg]")
print()
# Get buoy data from CDIP website
# Link: https://cdip.ucsd.edu/themes/?d2=p70:s:201&zoom=auto&pub_set=public®ions=california_south&tz=UTC&units=standard
# Calculate average heading using my own graph
total_sum = 0
for i in heading_azimuth:
total_sum += i
my_average = total_sum/len(heading_azimuth)
expected_average = 282.41
print("My calculated average heading (deg) is:")
print(my_average)
print("\nThe expected heading (deg) is:")
print(expected_average)
# Calculate error in findings
error_percent = (my_average - expected_average) / expected_average
print("\nThe percentage error in my model is:")
print(error_percent)My calculated average heading (deg) is:
286.95832871711616
The expected heading (deg) is:
282.41
The percentage error in my model is:
0.016105409571602048