Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deepdish deprecation and borealis_gaps update #12

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Deepdish deprecation and borealis_gaps update #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
830a802
Using h5py for reading in files in borealis_gaps.py
RemingtonRohel Sep 24, 2024
049519d
Minor bugfixes to grab the correct files and print nicely
RemingtonRohel Sep 24, 2024
42481e3
Fixed bad glob and missing downtimes at end of single-day search
RemingtonRohel Sep 24, 2024
26d77eb
Replacing deepdish I/O with h5py
RemingtonRohel Sep 24, 2024
1e6980a
Merge branch 'main' into dd_deprecation
RemingtonRohel Sep 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 10 additions & 13 deletions SNR/hdf5_rtplot_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import matplotlib.pyplot as plt
import numpy as np
import os
import deepdish as dd
import h5py

from pydarnio import BorealisRead
from multiprocessing import Process
Expand Down Expand Up @@ -120,7 +120,7 @@ def plot_unaveraged_range_time_data(data_array, num_sequences_array, timestamps_
new_power_array = np.transpose(power_array)

kw = {'width_ratios': [95, 5], 'height_ratios': [1, 3]}
fig, ((ax1, cax1), (ax2, cax2)) = plt.subplots(2, 2, figsize=figsize, gridspec_kw=kw)
fig, ((ax1, cax1), (ax2, cax2)) = plt.subplots(2, 2, figsize=figsize, gridspec_kw=kw, sharex='col')
fig.suptitle(f'{dataset_descriptor} Raw Power Sequence Time {start_time.strftime("%Y%m%d")} '
f'{start_time.strftime("%H:%M:%S")} to {end_time.strftime("%H:%M:%S")} UT vs Range')

Expand All @@ -138,7 +138,6 @@ def plot_unaveraged_range_time_data(data_array, num_sequences_array, timestamps_
fig.colorbar(img, cax=cax2, label='Raw Power (dB)')
cax1.axis('off')

ax2.sharex(ax1)
print(plot_filename)
plt.savefig(plot_filename)
plt.close()
Expand Down Expand Up @@ -202,7 +201,7 @@ def plot_antennas_range_time(antennas_iq_file, antenna_nums=None, num_processes=
if is_site_file:
arrays, antenna_names, antenna_indices = antennas_iq_site_to_array(antennas_iq_file, antenna_nums)
else:
reader = BorealisRead(antennas_iq_file, 'antennas_iq', 'array')
reader = BorealisRead(antennas_iq_file, 'antennas_iq')
arrays = reader.arrays

(num_records, num_antennas, max_num_sequences, num_samps) = arrays['data'].shape
Expand Down Expand Up @@ -458,7 +457,7 @@ def plot_averaged_range_time_data(data_array, timestamps_array, dataset_descript
end_time = datetime.datetime.utcfromtimestamp(timestamps[-1])

kw = {'width_ratios': [95, 5], 'height_ratios': [1, 3]}
fig, ((ax1, cax1), (ax2, cax2)) = plt.subplots(2, 2, figsize=figsize, gridspec_kw=kw)
fig, ((ax1, cax1), (ax2, cax2)) = plt.subplots(2, 2, figsize=figsize, gridspec_kw=kw, sharex='col')
fig.suptitle(f'{dataset_descriptor} PWR Time {start_time.strftime("%Y%m%d")} {start_time.strftime("%H:%M:%S")} to '
f'{end_time.strftime("%H:%M:%S")} UT vs Range')

Expand All @@ -475,7 +474,6 @@ def plot_averaged_range_time_data(data_array, timestamps_array, dataset_descript
fig.colorbar(img, cax=cax2, label='Raw Power (dB)')
cax1.axis('off')

ax2.get_shared_x_axes().join(ax1, ax2)
print(plot_filename)
plt.savefig(plot_filename)
plt.close()
Expand Down Expand Up @@ -670,17 +668,17 @@ def plot_rawrf_data(rawrf_file, antenna_nums=None, num_processes=3, sequence_num

time_of_plot = '.'.join(basename.split('.')[0:6])

records = dd.io.load(rawrf_file)
record_keys = sorted(list(records.keys()))
f = h5py.File(rawrf_file, 'r')
record_keys = sorted(list(f.keys()))

record = records[record_keys[0]]
record = f[record_keys[0]]

# This little hack was made to deal with rawrf files afflicted by Issue #258 on the Borealis GitHub, which
# has since been solved. It should work for all rawrf files regardless.
num_sequences, num_antennas, num_samps = record['data_dimensions']
num_sequences, num_antennas, num_samps = record.attrs['data_dimensions']
total_samples = record['data'].size
sequences_stored = int(total_samples / num_samps / num_antennas)
data = record['data'].reshape((sequences_stored, num_antennas, num_samps))
data = record['data'][()].reshape((sequences_stored, num_antennas, num_samps))

# typically, antenna names and antenna indices are the same except
# where certain antennas were skipped in data writing for any reason.
Expand Down Expand Up @@ -768,7 +766,7 @@ def plot_iq_data(voltage_samples, timestamps_array, dataset_descriptor, plot_fil
start_time = datetime.datetime.utcfromtimestamp(timestamps_array[0])
end_time = datetime.datetime.utcfromtimestamp(timestamps_array[-1])

fig, (ax1, ax2) = plt.subplots(2, 1, figsize=figsize)
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=figsize, sharex='col')
fig.suptitle(f'{dataset_descriptor} Raw Voltage Sequence Time {start_time.strftime("%Y%m%d")} '
f'{start_time.strftime("%H:%M:%S")} to {end_time.strftime("%H:%M:%S")} UT')

Expand All @@ -783,7 +781,6 @@ def plot_iq_data(voltage_samples, timestamps_array, dataset_descriptor, plot_fil
ax2.plot(np.arange(0, len(voltage_samples[0, :]))/sample_rate*1e6, 10 * np.log10(np.abs(voltage_samples[0, :])))
ax2.set_ylabel('Power (dB)')
ax2.set_xlabel('Time (us)')
ax2.get_shared_x_axes().join(ax1, ax2)

plot_name = plot_filename_prefix + '_time.jpg'
print(plot_name)
Expand Down
74 changes: 36 additions & 38 deletions SNR/minimal_restructuring.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
Functions
---------
"""
import copy
import h5py
import numpy as np
import os
import deepdish as dd


def antennas_iq_site_to_array(antennas_iq_file, antenna_nums=None):
Expand All @@ -37,48 +35,48 @@ def antennas_iq_site_to_array(antennas_iq_file, antenna_nums=None):
# Dictionary of fields that will be returned
arrays = {}

group = dd.io.load(antennas_iq_file)
timestamps = sorted(list(group.keys()))
with h5py.File(antennas_iq_file, 'r') as f:
timestamps = sorted(list(f.keys()))

antenna_arrays_order = group[timestamps[0]]['antenna_arrays_order']
arrays['antenna_arrays_order'] = antenna_arrays_order
antenna_arrays_order = f[timestamps[0]]['antenna_arrays_order'][:]
arrays['antenna_arrays_order'] = antenna_arrays_order

num_records = len(timestamps)
num_sequences_array = np.zeros(num_records, dtype=np.uint32)
num_records = len(timestamps)
num_sequences_array = np.zeros(num_records, dtype=np.uint32)

# typically, antenna names and antenna indices are the same except
# where certain antennas were skipped in data writing for any reason.
if antenna_nums is None or len(antenna_nums) == 0:
antenna_indices = list(range(0, antenna_arrays_order.size))
antenna_names = list(antenna_arrays_order)
else:
antenna_indices = [list(antenna_arrays_order).index('antenna_' + str(i)) for i in antenna_nums]
antenna_names = [f'antenna_{a}' for a in antenna_nums]
# typically, antenna names and antenna indices are the same except
# where certain antennas were skipped in data writing for any reason.
if antenna_nums is None or len(antenna_nums) == 0:
antenna_indices = list(range(0, antenna_arrays_order.size))
antenna_names = list(antenna_arrays_order)
else:
antenna_indices = [list(antenna_arrays_order).index(bytes(f'antenna_{i}', 'utf-8')) for i in antenna_nums]
antenna_names = [f'antenna_{a}' for a in antenna_nums]

# Get the maximum number of sequences for a record in the file
max_sequences = 0
for i, timestamp in enumerate(timestamps):
record = group[timestamp]
_, num_seqs, _ = record['data_dimensions']
num_sequences_array[i] = num_seqs
if num_seqs > max_sequences:
max_sequences = num_seqs
# Get the maximum number of sequences for a record in the file
max_sequences = 0
for i, timestamp in enumerate(timestamps):
record = f[timestamp]
_, num_seqs, _ = record['data_dimensions'][:]
num_sequences_array[i] = num_seqs
if num_seqs > max_sequences:
max_sequences = num_seqs

# We will need these dimensions for pre-allocating arrays
data_dimensions = group[timestamps[0]]['data_dimensions']
# We will need these dimensions for pre-allocating arrays
data_dimensions = f[timestamps[0]]['data_dimensions'][:]

# Create arrays for each of the needed fields
timestamps_data = np.zeros((num_records, max_sequences), dtype=np.float64)
data_array = np.zeros((num_records, len(antenna_indices), max_sequences, data_dimensions[2]), dtype=np.complex64)
# Create arrays for each of the needed fields
timestamps_data = np.zeros((num_records, max_sequences), dtype=np.float64)
data_array = np.zeros((num_records, len(antenna_indices), max_sequences, data_dimensions[2]), dtype=np.complex64)

# Copy the relevant data into the arrays
for i, timestamp in enumerate(timestamps):
record = group[timestamp]
num_antennas, num_seqs, num_samps = record['data_dimensions']
timestamps_data[i, :num_seqs] = record['sqn_timestamps']
data = record['data'].reshape(record['data_dimensions'])
data_array[i, :, :num_seqs, :] = \
data[antenna_indices, ...]
# Copy the relevant data into the arrays
for i, timestamp in enumerate(timestamps):
record = f[timestamp]
num_antennas, num_seqs, num_samps = record['data_dimensions'][:]
timestamps_data[i, :num_seqs] = record['sqn_timestamps'][:]
data = record['data'][:].reshape((num_antennas, num_seqs, num_samps))
data_array[i, :, :num_seqs, :] = \
data[antenna_indices, ...]

# Copy to dictionary and return
arrays['data'] = data_array
Expand Down
Loading