Skip to content

Commit

Permalink
Fix chunking requests and handling empty requests
Browse files Browse the repository at this point in the history
  • Loading branch information
fre171csiro committed Sep 10, 2024
1 parent 8f66423 commit 3302086
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 3 deletions.
2 changes: 2 additions & 0 deletions poetry.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[virtualenvs]
in-project = true
11 changes: 8 additions & 3 deletions pybomwater/bom_water.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,10 @@ def define_request_chunking_size(self, features, property, procedure, start_date
values_count = self.values_count(response)
# size = request_values_limit/(values_count)#*len(features)
sample_sizes.append(values_count)
return math.floor(request_values_limit/max(sample_sizes)*0.9)
if max(sample_sizes) > 0:
return math.floor((request_values_limit/max(sample_sizes)*0.9)/2)
else:
return 3
return size

def get_spatially_filtered_observations(self, features, spatial_path, bbox, property, procedure, t_begin, t_end):
Expand Down Expand Up @@ -407,15 +410,17 @@ def parse_node_attributes(self, node):

def values_count(self, response):
root = ET.fromstring(response.text)
value_nodes = []
# Unit and default quality code
prefix = './/{http://www.opengis.net/waterml/2.0}'
sos_prefix = './/{http://www.opengis.net/sos/2.0}'
# Parse time series data
query_observationData = f'{sos_prefix}observationData'
query_measurement = f'{prefix}MeasurementTVP'
observations = root.findall(query_observationData)
for obs in root.findall(query_observationData):
#Measurement values and associated metadata
value_nodes = observations[0].findall(query_measurement)
value_nodes = obs[0].findall(query_measurement)
break
return len(value_nodes)

def parse_data(self, response, stations=None):
Expand Down
25 changes: 25 additions & 0 deletions test/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,31 @@ def test_get_features_of_interest(self):
else:
assert False, "Feature not found"

def test_list_index_out_of_range(self):
_bm = bm.BomWater()
procedure = _bm.procedures.Pat1_C_B_1_DailyMean
prop = _bm.properties.Water_Temperature

# Setup bounding box for BoM api spatial filter query
low_left_lat = -37.505032
low_left_long = 138.00
upper_right_lat = -24.00
upper_right_long = 154.00

lower_left_coords = f'{low_left_lat} {low_left_long}'
upper_right_coords = f'{upper_right_lat} {upper_right_long}'
coords = tuple((lower_left_coords, upper_right_coords))

t_begin = "1800-01-01T00:00:00+10"
t_end = "2030-12-31T00:00:00+10"

spat_dir = r'.\\test\\test_data\\Spatial'
spatial_path = os.path.join( spat_dir, 'mdb_buffer_1km.shp')#'./test/test_data/Spatial/mdb_buffer_1km.shp'#
assert os.path.exists(spatial_path)

results = _bm.get_spatially_filtered_observations( None, str(spatial_path), coords, prop, procedure, t_begin, t_end)
assert len(results) > 0, 'No results found'

def divide_chunks(self, l, n):
# looping till length l
for i in range(0, len(l), n):
Expand Down

0 comments on commit 3302086

Please sign in to comment.