-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #362 from e-picsa/feat/forecasts-server-2
refactor: forecast api
- Loading branch information
Showing
15 changed files
with
1,228 additions
and
1,045 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100 changes: 100 additions & 0 deletions
100
apps/picsa-server/supabase/functions/dashboard/forecast-storage.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| import { getClient } from '../_shared/client.ts'; | ||
| import { getJsonData } from '../_shared/request.ts'; | ||
| import { JSONResponse } from '../_shared/response.ts'; | ||
| import { apiClient } from './forecast-db.ts'; | ||
| import { IDBClimateForecastRow } from './types.ts'; | ||
|
|
||
| interface IReqParams { | ||
| /** | ||
| * Max number of documents to retrieve. As requests are run in parallel smaller number | ||
| * reduces server workload. Default 5 | ||
| */ | ||
| limit?: number; | ||
| } | ||
|
|
||
| /** | ||
| * Handle downloading forecast files from api and populating to supabase storage entry | ||
| * Checks DB for any entries without storage files and attempts to update | ||
| */ | ||
| export const forecastStorage = async (req: Request) => { | ||
| // ensure api up and running before sending batched requests | ||
| await apiClient.GET('/v1/status/'); | ||
| const params = await getJsonData<IReqParams>(req); | ||
| const res = await new ForecastStorageUpdate().populateStorageFiles(params); | ||
| return JSONResponse(res); | ||
| }; | ||
|
|
||
| class ForecastStorageUpdate { | ||
| supabaseClient = getClient(); | ||
|
|
||
| private get table() { | ||
| return this.supabaseClient.from('forecasts'); | ||
| } | ||
|
|
||
| async populateStorageFiles(params: IReqParams) { | ||
| const { limit = 5 } = params; | ||
| const pending = await this.listPendingFiles(limit); | ||
|
|
||
| const updates: IDBClimateForecastRow[] = []; | ||
| const errors: any[] = []; | ||
| // TODO - make parallel and allow failure | ||
| for (const { country_code, id } of pending) { | ||
| const { data, error } = await this.storeForecast(country_code, id); | ||
| if (error) { | ||
| errors.push(error); | ||
| } | ||
| if (data) { | ||
| updates.push(data); | ||
| } | ||
| } | ||
| return { data: updates, error: errors }; | ||
| } | ||
|
|
||
| /** Check all climate forecast db entries for any that are missing corresponding storage files */ | ||
| private async listPendingFiles(limit = 5) { | ||
| const query = this.table.select('*').is('storage_file', null).order('id', { ascending: false }).limit(limit); | ||
| const { data, error } = await query; | ||
| if (error) { | ||
| throw error; | ||
| } | ||
| return data; | ||
| } | ||
|
|
||
| /** Retrieve forecast data from API, store to supabase storage and update DB */ | ||
| private async storeForecast( | ||
| country_code: string, | ||
| id: string | ||
| ): Promise<{ data?: IDBClimateForecastRow; error?: any }> { | ||
| const supabaseClient = getClient(); | ||
| // download from api | ||
| const req = apiClient.GET('/v1/documents/{country}/{filepath}', { | ||
| params: { path: { country: country_code as any, filepath: id } }, | ||
| parseAs: 'blob', | ||
| }); | ||
| const { data: fileData, response: apiResponse, error: apiError } = await req; | ||
| if (apiError) { | ||
| return { error: apiError }; | ||
| } | ||
| if (fileData) { | ||
| // upload to supabase storage | ||
| const contentType = apiResponse.headers.get('content-type') as string; | ||
| const { data: uploadData, error: uploadError } = await supabaseClient.storage | ||
| .from(country_code) | ||
| .upload(`climate/forecasts/${id}`, fileData, { contentType, upsert: true }); | ||
| if (uploadError) { | ||
| return { error: uploadError }; | ||
| } | ||
| // update db entry | ||
| const { data: updateData, error: updateError } = await this.updateForecastDBStorageEntry(id, uploadData.fullPath); | ||
| if (updateError) { | ||
| return { error: updateError }; | ||
| } | ||
| return { data: updateData?.[0] }; | ||
| } | ||
| return { error: `No filedata found for ${id}` }; | ||
| } | ||
|
|
||
| private updateForecastDBStorageEntry(id: string, storage_file: string) { | ||
| return this.table.update({ storage_file }).eq('id', id).select(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| import type { Database } from '../../types/db.types.ts'; | ||
| import type { paths, components } from '../../../../picsa-apps/dashboard/src/app/modules/climate/types/api.d.ts'; | ||
|
|
||
| export type climateApiPaths = paths; | ||
| export type climateApiComponents = components; | ||
|
|
||
| export type IDBClimateForecastRow = Database['public']['Tables']['forecasts']['Row']; | ||
| export type IDBClimateForecastInsert = Database['public']['Tables']['forecasts']['Insert']; | ||
|
|
||
| export type IApiClimateForecast = climateApiComponents['schemas']['DocumentMetadata']; | ||
|
|
||
| /********* Api Responses ************/ | ||
|
|
||
| export type IForecastDBAPIResponse = { | ||
| [country_code: string]: IDBClimateForecastInsert[]; | ||
| }; |
Oops, something went wrong.