Skip to content

Commit

Permalink
Promote IRVE DataFrame to app code
Browse files Browse the repository at this point in the history
  • Loading branch information
thbar committed Nov 4, 2024
1 parent 1f2849d commit a5d3888
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 30 deletions.
37 changes: 37 additions & 0 deletions apps/transport/lib/irve/data_frame.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
defmodule Transport.IRVE.DataFrame do
@moduledoc """
A module providing programmatic access to the static IRVE schema,
as stored in the source code.
"""

# TODO: move to dedicated module.
# TODO: consider grouping this (see unlock dynamic equivalent)
def schema_content do
__ENV__.file
|> Path.join("../../../../shared/meta/schema-irve-statique.json")
|> Path.expand()
|> File.read!()
|> Jason.decode!()
end

def remap_schema_type(input_type) do
case input_type do
# TODO: extract individual coordinates
:geopoint -> :string
# works for this specific case
:number -> {:u, 16}
type -> type
end
end

def dataframe_from_csv_body!(body) do
dtypes =
schema_content()
|> Map.fetch!("fields")
|> Enum.map(fn %{"name" => name, "type" => type} ->
{String.to_atom(name), String.to_atom(type) |> Transport.IRVE.DataFrame.remap_schema_type()}
end)

Explorer.DataFrame.load_csv!(body, dtypes: dtypes)
end
end
32 changes: 2 additions & 30 deletions scripts/irve/data-frame.exs
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
IO.puts("OK")

# https://www.data.gouv.fr/fr/datasets/623ca46c13130c3228abd018/ - Electra dataset (mid-sized)
# https://www.data.gouv.fr/fr/datasets/623ca46c13130c3228abd018/#/resources/e9bb3424-77cd-40ba-8bbd-5a19362d0365

Expand All @@ -9,31 +7,5 @@ sample_url = "https://www.data.gouv.fr/fr/datasets/r/e9bb3424-77cd-40ba-8bbd-5a1
%Req.Response{status: 200, body: body} =
Transport.IRVE.Fetcher.get!(sample_url, compressed: false, decode_body: false)

schema_file = "apps/shared/meta/schema-irve-statique.json"

# temporary mapper
type_mapper = fn
# TODO: extract individual coordinates
:geopoint -> :string
# works for this specific case
:number -> {:u, 16}
type -> type
end

dtypes =
schema_file
|> File.read!()
|> Jason.decode!()
|> Map.fetch!("fields")
|> Enum.map(fn %{"name" => name, "type" => type} ->
{String.to_atom(name), String.to_atom(type) |> type_mapper.()}
end)

IO.inspect(dtypes, IEx.inspect_opts())

df =
Explorer.DataFrame.load_csv!(body,
dtypes: dtypes
)

IO.inspect(df, IEx.inspect_opts())
Transport.IRVE.DataFrame.dataframe_from_csv_body!(body)
|> IO.inspect(IEx.inspect_opts)

0 comments on commit a5d3888

Please sign in to comment.