Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change :float to {:f, 64} and add {:f, 32} dtype #739

Merged
merged 9 commits into from
Nov 24, 2023
Merged

Change :float to {:f, 64} and add {:f, 32} dtype #739

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
be66375
Change `:float` to `{:f, 64}` and add `{:f, 32}` dtype
philss Nov 23, 2023
ed019b9
Add support for fetching from f32 series
philss Nov 23, 2023
c75c2ba
Make `Series.to_iovec/1` work for f32 series
philss Nov 23, 2023
911b623
Make `Series.from_binary/3` work with "f32 binaries"
philss Nov 23, 2023
c9505c2
Fix problems after rebase
philss Nov 24, 2023
2e1ed88
Fix mentions of `:float`
philss Nov 24, 2023
8d36451
Normalise dtypes instead of just validating
philss Nov 24, 2023
fae2366
Make inspection of float series similar to Nx
philss Nov 24, 2023
3605cfc
Doc that `:float` can be used as alias for `{:f, 64}`
philss Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ data exploration to Elixir.

Explorer high-level features are:

- Simply typed series: `:binary`, `:boolean`, `:category`, `:date`, `:datetime`, `:duration`, `:float`, `:integer`, `:string`, and `:time`.
- Simply typed series: `:binary`, `:boolean`, `:category`, `:date`, `:datetime`, `:duration`, `{:f, 32}`, `{:f, 64}`, `:integer`, `:string`, and `:time`.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we shouldn't do something like:

Suggested change
- Simply typed series: `:binary`, `:boolean`, `:category`, `:date`, `:datetime`, `:duration`, `{:f, 32}`, `{:f, 64}`, `:integer`, `:string`, and `:time`.
- Simply typed series: `:binary`, `:boolean`, `:category`, `:date`, `:datetime`, `:duration`, `:float` (`{:f, 32}` or `{:f, 64}`), `:integer`, `:string`, and `:time`.

WDYT?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed! I added more mentions of it in other places as well: 3605cfc


- A powerful but constrained and opinionated API, so you spend less time looking
for the right function and more time doing data manipulation.
Expand Down
2 changes: 1 addition & 1 deletion lib/explorer/backend/lazy_frame.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ defmodule Explorer.Backend.LazyFrame do
concat([
line(),
color("#{name} ", :map, opts),
color("#{dtypes[name]}", :atom, opts)
color("#{Explorer.Shared.dtype_to_string(dtypes[name])}", :atom, opts)
])
end

Expand Down
98 changes: 49 additions & 49 deletions lib/explorer/backend/lazy_series.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ defmodule Explorer.Backend.LazySeries do
@impl true
def fill_missing_with_strategy(%Series{} = s, strategy) do
args = [lazy_series!(s), strategy]
dtype = if strategy == :mean, do: :float, else: s.dtype
dtype = if strategy == :mean, do: {:f, 64}, else: s.dtype
data = new(:fill_missing_with_strategy, args, dtype, aggregations?(args))
Backend.Series.new(data, dtype)
end
Expand Down Expand Up @@ -437,9 +437,9 @@ defmodule Explorer.Backend.LazySeries do

if aggregations?(args), do: raise_agg_inside_window(:window_standard_deviation)

data = new(:window_standard_deviation, args, :float, false)
data = new(:window_standard_deviation, args, {:f, 64}, false)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

for op <- @cumulative_operations do
Expand Down Expand Up @@ -488,49 +488,49 @@ defmodule Explorer.Backend.LazySeries do
@impl true
def skew(%Series{} = series, bias) do
args = [series_or_lazy_series!(series), bias]
data = new(:skew, args, :float, true)
data = new(:skew, args, {:f, 64}, true)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def correlation(%Series{} = left, %Series{} = right, ddof) do
args = [series_or_lazy_series!(left), series_or_lazy_series!(right), ddof]
data = new(:correlation, args, :float, true)
data = new(:correlation, args, {:f, 64}, true)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def covariance(%Series{} = left, %Series{} = right) do
args = [series_or_lazy_series!(left), series_or_lazy_series!(right)]
data = new(:covariance, args, :float, true)
data = new(:covariance, args, {:f, 64}, true)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def variance(%Series{} = s, ddof \\ 1) do
args = [series_or_lazy_series!(s), ddof]
data = new(:variance, args, :float, true)
data = new(:variance, args, {:f, 64}, true)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def standard_deviation(%Series{} = s, ddof \\ 1) do
args = [series_or_lazy_series!(s), ddof]
data = new(:standard_deviation, args, :float, true)
data = new(:standard_deviation, args, {:f, 64}, true)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def coalesce(%Series{} = left, %Series{} = right) do
args = [series_or_lazy_series!(left), series_or_lazy_series!(right)]

dtype =
if left.dtype in [:float, :integer] do
if left.dtype in [{:f, 32}, {:f, 64}, :integer] do
resolve_numeric_dtype([left, right])
else
left.dtype
Expand All @@ -546,7 +546,7 @@ defmodule Explorer.Backend.LazySeries do
args = [series_or_lazy_series!(predicate) | binary_args(on_true, on_false)]

dtype =
if dtype in [:float, :integer] do
if dtype in [{:f, 32}, {:f, 64}, :integer] do
resolve_numeric_dtype([on_true, on_false])
else
dtype
Expand Down Expand Up @@ -635,9 +635,9 @@ defmodule Explorer.Backend.LazySeries do

if aggregations?(args), do: raise_agg_inside_window(:ewm_mean)

data = new(:ewm_mean, args, :float, false)
data = new(:ewm_mean, args, {:f, 64}, false)

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

defp dtype_for_agg_operation(op, _) when op in [:count, :nil_count, :n_distinct], do: :integer
Expand All @@ -646,7 +646,7 @@ defmodule Explorer.Backend.LazySeries do
when op in [:first, :last, :sum, :min, :max, :argmin, :argmax],
do: series.dtype

defp dtype_for_agg_operation(_, _), do: :float
defp dtype_for_agg_operation(_, _), do: {:f, 64}

defp resolve_numeric_dtype(items) do
dtypes =
Expand All @@ -658,12 +658,12 @@ defmodule Explorer.Backend.LazySeries do
end

case dtypes do
[dtype] when dtype in [:integer, :float] -> dtype
[_, _] -> :float
[dtype] when dtype in [:integer, {:f, 32}, {:f, 64}] -> dtype
[_, _] -> {:f, 64}
end
end

defp resolve_numeric_dtype(:window_mean, _items), do: :float
defp resolve_numeric_dtype(:window_mean, _items), do: {:f, 64}
defp resolve_numeric_dtype(_op, items), do: resolve_numeric_dtype(items)

# Returns the inner `data` if it's a lazy series. Otherwise raises an error.
Expand Down Expand Up @@ -720,30 +720,30 @@ defmodule Explorer.Backend.LazySeries do

@impl true
def log(%Series{} = series) do
data = new(:log, [lazy_series!(series)], :float)
data = new(:log, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def log(%Series{} = series, base) do
data = new(:log, [lazy_series!(series), base], :float)
data = new(:log, [lazy_series!(series), base], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def exp(%Series{} = series) do
data = new(:exp, [lazy_series!(series)], :float)
data = new(:exp, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def abs(%Series{} = series) do
data = new(:abs, [lazy_series!(series)], :float)
data = new(:abs, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
Expand Down Expand Up @@ -771,51 +771,51 @@ defmodule Explorer.Backend.LazySeries do
end

def clip(%Series{} = series, min, max) do
data = new(:clip_float, [lazy_series!(series), min * 1.0, max * 1.0], :float)
data = new(:clip_float, [lazy_series!(series), min * 1.0, max * 1.0], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def sin(%Series{} = series) do
data = new(:sin, [lazy_series!(series)], :float)
data = new(:sin, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def cos(%Series{} = series) do
data = new(:cos, [lazy_series!(series)], :float)
data = new(:cos, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def tan(%Series{} = series) do
data = new(:tan, [lazy_series!(series)], :float)
data = new(:tan, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def asin(%Series{} = series) do
data = new(:asin, [lazy_series!(series)], :float)
data = new(:asin, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def acos(%Series{} = series) do
data = new(:acos, [lazy_series!(series)], :float)
data = new(:acos, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def atan(%Series{} = series) do
data = new(:atan, [lazy_series!(series)], :float)
data = new(:atan, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
Expand Down Expand Up @@ -964,23 +964,23 @@ defmodule Explorer.Backend.LazySeries do

@impl true
def round(series, decimals) when is_integer(decimals) and decimals >= 0 do
data = new(:round, [lazy_series!(series), decimals], :float)
data = new(:round, [lazy_series!(series), decimals], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def floor(series) do
data = new(:floor, [lazy_series!(series)], :float)
data = new(:floor, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
def ceil(series) do
data = new(:ceil, [lazy_series!(series)], :float)
data = new(:ceil, [lazy_series!(series)], {:f, 64})

Backend.Series.new(data, :float)
Backend.Series.new(data, {:f, 64})
end

@impl true
Expand Down
Loading
Loading