Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move dict_without_keys to dagster.utils #2837

Merged
merged 1 commit into from
Sep 9, 2020
Merged

Move dict_without_keys to dagster.utils #2837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions python_modules/dagster/dagster/utils/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -517,3 +517,7 @@ def compose(*args):
# reduce using functional composition over all the arguments, with the identity function as
# initializer
return functools.reduce(lambda f, g: lambda x: f(g(x)), args, lambda x: x)


def dict_without_keys(ddict, *keys):
return {key: value for key, value in ddict.items() if key not in set(keys)}
5 changes: 1 addition & 4 deletions python_modules/libraries/dagster-pandas/dagster_pandas/data_frame.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,12 @@
dagster_type_materializer,
)
from dagster.config.field_utils import Selector
from dagster.utils import dict_without_keys
from dagster.utils.backcompat import canonicalize_backcompat_args, experimental

CONSTRAINT_BLACKLIST = {ColumnDTypeFnConstraint, ColumnDTypeInSetConstraint}


def dict_without_keys(ddict, *keys):
return {key: value for key, value in ddict.items() if key not in set(keys)}


@dagster_type_materializer(
Selector(
{
Expand Down
5 changes: 1 addition & 4 deletions python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from dagster.config.field_utils import Selector
from dagster.core.storage.system_storage import fs_intermediate_storage, fs_system_storage
from dagster.core.storage.type_storage import TypeStoragePlugin
from dagster.utils import dict_without_keys

WriteModeOptions = Enum(
"WriteMode",
Expand Down Expand Up @@ -75,10 +76,6 @@
)


def dict_without_keys(ddict, *keys):
return {key: value for key, value in ddict.items() if key not in set(keys)}


@dagster_type_materializer(
Selector(
{
Expand Down
5 changes: 1 addition & 4 deletions python_modules/libraries/dagster-pyspark/dagster_pyspark_tests/test_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
file_relative_path,
solid,
)
from dagster.utils import dict_without_keys
from dagster.utils.test import get_temp_dir

spark = SparkSession.builder.getOrCreate()
Expand All @@ -28,10 +29,6 @@
]


def dict_without_keys(ddict, *keys):
return {key: value for key, value in ddict.items() if key not in set(keys)}


def create_pyspark_df():
data = [Row(_c0=str(i), _c1=str(i)) for i in range(100)]
return spark.createDataFrame(data)
Expand Down