Skip to content

[SPARK-26822]Upgrade the deprecated module 'optparse' #23730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cchung100m wants to merge 5 commits into from
Closed

[SPARK-26822]Upgrade the deprecated module 'optparse' #23730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5b5df4e
solve_deprecated_module_optparse
cchung100m Feb 3, 2019
6e7ba67
Complete PySpark unit tests
cchung100m Feb 9, 2019
b0565e7
update dev/run-tests.py
cchung100m Feb 9, 2019
8fac90d
update dev/run-tests.py
cchung100m Feb 9, 2019
47e9163
Re-trigger the test build
cchung100m Feb 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions dev/run-tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from __future__ import print_function
import itertools
from optparse import OptionParser
from argparse import ArgumentParser
import os
import random
import re
Expand Down Expand Up @@ -484,20 +484,20 @@ def run_sparkr_tests():


def parse_opts():
parser = OptionParser(
parser = ArgumentParser(
prog="run-tests"
)
parser.add_option(
"-p", "--parallelism", type="int", default=8,
help="The number of suites to test in parallel (default %default)"
parser.add_argument(
"-p", "--parallelism", type=int, default=8,
help="The number of suites to test in parallel (default %(default)d)"
)

(opts, args) = parser.parse_args()
if args:
parser.error("Unsupported arguments: %s" % ' '.join(args))
if opts.parallelism < 1:
args, unknown = parser.parse_known_args()
if unknown:
parser.error("Unsupported arguments: %s" % ' '.join(unknown))
if args.parallelism < 1:
parser.error("Parallelism cannot be less than 1")
return opts
return args


def main():
Expand Down Expand Up @@ -636,6 +636,7 @@ def _test():
if failure_count:
sys.exit(-1)


Copy link
Member

@HyukjinKwon HyukjinKwon Feb 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's remove this newline.

Copy link
Author

@cchung100m cchung100m Feb 9, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @HyukjinKwon

I add the newline to solve PEP8: expected 2 blank lines after class or function definition, found 1, can we keep the newline here?

Copy link
Member

@srowen srowen Feb 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it's for PEP8, keep it.

Copy link
Member

@HyukjinKwon HyukjinKwon Feb 9, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea that's fine. I wonder why it didn't fail by pep8 check before tho.

if __name__ == "__main__":
_test()
main()
41 changes: 20 additions & 21 deletions python/run-tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

from __future__ import print_function
import logging
from optparse import OptionParser, OptionGroup
from argparse import ArgumentParser
import os
import re
import shutil
Expand Down Expand Up @@ -168,30 +168,30 @@ def get_default_python_executables():


def parse_opts():
parser = OptionParser(
parser = ArgumentParser(
prog="run-tests"
)
parser.add_option(
"--python-executables", type="string", default=','.join(get_default_python_executables()),
help="A comma-separated list of Python executables to test against (default: %default)"
parser.add_argument(
"--python-executables", type=str, default=','.join(get_default_python_executables()),
help="A comma-separated list of Python executables to test against (default: %(default)s)"
)
parser.add_option(
"--modules", type="string",
parser.add_argument(
"--modules", type=str,
default=",".join(sorted(python_modules.keys())),
help="A comma-separated list of Python modules to test (default: %default)"
help="A comma-separated list of Python modules to test (default: %(default)s)"
)
parser.add_option(
"-p", "--parallelism", type="int", default=4,
help="The number of suites to test in parallel (default %default)"
parser.add_argument(
"-p", "--parallelism", type=int, default=4,
help="The number of suites to test in parallel (default %(default)d)"
)
parser.add_option(
parser.add_argument(
"--verbose", action="store_true",
help="Enable additional debug logging"
)

group = OptionGroup(parser, "Developer Options")
group.add_option(
"--testnames", type="string",
group = parser.add_argument_group("Developer Options")
group.add_argument(
"--testnames", type=str,
default=None,
help=(
"A comma-separated list of specific modules, classes and functions of doctest "
Expand All @@ -201,14 +201,13 @@ def parse_opts():
"'pyspark.sql.tests FooTests.test_foo' to run the specific unittest in the class. "
"'--modules' option is ignored if they are given.")
)
parser.add_option_group(group)

(opts, args) = parser.parse_args()
if args:
parser.error("Unsupported arguments: %s" % ' '.join(args))
if opts.parallelism < 1:
args, unknown = parser.parse_known_args()
if unknown:
parser.error("Unsupported arguments: %s" % ' '.join(unknown))
if args.parallelism < 1:
parser.error("Parallelism cannot be less than 1")
return opts
return args


def _check_coverage(python_exec):
Expand Down