Skip to content

Remove isort dependency, migrate to ruff equivalent #39411

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jrmccluskey wants to merge 2 commits into
base: master
Choose a base branch
from
+54 −89
Open

Remove isort dependency, migrate to ruff equivalent #39411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f06302
Remove isort dependency, migrate to ruff equivalent
jrmccluskey Jul 21, 2026
7716829
Clean up old gen clients from the ignore list
jrmccluskey Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 53 additions & 52 deletions sdks/python/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,57 +45,6 @@ requires = [
# legacy installation is needed to generate `apache_beam.portability.api` package.
build-backend = "setuptools.build_meta"

[tool.isort]
py_version = 310
line_length = 120
old_finders = true
order_by_type = true
force_single_line = true
combine_star = true
src_paths = ["apache_beam"]
extra_standard_library = ["dataclasses"]
known_third_party = ["yaml"]
skip = [
"apiclient.py",
"avroio_test.py",
"cloudpickle.py",
"datastore_wordcount.py",
"datastoreio_test.py",
"doctests_test.py",
"fast_coders_test.py",
"hadoopfilesystem.py",
"iobase_test.py",
"main_test.py",
"model.py",
"preprocess.py",
"process_tfma.py",
"render_test.py",
"slow_coders_test.py",
"taxi.py",
"tfdv_analyze_and_validate.py",
"yaml/main.py",
"yaml_testing_test.py",
"bigquery_v2_client.py",
"bigquery_v2_messages.py",
"dataflow_v1b3_client.py",
"dataflow_v1b3_messages.py",
"storage_v1_client.py",
"storage_v1_messages.py",
"proto2_coder_test_messages_pb2.py",
"rate_limit_pb2.py",
"cloudbuild_v1_client.py",
"cloudbuild_v1_messages.py",
"boto3_client.py",
]
skip_glob = [
"*.pxd",
"*.pyx",
"*pb2*.py",
"**/examples/**/*.py",
"**/portability/api/**/*.py",
"**/portability/api/__init__.py",
]

[tool.ruff]
exclude = [
".bzr",
Expand Down Expand Up @@ -127,16 +76,19 @@ exclude = [
"*.pxd",
"*.pyx",
"*pb2*.py",
"*.ipynb",
"**/*.ipynb",
"**/examples/**/*.py",
"**/examples/**/*.ipynb",
"**/portability/api/**/*.py",
"**/portability/api/__init__.py",
]
target-version = "py310"
src = ["apache_beam"]
line-length = 120

@tvalentyn tvalentyn Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is not new change, is it? i noticed ruff didn't reformat lines longer than 80 characters recently.

@jrmccluskey jrmccluskey Jul 22, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a carry-over from the fact that our isort configuration has a line length of 120 to allow for long imports on single lines but our typical formatting directive is an 80 character line. At the moment ruff doesn't allow per-linter line length settings (see astral-sh/ruff#3206) but we also don't use ruff for formatting, only the linting. yapf is still the tool we use to catch formatting mismatches (for now) so this shouldn't alter what we allow in the code base


[tool.ruff.lint]
select = ["E9", "PL", "F821", "F822", "F823", "UP006"]
select = ["E9", "PL", "F821", "F822", "F823", "UP006", "I"]
ignore = [
# Ignored Pylint Checks
"PLC0415", # import-outside-toplevel
Expand Down Expand Up @@ -175,6 +127,55 @@ unfixable = []
# Allow unused variables when underscore-prefixed.
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"

[tool.ruff.lint.isort]
force-single-line = true
order-by-type = true
extra-standard-library = ["dataclasses"]
known-third-party = ["yaml"]
known-first-party = ["apache_beam"]

[tool.ruff.lint.per-file-ignores]
"**/apiclient.py" = ["I"]
"**/avroio_test.py" = ["I"]
"**/cloudpickle.py" = ["I"]
"**/datastore_wordcount.py" = ["I"]
"**/datastoreio_test.py" = ["I"]
"**/doctests_test.py" = ["I"]
"**/fast_coders_test.py" = ["I"]
"**/hadoopfilesystem.py" = ["I"]
"**/iobase_test.py" = ["I"]
"**/main_test.py" = ["I"]
"**/model.py" = ["I"]
"**/preprocess.py" = ["I"]
"**/process_tfma.py" = ["I"]
"**/render_test.py" = ["I"]
"**/slow_coders_test.py" = ["I"]
"**/taxi.py" = ["I"]
"**/tfdv_analyze_and_validate.py" = ["I"]
"**/yaml/main.py" = ["I"]
"**/yaml_testing_test.py" = ["I"]
"**/bigquery_v2_client.py" = ["I"]
"**/bigquery_v2_messages.py" = ["I"]
"**/proto2_coder_test_messages_pb2.py" = ["I"]
"**/rate_limit_pb2.py" = ["I"]
"**/boto3_client.py" = ["I"]
"**/coders_property_based_test.py" = ["I"]
"**/filebasedsource_test.py" = ["I"]
"**/bigtableio_it_test.py" = ["I"]
"**/bigtableio_test.py" = ["I"]
"**/mongodbio.py" = ["I"]
"**/textio_test.py" = ["I"]
"**/streaming_cache.py" = ["I"]
"**/pcoll_visualization.py" = ["I"]
"**/interactive_environment.py" = ["I"]
"**/pipeline_instrument_test.py" = ["I"]
"**/worker_handlers.py" = ["I"]
"**/feast_feature_store_it_test.py" = ["I"]
"**/vertex_ai_feature_store_it_test.py" = ["I"]
"**/vertex_ai_feature_store_test.py" = ["I"]
"setup.py" = ["I"]
"container/**" = ["I"]

[tool.pyrefly]
project-includes = ["apache_beam"]
python-version = "3.10.0"
Expand Down
37 changes: 1 addition & 36 deletions sdks/python/scripts/run_lint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,9 @@
# limitations under the License.
#

# This script will run ruff and isort on all module files.
# This script will run ruff on all module files.
#
# Use "ruff check apache_beam" to run ruff all files.
# Use "isort apache_beam" to run isort all files.
#
# The exit-code of the script indicates success or a failure.

Expand Down Expand Up @@ -82,40 +81,6 @@ echo -e "Linting modules:\n${MODULE}"
echo "Running ruff..."
ruff check ${MODULE} --extend-exclude="$FILES_TO_IGNORE"

echo "Running isort..."
# Skip files where isort is behaving weirdly
ISORT_EXCLUDED=(
"apiclient.py"
"avroio_test.py"
"cloudpickle.py"
"datastore_wordcount.py"
"datastoreio_test.py"
"doctests_test.py"
"fast_coders_test.py"
"hadoopfilesystem.py"
"iobase_test.py"
"main_test.py"
"model.py"
"preprocess.py"
"process_tfma.py"
"render_test.py"
"slow_coders_test.py"
"taxi.py"
"tfdv_analyze_and_validate.py"
"yaml/main.py"
"main_test.py"
"yaml_testing_test.py"
)
SKIP_PARAM=""
for file in "${ISORT_EXCLUDED[@]}"; do
SKIP_PARAM="$SKIP_PARAM --skip $file"
done
for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
SKIP_PARAM="$SKIP_PARAM --skip $(basename $file)"
done
isort ${MODULE} -p apache_beam --line-width 120 --check-only --order-by-type \
--combine-star --force-single-line-imports --diff --magic-placement ${SKIP_PARAM}

echo "Checking unittest.main..."
TESTS_MISSING_MAIN=$(
find ${MODULE} \
Expand Down
1 change: 0 additions & 1 deletion sdks/python/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,6 @@ def get_portability_package_data():
# BEAM-8840: Do NOT use tests_require or setup_requires.
extras_require={
'dev': [
'isort==7.0.0',
'pyrefly==0.54.0',
'ruff==0.15.7',
'yapf==0.43.0',
Expand Down
Loading