已合并
Static checking tools and rules are adopted directly from pytorch. #33435
Jingwei Huang创建于 4月9日
Static checking tools and rules are adopted directly from pytorch. #33435
已合并
共 61 个文件变更+12338-10
| @@ -0,0 +1,127 @@ | |||
| 1 | +--- | ||
| 2 | +AccessModifierOffset: -1 | ||
| 3 | +AlignAfterOpenBracket: AlwaysBreak | ||
| 4 | +AlignConsecutiveAssignments: false | ||
| 5 | +AlignConsecutiveDeclarations: false | ||
| 6 | +AlignEscapedNewlinesLeft: true | ||
| 7 | +AlignOperands: false | ||
| 8 | +AlignTrailingComments: false | ||
| 9 | +AllowAllParametersOfDeclarationOnNextLine: false | ||
| 10 | +AllowShortBlocksOnASingleLine: false | ||
| 11 | +AllowShortCaseLabelsOnASingleLine: false | ||
| 12 | +AllowShortFunctionsOnASingleLine: Empty | ||
| 13 | +AllowShortIfStatementsOnASingleLine: false | ||
| 14 | +AllowShortLoopsOnASingleLine: false | ||
| 15 | +AlwaysBreakAfterReturnType: None | ||
| 16 | +AlwaysBreakBeforeMultilineStrings: true | ||
| 17 | +AlwaysBreakTemplateDeclarations: true | ||
| 18 | +BinPackArguments: false | ||
| 19 | +BinPackParameters: false | ||
| 20 | +BraceWrapping: | ||
| 21 | + AfterClass: false | ||
| 22 | + AfterControlStatement: false | ||
| 23 | + AfterEnum: false | ||
| 24 | + AfterFunction: false | ||
| 25 | + AfterNamespace: false | ||
| 26 | + AfterObjCDeclaration: false | ||
| 27 | + AfterStruct: false | ||
| 28 | + AfterUnion: false | ||
| 29 | + BeforeCatch: false | ||
| 30 | + BeforeElse: false | ||
| 31 | + IndentBraces: false | ||
| 32 | +BreakBeforeBinaryOperators: None | ||
| 33 | +BreakBeforeBraces: Attach | ||
| 34 | +BreakBeforeTernaryOperators: true | ||
| 35 | +BreakConstructorInitializersBeforeComma: false | ||
| 36 | +BreakAfterJavaFieldAnnotations: false | ||
| 37 | +BreakStringLiterals: false | ||
| 38 | +ColumnLimit: 80 | ||
| 39 | +CommentPragmas: '^ IWYU pragma:' | ||
| 40 | +CompactNamespaces: false | ||
| 41 | +ConstructorInitializerAllOnOneLineOrOnePerLine: true | ||
| 42 | +ConstructorInitializerIndentWidth: 4 | ||
| 43 | +ContinuationIndentWidth: 4 | ||
| 44 | +Cpp11BracedListStyle: true | ||
| 45 | +DerivePointerAlignment: false | ||
| 46 | +DisableFormat: false | ||
| 47 | +ForEachMacros: | ||
| 48 | + - FOR_EACH_RANGE | ||
| 49 | + - FOR_EACH | ||
| 50 | +IncludeCategories: | ||
| 51 | + - Regex: '^<.*\.h(pp)?>' | ||
| 52 | + Priority: 1 | ||
| 53 | + - Regex: '^<.*' | ||
| 54 | + Priority: 2 | ||
| 55 | + - Regex: '.*' | ||
| 56 | + Priority: 3 | ||
| 57 | +IndentCaseLabels: true | ||
| 58 | +IndentWidth: 2 | ||
| 59 | +IndentWrappedFunctionNames: false | ||
| 60 | +KeepEmptyLinesAtTheStartOfBlocks: false | ||
| 61 | +MacroBlockBegin: '' | ||
| 62 | +MacroBlockEnd: '' | ||
| 63 | +Macros: | ||
| 64 | + - >- | ||
| 65 | + PyObject_HEAD_INIT(type)={ | ||
| 66 | + /* this is not exactly match with PyObject_HEAD_INIT in Python source code | ||
| 67 | + * but it is enough for clang-format */ | ||
| 68 | + { 0xFFFFFFFF }, | ||
| 69 | + (type) | ||
| 70 | + }, | ||
| 71 | + - >- | ||
| 72 | + PyVarObject_HEAD_INIT(type, size)={ | ||
| 73 | + { | ||
| 74 | + /* manually expand PyObject_HEAD_INIT(type) above | ||
| 75 | + * because clang-format do not support recursive expansion */ | ||
| 76 | + { 0xFFFFFFFF }, | ||
| 77 | + (type) | ||
| 78 | + }, | ||
| 79 | + (size) | ||
| 80 | + }, | ||
| 81 | +MaxEmptyLinesToKeep: 1 | ||
| 82 | +NamespaceIndentation: None | ||
| 83 | +PenaltyBreakBeforeFirstCallParameter: 1 | ||
| 84 | +PenaltyBreakComment: 300 | ||
| 85 | +PenaltyBreakFirstLessLess: 120 | ||
| 86 | +PenaltyBreakString: 1000 | ||
| 87 | +PenaltyExcessCharacter: 1000000 | ||
| 88 | +PenaltyReturnTypeOnItsOwnLine: 2000000 | ||
| 89 | +PointerAlignment: Left | ||
| 90 | +ReflowComments: true | ||
| 91 | +SortIncludes: true | ||
| 92 | +SpaceAfterCStyleCast: false | ||
| 93 | +SpaceBeforeAssignmentOperators: true | ||
| 94 | +SpaceBeforeParens: ControlStatements | ||
| 95 | +SpaceInEmptyParentheses: false | ||
| 96 | +SpacesBeforeTrailingComments: 1 | ||
| 97 | +SpacesInAngles: false | ||
| 98 | +SpacesInContainerLiterals: true | ||
| 99 | +SpacesInCStyleCastParentheses: false | ||
| 100 | +SpacesInParentheses: false | ||
| 101 | +SpacesInSquareBrackets: false | ||
| 102 | +Standard: c++17 | ||
| 103 | +StatementMacros: | ||
| 104 | + - C10_DEFINE_bool | ||
| 105 | + - C10_DEFINE_int | ||
| 106 | + - C10_DEFINE_int32 | ||
| 107 | + - C10_DEFINE_int64 | ||
| 108 | + - C10_DEFINE_string | ||
| 109 | + - C10_DEFINE_REGISTRY_WITHOUT_WARNING | ||
| 110 | + - C10_REGISTER_CREATOR | ||
| 111 | + - DEFINE_BINARY | ||
| 112 | + - PyObject_HEAD | ||
| 113 | + - PyObject_VAR_HEAD | ||
| 114 | + - PyException_HEAD | ||
| 115 | + - TORCH_DECLARE_bool | ||
| 116 | + | ||
| 117 | +TabWidth: 8 | ||
| 118 | +UseTab: Never | ||
| 119 | +--- | ||
| 120 | +Language: ObjC | ||
| 121 | +ColumnLimit: 120 | ||
| 122 | +AlignAfterOpenBracket: Align | ||
| 123 | +IndentWidth: 2 | ||
| 124 | +ObjCBlockIndentWidth: 2 | ||
| 125 | +ObjCSpaceAfterProperty: false | ||
| 126 | +ObjCSpaceBeforeProtocolList: false | ||
| 127 | +... | ||
| @@ -0,0 +1,81 @@ | |||
| 1 | +--- | ||
| 2 | +# NOTE there must be no spaces before the '-', so put the comma last. | ||
| 3 | +# The check bugprone-unchecked-optional-access is also turned on. | ||
| 4 | +# Note that it can cause clang-tidy to hang randomly. The tracking issue | ||
| 5 | +# can be found at https://github.com/llvm/llvm-project/issues/69369. | ||
| 6 | +# When that happens, we can disable it on the problematic code by NOLINT. | ||
| 7 | +InheritParentConfig: true | ||
| 8 | +Checks: ' | ||
| 9 | +bugprone-*, | ||
| 10 | +-bugprone-easily-swappable-parameters, | ||
| 11 | +-bugprone-forward-declaration-namespace, | ||
| 12 | +-bugprone-macro-parentheses, | ||
| 13 | +-bugprone-lambda-function-name, | ||
| 14 | +-bugprone-reserved-identifier, | ||
| 15 | +-bugprone-return-const-ref-from-parameter, | ||
| 16 | +-bugprone-swapped-arguments, | ||
| 17 | +clang-analyzer-core.*, | ||
| 18 | +clang-analyzer-cplusplus.*, | ||
| 19 | +clang-analyzer-nullability.*, | ||
| 20 | +clang-analyzer-deadcode.*, | ||
| 21 | +clang-diagnostic-missing-prototypes, | ||
| 22 | +cppcoreguidelines-*, | ||
| 23 | +-cppcoreguidelines-avoid-do-while, | ||
| 24 | +-cppcoreguidelines-avoid-magic-numbers, | ||
| 25 | +-cppcoreguidelines-avoid-non-const-global-variables, | ||
| 26 | +-cppcoreguidelines-interfaces-global-init, | ||
| 27 | +-cppcoreguidelines-macro-usage, | ||
| 28 | +-cppcoreguidelines-macro-to-enum, | ||
| 29 | +-cppcoreguidelines-owning-memory, | ||
| 30 | +-cppcoreguidelines-pro-bounds-array-to-pointer-decay, | ||
| 31 | +-cppcoreguidelines-pro-bounds-constant-array-index, | ||
| 32 | +-cppcoreguidelines-pro-bounds-pointer-arithmetic, | ||
| 33 | +-cppcoreguidelines-pro-type-cstyle-cast, | ||
| 34 | +-cppcoreguidelines-pro-type-reinterpret-cast, | ||
| 35 | +-cppcoreguidelines-pro-type-static-cast-downcast, | ||
| 36 | +-cppcoreguidelines-pro-type-union-access, | ||
| 37 | +-cppcoreguidelines-pro-type-vararg, | ||
| 38 | +-cppcoreguidelines-non-private-member-variables-in-classes, | ||
| 39 | +-facebook-hte-RelativeInclude, | ||
| 40 | +hicpp-exception-baseclass, | ||
| 41 | +hicpp-avoid-goto, | ||
| 42 | +misc-*, | ||
| 43 | +-misc-confusable-identifiers, | ||
| 44 | +-misc-const-correctness, | ||
| 45 | +-misc-include-cleaner, | ||
| 46 | +-misc-use-anonymous-namespace, | ||
| 47 | +-misc-unused-parameters, | ||
| 48 | +-misc-no-recursion, | ||
| 49 | +-misc-non-private-member-variables-in-classes, | ||
| 50 | +-misc-unused-using-decls, | ||
| 51 | +modernize-*, | ||
| 52 | +-modernize-macro-to-enum, | ||
| 53 | +-modernize-return-braced-init-list, | ||
| 54 | +-modernize-use-auto, | ||
| 55 | +-modernize-use-using, | ||
| 56 | +-modernize-use-trailing-return-type, | ||
| 57 | +-modernize-use-nodiscard, | ||
| 58 | +performance-*, | ||
| 59 | +-performance-enum-size, | ||
| 60 | +readability-container-size-empty, | ||
| 61 | +readability-delete-null-pointer, | ||
| 62 | +readability-duplicate-include, | ||
| 63 | +readability-named-parameter, | ||
| 64 | +readability-misplaced-array-index, | ||
| 65 | +readability-redundant*, | ||
| 66 | +readability-simplify-subscript-expr, | ||
| 67 | +readability-static-definition-in-anonymous-namespace | ||
| 68 | +readability-string-compare, | ||
| 69 | +-readability-redundant-access-specifiers, | ||
| 70 | +-readability-redundant-control-flow, | ||
| 71 | +-readability-redundant-inline-specifier, | ||
| 72 | +' | ||
| 73 | +HeaderFilterRegex: '^(aten/|c10/|torch/).*$' | ||
| 74 | +WarningsAsErrors: '*' | ||
| 75 | +LineFilter: | ||
| 76 | + - name: '/usr/include/.*' | ||
| 77 | +CheckOptions: | ||
| 78 | + cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor: true | ||
| 79 | + cppcoreguidelines-special-member-functions.AllowImplicitlyDeletedCopyOrMove: true | ||
| 80 | + misc-header-include-cycle.IgnoredFilesList: 'format.h;ivalue.h;custom_class.h;Dict.h;List.h;IListRef.h' | ||
| 81 | +... | ||
| @@ -0,0 +1 @@ | |||
| 1 | +filter=-convention/filename,-linelength,-package/consistency,-readability/logic,-readability/mixedcase,-readability/wonkycase,-syntax,-whitespace/eol,+whitespace/extra,-whitespace/indent,-whitespace/mismatch,-whitespace/newline,-whitespace/tabs | ||
| @@ -0,0 +1,36 @@ | |||
| 1 | +root = true | ||
| 2 | + | ||
| 3 | +[*] | ||
| 4 | +charset = utf-8 | ||
| 5 | +end_of_line = lf | ||
| 6 | +insert_final_newline = true | ||
| 7 | + | ||
| 8 | +# Python | ||
| 9 | +[*.{py,pyi,py.in,pyi.in}] | ||
| 10 | +indent_style = space | ||
| 11 | +indent_size = 4 | ||
| 12 | + | ||
| 13 | +# C/C++/CUDA | ||
| 14 | +[*.{cpp,hpp,cxx,cc,c,h,cu,cuh}] | ||
| 15 | +indent_style = space | ||
| 16 | +indent_size = 2 | ||
| 17 | + | ||
| 18 | +# Objective-C | ||
| 19 | +[*.{mm,m,M}] | ||
| 20 | +indent_style = space | ||
| 21 | +indent_size = 2 | ||
| 22 | + | ||
| 23 | +# Clang tools | ||
| 24 | +[.clang-{format,tidy}] | ||
| 25 | +indent_style = space | ||
| 26 | +indent_size = 2 | ||
| 27 | + | ||
| 28 | +# Make | ||
| 29 | +[Makefile] | ||
| 30 | +indent_style = tab | ||
| 31 | + | ||
| 32 | +# Batch file | ||
| 33 | +[*.bat] | ||
| 34 | +indent_style = space | ||
| 35 | +indent_size = 2 | ||
| 36 | +end_of_line = crlf | ||
| @@ -0,0 +1,53 @@ | |||
| 1 | +[flake8] | ||
| 2 | +# NOTE: **Mirror any changes** to this file the [tool.ruff] config in pyproject.toml | ||
| 3 | +# before we can fully move to use ruff | ||
| 4 | +enable-extensions = G | ||
| 5 | +select = B,C,E,F,G,P,SIM1,SIM911,T4,W,B9 | ||
| 6 | +max-line-length = 120 | ||
| 7 | +# C408 ignored because we like the dict keyword argument syntax | ||
| 8 | +# E501 is not flexible enough, we're using B950 instead | ||
| 9 | +ignore = | ||
| 10 | + E203,E305,E402,E501,E704,E741,F405,F841,F999,W503,W504,C408,E302,W291,E303,F824, | ||
| 11 | + # shebang has extra meaning in fbcode lints, so I think it's not worth trying | ||
| 12 | + # to line this up with executable bit | ||
| 13 | + EXE001, | ||
| 14 | + # these ignores are from flake8-bugbear; please fix! | ||
| 15 | + B007,B008,B017,B019,B023,B028,B903,B905,B906,B907,B908,B910 | ||
| 16 | + # these ignores are from flake8-simplify. please fix or ignore with commented reason | ||
| 17 | + SIM105,SIM108,SIM110,SIM111,SIM113,SIM114,SIM115,SIM116,SIM117,SIM118,SIM119,SIM12, | ||
| 18 | + # SIM104 is already covered by pyupgrade ruff | ||
| 19 | + SIM104, | ||
| 20 | + # flake8-simplify code styles | ||
| 21 | + SIM102,SIM103,SIM106,SIM112 | ||
| 22 | +per-file-ignores = | ||
| 23 | + __init__.py: F401 | ||
| 24 | + test/**: F821 | ||
| 25 | + test/**/__init__.py: F401,F821 | ||
| 26 | + torch/utils/cpp_extension.py: B950 | ||
| 27 | + torchgen/api/types/__init__.py: F401,F403 | ||
| 28 | + torchgen/executorch/api/types/__init__.py: F401,F403 | ||
| 29 | + test/dynamo/test_higher_order_ops.py: B950 | ||
| 30 | + test/dynamo/test_error_messages.py: B950 | ||
| 31 | + torch/testing/_internal/dynamo_test_failures.py: B950 | ||
| 32 | + torch/__init__.py: F401 | ||
| 33 | + torch/_prims/__init__.py: F401 | ||
| 34 | +optional-ascii-coding = True | ||
| 35 | +exclude = | ||
| 36 | + ./.git, | ||
| 37 | + ./build_test_custom_build, | ||
| 38 | + ./build, | ||
| 39 | + ./caffe2, | ||
| 40 | + ./docs/caffe2, | ||
| 41 | + ./docs/cpp/src, | ||
| 42 | + ./docs/src, | ||
| 43 | + ./functorch/docs, | ||
| 44 | + ./functorch/examples, | ||
| 45 | + ./functorch/docs/source/tutorials, | ||
| 46 | + ./torch/testing/_internal/py312_intrinsics.py | ||
| 47 | + ./scripts, | ||
| 48 | + ./test/generated_type_hints_smoketest.py, | ||
| 49 | + ./third_party, | ||
| 50 | + ./torch/include, | ||
| 51 | + ./torch/lib, | ||
| 52 | + ./venv, | ||
| 53 | + *.pyi | ||
| @@ -0,0 +1,1806 @@ | |||
| 1 | +[[linter]] | ||
| 2 | +code = 'FLAKE8' | ||
| 3 | +include_patterns = ['**/*.py'] | ||
| 4 | +exclude_patterns = [ | ||
| 5 | + '.git/**', | ||
| 6 | + 'build_test_custom_build/**', | ||
| 7 | + 'build/**', | ||
| 8 | + 'caffe2/**', | ||
| 9 | + 'docs/caffe2/**', | ||
| 10 | + 'docs/cpp/src/**', | ||
| 11 | + 'docs/src/**', | ||
| 12 | + 'fb/**', | ||
| 13 | + '**/fb/**', | ||
| 14 | + 'functorch/docs/**', | ||
| 15 | + 'functorch/examples/**', | ||
| 16 | + 'functorch/docs/source/tutorials/**', | ||
| 17 | + 'torch/_inductor/fx_passes/serialized_patterns/**', | ||
| 18 | + 'torch/_inductor/autoheuristic/artifacts/**', | ||
| 19 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**', | ||
| 20 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py', | ||
| 21 | + 'scripts/**', | ||
| 22 | + 'test/generated_type_hints_smoketest.py', | ||
| 23 | + 'test/test_torchfuzz_repros.py', | ||
| 24 | + # CPython tests | ||
| 25 | + 'test/dynamo/cpython/**', | ||
| 26 | + # Tests from the NumPy test suite | ||
| 27 | + 'test/torch_np/numpy_test/**/*.py', | ||
| 28 | + 'third_party/**', | ||
| 29 | + 'torch/include/**', | ||
| 30 | + 'torch/lib/**', | ||
| 31 | + 'venv/**', | ||
| 32 | + '**/*.pyi', | ||
| 33 | + "tools/experimental/torchfuzz/**", | ||
| 34 | + 'tools/test/test_selective_build.py', | ||
| 35 | +] | ||
| 36 | +command = [ | ||
| 37 | + 'uv', | ||
| 38 | + 'run', | ||
| 39 | + '--script', | ||
| 40 | + 'tools/linter/adapters/flake8_linter.py', | ||
| 41 | + '--', | ||
| 42 | + '@{{PATHSFILE}}' | ||
| 43 | +] | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +[[linter]] | ||
| 47 | +code = 'CLANGFORMAT' | ||
| 48 | +include_patterns = [ | ||
| 49 | + 'aten/src/ATen/*.h', | ||
| 50 | + 'aten/src/ATen/cpu/vec/**/*.h', | ||
| 51 | + 'aten/src/ATen/accelerator/**/*.h', | ||
| 52 | + 'aten/src/ATen/accelerator/**/*.cpp', | ||
| 53 | + 'aten/src/ATen/mps/**/*.mm', | ||
| 54 | + 'aten/src/ATen/mps/**/*.h', | ||
| 55 | + 'aten/src/ATen/xpu/**/*.h', | ||
| 56 | + 'aten/src/ATen/xpu/**/*.cpp', | ||
| 57 | + 'aten/src/ATen/core/boxing/**/*.h', | ||
| 58 | + 'aten/src/ATen/core/dispatch/**/*.h', | ||
| 59 | + 'aten/src/ATen/core/Formatting.cpp', | ||
| 60 | + 'aten/src/ATen/native/mps/**/*.metal', | ||
| 61 | + 'aten/src/ATen/native/mps/**/*.mm', | ||
| 62 | + 'aten/src/ATen/native/mps/**/*.h', | ||
| 63 | + 'aten/src/ATen/native/vulkan/**/*.h', | ||
| 64 | + 'aten/src/ATen/native/vulkan/**/*.cpp', | ||
| 65 | + 'aten/src/ATen/native/cuda/MultiTensorApply.cuh', | ||
| 66 | + 'aten/src/ATen/native/**/Foreach*.*', | ||
| 67 | + 'aten/src/ATen/native/cuda/fused*.*', | ||
| 68 | + 'aten/src/ATen/native/cuda/Fused*.cu', | ||
| 69 | + 'aten/src/ATen/native/cudnn/*.h', | ||
| 70 | + 'aten/src/ATen/native/cudnn/*.cpp', | ||
| 71 | + 'aten/src/ATen/native/mkldnn/xpu/**/*.h', | ||
| 72 | + 'aten/src/ATen/native/mkldnn/xpu/**/*.cpp', | ||
| 73 | + 'aten/src/ATen/native/Tensor*.h', | ||
| 74 | + 'aten/src/ATen/native/Tensor*.cpp', | ||
| 75 | + 'c10/**/*.h', | ||
| 76 | + 'c10/**/*.cpp', | ||
| 77 | + 'torch/csrc/**/*.h', | ||
| 78 | + 'torch/csrc/**/*.hpp', | ||
| 79 | + 'torch/csrc/**/*.cpp', | ||
| 80 | + 'torch/nativert/**/*.h', | ||
| 81 | + 'torch/nativert/**/*.cpp', | ||
| 82 | + 'torch/headeronly/**/*.h', | ||
| 83 | + 'test/cpp/**/*.h', | ||
| 84 | + 'test/cpp/**/*.cpp', | ||
| 85 | +] | ||
| 86 | +exclude_patterns = [ | ||
| 87 | + 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', | ||
| 88 | + 'aten/src/ATen/native/mps/kernels/Quantized.metal', | ||
| 89 | + 'c10/util/strong_type.h', | ||
| 90 | + '**/fb/**', | ||
| 91 | + 'torch/csrc/inductor/aoti_torch/generated/**', | ||
| 92 | + 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', | ||
| 93 | + 'torch/csrc/utils/pythoncapi_compat.h', | ||
| 94 | + 'aten/src/ATen/dlpack.h', | ||
| 95 | +] | ||
| 96 | +init_command = [ | ||
| 97 | + 'python3', | ||
| 98 | + 'tools/linter/adapters/s3_init.py', | ||
| 99 | + '--config-json=tools/linter/adapters/s3_init_config.json', | ||
| 100 | + '--linter=clang-format', | ||
| 101 | + '--dry-run={{DRYRUN}}', | ||
| 102 | + '--output-dir=.lintbin', | ||
| 103 | + '--output-name=clang-format', | ||
| 104 | +] | ||
| 105 | +command = [ | ||
| 106 | + 'python3', | ||
| 107 | + 'tools/linter/adapters/clangformat_linter.py', | ||
| 108 | + '--binary=.lintbin/clang-format', | ||
| 109 | + '--', | ||
| 110 | + '@{{PATHSFILE}}' | ||
| 111 | +] | ||
| 112 | +is_formatter = true | ||
| 113 | + | ||
| 114 | + | ||
| 115 | +[[linter]] | ||
| 116 | +code = 'PYREFLY' | ||
| 117 | +include_patterns = [ | ||
| 118 | + 'torch/**/*.py', | ||
| 119 | + 'torch/**/*.pyi', | ||
| 120 | + 'torchgen/**/*.py', | ||
| 121 | + 'torchgen/**/*.pyi', | ||
| 122 | + 'functorch/**/*.py', | ||
| 123 | + 'functorch/**/*.pyi', | ||
| 124 | + 'tools/**/*.py', | ||
| 125 | + 'tools/**/*.pyi', | ||
| 126 | +] | ||
| 127 | +exclude_patterns = [ | ||
| 128 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**', | ||
| 129 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py', | ||
| 130 | +] | ||
| 131 | +command = [ | ||
| 132 | + 'uv', | ||
| 133 | + 'run', | ||
| 134 | + '--script', | ||
| 135 | + 'tools/linter/adapters/pyrefly_linter.py', | ||
| 136 | + '--config=pyrefly.toml', | ||
| 137 | +] | ||
| 138 | + | ||
| 139 | +[[linter]] | ||
| 140 | +code = 'CLANGTIDY' | ||
| 141 | +include_patterns = [ | ||
| 142 | + # Enable coverage of headers in aten/src/ATen | ||
| 143 | + # and excluding most sub-directories for now. | ||
| 144 | + 'aten/src/ATen/*.h', | ||
| 145 | + 'aten/src/ATen/*.cpp', | ||
| 146 | + 'aten/src/ATen/cuda/*.cpp', | ||
| 147 | + 'aten/src/ATen/cpu/*.h', | ||
| 148 | + 'aten/src/ATen/cpu/*.cpp', | ||
| 149 | + 'aten/src/ATen/accelerator/**/*.h', | ||
| 150 | + 'aten/src/ATen/accelerator/**/*.cpp', | ||
| 151 | + 'aten/src/ATen/core/*.h', | ||
| 152 | + 'aten/src/ATen/core/*.cpp', | ||
| 153 | + 'aten/src/ATen/cudnn/*.h', | ||
| 154 | + 'aten/src/ATen/cudnn/*.cpp', | ||
| 155 | + 'aten/src/ATen/native/mkldnn/xpu/**/*.h', | ||
| 156 | + 'aten/src/ATen/native/mkldnn/xpu/**/*.cpp', | ||
| 157 | + 'aten/src/ATen/detail/*', | ||
| 158 | + 'aten/src/ATen/functorch/*.h', | ||
| 159 | + 'aten/src/ATen/functorch/*.cpp', | ||
| 160 | + 'aten/src/ATen/native/nested/cuda/*.cpp', | ||
| 161 | + 'aten/src/ATen/native/nested/cuda/*.h', | ||
| 162 | + 'aten/src/ATen/native/nested/*.cpp', | ||
| 163 | + 'aten/src/ATen/native/nested/*.h', | ||
| 164 | + 'aten/src/ATen/xpu/**/*.h', | ||
| 165 | + 'aten/src/ATen/xpu/**/*.cpp', | ||
| 166 | + 'c10/**/*.cpp', | ||
| 167 | + 'c10/**/*.h', | ||
| 168 | + 'torch/*.h', | ||
| 169 | + 'torch/_inductor/codegen/aoti_runtime/*.h', | ||
| 170 | + 'torch/_inductor/codegen/aoti_runtime/*.cpp', | ||
| 171 | + 'torch/csrc/*.h', | ||
| 172 | + 'torch/csrc/*.cpp', | ||
| 173 | + 'torch/csrc/**/*.h', | ||
| 174 | + 'torch/csrc/**/*.cpp', | ||
| 175 | + 'torch/csrc/jit/serialization/*.h', | ||
| 176 | + 'torch/csrc/jit/serialization/*.cpp', | ||
| 177 | + 'torch/nativert/*.h', | ||
| 178 | + 'torch/nativert/*.cpp', | ||
| 179 | + 'torch/nativert/**/*.h', | ||
| 180 | + 'torch/nativert/**/*.cpp', | ||
| 181 | + 'torch/headeronly/**/*.h', | ||
| 182 | +] | ||
| 183 | +exclude_patterns = [ | ||
| 184 | + # The negative filters below are to exclude files that include onnx_pb.h or | ||
| 185 | + # caffe2_pb.h, otherwise we'd have to build protos as part of this CI job. | ||
| 186 | + # CUDA files are also excluded. | ||
| 187 | + '**/fb/**', | ||
| 188 | + '**/generated/**', | ||
| 189 | + '**/*pb.h', | ||
| 190 | + '**/*inl.h', | ||
| 191 | + 'aten/src/ATen/cpu/FlushDenormal.cpp', | ||
| 192 | + 'aten/src/ATen/cpu/vml.h', | ||
| 193 | + 'aten/src/ATen/CPUFixedAllocator.h', | ||
| 194 | + 'aten/src/ATen/Parallel*.h', | ||
| 195 | + 'c10/xpu/**/*.h', | ||
| 196 | + 'c10/xpu/**/*.cpp', | ||
| 197 | + 'c10/benchmark/intrusive_ptr_benchmark.cpp', | ||
| 198 | + 'c10/cuda/CUDAAlgorithm.h', | ||
| 199 | + 'c10/util/complex_math.h', | ||
| 200 | + 'c10/util/complex_utils.h', | ||
| 201 | + 'c10/util/flat_hash_map.h', | ||
| 202 | + 'c10/util/logging*.h', | ||
| 203 | + 'c10/metal/*.h', | ||
| 204 | + 'c10/util/hash.h', | ||
| 205 | + 'c10/util/strong_type.h', | ||
| 206 | + 'c10/util/SmallVector.h', | ||
| 207 | + 'c10/util/win32-headers.h', | ||
| 208 | + 'c10/test/**/*.h', | ||
| 209 | + 'c10/test/**/*.cpp', | ||
| 210 | + 'third_party/**/*', | ||
| 211 | + 'torch/csrc/autograd/generated/**', | ||
| 212 | + 'torch/csrc/distributed/**/*.cu', | ||
| 213 | + 'torch/csrc/distributed/c10d/WinSockUtils.hpp', | ||
| 214 | + 'torch/csrc/distributed/c10d/quantization/quantization_gpu.h', | ||
| 215 | + 'torch/csrc/dynamo/eval_frame.h', | ||
| 216 | + 'torch/csrc/inductor/aoti_torch/c/shim.h', | ||
| 217 | + 'torch/csrc/jit/**/*', | ||
| 218 | + 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', | ||
| 219 | + 'torch/csrc/utils/generated_serialization_types.h', | ||
| 220 | + 'torch/csrc/utils/pythoncapi_compat.h', | ||
| 221 | + 'torch/csrc/inductor/aoti_runtime/sycl_runtime_wrappers.h', | ||
| 222 | +] | ||
| 223 | +init_command = [ | ||
| 224 | + 'python3', | ||
| 225 | + 'tools/linter/adapters/s3_init.py', | ||
| 226 | + '--config-json=tools/linter/adapters/s3_init_config.json', | ||
| 227 | + '--linter=clang-tidy', | ||
| 228 | + '--dry-run={{DRYRUN}}', | ||
| 229 | + '--output-dir=.lintbin', | ||
| 230 | + '--output-name=clang-tidy', | ||
| 231 | +] | ||
| 232 | +command = [ | ||
| 233 | + 'python3', | ||
| 234 | + 'tools/linter/adapters/clangtidy_linter.py', | ||
| 235 | + '--binary=.lintbin/clang-tidy', | ||
| 236 | + '--build_dir=./build', | ||
| 237 | + '--', | ||
| 238 | + '@{{PATHSFILE}}' | ||
| 239 | +] | ||
| 240 | + | ||
| 241 | +[[linter]] | ||
| 242 | +code = 'CLANGTIDY_EXECUTORCH_COMPATIBILITY' | ||
| 243 | +include_patterns = [ | ||
| 244 | + # c10 headers that must be C++17 compatible | ||
| 245 | + 'c10/macros/Export.h', | ||
| 246 | + 'c10/macros/Macros.h', | ||
| 247 | + 'c10/macros/cmake_macros.h', | ||
| 248 | + 'c10/util/BFloat16.h', | ||
| 249 | + 'c10/util/BFloat16-inl.h', | ||
| 250 | + 'c10/util/BFloat16-math.h', | ||
| 251 | + 'c10/util/Half.h', | ||
| 252 | + 'c10/util/Half-inl.h', | ||
| 253 | + 'c10/util/TypeSafeSignMath.h', | ||
| 254 | + 'c10/util/bit_cast.h', | ||
| 255 | + 'c10/util/complex.h', | ||
| 256 | + 'c10/util/floating_point_utils.h', | ||
| 257 | + 'c10/util/irange.h', | ||
| 258 | + 'c10/util/llvmMathExtras.h', | ||
| 259 | + 'c10/util/overflows.h', | ||
| 260 | + 'c10/util/safe_numerics.h', | ||
| 261 | + # torch/headeronly headers that must be C++17 compatible | ||
| 262 | + 'torch/headeronly/macros/cmake_macros.h', | ||
| 263 | + 'torch/headeronly/macros/Export.h', | ||
| 264 | + 'torch/headeronly/macros/Macros.h', | ||
| 265 | + 'torch/headeronly/util/BFloat16.h', | ||
| 266 | + 'torch/headeronly/util/Half.h', | ||
| 267 | + 'torch/headeronly/util/TypeSafeSignMath.h', | ||
| 268 | + 'torch/headeronly/util/bit_cast.h', | ||
| 269 | + 'torch/headeronly/util/complex.h', | ||
| 270 | + 'torch/headeronly/util/floating_point_utils.h', | ||
| 271 | +] | ||
| 272 | +exclude_patterns = [ | ||
| 273 | + '**/fb/**', | ||
| 274 | +] | ||
| 275 | +init_command = [ | ||
| 276 | + 'python3', | ||
| 277 | + 'tools/linter/adapters/s3_init.py', | ||
| 278 | + '--config-json=tools/linter/adapters/s3_init_config.json', | ||
| 279 | + '--linter=clang-tidy', | ||
| 280 | + '--dry-run={{DRYRUN}}', | ||
| 281 | + '--output-dir=.lintbin', | ||
| 282 | + '--output-name=clang-tidy', | ||
| 283 | +] | ||
| 284 | +command = [ | ||
| 285 | + 'python3', | ||
| 286 | + 'tools/linter/adapters/clangtidy_linter.py', | ||
| 287 | + '--binary=.lintbin/clang-tidy', | ||
| 288 | + '--build_dir=./build', | ||
| 289 | + '--std=c++17', | ||
| 290 | + '--', | ||
| 291 | + '@{{PATHSFILE}}' | ||
| 292 | +] | ||
| 293 | + | ||
| 294 | +[[linter]] | ||
| 295 | +code = 'TYPEIGNORE' | ||
| 296 | +include_patterns = ['**/*.py', '**/*.pyi'] | ||
| 297 | +exclude_patterns = [ | ||
| 298 | + 'fb/**', | ||
| 299 | + '**/fb/**', | ||
| 300 | + 'test/test_jit.py', | ||
| 301 | +] | ||
| 302 | +command = [ | ||
| 303 | + 'python3', | ||
| 304 | + 'tools/linter/adapters/grep_linter.py', | ||
| 305 | + '--pattern=# type:\s*ignore([^\[]|$)', | ||
| 306 | + '--linter-name=TYPEIGNORE', | ||
| 307 | + '--error-name=unqualified type: ignore', | ||
| 308 | + """--error-description=\ | ||
| 309 | + This line has an unqualified `type: ignore`; \ | ||
| 310 | + please convert it to `type: ignore[xxxx]`\ | ||
| 311 | + """, | ||
| 312 | + '--', | ||
| 313 | + '@{{PATHSFILE}}' | ||
| 314 | +] | ||
| 315 | + | ||
| 316 | +[[linter]] | ||
| 317 | +code = 'TYPENOSKIP' | ||
| 318 | +include_patterns = ['mypy.ini'] | ||
| 319 | +command = [ | ||
| 320 | + 'python3', | ||
| 321 | + 'tools/linter/adapters/grep_linter.py', | ||
| 322 | + '--pattern=follow_imports\s*=\s*skip', | ||
| 323 | + '--linter-name=TYPENOSKIP', | ||
| 324 | + '--error-name=use of follow_imports = skip', | ||
| 325 | + """--error-description=\ | ||
| 326 | + follow_imports = skip is forbidden from mypy.ini configuration as it \ | ||
| 327 | + is extremely easy to accidentally turn off type checking unintentionally. If \ | ||
| 328 | + you need to suppress type errors, use a top level # mypy: ignore-errors. \ | ||
| 329 | + Do not rely on automatic Any substitution; instead, manually # type: ignore \ | ||
| 330 | + at use sites or define a pyi type stub with more relaxed types. \ | ||
| 331 | + """, | ||
| 332 | + '--', | ||
| 333 | + '@{{PATHSFILE}}' | ||
| 334 | +] | ||
| 335 | + | ||
| 336 | +[[linter]] | ||
| 337 | +code = 'NOQA' | ||
| 338 | +include_patterns = ['**/*.py', '**/*.pyi'] | ||
| 339 | +exclude_patterns = [ | ||
| 340 | + 'caffe2/**', | ||
| 341 | + 'fb/**', | ||
| 342 | + '**/fb/**' | ||
| 343 | + ] | ||
| 344 | +command = [ | ||
| 345 | + 'python3', | ||
| 346 | + 'tools/linter/adapters/grep_linter.py', | ||
| 347 | + '--pattern=# noqa([^:]|$)', | ||
| 348 | + '--linter-name=NOQA', | ||
| 349 | + '--error-name=unqualified noqa', | ||
| 350 | + """--error-description=\ | ||
| 351 | + This line has an unqualified `noqa`; \ | ||
| 352 | + please convert it to `noqa: XXXX`\ | ||
| 353 | + """, | ||
| 354 | + '--', | ||
| 355 | + '@{{PATHSFILE}}' | ||
| 356 | +] | ||
| 357 | + | ||
| 358 | +[[linter]] | ||
| 359 | +code = 'NATIVEFUNCTIONS' | ||
| 360 | +include_patterns=['aten/src/ATen/native/native_functions.yaml'] | ||
| 361 | +command = [ | ||
| 362 | + 'uv', | ||
| 363 | + 'run', | ||
| 364 | + '--script', | ||
| 365 | + 'tools/linter/adapters/nativefunctions_linter.py', | ||
| 366 | + '--native-functions-yml=aten/src/ATen/native/native_functions.yaml', | ||
| 367 | +] | ||
| 368 | +is_formatter = true | ||
| 369 | + | ||
| 370 | +[[linter]] | ||
| 371 | +code = 'GHA' | ||
| 372 | +include_patterns=['.github/workflows/**/*.yml'] | ||
| 373 | +command = [ | ||
| 374 | + 'uv', | ||
| 375 | + 'run', | ||
| 376 | + '--script', | ||
| 377 | + 'tools/linter/adapters/gha_linter.py', | ||
| 378 | + '--', | ||
| 379 | + '@{{PATHSFILE}}' | ||
| 380 | +] | ||
| 381 | + | ||
| 382 | +[[linter]] | ||
| 383 | +code = 'NEWLINE' | ||
| 384 | +include_patterns=['**'] | ||
| 385 | +exclude_patterns=[ | ||
| 386 | + '**/contrib/**', | ||
| 387 | + 'third_party/**', | ||
| 388 | + '**/*.bat', | ||
| 389 | + '**/*.expect', | ||
| 390 | + '**/*.ipynb', | ||
| 391 | + '**/*.ps1', | ||
| 392 | + '**/*.ptl', | ||
| 393 | + 'fb/**', | ||
| 394 | + '**/fb/**', | ||
| 395 | + 'tools/clang_format_hash/**', | ||
| 396 | + 'test/cpp/jit/upgrader_models/*.ptl', | ||
| 397 | + 'test/cpp/jit/upgrader_models/*.ptl.ff', | ||
| 398 | + 'test/dynamo/cpython/**', | ||
| 399 | + '**/*.png', | ||
| 400 | + '**/*.gz', | ||
| 401 | + '**/*.patch', | ||
| 402 | +] | ||
| 403 | +command = [ | ||
| 404 | + 'python3', | ||
| 405 | + 'tools/linter/adapters/newlines_linter.py', | ||
| 406 | + '--', | ||
| 407 | + '@{{PATHSFILE}}', | ||
| 408 | +] | ||
| 409 | +is_formatter = true | ||
| 410 | + | ||
| 411 | +[[linter]] | ||
| 412 | +code = 'SPACES' | ||
| 413 | +include_patterns = ['**'] | ||
| 414 | +exclude_patterns = [ | ||
| 415 | + '**/contrib/**', | ||
| 416 | + '**/*.diff', | ||
| 417 | + '**/*.patch', | ||
| 418 | + 'third_party/**', | ||
| 419 | + 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', | ||
| 420 | + 'fb/**', | ||
| 421 | + '**/fb/**', | ||
| 422 | + 'test/cpp/jit/upgrader_models/*.ptl', | ||
| 423 | + 'test/cpp/jit/upgrader_models/*.ptl.ff', | ||
| 424 | +] | ||
| 425 | +command = [ | ||
| 426 | + 'python3', | ||
| 427 | + 'tools/linter/adapters/grep_linter.py', | ||
| 428 | + '--pattern=[[:blank:]]$', | ||
| 429 | + '--linter-name=SPACES', | ||
| 430 | + '--error-name=trailing spaces', | ||
| 431 | + '--replace-pattern=s/[[:blank:]]+$//', | ||
| 432 | + """--error-description=\ | ||
| 433 | + This line has trailing spaces; please remove them.\ | ||
| 434 | + """, | ||
| 435 | + '--', | ||
| 436 | + '@{{PATHSFILE}}' | ||
| 437 | +] | ||
| 438 | + | ||
| 439 | +[[linter]] | ||
| 440 | +code = 'TABS' | ||
| 441 | +include_patterns = ['**'] | ||
| 442 | +exclude_patterns = [ | ||
| 443 | + '**/*.svg', | ||
| 444 | + '**/*Makefile', | ||
| 445 | + '**/contrib/**', | ||
| 446 | + 'third_party/**', | ||
| 447 | + '**/.gitattributes', | ||
| 448 | + '**/.gitmodules', | ||
| 449 | + 'fb/**', | ||
| 450 | + '**/fb/**', | ||
| 451 | + 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', | ||
| 452 | + 'test/cpp/jit/upgrader_models/*.ptl', | ||
| 453 | + 'test/cpp/jit/upgrader_models/*.ptl.ff', | ||
| 454 | + '.ci/docker/common/install_rocm_drm.sh', | ||
| 455 | + '.lintrunner.toml', | ||
| 456 | + '**/*.patch', | ||
| 457 | +] | ||
| 458 | +command = [ | ||
| 459 | + 'python3', | ||
| 460 | + 'tools/linter/adapters/grep_linter.py', | ||
| 461 | + # @lint-ignore TXT2 | ||
| 462 | + '--pattern= ', | ||
| 463 | + '--linter-name=TABS', | ||
| 464 | + '--error-name=saw some tabs', | ||
| 465 | + '--replace-pattern=s/\t/ /', | ||
| 466 | + """--error-description=\ | ||
| 467 | + This line has tabs; please replace them with spaces.\ | ||
| 468 | + """, | ||
| 469 | + '--', | ||
| 470 | + '@{{PATHSFILE}}' | ||
| 471 | +] | ||
| 472 | + | ||
| 473 | +[[linter]] | ||
| 474 | +code = 'C10_UNUSED' | ||
| 475 | +include_patterns = [ | ||
| 476 | + '**/*.cpp', | ||
| 477 | + '**/*.h', | ||
| 478 | +] | ||
| 479 | +exclude_patterns = [ | ||
| 480 | + 'torch/headeronly/macros/Macros.h', | ||
| 481 | +] | ||
| 482 | +command = [ | ||
| 483 | + 'python3', | ||
| 484 | + 'tools/linter/adapters/grep_linter.py', | ||
| 485 | + '--pattern=C10_UNUSED', | ||
| 486 | + '--linter-name=C10_UNUSED', | ||
| 487 | + '--error-name=deprecated C10_UNUSED macro', | ||
| 488 | + '--replace-pattern=s/C10_UNUSED/[[maybe_unused]]/', | ||
| 489 | + """--error-description=\ | ||
| 490 | + Deprecated macro, use [[maybe_unused]] directly\ | ||
| 491 | + """, | ||
| 492 | + '--', | ||
| 493 | + '@{{PATHSFILE}}' | ||
| 494 | +] | ||
| 495 | + | ||
| 496 | +[[linter]] | ||
| 497 | +code = 'C10_NODISCARD' | ||
| 498 | +include_patterns = [ | ||
| 499 | + '**/*.cpp', | ||
| 500 | + '**/*.h', | ||
| 501 | +] | ||
| 502 | +exclude_patterns = [ | ||
| 503 | + 'torch/headeronly/macros/Macros.h', | ||
| 504 | +] | ||
| 505 | +command = [ | ||
| 506 | + 'python3', | ||
| 507 | + 'tools/linter/adapters/grep_linter.py', | ||
| 508 | + '--pattern=C10_NODISCARD', | ||
| 509 | + '--linter-name=C10_NODISCARD', | ||
| 510 | + '--error-name=deprecated C10_NODISCARD macro', | ||
| 511 | + '--replace-pattern=s/C10_NODISCARD/[[nodiscard]]/', | ||
| 512 | + """--error-description=\ | ||
| 513 | + Deprecated macro, use [[nodiscard]] directly\ | ||
| 514 | + """, | ||
| 515 | + '--', | ||
| 516 | + '@{{PATHSFILE}}' | ||
| 517 | +] | ||
| 518 | + | ||
| 519 | +[[linter]] | ||
| 520 | +code = 'RAWTHROW' | ||
| 521 | +include_patterns = [ | ||
| 522 | + '**/*.cpp', | ||
| 523 | + '**/*.h', | ||
| 524 | +] | ||
| 525 | +exclude_patterns = [ | ||
| 526 | + # Vendored code (do not modify) | ||
| 527 | + 'c10/util/flat_hash_map.h', | ||
| 528 | + 'c10/util/order_preserving_flat_hash_map.h', | ||
| 529 | + 'aten/src/ATen/native/quantized/cpu/qnnpack/**', | ||
| 530 | + 'aten/src/ATen/native/transformers/cuda/mem_eff_attention/**', | ||
| 531 | + # Standalone code that cannot depend on c10 | ||
| 532 | + 'torch/csrc/inductor/aoti_runtime/**', | ||
| 533 | + # Test files use EXPECT_THROW which is a gtest macro | ||
| 534 | + 'test/cpp/**/*.cpp', | ||
| 535 | +] | ||
| 536 | +command = [ | ||
| 537 | + 'python3', | ||
| 538 | + 'tools/linter/adapters/grep_linter.py', | ||
| 539 | + '--pattern=\bthrow\b', | ||
| 540 | + '--allowlist-pattern=@allow-raw-throw', | ||
| 541 | + '--linter-name=RAWTHROW', | ||
| 542 | + '--error-name=raw throw statement', | ||
| 543 | + """--error-description=\ | ||
| 544 | + Do not use raw `throw` in C++ code. Use TORCH_CHECK, \ | ||
| 545 | + TORCH_CHECK_WITH, C10_THROW_ERROR, or other error-reporting macros \ | ||
| 546 | + instead. See c10/util/Exception.h for available macros. If this is a \ | ||
| 547 | + re-throw (throw;), restructure the code to avoid try/catch/throw or \ | ||
| 548 | + add the file to the RAWTHROW exclude list in .lintrunner.toml with \ | ||
| 549 | + justification.\ | ||
| 550 | + """, | ||
| 551 | + '--', | ||
| 552 | + '@{{PATHSFILE}}' | ||
| 553 | +] | ||
| 554 | + | ||
| 555 | +[[linter]] | ||
| 556 | +code = 'INCLUDE' | ||
| 557 | +include_patterns = [ | ||
| 558 | + 'c10/**', | ||
| 559 | + 'aten/**', | ||
| 560 | + 'torch/csrc/**', | ||
| 561 | + 'torch/nativert/**', | ||
| 562 | +] | ||
| 563 | +exclude_patterns = [ | ||
| 564 | + 'aten/src/ATen/native/quantized/cpu/qnnpack/**', | ||
| 565 | + 'aten/src/ATen/native/vulkan/api/vk_mem_alloc.h', | ||
| 566 | + 'aten/src/ATen/native/vulkan/glsl/**', | ||
| 567 | + '**/fb/**', | ||
| 568 | + 'torch/csrc/jit/serialization/mobile_bytecode_generated.h', | ||
| 569 | + 'torch/csrc/utils/pythoncapi_compat.h', | ||
| 570 | +] | ||
| 571 | +command = [ | ||
| 572 | + 'python3', | ||
| 573 | + 'tools/linter/adapters/grep_linter.py', | ||
| 574 | + '--pattern=#include "', | ||
| 575 | + '--linter-name=INCLUDE', | ||
| 576 | + '--error-name=quoted include', | ||
| 577 | + '--replace-pattern=s/#include "(.*)"$/#include <\1>/', | ||
| 578 | + """--error-description=\ | ||
| 579 | + This #include uses quotes; please convert it to #include <xxxx>\ | ||
| 580 | + """, | ||
| 581 | + '--', | ||
| 582 | + '@{{PATHSFILE}}' | ||
| 583 | +] | ||
| 584 | + | ||
| 585 | +[[linter]] | ||
| 586 | +code = 'PYBIND11_INCLUDE' | ||
| 587 | +include_patterns = [ | ||
| 588 | + '**/*.cpp', | ||
| 589 | + '**/*.h', | ||
| 590 | +] | ||
| 591 | +exclude_patterns = [ | ||
| 592 | + 'torch/csrc/utils/pybind.h', | ||
| 593 | + 'torch/utils/benchmark/utils/valgrind_wrapper/compat_bindings.cpp', | ||
| 594 | + 'caffe2/**/*', | ||
| 595 | +] | ||
| 596 | +command = [ | ||
| 597 | + 'python3', | ||
| 598 | + 'tools/linter/adapters/grep_linter.py', | ||
| 599 | + '--pattern=#include <pybind11\/(^|[^(gil_simple\.h)])', | ||
| 600 | + '--allowlist-pattern=#include <torch\/csrc\/utils\/pybind.h>', | ||
| 601 | + '--linter-name=PYBIND11_INCLUDE', | ||
| 602 | + '--match-first-only', | ||
| 603 | + '--error-name=direct include of pybind11', | ||
| 604 | + # https://stackoverflow.com/a/33416489/23845 | ||
| 605 | + # NB: this won't work if the pybind11 include is on the first line; | ||
| 606 | + # but that's fine because it will just mean the lint will still fail | ||
| 607 | + # after applying the change and you will have to fix it manually | ||
| 608 | + '--replace-pattern=1,/(#include <pybind11\/)/ s/(#include <pybind11\/)/#include <torch\/csrc\/utils\/pybind.h>\n\1/', | ||
| 609 | + """--error-description=\ | ||
| 610 | + This #include directly includes pybind11 without also including \ | ||
| 611 | + #include <torch/csrc/utils/pybind.h>; this means some important \ | ||
| 612 | + specializations may not be included.\ | ||
| 613 | + """, | ||
| 614 | + '--', | ||
| 615 | + '@{{PATHSFILE}}' | ||
| 616 | +] | ||
| 617 | + | ||
| 618 | +[[linter]] | ||
| 619 | +code = 'ERROR_PRONE_ISINSTANCE' | ||
| 620 | +include_patterns = [ | ||
| 621 | + 'torch/_refs/**/*.py', | ||
| 622 | + 'torch/_prims/**/*.py', | ||
| 623 | + 'torch/_prims_common/**/*.py', | ||
| 624 | + 'torch/_decomp/**/*.py', | ||
| 625 | + 'torch/_meta_registrations.py', | ||
| 626 | +] | ||
| 627 | +exclude_patterns = [ | ||
| 628 | + '**/fb/**', | ||
| 629 | +] | ||
| 630 | +command = [ | ||
| 631 | + 'python3', | ||
| 632 | + 'tools/linter/adapters/grep_linter.py', | ||
| 633 | + '--pattern=isinstance\([^)]+(int|float)\)', | ||
| 634 | + '--linter-name=ERROR_PRONE_ISINSTANCE', | ||
| 635 | + '--error-name=error prone isinstance', | ||
| 636 | + """--error-description=\ | ||
| 637 | + This line has an isinstance call that directly refers to \ | ||
| 638 | + int or float. This is error-prone because you may also \ | ||
| 639 | + have wanted to allow SymInt or SymFloat in your test. \ | ||
| 640 | + To suppress this lint, use an appropriate type alias defined \ | ||
| 641 | + in torch._prims_common; use IntLike/FloatLike when you would accept \ | ||
| 642 | + both regular and symbolic numbers, Dim for ints representing \ | ||
| 643 | + dimensions, or IntWithoutSymInt/FloatWithoutSymFloat if you really \ | ||
| 644 | + meant to exclude symbolic numbers. | ||
| 645 | + """, | ||
| 646 | + '--', | ||
| 647 | + '@{{PATHSFILE}}' | ||
| 648 | +] | ||
| 649 | + | ||
| 650 | +[[linter]] | ||
| 651 | +code = 'PYBIND11_SPECIALIZATION' | ||
| 652 | +include_patterns = [ | ||
| 653 | + '**/*.cpp', | ||
| 654 | + '**/*.h', | ||
| 655 | +] | ||
| 656 | +exclude_patterns = [ | ||
| 657 | + # The place for all orphan specializations | ||
| 658 | + 'torch/csrc/utils/pybind.h', | ||
| 659 | + # These specializations are non-orphan | ||
| 660 | + 'torch/csrc/distributed/c10d/init.cpp', | ||
| 661 | + 'torch/csrc/jit/python/pybind.h', | ||
| 662 | + 'fb/**', | ||
| 663 | + '**/fb/**', | ||
| 664 | + # These are safe to exclude as they do not have Python | ||
| 665 | + 'c10/**/*', | ||
| 666 | +] | ||
| 667 | +command = [ | ||
| 668 | + 'python3', | ||
| 669 | + 'tools/linter/adapters/grep_linter.py', | ||
| 670 | + '--pattern=PYBIND11_DECLARE_HOLDER_TYPE', | ||
| 671 | + '--linter-name=PYBIND11_SPECIALIZATION', | ||
| 672 | + '--error-name=pybind11 specialization in non-standard location', | ||
| 673 | + """--error-description=\ | ||
| 674 | + This pybind11 specialization (PYBIND11_DECLARE_HOLDER_TYPE) should \ | ||
| 675 | + be placed in torch/csrc/utils/pybind.h so that it is guaranteed to be \ | ||
| 676 | + included at any site that may potentially make use of it via py::cast. \ | ||
| 677 | + If your specialization is in the same header file as the definition \ | ||
| 678 | + of the holder type, you can ignore this lint by adding your header to \ | ||
| 679 | + the exclude_patterns for this lint in .lintrunner.toml. For more \ | ||
| 680 | + information see https://github.com/pybind/pybind11/issues/4099 \ | ||
| 681 | + """, | ||
| 682 | + '--', | ||
| 683 | + '@{{PATHSFILE}}' | ||
| 684 | +] | ||
| 685 | + | ||
| 686 | +[[linter]] | ||
| 687 | +code = 'PYPIDEP' | ||
| 688 | +include_patterns = ['.github/**'] | ||
| 689 | +exclude_patterns = [ | ||
| 690 | + '**/*.rst', | ||
| 691 | + '**/*.py', | ||
| 692 | + '**/*.md', | ||
| 693 | + '**/*.diff', | ||
| 694 | + '**/fb/**', | ||
| 695 | +] | ||
| 696 | +command = [ | ||
| 697 | + 'python3', | ||
| 698 | + 'tools/linter/adapters/grep_linter.py', | ||
| 699 | + """--pattern=\ | ||
| 700 | + (pip|pip3|python -m pip|python3 -m pip|python3 -mpip|python -mpip) \ | ||
| 701 | + install ([a-zA-Z0-9][A-Za-z0-9\\._\\-]+)([^/=<>~!]+)[A-Za-z0-9\\._\\-\\*\\+\\!]*$\ | ||
| 702 | + """, | ||
| 703 | + '--linter-name=PYPIDEP', | ||
| 704 | + '--error-name=unpinned PyPI install', | ||
| 705 | + """--error-description=\ | ||
| 706 | + This line has unpinned PyPi installs; \ | ||
| 707 | + please pin them to a specific version: e.g. 'thepackage==1.2'\ | ||
| 708 | + """, | ||
| 709 | + '--', | ||
| 710 | + '@{{PATHSFILE}}' | ||
| 711 | +] | ||
| 712 | + | ||
| 713 | +[[linter]] | ||
| 714 | +code = 'EXEC' | ||
| 715 | +include_patterns = ['**'] | ||
| 716 | +exclude_patterns = [ | ||
| 717 | + 'third_party/**', | ||
| 718 | + 'torch/bin/**', | ||
| 719 | + '**/*.so', | ||
| 720 | + '**/*.py', | ||
| 721 | + '**/*.sh', | ||
| 722 | + '**/*.bash', | ||
| 723 | + '**/git-pre-commit', | ||
| 724 | + '**/git-clang-format', | ||
| 725 | + '**/gradlew', | ||
| 726 | + 'fb/**', | ||
| 727 | + '**/fb/**', | ||
| 728 | +] | ||
| 729 | +command = [ | ||
| 730 | + 'python3', | ||
| 731 | + 'tools/linter/adapters/exec_linter.py', | ||
| 732 | + '--', | ||
| 733 | + '@{{PATHSFILE}}', | ||
| 734 | +] | ||
| 735 | + | ||
| 736 | +[[linter]] | ||
| 737 | +code = 'CUBINCLUDE' | ||
| 738 | +include_patterns = ['aten/**'] | ||
| 739 | +exclude_patterns = [ | ||
| 740 | + 'aten/src/ATen/cuda/cub*.cuh', | ||
| 741 | + '**/fb/**', | ||
| 742 | +] | ||
| 743 | +command = [ | ||
| 744 | + 'python3', | ||
| 745 | + 'tools/linter/adapters/grep_linter.py', | ||
| 746 | + '--pattern=#include <cub/', | ||
| 747 | + '--linter-name=CUBINCLUDE', | ||
| 748 | + '--error-name=direct cub include', | ||
| 749 | + """--error-description=\ | ||
| 750 | + This line has a direct cub include; please include \ | ||
| 751 | + ATen/cuda/cub.cuh instead and wrap your cub calls in \ | ||
| 752 | + at::native namespace if necessary. | ||
| 753 | + """, | ||
| 754 | + '--', | ||
| 755 | + '@{{PATHSFILE}}' | ||
| 756 | +] | ||
| 757 | + | ||
| 758 | +[[linter]] | ||
| 759 | +code = 'RAWCUDA' | ||
| 760 | +include_patterns = [ | ||
| 761 | + 'aten/**', | ||
| 762 | + 'c10/**', | ||
| 763 | +] | ||
| 764 | +exclude_patterns = [ | ||
| 765 | + 'aten/src/ATen/test/**', | ||
| 766 | + 'c10/cuda/CUDAFunctions.h', | ||
| 767 | + 'c10/cuda/CUDACachingAllocator.cpp', | ||
| 768 | + '**/fb/**', | ||
| 769 | +] | ||
| 770 | +command = [ | ||
| 771 | + 'python3', | ||
| 772 | + 'tools/linter/adapters/grep_linter.py', | ||
| 773 | + '--pattern=cudaStreamSynchronize', | ||
| 774 | + '--linter-name=RAWCUDA', | ||
| 775 | + '--error-name=raw CUDA API usage', | ||
| 776 | + """--error-description=\ | ||
| 777 | + This line calls raw CUDA APIs directly; please use at::cuda wrappers instead. | ||
| 778 | + """, | ||
| 779 | + '--', | ||
| 780 | + '@{{PATHSFILE}}' | ||
| 781 | +] | ||
| 782 | + | ||
| 783 | +[[linter]] | ||
| 784 | +code = 'RAWCUDADEVICE' | ||
| 785 | +include_patterns = [ | ||
| 786 | + 'aten/**', | ||
| 787 | + 'c10/**', | ||
| 788 | + 'torch/csrc/**', | ||
| 789 | + 'torch/nativert/**', | ||
| 790 | +] | ||
| 791 | +exclude_patterns = [ | ||
| 792 | + 'aten/src/ATen/cuda/CUDAContext.cpp', | ||
| 793 | + 'aten/src/ATen/cuda/CUDAGeneratorImpl.cpp', | ||
| 794 | + 'aten/src/ATen/test/**', | ||
| 795 | + 'c10/core/impl/InlineDeviceGuard.h', | ||
| 796 | + 'c10/cuda/CUDAFunctions.cpp', | ||
| 797 | + 'c10/cuda/CUDAGuard.h', | ||
| 798 | + 'c10/cuda/impl/CUDATest.cpp', | ||
| 799 | + 'torch/csrc/cuda/nccl.cpp', | ||
| 800 | + '**/fb/**', | ||
| 801 | +] | ||
| 802 | +command = [ | ||
| 803 | + 'python3', | ||
| 804 | + 'tools/linter/adapters/grep_linter.py', | ||
| 805 | + '--pattern=(cudaSetDevice|cudaGetDevice)\\(', | ||
| 806 | + '--linter-name=RAWCUDADEVICE', | ||
| 807 | + '--error-name=raw CUDA API usage', | ||
| 808 | + """--error-description=\ | ||
| 809 | + This line calls raw CUDA APIs directly; please use c10::cuda wrappers instead. | ||
| 810 | + """, | ||
| 811 | + '--', | ||
| 812 | + '@{{PATHSFILE}}' | ||
| 813 | +] | ||
| 814 | + | ||
| 815 | +[[linter]] | ||
| 816 | +code = 'ROOT_LOGGING' | ||
| 817 | +include_patterns = [ | ||
| 818 | + '**/*.py', | ||
| 819 | +] | ||
| 820 | +# These are not library code, but scripts in their own right, and so | ||
| 821 | +# therefore are permitted to use logging | ||
| 822 | +exclude_patterns = [ | ||
| 823 | + 'tools/**', | ||
| 824 | + 'test/**', | ||
| 825 | + 'benchmarks/**', | ||
| 826 | + 'torch/distributed/run.py', | ||
| 827 | + 'functorch/benchmarks/**', | ||
| 828 | + # Grandfathered in | ||
| 829 | + 'caffe2/**', | ||
| 830 | + 'fb/**', | ||
| 831 | + '**/fb/**', | ||
| 832 | +] | ||
| 833 | +command = [ | ||
| 834 | + 'python3', | ||
| 835 | + 'tools/linter/adapters/grep_linter.py', | ||
| 836 | + '--pattern=logging\.(debug|info|warn|warning|error|critical|log|exception)\(', | ||
| 837 | + '--replace-pattern=s/logging\.(debug|info|warn|warning|error|critical|log|exception)\(/log.\1(/', | ||
| 838 | + '--linter-name=ROOT_LOGGING', | ||
| 839 | + '--error-name=use of root logger', | ||
| 840 | + """--error-description=\ | ||
| 841 | + Do not use root logger (logging.info, etc) directly; instead \ | ||
| 842 | + define 'log = logging.getLogger(__name__)' and call, e.g., log.info(). | ||
| 843 | + """, | ||
| 844 | + '--', | ||
| 845 | + '@{{PATHSFILE}}' | ||
| 846 | +] | ||
| 847 | + | ||
| 848 | +[[linter]] | ||
| 849 | +code = 'DEPLOY_DETECTION' | ||
| 850 | +include_patterns = [ | ||
| 851 | + '**/*.py', | ||
| 852 | +] | ||
| 853 | +command = [ | ||
| 854 | + 'python3', | ||
| 855 | + 'tools/linter/adapters/grep_linter.py', | ||
| 856 | + '--pattern=sys\.executable == .torch_deploy.', | ||
| 857 | + '--replace-pattern=s/sys\.executable == .torch_deploy./torch._running_with_deploy\(\)/', | ||
| 858 | + '--linter-name=DEPLOY_DETECTION', | ||
| 859 | + '--error-name=properly detect deploy runner', | ||
| 860 | + """--error-description=\ | ||
| 861 | + Do not use sys.executable to detect if running within deploy/multipy, use torch._running_with_deploy(). | ||
| 862 | + """, | ||
| 863 | + '--', | ||
| 864 | + '@{{PATHSFILE}}' | ||
| 865 | +] | ||
| 866 | + | ||
| 867 | +[[linter]] | ||
| 868 | +code = 'CMAKE' | ||
| 869 | +include_patterns = [ | ||
| 870 | + "**/*.cmake", | ||
| 871 | + "**/*.cmake.in", | ||
| 872 | + "**/CMakeLists.txt", | ||
| 873 | +] | ||
| 874 | +exclude_patterns = [ | ||
| 875 | + 'cmake/Modules/**', | ||
| 876 | + 'cmake/Modules_CUDA_fix/**', | ||
| 877 | + 'cmake/Caffe2Config.cmake.in', | ||
| 878 | + 'aten/src/ATen/ATenConfig.cmake.in', | ||
| 879 | + 'cmake/TorchConfig.cmake.in', | ||
| 880 | + 'cmake/TorchConfigVersion.cmake.in', | ||
| 881 | + 'cmake/cmake_uninstall.cmake.i', | ||
| 882 | + 'fb/**', | ||
| 883 | + '**/fb/**', | ||
| 884 | +] | ||
| 885 | +command = [ | ||
| 886 | + 'uv', | ||
| 887 | + 'run', | ||
| 888 | + '--script', | ||
| 889 | + 'tools/linter/adapters/cmake_linter.py', | ||
| 890 | + '--config=.cmakelintrc', | ||
| 891 | + '--', | ||
| 892 | + '@{{PATHSFILE}}', | ||
| 893 | +] | ||
| 894 | + | ||
| 895 | +[[linter]] | ||
| 896 | +code = 'SHELLCHECK' | ||
| 897 | +include_patterns = [ | ||
| 898 | + '.ci/pytorch/**/*.sh' | ||
| 899 | +] | ||
| 900 | +exclude_patterns = [ | ||
| 901 | + '**/fb/**', | ||
| 902 | +] | ||
| 903 | +command = [ | ||
| 904 | + 'uv', | ||
| 905 | + 'run', | ||
| 906 | + '--script', | ||
| 907 | + 'tools/linter/adapters/shellcheck_linter.py', | ||
| 908 | + '--', | ||
| 909 | + '@{{PATHSFILE}}', | ||
| 910 | +] | ||
| 911 | + | ||
| 912 | +[[linter]] | ||
| 913 | +code = 'ACTIONLINT' | ||
| 914 | +include_patterns = [ | ||
| 915 | + '.github/workflows/*.yml', | ||
| 916 | + '.github/workflows/*.yaml', | ||
| 917 | + # actionlint does not support composite actions yet | ||
| 918 | + # '.github/actions/**/*.yml', | ||
| 919 | + # '.github/actions/**/*.yaml', | ||
| 920 | +] | ||
| 921 | +exclude_patterns = [ | ||
| 922 | + '**/fb/**', | ||
| 923 | +] | ||
| 924 | +command = [ | ||
| 925 | + 'python3', | ||
| 926 | + 'tools/linter/adapters/actionlint_linter.py', | ||
| 927 | + '--binary=.lintbin/actionlint', | ||
| 928 | + '--', | ||
| 929 | + '@{{PATHSFILE}}', | ||
| 930 | +] | ||
| 931 | +init_command = [ | ||
| 932 | + 'python3', | ||
| 933 | + 'tools/linter/adapters/s3_init.py', | ||
| 934 | + '--config-json=tools/linter/adapters/s3_init_config.json', | ||
| 935 | + '--linter=actionlint', | ||
| 936 | + '--dry-run={{DRYRUN}}', | ||
| 937 | + '--output-dir=.lintbin', | ||
| 938 | + '--output-name=actionlint', | ||
| 939 | +] | ||
| 940 | + | ||
| 941 | +[[linter]] | ||
| 942 | +code = 'TESTOWNERS' | ||
| 943 | +include_patterns = [ | ||
| 944 | + 'test/**/test_*.py', | ||
| 945 | + 'test/**/*_test.py', | ||
| 946 | +] | ||
| 947 | +exclude_patterns = [ | ||
| 948 | + 'test/run_test.py', | ||
| 949 | + '**/fb/**', | ||
| 950 | +] | ||
| 951 | +command = [ | ||
| 952 | + 'python3', | ||
| 953 | + 'tools/linter/adapters/testowners_linter.py', | ||
| 954 | + '--', | ||
| 955 | + '@{{PATHSFILE}}', | ||
| 956 | +] | ||
| 957 | + | ||
| 958 | +[[linter]] | ||
| 959 | +code = 'TEST_HAS_MAIN' | ||
| 960 | +include_patterns = [ | ||
| 961 | + 'test/**/test_*.py', | ||
| 962 | +] | ||
| 963 | +exclude_patterns = [ | ||
| 964 | + 'test/run_test.py', | ||
| 965 | + '**/fb/**', | ||
| 966 | + 'test/dynamo/cpython/3.13/**', | ||
| 967 | + 'test/quantization/**', # should be run through test/test_quantization.py | ||
| 968 | + 'test/jit/**', # should be run through test/test_jit.py | ||
| 969 | + 'test/ao/sparsity/**', # should be run through test/test_ao_sparsity.py | ||
| 970 | + 'test/fx/**', # should be run through test/test_fx.py | ||
| 971 | + 'test/package/**', # excluded by test/run_test.py | ||
| 972 | + 'test/distributed/argparse_util_test.py', | ||
| 973 | + 'test/distributed/bin/test_script.py', | ||
| 974 | + 'test/distributed/elastic/agent/server/test/local_elastic_agent_test.py', | ||
| 975 | + 'test/distributed/elastic/multiprocessing/bin/test_script.py', | ||
| 976 | + 'test/distributed/elastic/multiprocessing/bin/zombie_test.py', | ||
| 977 | + 'test/distributed/elastic/multiprocessing/errors/api_test.py', | ||
| 978 | + 'test/distributed/elastic/multiprocessing/errors/error_handler_test.py', | ||
| 979 | + 'test/distributed/elastic/multiprocessing/redirects_test.py', | ||
| 980 | + 'test/distributed/elastic/multiprocessing/tail_log_test.py', | ||
| 981 | + 'test/distributed/elastic/rendezvous/api_test.py', | ||
| 982 | + 'test/distributed/elastic/rendezvous/c10d_rendezvous_backend_test.py', | ||
| 983 | + 'test/distributed/elastic/rendezvous/dynamic_rendezvous_test.py', | ||
| 984 | + 'test/distributed/elastic/rendezvous/etcd_rendezvous_backend_test.py', | ||
| 985 | + 'test/distributed/elastic/rendezvous/etcd_rendezvous_test.py', | ||
| 986 | + 'test/distributed/elastic/rendezvous/etcd_server_test.py', | ||
| 987 | + 'test/distributed/elastic/rendezvous/rendezvous_backend_test.py', | ||
| 988 | + 'test/distributed/elastic/rendezvous/static_rendezvous_test.py', | ||
| 989 | + 'test/distributed/elastic/rendezvous/utils_test.py', | ||
| 990 | + 'test/distributed/elastic/timer/api_test.py', | ||
| 991 | + 'test/distributed/elastic/utils/data/cycling_iterator_test.py', | ||
| 992 | + 'test/distributed/launcher/api_test.py', | ||
| 993 | + 'test/distributed/launcher/bin/test_script.py', | ||
| 994 | + 'test/distributed/launcher/bin/test_script_init_method.py', | ||
| 995 | + 'test/distributed/launcher/bin/test_script_is_torchelastic_launched.py', | ||
| 996 | + 'test/distributed/launcher/bin/test_script_local_rank.py', | ||
| 997 | + 'test/distributed/launcher/launch_test.py', | ||
| 998 | + 'test/distributed/launcher/run_test.py', | ||
| 999 | + 'test/distributed/optim/test_apply_optimizer_in_backward.py', | ||
| 1000 | + 'test/distributed/optim/test_named_optimizer.py', | ||
| 1001 | + 'test/distributed/test_c10d_spawn.py', | ||
| 1002 | + 'test/distributed/test_collective_utils.py', | ||
| 1003 | + 'test/distributions/test_distributions.py', | ||
| 1004 | + 'test/inductor/test_aot_inductor_utils.py', | ||
| 1005 | + 'test/lazy/test_bindings.py', | ||
| 1006 | + 'test/lazy/test_extract_compiled_graph.py', | ||
| 1007 | + 'test/lazy/test_meta_kernel.py', | ||
| 1008 | + 'test/nn/test_init.py', | ||
| 1009 | + 'test/onnx/model_defs/op_test.py', | ||
| 1010 | + 'test/onnx/test_models_quantized_onnxruntime.py', | ||
| 1011 | + 'test/onnx/test_onnxscript_no_runtime.py', | ||
| 1012 | + 'test/onnx_caffe2/test_caffe2_common.py', | ||
| 1013 | + 'test/optim/test_lrscheduler.py', | ||
| 1014 | + 'test/optim/test_optim.py', | ||
| 1015 | + 'test/optim/test_swa_utils.py', | ||
| 1016 | + 'test/run_test.py', | ||
| 1017 | + 'test/test_bundled_images.py', | ||
| 1018 | + 'test/test_cuda_expandable_segments.py', | ||
| 1019 | + 'test/test_hub.py', | ||
| 1020 | +] | ||
| 1021 | +command = [ | ||
| 1022 | + 'uv', | ||
| 1023 | + 'run', | ||
| 1024 | + '--script', | ||
| 1025 | + 'tools/linter/adapters/test_has_main_linter.py', | ||
| 1026 | + '--', | ||
| 1027 | + '@{{PATHSFILE}}', | ||
| 1028 | +] | ||
| 1029 | + | ||
| 1030 | +[[linter]] | ||
| 1031 | +code = 'CALL_ONCE' | ||
| 1032 | +include_patterns = [ | ||
| 1033 | + 'c10/**', | ||
| 1034 | + 'aten/**', | ||
| 1035 | + 'torch/csrc/**', | ||
| 1036 | + 'torch/nativert/**', | ||
| 1037 | +] | ||
| 1038 | +exclude_patterns = [ | ||
| 1039 | + 'c10/util/CallOnce.h', | ||
| 1040 | + '**/fb/**', | ||
| 1041 | +] | ||
| 1042 | +command = [ | ||
| 1043 | + 'python3', | ||
| 1044 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1045 | + '--pattern=std::call_once', | ||
| 1046 | + '--linter-name=CALL_ONCE', | ||
| 1047 | + '--error-name=invalid call_once', | ||
| 1048 | + '--replace-pattern=s/std::call_once/c10::call_once/', | ||
| 1049 | + """--error-description=\ | ||
| 1050 | + Use of std::call_once is forbidden and should be replaced with c10::call_once\ | ||
| 1051 | + """, | ||
| 1052 | + '--', | ||
| 1053 | + '@{{PATHSFILE}}' | ||
| 1054 | +] | ||
| 1055 | + | ||
| 1056 | +[[linter]] | ||
| 1057 | +code = 'CONTEXT_DECORATOR' | ||
| 1058 | +include_patterns = [ | ||
| 1059 | + 'torch/**', | ||
| 1060 | +] | ||
| 1061 | +command = [ | ||
| 1062 | + 'python3', | ||
| 1063 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1064 | + '--pattern=@.*(dynamo_timed|preserve_rng_state|clear_frame|with_fresh_cache_if_config|use_lazy_graph_module|_disable_current_modes)', | ||
| 1065 | + '--allowlist-pattern=clear_frames', | ||
| 1066 | + '--linter-name=CONTEXT_DECORATOR', | ||
| 1067 | + '--error-name=avoid context decorator', | ||
| 1068 | + """--error-description=\ | ||
| 1069 | + Do not use context manager as decorator as it breaks cProfile traces. Use it as \ | ||
| 1070 | + a context manager instead\ | ||
| 1071 | + """, | ||
| 1072 | + '--', | ||
| 1073 | + '@{{PATHSFILE}}' | ||
| 1074 | +] | ||
| 1075 | + | ||
| 1076 | +[[linter]] | ||
| 1077 | +code = 'ONCE_FLAG' | ||
| 1078 | +include_patterns = [ | ||
| 1079 | + 'c10/**', | ||
| 1080 | + 'aten/**', | ||
| 1081 | + 'torch/csrc/**', | ||
| 1082 | + 'torch/nativert/**', | ||
| 1083 | +] | ||
| 1084 | +exclude_patterns = [ | ||
| 1085 | + '**/fb/**', | ||
| 1086 | +] | ||
| 1087 | +command = [ | ||
| 1088 | + 'python3', | ||
| 1089 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1090 | + '--pattern=std::once_flag', | ||
| 1091 | + '--linter-name=ONCE_FLAG', | ||
| 1092 | + '--error-name=invalid once_flag', | ||
| 1093 | + '--replace-pattern=s/std::once_flag/c10::once_flag/', | ||
| 1094 | + """--error-description=\ | ||
| 1095 | + Use of std::once_flag is forbidden and should be replaced with c10::once_flag\ | ||
| 1096 | + """, | ||
| 1097 | + '--', | ||
| 1098 | + '@{{PATHSFILE}}' | ||
| 1099 | +] | ||
| 1100 | + | ||
| 1101 | +[[linter]] | ||
| 1102 | +code = 'WORKFLOWSYNC' | ||
| 1103 | +include_patterns = [ | ||
| 1104 | + '.github/workflows/*.yml', | ||
| 1105 | + '.github/workflows/*.yaml', | ||
| 1106 | +] | ||
| 1107 | +command = [ | ||
| 1108 | + 'uv', | ||
| 1109 | + 'run', | ||
| 1110 | + '--script', | ||
| 1111 | + 'tools/linter/adapters/workflow_consistency_linter.py', | ||
| 1112 | + '--', | ||
| 1113 | + '@{{PATHSFILE}}' | ||
| 1114 | +] | ||
| 1115 | + | ||
| 1116 | +[[linter]] | ||
| 1117 | +code = 'NO_WORKFLOWS_ON_FORK' | ||
| 1118 | +include_patterns = [ | ||
| 1119 | + '.github/**/*.yml', | ||
| 1120 | + '.github/**/*.yaml', | ||
| 1121 | +] | ||
| 1122 | +exclude_patterns = [ | ||
| 1123 | + '**/fb/**', | ||
| 1124 | +] | ||
| 1125 | +command = [ | ||
| 1126 | + 'uv', | ||
| 1127 | + 'run', | ||
| 1128 | + '--script', | ||
| 1129 | + 'tools/linter/adapters/no_workflows_on_fork.py', | ||
| 1130 | + '--', | ||
| 1131 | + '@{{PATHSFILE}}', | ||
| 1132 | +] | ||
| 1133 | + | ||
| 1134 | +[[linter]] | ||
| 1135 | +code = 'CODESPELL' | ||
| 1136 | +command = [ | ||
| 1137 | + 'uv', | ||
| 1138 | + 'run', | ||
| 1139 | + '--script', | ||
| 1140 | + 'tools/linter/adapters/codespell_linter.py', | ||
| 1141 | + '--', | ||
| 1142 | + '@{{PATHSFILE}}' | ||
| 1143 | +] | ||
| 1144 | +include_patterns = [ | ||
| 1145 | + '**', | ||
| 1146 | +] | ||
| 1147 | +exclude_patterns = [ | ||
| 1148 | + # We don't care too much about files in this directory, don't enforce | ||
| 1149 | + # spelling on them | ||
| 1150 | + 'caffe2/**', | ||
| 1151 | + 'fb/**', | ||
| 1152 | + '**/fb/**', | ||
| 1153 | + 'third_party/**', | ||
| 1154 | + 'test/dynamo/cpython/**', | ||
| 1155 | + 'torch/_vendor/**', | ||
| 1156 | + 'torch/_inductor/fx_passes/serialized_patterns/**', | ||
| 1157 | + 'torch/_inductor/autoheuristic/artifacts/**', | ||
| 1158 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**', | ||
| 1159 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py', | ||
| 1160 | + 'torch/utils/model_dump/preact.mjs', | ||
| 1161 | +] | ||
| 1162 | +is_formatter = true | ||
| 1163 | + | ||
| 1164 | +# usort + ruff-format | ||
| 1165 | +[[linter]] | ||
| 1166 | +code = 'PYFMT' | ||
| 1167 | +include_patterns = [ | ||
| 1168 | + '**/*.py', | ||
| 1169 | + '**/*.pyi', | ||
| 1170 | +] | ||
| 1171 | +command = [ | ||
| 1172 | + 'uv', | ||
| 1173 | + 'run', | ||
| 1174 | + '--script', | ||
| 1175 | + 'tools/linter/adapters/pyfmt_linter.py', | ||
| 1176 | + '--', | ||
| 1177 | + '@{{PATHSFILE}}' | ||
| 1178 | +] | ||
| 1179 | +exclude_patterns = [ | ||
| 1180 | + 'tools/gen_vulkan_spv.py', | ||
| 1181 | + # We don't care too much about files in this directory, don't enforce | ||
| 1182 | + # formatting on them | ||
| 1183 | + 'caffe2/**/*.py', | ||
| 1184 | + 'caffe2/**/*.pyi', | ||
| 1185 | + 'fb/**', | ||
| 1186 | + '**/fb/**', | ||
| 1187 | + 'test/dynamo/cpython/**', | ||
| 1188 | + 'third_party/**/*.py', | ||
| 1189 | + 'third_party/**/*.pyi', | ||
| 1190 | + 'torch/_vendor/**', | ||
| 1191 | + 'torch/_inductor/fx_passes/serialized_patterns/**', | ||
| 1192 | + 'torch/_inductor/autoheuristic/artifacts/**', | ||
| 1193 | + # Needs Python 3.12+ | ||
| 1194 | + 'torch/testing/_internal/py312_intrinsics.py', | ||
| 1195 | + # These files are all grandfathered in, feel free to remove from this list | ||
| 1196 | + # as necessary | ||
| 1197 | + 'test/quantization/__init__.py', | ||
| 1198 | + 'test/quantization/core/__init__.py', | ||
| 1199 | + 'test/quantization/core/experimental/apot_fx_graph_mode_ptq.py', | ||
| 1200 | + 'test/quantization/core/experimental/apot_fx_graph_mode_qat.py', | ||
| 1201 | + 'test/quantization/core/experimental/quantization_util.py', | ||
| 1202 | + 'test/quantization/core/experimental/test_bits.py', | ||
| 1203 | + 'test/quantization/core/experimental/test_fake_quantize.py', | ||
| 1204 | + 'test/quantization/core/experimental/test_linear.py', | ||
| 1205 | + 'test/quantization/core/experimental/test_nonuniform_observer.py', | ||
| 1206 | + 'test/quantization/core/experimental/test_quantized_tensor.py', | ||
| 1207 | + 'test/quantization/core/experimental/test_quantizer.py', | ||
| 1208 | + 'test/quantization/core/test_backend_config.py', | ||
| 1209 | + 'test/quantization/core/test_docs.py', | ||
| 1210 | + 'test/quantization/core/test_quantized_functional.py', | ||
| 1211 | + 'test/quantization/core/test_quantized_module.py', | ||
| 1212 | + 'test/quantization/core/test_quantized_op.py', | ||
| 1213 | + 'test/quantization/core/test_quantized_tensor.py', | ||
| 1214 | + 'test/quantization/core/test_top_level_apis.py', | ||
| 1215 | + 'test/quantization/core/test_utils.py', | ||
| 1216 | + 'test/quantization/core/test_workflow_module.py', | ||
| 1217 | + 'test/quantization/core/test_workflow_ops.py', | ||
| 1218 | + 'test/quantization/fx/__init__.py', | ||
| 1219 | + 'test/quantization/fx/test_equalize_fx.py', | ||
| 1220 | + 'test/quantization/fx/test_model_report_fx.py', | ||
| 1221 | + 'test/quantization/fx/test_numeric_suite_fx.py', | ||
| 1222 | + 'test/quantization/fx/test_quantize_fx.py', | ||
| 1223 | + 'test/quantization/fx/test_subgraph_rewriter.py', | ||
| 1224 | + 'test/test_function_schema.py', | ||
| 1225 | + 'test/test_functional_autograd_benchmark.py', | ||
| 1226 | + 'test/test_functional_optim.py', | ||
| 1227 | + 'test/test_functionalization_of_rng_ops.py', | ||
| 1228 | + 'test/test_datapipe.py', | ||
| 1229 | + 'test/test_futures.py', | ||
| 1230 | + 'test/test_fx.py', | ||
| 1231 | + 'test/test_fx_experimental.py', | ||
| 1232 | + 'test/test_fx_passes.py', | ||
| 1233 | + 'test/test_fx_reinplace_pass.py', | ||
| 1234 | + 'test/test_import_stats.py', | ||
| 1235 | + 'test/test_itt.py', | ||
| 1236 | + 'test/test_jit.py', | ||
| 1237 | + 'test/test_jit_autocast.py', | ||
| 1238 | + 'test/test_jit_cuda_fuser.py', | ||
| 1239 | + 'test/test_jit_disabled.py', | ||
| 1240 | + 'test/test_jit_fuser.py', | ||
| 1241 | + 'test/test_jit_fuser_legacy.py', | ||
| 1242 | + 'test/test_jit_legacy.py', | ||
| 1243 | + 'test/test_jit_llga_fuser.py', | ||
| 1244 | + 'test/test_jit_profiling.py', | ||
| 1245 | + 'test/test_jit_simple.py', | ||
| 1246 | + 'test/test_jit_string.py', | ||
| 1247 | + 'test/test_jiterator.py', | ||
| 1248 | + 'test/test_kernel_launch_checks.py', | ||
| 1249 | + 'test/test_linalg.py', | ||
| 1250 | + 'test/test_masked.py', | ||
| 1251 | + 'test/test_maskedtensor.py', | ||
| 1252 | + 'test/test_matmul_cuda.py', | ||
| 1253 | + 'test/test_scaled_matmul_cuda.py', | ||
| 1254 | + 'test/test_meta.py', | ||
| 1255 | + 'test/test_metal.py', | ||
| 1256 | + 'test/test_mkl_verbose.py', | ||
| 1257 | + 'test/test_mkldnn.py', | ||
| 1258 | + 'test/test_mkldnn_fusion.py', | ||
| 1259 | + 'test/test_mkldnn_verbose.py', | ||
| 1260 | + 'test/test_mobile_optimizer.py', | ||
| 1261 | + 'test/test_model_dump.py', | ||
| 1262 | + 'test/test_modules.py', | ||
| 1263 | + 'test/test_monitor.py', | ||
| 1264 | + 'test/test_mps.py', | ||
| 1265 | + 'test/test_multiprocessing_spawn.py', | ||
| 1266 | + 'test/test_namedtensor.py', | ||
| 1267 | + 'test/test_namedtuple_return_api.py', | ||
| 1268 | + 'test/test_native_functions.py', | ||
| 1269 | + 'test/test_native_mha.py', | ||
| 1270 | + 'test/test_nn.py', | ||
| 1271 | + 'test/test_out_dtype_op.py', | ||
| 1272 | + 'test/test_overrides.py', | ||
| 1273 | + 'test/test_prims.py', | ||
| 1274 | + 'test/test_proxy_tensor.py', | ||
| 1275 | + 'test/test_pruning_op.py', | ||
| 1276 | + 'test/test_quantization.py', | ||
| 1277 | + 'test/test_reductions.py', | ||
| 1278 | + 'test/test_scatter_gather_ops.py', | ||
| 1279 | + 'test/test_schema_check.py', | ||
| 1280 | + 'test/test_segment_reductions.py', | ||
| 1281 | + 'test/test_serialization.py', | ||
| 1282 | + 'test/test_set_default_mobile_cpu_allocator.py', | ||
| 1283 | + 'test/test_sparse.py', | ||
| 1284 | + 'test/test_sparse_csr.py', | ||
| 1285 | + 'test/test_sparse_semi_structured.py', | ||
| 1286 | + 'test/test_spectral_ops.py', | ||
| 1287 | + 'test/test_stateless.py', | ||
| 1288 | + 'test/test_static_runtime.py', | ||
| 1289 | + 'test/test_subclass.py', | ||
| 1290 | + 'test/test_sympy_utils.py', | ||
| 1291 | + 'test/test_tensor_creation_ops.py', | ||
| 1292 | + 'test/test_tensorboard.py', | ||
| 1293 | + 'test/test_tensorexpr.py', | ||
| 1294 | + 'test/test_tensorexpr_pybind.py', | ||
| 1295 | + 'test/test_testing.py', | ||
| 1296 | + 'test/test_torch.py', | ||
| 1297 | + 'test/test_transformers.py', | ||
| 1298 | + 'test/test_type_promotion.py', | ||
| 1299 | + 'test/test_unary_ufuncs.py', | ||
| 1300 | + 'test/test_vulkan.py', | ||
| 1301 | + 'torch/_awaits/__init__.py', | ||
| 1302 | + 'torch/_export/__init__.py', | ||
| 1303 | + 'torch/_export/constraints.py', | ||
| 1304 | + 'torch/_export/db/__init__.py', | ||
| 1305 | + 'torch/_export/db/case.py', | ||
| 1306 | + 'torch/_export/db/examples/__init__.py', | ||
| 1307 | + 'torch/_export/db/examples/assume_constant_result.py', | ||
| 1308 | + 'torch/_export/db/examples/autograd_function.py', | ||
| 1309 | + 'torch/_export/db/examples/class_method.py', | ||
| 1310 | + 'torch/_export/db/examples/cond_branch_class_method.py', | ||
| 1311 | + 'torch/_export/db/examples/cond_branch_nested_function.py', | ||
| 1312 | + 'torch/_export/db/examples/cond_branch_nonlocal_variables.py', | ||
| 1313 | + 'torch/_export/db/examples/cond_closed_over_variable.py', | ||
| 1314 | + 'torch/_export/db/examples/cond_operands.py', | ||
| 1315 | + 'torch/_export/db/examples/cond_predicate.py', | ||
| 1316 | + 'torch/_export/db/examples/decorator.py', | ||
| 1317 | + 'torch/_export/db/examples/dictionary.py', | ||
| 1318 | + 'torch/_export/db/examples/dynamic_shape_assert.py', | ||
| 1319 | + 'torch/_export/db/examples/dynamic_shape_constructor.py', | ||
| 1320 | + 'torch/_export/db/examples/dynamic_shape_if_guard.py', | ||
| 1321 | + 'torch/_export/db/examples/dynamic_shape_map.py', | ||
| 1322 | + 'torch/_export/db/examples/dynamic_shape_round.py', | ||
| 1323 | + 'torch/_export/db/examples/dynamic_shape_slicing.py', | ||
| 1324 | + 'torch/_export/db/examples/dynamic_shape_view.py', | ||
| 1325 | + 'torch/_export/db/examples/fn_with_kwargs.py', | ||
| 1326 | + 'torch/_export/db/examples/list_contains.py', | ||
| 1327 | + 'torch/_export/db/examples/list_unpack.py', | ||
| 1328 | + 'torch/_export/db/examples/nested_function.py', | ||
| 1329 | + 'torch/_export/db/examples/null_context_manager.py', | ||
| 1330 | + 'torch/_export/db/examples/pytree_flatten.py', | ||
| 1331 | + 'torch/_export/db/examples/scalar_output.py', | ||
| 1332 | + 'torch/_export/db/examples/specialized_attribute.py', | ||
| 1333 | + 'torch/_export/db/examples/static_for_loop.py', | ||
| 1334 | + 'torch/_export/db/examples/static_if.py', | ||
| 1335 | + 'torch/_export/db/examples/tensor_setattr.py', | ||
| 1336 | + 'torch/_export/db/examples/type_reflection_method.py', | ||
| 1337 | + 'torch/_export/db/gen_example.py', | ||
| 1338 | + 'torch/_export/db/logging.py', | ||
| 1339 | + 'torch/testing/_internal/__init__.py', | ||
| 1340 | + 'torch/testing/_internal/autocast_test_lists.py', | ||
| 1341 | + 'torch/testing/_internal/autograd_function_db.py', | ||
| 1342 | + 'torch/testing/_internal/check_kernel_launches.py', | ||
| 1343 | + 'torch/testing/_internal/codegen/__init__.py', | ||
| 1344 | + 'torch/testing/_internal/codegen/random_topo_test.py', | ||
| 1345 | + 'torch/testing/_internal/common_cuda.py', | ||
| 1346 | + 'torch/testing/_internal/common_jit.py', | ||
| 1347 | + 'torch/testing/_internal/common_methods_invocations.py', | ||
| 1348 | + 'torch/testing/_internal/common_modules.py', | ||
| 1349 | + 'torch/testing/_internal/common_nn.py', | ||
| 1350 | + 'torch/testing/_internal/common_pruning.py', | ||
| 1351 | + 'torch/testing/_internal/common_quantization.py', | ||
| 1352 | + 'torch/testing/_internal/common_quantized.py', | ||
| 1353 | + 'torch/testing/_internal/common_subclass.py', | ||
| 1354 | + 'torch/testing/_internal/common_utils.py', | ||
| 1355 | + 'torch/testing/_internal/composite_compliance.py', | ||
| 1356 | + 'torch/testing/_internal/hop_db.py', | ||
| 1357 | + 'torch/testing/_internal/custom_op_db.py', | ||
| 1358 | + 'torch/testing/_internal/data/__init__.py', | ||
| 1359 | + 'torch/testing/_internal/data/network1.py', | ||
| 1360 | + 'torch/testing/_internal/data/network2.py', | ||
| 1361 | + 'torch/testing/_internal/dist_utils.py', | ||
| 1362 | + 'torch/testing/_internal/generated/__init__.py', | ||
| 1363 | + 'torch/testing/_internal/hypothesis_utils.py', | ||
| 1364 | + 'torch/testing/_internal/inductor_utils.py', | ||
| 1365 | + 'torch/testing/_internal/jit_metaprogramming_utils.py', | ||
| 1366 | + 'torch/testing/_internal/jit_utils.py', | ||
| 1367 | + 'torch/testing/_internal/logging_tensor.py', | ||
| 1368 | + 'torch/testing/_internal/logging_utils.py', | ||
| 1369 | + 'torch/testing/_internal/optests/__init__.py', | ||
| 1370 | + 'torch/testing/_internal/optests/aot_autograd.py', | ||
| 1371 | + 'torch/testing/_internal/optests/compile_check.py', | ||
| 1372 | + 'torch/testing/_internal/optests/fake_tensor.py', | ||
| 1373 | + 'torch/testing/_internal/optests/make_fx.py', | ||
| 1374 | + 'torch/testing/_internal/quantization_torch_package_models.py', | ||
| 1375 | + 'torch/testing/_internal/test_module/__init__.py', | ||
| 1376 | + 'torch/testing/_internal/test_module/future_div.py', | ||
| 1377 | + 'torch/testing/_internal/test_module/no_future_div.py', | ||
| 1378 | + 'torch/utils/benchmark/__init__.py', | ||
| 1379 | + 'torch/utils/benchmark/examples/__init__.py', | ||
| 1380 | + 'torch/utils/benchmark/examples/compare.py', | ||
| 1381 | + 'torch/utils/benchmark/examples/fuzzer.py', | ||
| 1382 | + 'torch/utils/benchmark/examples/op_benchmark.py', | ||
| 1383 | + 'torch/utils/benchmark/examples/simple_timeit.py', | ||
| 1384 | + 'torch/utils/benchmark/examples/sparse/compare.py', | ||
| 1385 | + 'torch/utils/benchmark/examples/sparse/fuzzer.py', | ||
| 1386 | + 'torch/utils/benchmark/examples/sparse/op_benchmark.py', | ||
| 1387 | + 'torch/utils/benchmark/examples/spectral_ops_fuzz_test.py', | ||
| 1388 | + 'torch/utils/benchmark/op_fuzzers/__init__.py', | ||
| 1389 | + 'torch/utils/benchmark/op_fuzzers/binary.py', | ||
| 1390 | + 'torch/utils/benchmark/op_fuzzers/sparse_binary.py', | ||
| 1391 | + 'torch/utils/benchmark/op_fuzzers/sparse_unary.py', | ||
| 1392 | + 'torch/utils/benchmark/op_fuzzers/spectral.py', | ||
| 1393 | + 'torch/utils/benchmark/op_fuzzers/unary.py', | ||
| 1394 | + 'torch/utils/benchmark/utils/__init__.py', | ||
| 1395 | + 'torch/utils/benchmark/utils/_stubs.py', | ||
| 1396 | + 'torch/utils/benchmark/utils/common.py', | ||
| 1397 | + 'torch/utils/benchmark/utils/compare.py', | ||
| 1398 | + 'torch/utils/benchmark/utils/compile.py', | ||
| 1399 | + 'torch/utils/benchmark/utils/cpp_jit.py', | ||
| 1400 | + 'torch/utils/benchmark/utils/fuzzer.py', | ||
| 1401 | + 'torch/utils/benchmark/utils/sparse_fuzzer.py', | ||
| 1402 | + 'torch/utils/benchmark/utils/timer.py', | ||
| 1403 | + 'torch/utils/benchmark/utils/valgrind_wrapper/__init__.py', | ||
| 1404 | + 'torch/utils/benchmark/utils/valgrind_wrapper/timer_interface.py', | ||
| 1405 | + 'torch/utils/bundled_inputs.py', | ||
| 1406 | + 'torch/utils/checkpoint.py', | ||
| 1407 | + 'torch/utils/collect_env.py', | ||
| 1408 | + 'torch/utils/cpp_backtrace.py', | ||
| 1409 | + 'torch/utils/cpp_extension.py', | ||
| 1410 | + 'torch/utils/dlpack.py', | ||
| 1411 | + 'torch/utils/file_baton.py', | ||
| 1412 | + 'torch/utils/flop_counter.py', | ||
| 1413 | + 'torch/utils/hipify/__init__.py', | ||
| 1414 | + 'torch/utils/hipify/constants.py', | ||
| 1415 | + 'torch/utils/hipify/cuda_to_hip_mappings.py', | ||
| 1416 | + 'torch/utils/hipify/hipify_python.py', | ||
| 1417 | + 'torch/utils/hipify/version.py', | ||
| 1418 | + 'torch/utils/hooks.py', | ||
| 1419 | + 'torch/utils/jit/__init__.py', | ||
| 1420 | + 'torch/utils/jit/log_extract.py', | ||
| 1421 | + 'torch/utils/mkldnn.py', | ||
| 1422 | + 'torch/utils/mobile_optimizer.py', | ||
| 1423 | + 'torch/utils/model_dump/__init__.py', | ||
| 1424 | + 'torch/utils/model_dump/__main__.py', | ||
| 1425 | + 'torch/utils/model_zoo.py', | ||
| 1426 | + 'torch/utils/show_pickle.py', | ||
| 1427 | + 'torch/utils/tensorboard/__init__.py', | ||
| 1428 | + 'torch/utils/tensorboard/_caffe2_graph.py', | ||
| 1429 | + 'torch/utils/tensorboard/_convert_np.py', | ||
| 1430 | + 'torch/utils/tensorboard/_embedding.py', | ||
| 1431 | + 'torch/utils/tensorboard/_onnx_graph.py', | ||
| 1432 | + 'torch/utils/tensorboard/_proto_graph.py', | ||
| 1433 | + 'torch/utils/tensorboard/_pytorch_graph.py', | ||
| 1434 | + 'torch/utils/tensorboard/_utils.py', | ||
| 1435 | + 'torch/utils/tensorboard/summary.py', | ||
| 1436 | + 'torch/utils/tensorboard/writer.py', | ||
| 1437 | + 'torch/utils/throughput_benchmark.py', | ||
| 1438 | + 'torch/utils/viz/__init__.py', | ||
| 1439 | + 'torch/utils/viz/_cycles.py', | ||
| 1440 | +] | ||
| 1441 | +is_formatter = true | ||
| 1442 | + | ||
| 1443 | +[[linter]] | ||
| 1444 | +code = 'PYPROJECT' | ||
| 1445 | +command = [ | ||
| 1446 | + 'uv', | ||
| 1447 | + 'run', | ||
| 1448 | + '--script', | ||
| 1449 | + 'tools/linter/adapters/pyproject_linter.py', | ||
| 1450 | + '--', | ||
| 1451 | + '@{{PATHSFILE}}' | ||
| 1452 | +] | ||
| 1453 | +include_patterns = [ | ||
| 1454 | + "**/pyproject.toml", | ||
| 1455 | +] | ||
| 1456 | + | ||
| 1457 | +[[linter]] | ||
| 1458 | +code = 'CMAKE_MINIMUM_REQUIRED' | ||
| 1459 | +command = [ | ||
| 1460 | + 'uv', | ||
| 1461 | + 'run', | ||
| 1462 | + '--script', | ||
| 1463 | + 'tools/linter/adapters/cmake_minimum_required_linter.py', | ||
| 1464 | + '--', | ||
| 1465 | + '@{{PATHSFILE}}' | ||
| 1466 | +] | ||
| 1467 | +include_patterns = [ | ||
| 1468 | + "**/pyproject.toml", | ||
| 1469 | + "**/CMakeLists.txt", | ||
| 1470 | + "**/CMakeLists.txt.in", | ||
| 1471 | + "**/*.cmake", | ||
| 1472 | + "**/*.cmake.in", | ||
| 1473 | + "**/*requirements*.txt", | ||
| 1474 | + "**/*requirements*.in", | ||
| 1475 | +] | ||
| 1476 | + | ||
| 1477 | +[[linter]] | ||
| 1478 | +code = 'COPYRIGHT' | ||
| 1479 | +include_patterns = ['**'] | ||
| 1480 | +exclude_patterns = [ | ||
| 1481 | + '.lintrunner.toml', | ||
| 1482 | + 'fb/**', | ||
| 1483 | + '**/fb/**', | ||
| 1484 | +] | ||
| 1485 | +command = [ | ||
| 1486 | + 'python3', | ||
| 1487 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1488 | + '--pattern=Confidential and proprietary', | ||
| 1489 | + '--linter-name=COPYRIGHT', | ||
| 1490 | + '--error-name=Confidential Code', | ||
| 1491 | + """--error-description=\ | ||
| 1492 | + Proprietary and confidential source code\ | ||
| 1493 | + should not be contributed to PyTorch codebase\ | ||
| 1494 | + """, | ||
| 1495 | + '--', | ||
| 1496 | + '@{{PATHSFILE}}' | ||
| 1497 | +] | ||
| 1498 | + | ||
| 1499 | +[[linter]] | ||
| 1500 | +code = 'BAZEL_LINTER' | ||
| 1501 | +include_patterns = ['WORKSPACE'] | ||
| 1502 | +command = [ | ||
| 1503 | + 'python3', | ||
| 1504 | + 'tools/linter/adapters/bazel_linter.py', | ||
| 1505 | + '--binary=.lintbin/bazel', | ||
| 1506 | + '--', | ||
| 1507 | + '@{{PATHSFILE}}' | ||
| 1508 | +] | ||
| 1509 | +init_command = [ | ||
| 1510 | + 'python3', | ||
| 1511 | + 'tools/linter/adapters/s3_init.py', | ||
| 1512 | + '--config-json=tools/linter/adapters/s3_init_config.json', | ||
| 1513 | + '--linter=bazel', | ||
| 1514 | + '--dry-run={{DRYRUN}}', | ||
| 1515 | + '--output-dir=.lintbin', | ||
| 1516 | + '--output-name=bazel', | ||
| 1517 | +] | ||
| 1518 | +is_formatter = true | ||
| 1519 | + | ||
| 1520 | +[[linter]] | ||
| 1521 | +code = 'LINTRUNNER_VERSION' | ||
| 1522 | +include_patterns = ['**'] | ||
| 1523 | +exclude_patterns = [ | ||
| 1524 | + 'fb/**', | ||
| 1525 | + '**/fb/**', | ||
| 1526 | +] | ||
| 1527 | +command = [ | ||
| 1528 | + 'python3', | ||
| 1529 | + 'tools/linter/adapters/lintrunner_version_linter.py' | ||
| 1530 | +] | ||
| 1531 | + | ||
| 1532 | +[[linter]] | ||
| 1533 | +code = 'RUFF' | ||
| 1534 | +include_patterns = [ | ||
| 1535 | + '**/*.py', | ||
| 1536 | + '**/*.pyi', | ||
| 1537 | + '**/*.ipynb', | ||
| 1538 | + 'pyproject.toml', | ||
| 1539 | +] | ||
| 1540 | +exclude_patterns = [ | ||
| 1541 | + 'caffe2/**', | ||
| 1542 | + 'functorch/docs/**', | ||
| 1543 | + 'torch/_inductor/fx_passes/serialized_patterns/**', | ||
| 1544 | + 'torch/_inductor/autoheuristic/artifacts/**', | ||
| 1545 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**', | ||
| 1546 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py', | ||
| 1547 | + 'test/dynamo/cpython/**', | ||
| 1548 | + 'test/test_torchfuzz_repros.py', | ||
| 1549 | + 'scripts/**', | ||
| 1550 | + 'third_party/**', | ||
| 1551 | + 'fb/**', | ||
| 1552 | + '**/fb/**', | ||
| 1553 | +] | ||
| 1554 | +command = [ | ||
| 1555 | + 'uv', | ||
| 1556 | + 'run', | ||
| 1557 | + '--script', | ||
| 1558 | + 'tools/linter/adapters/ruff_linter.py', | ||
| 1559 | + '--config=pyproject.toml', | ||
| 1560 | + '--show-disable', | ||
| 1561 | + '--', | ||
| 1562 | + '@{{PATHSFILE}}' | ||
| 1563 | +] | ||
| 1564 | +is_formatter = true | ||
| 1565 | + | ||
| 1566 | +# This linter prevents merge conflicts in csv files in pytorch by enforcing | ||
| 1567 | +# three lines of whitespace between entries such that unless people are modifying | ||
| 1568 | +# the same line, merge conflicts should not arise in git or hg | ||
| 1569 | +[[linter]] | ||
| 1570 | +code = 'MERGE_CONFLICTLESS_CSV' | ||
| 1571 | +include_patterns = [ | ||
| 1572 | + 'benchmarks/dynamo/ci_expected_accuracy/*.csv', | ||
| 1573 | + 'benchmarks/dynamo/pr_time_benchmarks/expected_results.csv', | ||
| 1574 | +] | ||
| 1575 | +command = [ | ||
| 1576 | + 'python3', | ||
| 1577 | + 'tools/linter/adapters/no_merge_conflict_csv_linter.py', | ||
| 1578 | + '--', | ||
| 1579 | + '@{{PATHSFILE}}' | ||
| 1580 | +] | ||
| 1581 | +is_formatter = true | ||
| 1582 | + | ||
| 1583 | + | ||
| 1584 | +[[linter]] | ||
| 1585 | +code = 'META_NO_CREATE_UNBACKED' | ||
| 1586 | +include_patterns = [ | ||
| 1587 | + "torch/_meta_registrations.py" | ||
| 1588 | +] | ||
| 1589 | +command = [ | ||
| 1590 | + 'python3', | ||
| 1591 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1592 | + '--pattern=create_unbacked', | ||
| 1593 | + '--linter-name=META_NO_CREATE_UNBACKED', | ||
| 1594 | + '--error-name=no create_unbacked in meta registrations', | ||
| 1595 | + """--error-description=\ | ||
| 1596 | + Data-dependent operators should have their meta \ | ||
| 1597 | + registration in torch/_subclasses/fake_impls.py, \ | ||
| 1598 | + not torch/_meta_registrations.py | ||
| 1599 | + """, | ||
| 1600 | + '--', | ||
| 1601 | + '@{{PATHSFILE}}' | ||
| 1602 | +] | ||
| 1603 | + | ||
| 1604 | +[[linter]] | ||
| 1605 | +code = 'ATEN_CPU_GPU_AGNOSTIC' | ||
| 1606 | +include_patterns = [ | ||
| 1607 | + # aten source | ||
| 1608 | + "aten/src/ATen/*.cpp", | ||
| 1609 | + "aten/src/ATen/cpu/*.cpp", | ||
| 1610 | + "aten/src/ATen/functorch/**/*.cpp", | ||
| 1611 | + "aten/src/ATen/nnapi/*.cpp", | ||
| 1612 | + "aten/src/ATen/quantized/*.cpp", | ||
| 1613 | + "aten/src/ATen/vulkan/*.cpp", | ||
| 1614 | + "aten/src/ATen/metal/*.cpp", | ||
| 1615 | + "aten/src/ATen/detail/CPUGuardImpl.cpp", | ||
| 1616 | + "aten/src/ATen/detail/MetaGuardImpl.cpp", | ||
| 1617 | + # aten native source | ||
| 1618 | + "aten/src/ATen/native/cpu/*.cpp", | ||
| 1619 | + "aten/src/ATen/native/ao_sparse/cpu/kernels/*.cpp", | ||
| 1620 | + "aten/src/ATen/native/ao_sparse/quantized/cpu/kernels/*.cpp", | ||
| 1621 | + "aten/src/ATen/native/quantized/cpu/kernels/*.cpp", | ||
| 1622 | + "aten/src/ATen/native/*.cpp", | ||
| 1623 | + "aten/src/ATen/native/cpu/**/*.cpp", | ||
| 1624 | + "aten/src/ATen/native/ao_sparse/*.cpp", | ||
| 1625 | + "aten/src/ATen/native/ao_sparse/**/*.cpp", | ||
| 1626 | + "aten/src/ATen/native/ao_sparse/quantized/*.cpp", | ||
| 1627 | + "aten/src/ATen/native/ao_sparse/quantized/**/*.cpp", | ||
| 1628 | + "aten/src/ATen/native/nested/*.cpp", | ||
| 1629 | + "aten/src/ATen/native/quantized/*.cpp", | ||
| 1630 | + "aten/src/ATen/native/quantized/**/*.cpp", | ||
| 1631 | + "aten/src/ATen/native/sparse/*.cpp", | ||
| 1632 | + "aten/src/ATen/native/transformers/*.cpp", | ||
| 1633 | + "aten/src/ATen/native/utils/*.cpp", | ||
| 1634 | + "aten/src/ATen/native/xnnpack/*.cpp", | ||
| 1635 | + "aten/src/ATen/native/metal/MetalPrepackOpRegister.cpp", | ||
| 1636 | + # aten headers | ||
| 1637 | + "aten/src/ATen/*.h", | ||
| 1638 | + "aten/src/ATen/functorch/**/*.h", | ||
| 1639 | + "aten/src/ATen/ops/*.h", | ||
| 1640 | + "aten/src/ATen/cpu/**/*.h", | ||
| 1641 | + "aten/src/ATen/nnapi/*.h", | ||
| 1642 | + "aten/src/ATen/quantized/*.h", | ||
| 1643 | + "aten/src/ATen/vulkan/*.h", | ||
| 1644 | + "aten/src/ATen/metal/*.h", | ||
| 1645 | + "aten/src/ATen/mps/*.h", | ||
| 1646 | + # aten native headers | ||
| 1647 | + "aten/src/ATen/native/*.h", | ||
| 1648 | + "aten/src/ATen/native/cpu/**/*.h", | ||
| 1649 | + "aten/src/ATen/native/nested/*.h", | ||
| 1650 | + "aten/src/ATen/native/sparse/*.h", | ||
| 1651 | + "aten/src/ATen/native/ao_sparse/*.h", | ||
| 1652 | + "aten/src/ATen/native/ao_sparse/cpu/*.h", | ||
| 1653 | + "aten/src/ATen/native/ao_sparse/quantized/*.h", | ||
| 1654 | + "aten/src/ATen/native/ao_sparse/quantized/cpu/*.h", | ||
| 1655 | + "aten/src/ATen/native/quantized/*.h", | ||
| 1656 | + "aten/src/ATen/native/quantized/cpu/*.h", | ||
| 1657 | + "aten/src/ATen/native/transformers/*.h", | ||
| 1658 | + "aten/src/ATen/native/quantized/cpu/qnnpack/include/*.h", | ||
| 1659 | + "aten/src/ATen/native/utils/*.h", | ||
| 1660 | + "aten/src/ATen/native/vulkan/ops/*.h", | ||
| 1661 | + "aten/src/ATen/native/xnnpack/*.h", | ||
| 1662 | + "aten/src/ATen/native/metal/MetalPrepackOpContext.h", | ||
| 1663 | + "aten/src/ATen/native/mps/Copy.h", | ||
| 1664 | + "aten/src/ATen/native/mkldnn/**/*.h", | ||
| 1665 | +] | ||
| 1666 | +exclude_patterns = [ | ||
| 1667 | + "aten/src/ATen/Context.h", | ||
| 1668 | + "aten/src/ATen/Context.cpp", | ||
| 1669 | + "aten/src/ATen/DLConvertor.cpp", | ||
| 1670 | + "aten/src/ATen/core/Array.h", | ||
| 1671 | + "aten/src/ATen/native/quantized/ConvUtils.h", | ||
| 1672 | + "aten/src/ATen/native/sparse/SparseBlasImpl.cpp", # triton implementation | ||
| 1673 | + "aten/src/ATen/native/transformers/attention.cpp", | ||
| 1674 | + "aten/src/ATen/native/**/cudnn/**", # cudnn is cuda specific | ||
| 1675 | +] | ||
| 1676 | +command = [ | ||
| 1677 | + 'python3', | ||
| 1678 | + 'tools/linter/adapters/grep_linter.py', | ||
| 1679 | + '--pattern=(^#if.*USE_ROCM.*)|(^#if.*USE_CUDA.*)', | ||
| 1680 | + '--linter-name=ATEN_CPU', | ||
| 1681 | + '--error-name=aten-cpu should be gpu agnostic', | ||
| 1682 | + """--error-description=\ | ||
| 1683 | + We strongly discourage the compile-time divergence \ | ||
| 1684 | + on ATen-CPU code for different GPU code. This \ | ||
| 1685 | + disallows sharing the same aten-cpu shared object \ | ||
| 1686 | + between different GPU backends \ | ||
| 1687 | + """, | ||
| 1688 | + '--', | ||
| 1689 | + '@{{PATHSFILE}}' | ||
| 1690 | +] | ||
| 1691 | +is_formatter = true | ||
| 1692 | + | ||
| 1693 | +# `set_linter` detects occurrences of built-in `set` in areas of Python code like | ||
| 1694 | +# _inductor where the instability of iteration in `set` has proven a problem. | ||
| 1695 | + | ||
| 1696 | +[[linter]] | ||
| 1697 | +code = 'SET_LINTER' | ||
| 1698 | +command = [ | ||
| 1699 | + 'python3', | ||
| 1700 | + 'tools/linter/adapters/set_linter.py', | ||
| 1701 | + '--lintrunner', | ||
| 1702 | + '--', | ||
| 1703 | + '@{{PATHSFILE}}' | ||
| 1704 | +] | ||
| 1705 | +include_patterns = [ | ||
| 1706 | + "torch/_inductor/**/*.py", | ||
| 1707 | + "torch/_functorch/partitioners.py", | ||
| 1708 | +] | ||
| 1709 | +is_formatter = true | ||
| 1710 | + | ||
| 1711 | +# `docstring_linter` reports on long Python classes, methods, and functions | ||
| 1712 | +# whose definitions have very small docstrings or none at all. | ||
| 1713 | +# | ||
| 1714 | +[[linter]] | ||
| 1715 | +code = 'DOCSTRING_LINTER' | ||
| 1716 | +command = [ | ||
| 1717 | + 'python3', | ||
| 1718 | + 'tools/linter/adapters/docstring_linter.py', | ||
| 1719 | + '--lintrunner', | ||
| 1720 | + '--', | ||
| 1721 | + '@{{PATHSFILE}}' | ||
| 1722 | +] | ||
| 1723 | +include_patterns = [ | ||
| 1724 | + 'torch/_inductor/**/*.py' | ||
| 1725 | +] | ||
| 1726 | +exclude_patterns = [ | ||
| 1727 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**', | ||
| 1728 | + 'torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py', | ||
| 1729 | +] | ||
| 1730 | +is_formatter = false | ||
| 1731 | + | ||
| 1732 | +# `import_linter` reports on importing disallowed third party libraries. | ||
| 1733 | +[[linter]] | ||
| 1734 | +code = 'IMPORT_LINTER' | ||
| 1735 | +command = [ | ||
| 1736 | + 'python3', | ||
| 1737 | + 'tools/linter/adapters/import_linter.py', | ||
| 1738 | + '--', | ||
| 1739 | + '@{{PATHSFILE}}' | ||
| 1740 | +] | ||
| 1741 | +include_patterns = [ | ||
| 1742 | + 'torch/_dynamo/**', | ||
| 1743 | +] | ||
| 1744 | +is_formatter = false | ||
| 1745 | + | ||
| 1746 | +[[linter]] | ||
| 1747 | +code = 'TEST_DEVICE_BIAS' | ||
| 1748 | +command = [ | ||
| 1749 | + 'python3', | ||
| 1750 | + 'tools/linter/adapters/test_device_bias_linter.py', | ||
| 1751 | + '--', | ||
| 1752 | + '@{{PATHSFILE}}', | ||
| 1753 | +] | ||
| 1754 | +include_patterns = [ | ||
| 1755 | + 'test/**/test_*.py', | ||
| 1756 | +] | ||
| 1757 | +exclude_patterns = [ | ||
| 1758 | + # CPython tests | ||
| 1759 | + 'test/dynamo/cpython/**', | ||
| 1760 | +] | ||
| 1761 | + | ||
| 1762 | +# 'header_only_linter' reports on properly testing header-only APIs. | ||
| 1763 | +[[linter]] | ||
| 1764 | +code = 'HEADER_ONLY_LINTER' | ||
| 1765 | +command = [ | ||
| 1766 | + 'python3', | ||
| 1767 | + 'tools/linter/adapters/header_only_linter.py', | ||
| 1768 | +] | ||
| 1769 | +include_patterns = [ | ||
| 1770 | + 'torch/header_only_apis.txt', | ||
| 1771 | +] | ||
| 1772 | +is_formatter = false | ||
| 1773 | + | ||
| 1774 | + | ||
| 1775 | +[[linter]] | ||
| 1776 | +code = "GB_REGISTRY" | ||
| 1777 | +include_patterns = ["torch/_dynamo/**/*.py"] | ||
| 1778 | +command = [ | ||
| 1779 | + "python3", | ||
| 1780 | + "tools/linter/adapters/gb_registry_linter.py", | ||
| 1781 | +] | ||
| 1782 | + | ||
| 1783 | +[[linter]] | ||
| 1784 | +code = 'STABLE_SHIM_VERSION' | ||
| 1785 | +include_patterns = [ | ||
| 1786 | + 'torch/csrc/stable/c/shim.h', | ||
| 1787 | + 'torch/csrc/inductor/aoti_torch/c/shim.h' | ||
| 1788 | +] | ||
| 1789 | +command = [ | ||
| 1790 | + 'python3', | ||
| 1791 | + 'tools/linter/adapters/stable_shim_version_linter.py', | ||
| 1792 | + '--', | ||
| 1793 | + '@{{PATHSFILE}}' | ||
| 1794 | +] | ||
| 1795 | + | ||
| 1796 | +[[linter]] | ||
| 1797 | +code = 'STABLE_SHIM_USAGE' | ||
| 1798 | +include_patterns = [ | ||
| 1799 | + 'torch/csrc/stable/**/*.h', | ||
| 1800 | +] | ||
| 1801 | +command = [ | ||
| 1802 | + 'python3', | ||
| 1803 | + 'tools/linter/adapters/stable_shim_usage_linter.py', | ||
| 1804 | + '--', | ||
| 1805 | + '@{{PATHSFILE}}' | ||
| 1806 | +] | ||
| @@ -10,7 +10,7 @@ | |||
| 10 | - 审查Pull Request并协助其他贡献者。 | 10 | - 审查Pull Request并协助其他贡献者。 |
| 11 | - 传播项目:在博客文章、社交媒体上分享PyTorch，或给仓库点个⭐。 | 11 | - 传播项目:在博客文章、社交媒体上分享PyTorch，或给仓库点个⭐。 |
| 12 | 12 | ||
| 13 | -## 寻找可贡献的问题 | 13 | +## 寻找可贡献的问题 |
| 14 | 14 | ||
| 15 | 您可以通过查看[Issues列表](https://gitcode.com/Ascend/pytorch/issues)了解项目的发展计划和路线图。 | 15 | 您可以通过查看[Issues列表](https://gitcode.com/Ascend/pytorch/issues)了解项目的发展计划和路线图。 |
| 16 | 16 | ||
| @@ -37,16 +37,16 @@ cd pytorch | |||
| 37 | 37 | ||
| 38 | ``` | 38 | ``` |
| 39 | 3. 在个人仓库进行代码开发 | 39 | 3. 在个人仓库进行代码开发 |
| 40 | - 代码开发请遵循 **[代码规范](#代码规范)** | 40 | + 代码开发请遵循 **[代码规范](#代码规范)** |
| 41 | 41 | ||
| 42 | 4. 代码测试 | 42 | 4. 代码测试 |
| 43 | - 参见 **[代码测试](https://gitcode.com/Ascend/pytorch/blob/master/test/README.md)** | 43 | + 参见 **[代码测试](https://gitcode.com/Ascend/pytorch/blob/master/test/README.md)** |
| 44 | 44 | ||
| 45 | -5. **[门禁异常处理](#门禁异常处理)** | 45 | +5. **[门禁异常处理](#门禁异常处理)** |
| 46 | 46 | ||
| 47 | -6. **[提交Pull Request](#提交Pull-Request)** | 47 | +6. **[提交Pull Request](#提交Pull-Request)** |
| 48 | 48 | ||
| 49 | -7. **[报告问题](#报告问题)** | 49 | +7. **[报告问题](#报告问题)** |
| 50 | 50 | ||
| 51 | #### 代码规范 | 51 | #### 代码规范 |
| 52 | 52 | ||
| @@ -54,7 +54,7 @@ cd pytorch | |||
| 54 | 54 | ||
| 55 | - 编码指南 | 55 | - 编码指南 |
| 56 | 56 | ||
| 57 | - 请在PyTorch社区使用规统一的编码分格，python建议的编码风格是[PEP 8编码样式](https://pep8.org/)，C++编码所建议的风格是 [Google C++编码指南](http://google.github.io/styleguide/cppguide.html) 。可以使用[CppLint](https://github.com/cpplint/cpplint)，[CppCheck](http://cppcheck.sourceforge.net/)，[CMakeLint](https://github.com/cmake-lint/cmake-lint)，[CodeSpell](https://github.com/codespell-project/codespell)， [Lizard](http://www.lizard.ws/)，[ShellCheck](https://github.com/koalaman/shellcheck)和[pylint](https://pylint.org/)检查代码的格式，建议在您的IDE中安装这些插件。 | 57 | + 请在PyTorch社区使用规统一的编码分格，python建议的编码风格是[PEP 8编码样式](https://pep8.org/)，C++编码所建议的风格是 [Google C++编码指南](http://google.github.io/styleguide/cppguide.html) 。执行代码检查，可参照[本地静态检查](#本地静态检查)。 |
| 58 | 58 | ||
| 59 | - 单元测试指南 | 59 | - 单元测试指南 |
| 60 | 60 | ||
| @@ -115,7 +115,7 @@ cd pytorch | |||
| 115 | - 功能验证 | 115 | - 功能验证 |
| 116 | - CheckList | 116 | - CheckList |
| 117 | 确认信息完整准确后提交Pull Request，等待代码审查 | 117 | 确认信息完整准确后提交Pull Request，等待代码审查 |
| 118 | - | 118 | + |
| 119 | #### 报告问题 | 119 | #### 报告问题 |
| 120 | 120 | ||
| 121 | 为项目做出贡献的一个好方法是在遇到问题时发送详细报告。我们总是很感激写得很好、彻底的错误报告，并会由此感谢您！ | 121 | 为项目做出贡献的一个好方法是在遇到问题时发送详细报告。我们总是很感激写得很好、彻底的错误报告，并会由此感谢您！ |
| @@ -135,6 +135,59 @@ cd pytorch | |||
| 135 | - 如果您发现一个未解决的问题，而这正是您要解决的问题，请对该问题发表一些评论，告诉其他人您将负责它。 | 135 | - 如果您发现一个未解决的问题，而这正是您要解决的问题，请对该问题发表一些评论，告诉其他人您将负责它。 |
| 136 | - 如果问题已打开一段时间，建议贡献者在解决该问题之前进行预检查。 | 136 | - 如果问题已打开一段时间，建议贡献者在解决该问题之前进行预检查。 |
| 137 | - 如果您解决了自己报告的问题，则还需要在关闭该问题之前让其他人知道。 | 137 | - 如果您解决了自己报告的问题，则还需要在关闭该问题之前让其他人知道。 |
| 138 | + | ||
| 139 | +### 本地静态检查 | ||
| 140 | + | ||
| 141 | +项目使用 [lintrunner](https://github.com/suo/lintrunner) 进行静态检查，支持在本地运行与 CI 完全一致的检查项， | ||
| 142 | +包括 Python 代码风格（Flake8、Ruff、PYFMT）、C++ 格式（ClangFormat、ClangTidy）、拼写检查（Codespell）等。 | ||
| 143 | + | ||
| 144 | +#### 安装依赖 | ||
| 145 | + | ||
| 146 | +```bash | ||
| 147 | +# 安装 lintrunner 及 uv（部分 linter 需要） | ||
| 148 | +pip install lintrunner | ||
| 149 | +pip install uv | ||
| 150 | +``` | ||
| 151 | + | ||
| 152 | +#### 初始化（首次使用或更新时执行一次） | ||
| 153 | + | ||
| 154 | +```bash | ||
| 155 | +# 下载 lintrunner 所需的外部二进制工具（clang-format、clang-tidy 等） | ||
| 156 | +lintrunner init | ||
| 157 | +``` | ||
| 158 | + | ||
| 159 | +#### 执行静态检查 | ||
| 160 | + | ||
| 161 | +```bash | ||
| 162 | +# 检查当前工作区改动和HEAD提交的文件增量（工作区 + HEAD） | ||
| 163 | +lintrunner | ||
| 164 | + | ||
| 165 | +# 仅运行指定检查项 | ||
| 166 | +lintrunner --take FLAKE8,RUFF,PYFMT,SPACES,TABS,NEWLINE | ||
| 167 | + | ||
| 168 | +# 自动修复可自动修复的问题（formatter 类 linter, 如忽略PYREFLY） | ||
| 169 | +lintrunner --skip PYREFLY -a | ||
| 170 | + | ||
| 171 | +# 仅检查当前工作区改动的文件增量 | ||
| 172 | +git diff --name-only HEAD | xargs lintrunner | ||
| 173 | +``` | ||
| 174 | + | ||
| 175 | +> **提示**：`--take` 参数可指定只运行部分检查项，常用项如下： | ||
| 176 | +> | ||
| 177 | +> | 代码 | 说明 | | ||
| 178 | +> |------|------| | ||
| 179 | +> | `FLAKE8` | Python 语法与风格检查 | | ||
| 180 | +> | `RUFF` | Python 快速 lint 与 import 排序 | | ||
| 181 | +> | `PYFMT` | Python 代码格式化（usort + ruff-format） | | ||
| 182 | +> | `CLANGFORMAT` | C++ 代码格式化 | | ||
| 183 | +> | `CLANGTIDY` | C++ 静态分析 | | ||
| 184 | +> | `SPACES` | 行尾空格检查 | | ||
| 185 | +> | `TABS` | Tab 字符检查 | | ||
| 186 | +> | `NEWLINE` | 文件末尾换行检查 | | ||
| 187 | +> | `CODESPELL` | 拼写检查，如果是误报可以将误报词按照字典序添加至 `tools/linter/dictionary.txt` 后再重新检查 | | ||
| 188 | + | ||
| 189 | +更多执行命令可参照[lintrunner wiki](https://github.com/pytorch/pytorch/wiki/lintrunner)。 | ||
| 190 | + | ||
| 138 | ## 社区准则 | 191 | ## 社区准则 |
| 139 | 192 | ||
| 140 | ### 行为准则 | 193 | ### 行为准则 |
| @@ -150,5 +203,3 @@ cd pytorch | |||
| 150 | 203 | ||
| 151 | - **Issues**:用于报告Bug、提出功能建议和讨论技术问题 | 204 | - **Issues**:用于报告Bug、提出功能建议和讨论技术问题 |
| 152 | - **Pull Requests**:用于代码审查和讨论具体实现 | 205 | - **Pull Requests**:用于代码审查和讨论具体实现 |
| 153 | - | ||
| 154 | - | ||
| @@ -0,0 +1,65 @@ | |||
| 1 | +# This is the PyTorch mypy-strict.ini file (note: don't change this line! - | ||
| 2 | +# test_run_mypy in test/test_type_hints.py uses this string) | ||
| 3 | + | ||
| 4 | +# Unlike mypy.ini, it enforces very strict typing rules. The intention is for | ||
| 5 | +# this config file to be used to ENFORCE that people are using mypy on codegen | ||
| 6 | +# files. | ||
| 7 | + | ||
| 8 | +[mypy] | ||
| 9 | +python_version = 3.10 | ||
| 10 | +plugins = mypy_plugins/check_mypy_version.py, numpy.typing.mypy_plugin | ||
| 11 | + | ||
| 12 | +cache_dir = .mypy_cache/strict | ||
| 13 | +allow_redefinition = True | ||
| 14 | +strict_optional = True | ||
| 15 | +show_error_codes = True | ||
| 16 | +show_column_numbers = True | ||
| 17 | +warn_no_return = True | ||
| 18 | +disallow_any_unimported = True | ||
| 19 | + | ||
| 20 | +strict = True | ||
| 21 | +implicit_reexport = False | ||
| 22 | + | ||
| 23 | +# do not re-enable this: | ||
| 24 | +# https://github.com/pytorch/pytorch/pull/60006#issuecomment-866130657 | ||
| 25 | +warn_unused_ignores = False | ||
| 26 | + | ||
| 27 | +files = | ||
| 28 | + .github, | ||
| 29 | + benchmarks/instruction_counts, | ||
| 30 | + tools, | ||
| 31 | + torch/profiler/_memory_profiler.py, | ||
| 32 | + torch/utils/_pytree.py, | ||
| 33 | + torch/utils/_cxx_pytree.py, | ||
| 34 | + torch/utils/benchmark/utils/common.py, | ||
| 35 | + torch/utils/benchmark/utils/timer.py, | ||
| 36 | + torch/utils/benchmark/utils/valgrind_wrapper | ||
| 37 | + | ||
| 38 | +# Specifically enable imports of benchmark utils. As more of `torch` becomes | ||
| 39 | +# strict compliant, those modules can be enabled as well. | ||
| 40 | +[mypy-torch.utils.benchmark.utils.*] | ||
| 41 | +follow_imports = normal | ||
| 42 | + | ||
| 43 | +# Don't follow imports as much of `torch` is not strict compliant. | ||
| 44 | +[mypy-torch] | ||
| 45 | +follow_imports = skip | ||
| 46 | + | ||
| 47 | +[mypy-torch.*] | ||
| 48 | +follow_imports = skip | ||
| 49 | + | ||
| 50 | +# Missing stubs. | ||
| 51 | + | ||
| 52 | +[mypy-numpy] | ||
| 53 | +ignore_missing_imports = True | ||
| 54 | + | ||
| 55 | +[mypy-sympy] | ||
| 56 | +ignore_missing_imports = True | ||
| 57 | + | ||
| 58 | +[mypy-sympy.*] | ||
| 59 | +ignore_missing_imports = True | ||
| 60 | + | ||
| 61 | +[mypy-mypy.*] | ||
| 62 | +ignore_missing_imports = True | ||
| 63 | + | ||
| 64 | +[mypy-usort.*] | ||
| 65 | +ignore_missing_imports = True | ||
| @@ -0,0 +1,311 @@ | |||
| 1 | +# This is the PyTorch mypy.ini file (note: don't change this line! - | ||
| 2 | +# test_run_mypy in test/test_type_hints.py uses this string) | ||
| 3 | + | ||
| 4 | +[mypy] | ||
| 5 | +plugins = mypy_plugins/check_mypy_version.py, mypy_plugins/sympy_mypy_plugin.py | ||
| 6 | + | ||
| 7 | +cache_dir = .mypy_cache/normal | ||
| 8 | +allow_redefinition = True | ||
| 9 | +warn_unused_configs = True | ||
| 10 | +warn_redundant_casts = True | ||
| 11 | +show_error_codes = True | ||
| 12 | +show_column_numbers = True | ||
| 13 | +check_untyped_defs = True | ||
| 14 | +disallow_untyped_defs = True | ||
| 15 | +disallow_untyped_decorators = True | ||
| 16 | +follow_imports = normal | ||
| 17 | +local_partial_types = True | ||
| 18 | +enable_error_code = possibly-undefined | ||
| 19 | + | ||
| 20 | +# do not re-enable this: | ||
| 21 | +# https://github.com/pytorch/pytorch/pull/60006#issuecomment-866130657 | ||
| 22 | +warn_unused_ignores = False | ||
| 23 | + | ||
| 24 | +# | ||
| 25 | +# Note: test/ still has syntax errors so can't be added | ||
| 26 | +# | ||
| 27 | +# Typing tests is low priority, but enabling type checking on the | ||
| 28 | +# untyped test functions (using `--check-untyped-defs`) is still | ||
| 29 | +# high-value because it helps test the typing. | ||
| 30 | +# | ||
| 31 | + | ||
| 32 | +files = | ||
| 33 | + torch, | ||
| 34 | + caffe2, | ||
| 35 | + test/test_bundled_images.py, | ||
| 36 | + test/test_bundled_inputs.py, | ||
| 37 | + test/test_complex.py, | ||
| 38 | + test/test_datapipe.py, | ||
| 39 | + test/test_futures.py, | ||
| 40 | + test/test_numpy_interop.py, | ||
| 41 | + test/test_torch.py, | ||
| 42 | + test/test_type_hints.py, | ||
| 43 | + test/test_type_info.py, | ||
| 44 | + test/test_utils.py | ||
| 45 | + | ||
| 46 | +# | ||
| 47 | +# `exclude` is a regex, not a list of paths like `files` (sigh) | ||
| 48 | +# | ||
| 49 | +exclude = torch/include/|torch/csrc/|torch/distributed/elastic/agent/server/api.py|torch/testing/_internal|torch/distributed/fsdp/fully_sharded_data_parallel.py | ||
| 50 | + | ||
| 51 | +python_version = 3.11 | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +# | ||
| 55 | +# Extension modules without stubs. | ||
| 56 | +# | ||
| 57 | + | ||
| 58 | +[mypy-torch.for_onnx.onnx] | ||
| 59 | +ignore_missing_imports = True | ||
| 60 | + | ||
| 61 | +[mypy-torch.ao.quantization.experimental.apot_utils] | ||
| 62 | +ignore_missing_imports = True | ||
| 63 | + | ||
| 64 | +[mypy-torch.ao.quantization.experimental.quantizer] | ||
| 65 | +ignore_missing_imports = True | ||
| 66 | + | ||
| 67 | +[mypy-torch.ao.quantization.experimental.observer] | ||
| 68 | +ignore_missing_imports = True | ||
| 69 | + | ||
| 70 | +[mypy-torch.ao.quantization.experimental.APoT_tensor] | ||
| 71 | +ignore_missing_imports = True | ||
| 72 | + | ||
| 73 | +[mypy-torch.ao.quantization.experimental.fake_quantize_function] | ||
| 74 | +ignore_missing_imports = True | ||
| 75 | + | ||
| 76 | +[mypy-torch.ao.quantization.experimental.fake_quantize] | ||
| 77 | +ignore_missing_imports = True | ||
| 78 | + | ||
| 79 | +[mypy-torch.ao.quantization.pt2e._affine_quantization] | ||
| 80 | +ignore_errors = True | ||
| 81 | + | ||
| 82 | +# | ||
| 83 | +# Files with various errors. Mostly real errors, possibly some false | ||
| 84 | +# positives as well. | ||
| 85 | +# | ||
| 86 | + | ||
| 87 | +[mypy-test_torch] | ||
| 88 | +check_untyped_defs = False | ||
| 89 | + | ||
| 90 | +# Excluded from mypy due to OpInfos being annoying to type | ||
| 91 | +[mypy-torch.testing._internal.common_methods_invocations.*] | ||
| 92 | +ignore_errors = True | ||
| 93 | + | ||
| 94 | +[mypy-torch.testing._internal.hypothesis_utils.*] | ||
| 95 | +ignore_errors = True | ||
| 96 | + | ||
| 97 | +[mypy-torch.testing._internal.common_quantization.*] | ||
| 98 | +ignore_errors = True | ||
| 99 | + | ||
| 100 | +[mypy-torch.testing._internal.generated.*] | ||
| 101 | +ignore_errors = True | ||
| 102 | + | ||
| 103 | +[mypy-torch.testing._internal.distributed.*] | ||
| 104 | +ignore_errors = True | ||
| 105 | + | ||
| 106 | +[mypy-torch.nn.modules.pooling] | ||
| 107 | +ignore_errors = True | ||
| 108 | + | ||
| 109 | +[mypy-torch.nn.parallel._functions] | ||
| 110 | +ignore_errors = True | ||
| 111 | + | ||
| 112 | +[mypy-torch._appdirs] | ||
| 113 | +ignore_errors = True | ||
| 114 | + | ||
| 115 | +[mypy-torch.multiprocessing.pool] | ||
| 116 | +ignore_errors = True | ||
| 117 | + | ||
| 118 | +[mypy-torch.overrides] | ||
| 119 | +ignore_errors = True | ||
| 120 | + | ||
| 121 | +# | ||
| 122 | +# Files with 'type: ignore' comments that are needed if checked with mypy-strict.ini | ||
| 123 | +# | ||
| 124 | + | ||
| 125 | +[mypy-tools.render_junit] | ||
| 126 | +warn_unused_ignores = False | ||
| 127 | + | ||
| 128 | +[mypy-tools.generate_torch_version] | ||
| 129 | +warn_unused_ignores = False | ||
| 130 | + | ||
| 131 | +# | ||
| 132 | +# Adding type annotations to caffe2 is probably not worth the effort | ||
| 133 | +# only work on this if you have a specific reason for it, otherwise | ||
| 134 | +# leave these ignores as they are. | ||
| 135 | +# | ||
| 136 | + | ||
| 137 | +[mypy-caffe2.python.*] | ||
| 138 | +ignore_errors = True | ||
| 139 | + | ||
| 140 | +[mypy-caffe2.proto.*] | ||
| 141 | +ignore_errors = True | ||
| 142 | + | ||
| 143 | +[mypy-caffe2.distributed.store_ops_test_util] | ||
| 144 | +ignore_errors = True | ||
| 145 | + | ||
| 146 | +[mypy-caffe2.experiments.*] | ||
| 147 | +ignore_errors = True | ||
| 148 | + | ||
| 149 | +[mypy-caffe2.contrib.*] | ||
| 150 | +ignore_errors = True | ||
| 151 | + | ||
| 152 | +[mypy-caffe2.quantization.server.*] | ||
| 153 | +ignore_errors = True | ||
| 154 | + | ||
| 155 | +# | ||
| 156 | +# Third party dependencies that don't have types. | ||
| 157 | +# | ||
| 158 | + | ||
| 159 | +[mypy-triton.*] | ||
| 160 | +ignore_missing_imports = True | ||
| 161 | + | ||
| 162 | +[mypy-tensorflow.*] | ||
| 163 | +ignore_missing_imports = True | ||
| 164 | + | ||
| 165 | +[mypy-tensorboard.*] | ||
| 166 | +ignore_missing_imports = True | ||
| 167 | + | ||
| 168 | +[mypy-matplotlib.*] | ||
| 169 | +ignore_missing_imports = True | ||
| 170 | + | ||
| 171 | +[mypy-numpy.*] | ||
| 172 | +ignore_missing_imports = True | ||
| 173 | + | ||
| 174 | +[mypy-sympy] | ||
| 175 | +ignore_missing_imports = True | ||
| 176 | + | ||
| 177 | +[mypy-sympy.*] | ||
| 178 | +ignore_missing_imports = True | ||
| 179 | + | ||
| 180 | +[mypy-hypothesis.*] | ||
| 181 | +ignore_missing_imports = True | ||
| 182 | + | ||
| 183 | +[mypy-tqdm.*] | ||
| 184 | +ignore_missing_imports = True | ||
| 185 | + | ||
| 186 | +[mypy-multiprocessing.*] | ||
| 187 | +ignore_missing_imports = True | ||
| 188 | + | ||
| 189 | +[mypy-setuptools.*] | ||
| 190 | +ignore_missing_imports = True | ||
| 191 | + | ||
| 192 | +[mypy-distutils.*] | ||
| 193 | +ignore_missing_imports = True | ||
| 194 | + | ||
| 195 | +[mypy-nvd3.*] | ||
| 196 | +ignore_missing_imports = True | ||
| 197 | + | ||
| 198 | +[mypy-future.utils] | ||
| 199 | +ignore_missing_imports = True | ||
| 200 | + | ||
| 201 | +[mypy-past.builtins] | ||
| 202 | +ignore_missing_imports = True | ||
| 203 | + | ||
| 204 | +[mypy-numba.*] | ||
| 205 | +ignore_missing_imports = True | ||
| 206 | + | ||
| 207 | +[mypy-PIL.*] | ||
| 208 | +ignore_missing_imports = True | ||
| 209 | + | ||
| 210 | +[mypy-moviepy.*] | ||
| 211 | +ignore_missing_imports = True | ||
| 212 | + | ||
| 213 | +[mypy-cv2.*] | ||
| 214 | +ignore_missing_imports = True | ||
| 215 | + | ||
| 216 | +[mypy-torchvision.*] | ||
| 217 | +ignore_missing_imports = True | ||
| 218 | + | ||
| 219 | +[mypy-pycuda.*] | ||
| 220 | +ignore_missing_imports = True | ||
| 221 | + | ||
| 222 | +[mypy-tensorrt.*] | ||
| 223 | +ignore_missing_imports = True | ||
| 224 | + | ||
| 225 | +[mypy-tornado.*] | ||
| 226 | +ignore_missing_imports = True | ||
| 227 | + | ||
| 228 | +[mypy-pydot.*] | ||
| 229 | +ignore_missing_imports = True | ||
| 230 | + | ||
| 231 | +[mypy-networkx.*] | ||
| 232 | +ignore_missing_imports = True | ||
| 233 | + | ||
| 234 | +[mypy-scipy.*] | ||
| 235 | +ignore_missing_imports = True | ||
| 236 | + | ||
| 237 | +[mypy-IPython.*] | ||
| 238 | +ignore_missing_imports = True | ||
| 239 | + | ||
| 240 | +[mypy-google.protobuf.textformat] | ||
| 241 | +ignore_missing_imports = True | ||
| 242 | + | ||
| 243 | +[mypy-lmdb.*] | ||
| 244 | +ignore_missing_imports = True | ||
| 245 | + | ||
| 246 | +[mypy-mpi4py.*] | ||
| 247 | +ignore_missing_imports = True | ||
| 248 | + | ||
| 249 | +[mypy-skimage.*] | ||
| 250 | +ignore_missing_imports = True | ||
| 251 | + | ||
| 252 | +[mypy-librosa.*] | ||
| 253 | +ignore_missing_imports = True | ||
| 254 | + | ||
| 255 | +[mypy-mypy.*] | ||
| 256 | +ignore_missing_imports = True | ||
| 257 | + | ||
| 258 | +[mypy-xml.*] | ||
| 259 | +ignore_missing_imports = True | ||
| 260 | + | ||
| 261 | +[mypy-boto3.*] | ||
| 262 | +ignore_missing_imports = True | ||
| 263 | + | ||
| 264 | +[mypy-dill.*] | ||
| 265 | +ignore_missing_imports = True | ||
| 266 | + | ||
| 267 | +[mypy-usort.*] | ||
| 268 | +ignore_missing_imports = True | ||
| 269 | + | ||
| 270 | +[mypy-torch._inductor.*] | ||
| 271 | +disallow_any_generics = True | ||
| 272 | + | ||
| 273 | +[mypy-torch._dynamo.*] | ||
| 274 | +disallow_any_generics = True | ||
| 275 | + | ||
| 276 | +[mypy-cutlass_library.*] | ||
| 277 | +ignore_missing_imports = True | ||
| 278 | + | ||
| 279 | +[mypy-deeplearning.*] | ||
| 280 | +ignore_missing_imports = True | ||
| 281 | + | ||
| 282 | +[mypy-einops.*] | ||
| 283 | +ignore_missing_imports = True | ||
| 284 | + | ||
| 285 | +[mypy-libfb.*] | ||
| 286 | +ignore_missing_imports = True | ||
| 287 | + | ||
| 288 | +[mypy-torch.*.fb.*] | ||
| 289 | +ignore_missing_imports = True | ||
| 290 | + | ||
| 291 | +[mypy-torch.fb.*] | ||
| 292 | +ignore_missing_imports = True | ||
| 293 | + | ||
| 294 | +[mypy-torch_xla.*] | ||
| 295 | +ignore_missing_imports = True | ||
| 296 | + | ||
| 297 | +# | ||
| 298 | +# Third party dependencies that are optional. | ||
| 299 | +# | ||
| 300 | + | ||
| 301 | +[mypy-onnx.*] | ||
| 302 | +ignore_missing_imports = True | ||
| 303 | + | ||
| 304 | +[mypy-onnxruntime.*] | ||
| 305 | +ignore_missing_imports = True | ||
| 306 | + | ||
| 307 | +[mypy-onnxscript.*] | ||
| 308 | +ignore_missing_imports = True | ||
| 309 | + | ||
| 310 | +[mypy-redis] | ||
| 311 | +ignore_missing_imports = True | ||
| @@ -0,0 +1,292 @@ | |||
| 1 | + # Package ###################################################################### | ||
| 2 | + | ||
| 3 | +[project] | ||
| 4 | +name = "torch_npu" | ||
| 5 | +dynamic = [ | ||
| 6 | + "classifiers", | ||
| 7 | + "entry-points", | ||
| 8 | + "dependencies", | ||
| 9 | + "scripts", | ||
| 10 | + "version", | ||
| 11 | + "description", | ||
| 12 | + "readme", | ||
| 13 | + "license", | ||
| 14 | +] | ||
| 15 | + | ||
| 16 | +[project.urls] | ||
| 17 | +Homepage = "https://pytorch.org" | ||
| 18 | +Repository = "https://github.com/pytorch/pytorch" | ||
| 19 | +Documentation = "https://pytorch.org/docs" | ||
| 20 | +"Issue Tracker" = "https://github.com/pytorch/pytorch/issues" | ||
| 21 | +Forum = "https://discuss.pytorch.org" | ||
| 22 | + | ||
| 23 | +[project.optional-dependencies] | ||
| 24 | +optree = ["optree>=0.13.0"] | ||
| 25 | +opt-einsum = ["opt-einsum>=3.3"] | ||
| 26 | +pyyaml = ["pyyaml"] | ||
| 27 | + | ||
| 28 | +# Linter tools ################################################################# | ||
| 29 | + | ||
| 30 | +[tool.isort] | ||
| 31 | +src_paths = ["caffe2", "torch", "torchgen", "functorch", "test"] | ||
| 32 | +extra_standard_library = ["typing_extensions"] | ||
| 33 | +skip_gitignore = true | ||
| 34 | +skip_glob = ["third_party/*"] | ||
| 35 | +atomic = true | ||
| 36 | +profile = "black" | ||
| 37 | +indent = 4 | ||
| 38 | +line_length = 88 | ||
| 39 | +lines_after_imports = 2 | ||
| 40 | +multi_line_output = 3 | ||
| 41 | +include_trailing_comma = true | ||
| 42 | +combine_as_imports = true | ||
| 43 | + | ||
| 44 | +[tool.usort] | ||
| 45 | +preserve_inline_comments = true | ||
| 46 | +collapse_blank_lines_in_category = false | ||
| 47 | + | ||
| 48 | +[tool.usort.known] | ||
| 49 | +first_party = ["caffe2", "torch", "torchgen", "functorch", "test"] | ||
| 50 | +standard_library = ["typing_extensions"] | ||
| 51 | + | ||
| 52 | +[tool.ruff] | ||
| 53 | +line-length = 88 | ||
| 54 | +src = ["caffe2", "torch", "torchgen", "functorch", "test"] | ||
| 55 | +extend-exclude = ["third_party", "test/dynamo/cpython"] | ||
| 56 | + | ||
| 57 | +[tool.ruff.per-file-target-version] | ||
| 58 | +"**/py312_intrinsics.py" = "py312" | ||
| 59 | + | ||
| 60 | +[tool.ruff.format] | ||
| 61 | +docstring-code-format = true | ||
| 62 | +quote-style = "double" | ||
| 63 | + | ||
| 64 | +[tool.ruff.lint] | ||
| 65 | +# NOTE: Synchoronize the ignores with .flake8 | ||
| 66 | +external = [ | ||
| 67 | + "B001", | ||
| 68 | + "B902", | ||
| 69 | + "B950", | ||
| 70 | + "E121", | ||
| 71 | + "E122", | ||
| 72 | + "E128", | ||
| 73 | + "E131", | ||
| 74 | + "E704", | ||
| 75 | + "E723", | ||
| 76 | + "F723", | ||
| 77 | + "F812", | ||
| 78 | + "P201", | ||
| 79 | + "P204", | ||
| 80 | + "T484", | ||
| 81 | + "TOR901", | ||
| 82 | +] | ||
| 83 | +ignore = [ | ||
| 84 | + # these ignores are from flake8-bugbear; please fix! | ||
| 85 | + "B007", "B008", "B017", | ||
| 86 | + "B018", # Useless expression | ||
| 87 | + "B023", | ||
| 88 | + "B028", # No explicit `stacklevel` keyword argument found | ||
| 89 | + "E402", | ||
| 90 | + "C408", # C408 ignored because we like the dict keyword argument syntax | ||
| 91 | + "E501", # E501 is not flexible enough, we're using B950 instead | ||
| 92 | + "E741", | ||
| 93 | + "EXE001", | ||
| 94 | + "F405", | ||
| 95 | + "FURB122", # writelines | ||
| 96 | + # these ignores are from ruff NPY; please fix! | ||
| 97 | + "NPY002", | ||
| 98 | + # these ignores are from ruff PERF; please fix! | ||
| 99 | + "PERF203", | ||
| 100 | + "PERF401", | ||
| 101 | + # these ignores are from PYI; please fix! | ||
| 102 | + "PYI024", | ||
| 103 | + "PYI036", | ||
| 104 | + "PYI041", | ||
| 105 | + "PYI056", | ||
| 106 | + "SIM102", "SIM103", "SIM112", # flake8-simplify code styles | ||
| 107 | + "SIM105", # these ignores are from flake8-simplify. please fix or ignore with commented reason | ||
| 108 | + "SIM108", # SIM108 ignored because we prefer if-else-block instead of ternary expression | ||
| 109 | + "SIM110", # Checks for for loops that can be replaced with a builtin function, like any or all. | ||
| 110 | + "SIM114", # Combine `if` branches using logical `or` operator | ||
| 111 | + "SIM116", # Disable Use a dictionary instead of consecutive `if` statements | ||
| 112 | + "SIM117", | ||
| 113 | + "SIM300", # Yoda condition detected | ||
| 114 | + "TC006", | ||
| 115 | + # TODO: Remove Python-3.10 specific suppressions | ||
| 116 | + "B905", | ||
| 117 | +] | ||
| 118 | +select = [ | ||
| 119 | + "B", | ||
| 120 | + "B904", # Re-raised error without specifying the cause via the from keyword | ||
| 121 | + "C4", | ||
| 122 | + "G", | ||
| 123 | + "E", | ||
| 124 | + "EXE", | ||
| 125 | + "F", | ||
| 126 | + "SIM", | ||
| 127 | + "W", | ||
| 128 | + # Not included in flake8 | ||
| 129 | + "FURB", | ||
| 130 | + "LOG", | ||
| 131 | + "NPY", | ||
| 132 | + "PERF", | ||
| 133 | + "PGH004", | ||
| 134 | + "PIE", | ||
| 135 | + "PLC0131", # type bivariance | ||
| 136 | + "PLC0132", # type param mismatch | ||
| 137 | + "PLC1802", # len({expression}) used as condition without comparison | ||
| 138 | + "PLC0205", # string as __slots__ | ||
| 139 | + "PLC3002", # unnecessary-direct-lambda-call | ||
| 140 | + "PLC0414", # Import alias does not rename original package | ||
| 141 | + "PLE", | ||
| 142 | + "PLR0133", # constant comparison | ||
| 143 | + "PLR0206", # property with params | ||
| 144 | + "PLR1722", # use sys exit | ||
| 145 | + "PLR1736", # unnecessary list index | ||
| 146 | + "PLW0127", # Self-assignment of variable | ||
| 147 | + "PLW0129", # assert on string literal | ||
| 148 | + "PLW0131", # named expr without context | ||
| 149 | + "PLW0133", # useless exception statement | ||
| 150 | + "PLW0245", # super without brackets | ||
| 151 | + "PLW0406", # import self | ||
| 152 | + "PLW0711", # binary op exception | ||
| 153 | + "PLW1501", # bad open mode | ||
| 154 | + "PLW1507", # shallow copy os.environ | ||
| 155 | + "PLW1509", # preexec_fn not safe with threads | ||
| 156 | + "PLW2101", # useless lock statement | ||
| 157 | + "PLW3301", # nested min max | ||
| 158 | + "PT006", # TODO: enable more PT rules | ||
| 159 | + "PT014", # duplicate parameterize case | ||
| 160 | + "PT022", | ||
| 161 | + "PT023", | ||
| 162 | + "PT024", | ||
| 163 | + "PT025", | ||
| 164 | + "PT026", | ||
| 165 | + "PYI", | ||
| 166 | + "Q003", # avoidable escaped quote | ||
| 167 | + "Q004", # unnecessary escaped quote | ||
| 168 | + "RSE", | ||
| 169 | + "RUF007", # pairwise over zip | ||
| 170 | + "RUF008", # mutable dataclass default | ||
| 171 | + "RUF013", # ban implicit optional | ||
| 172 | + "RUF015", # access first ele in constant time | ||
| 173 | + "RUF016", # type error non-integer index | ||
| 174 | + "RUF017", | ||
| 175 | + "RUF018", # no assignment in assert | ||
| 176 | + "RUF019", # unnecessary-key-check | ||
| 177 | + "RUF020", # never union | ||
| 178 | + "RUF024", # from keys mutable | ||
| 179 | + "RUF026", # default factory kwarg | ||
| 180 | + "RUF030", # No print statement in assert | ||
| 181 | + "RUF033", # default values __post_init__ dataclass | ||
| 182 | + "RUF041", # simplify nested Literal | ||
| 183 | + "RUF048", # properly parse `__version__` | ||
| 184 | + "RUF200", # validate pyproject.toml | ||
| 185 | + "S324", # for hashlib FIPS compliance | ||
| 186 | + "SLOT", | ||
| 187 | + "TC", | ||
| 188 | + "TRY002", # ban vanilla raise (todo fix NOQAs) | ||
| 189 | + "TRY203", | ||
| 190 | + "TRY401", # verbose-log-message | ||
| 191 | + "UP", | ||
| 192 | + "YTT", | ||
| 193 | + "S101", | ||
| 194 | +] | ||
| 195 | + | ||
| 196 | +[tool.ruff.lint.per-file-ignores] | ||
| 197 | +"__init__.py" = [ | ||
| 198 | + "F401", | ||
| 199 | +] | ||
| 200 | +"*.pyi" = [ | ||
| 201 | + "PYI011", # typed-argument-default-in-stub | ||
| 202 | + "PYI021", # docstring-in-stub | ||
| 203 | + "PYI053", # string-or-bytes-too-long | ||
| 204 | +] | ||
| 205 | +"functorch/docs/source/tutorials/**" = [ | ||
| 206 | + "F401", | ||
| 207 | +] | ||
| 208 | +"test/export/**" = [ | ||
| 209 | + "PGH004", | ||
| 210 | +] | ||
| 211 | +"test/typing/**" = [ | ||
| 212 | + "PGH004" | ||
| 213 | +] | ||
| 214 | +"test/typing/reveal/**" = [ | ||
| 215 | + "F821", | ||
| 216 | +] | ||
| 217 | +"test/torch_np/numpy_tests/**" = [ | ||
| 218 | + "F821", | ||
| 219 | + "NPY201", | ||
| 220 | +] | ||
| 221 | +"test/dynamo/test_bytecode_utils.py" = [ | ||
| 222 | + "F821", | ||
| 223 | +] | ||
| 224 | +"test/dynamo/test_debug_utils.py" = [ | ||
| 225 | + "UP037", | ||
| 226 | +] | ||
| 227 | +"test/dynamo/test_misc.py" = [ | ||
| 228 | + "PGH004", | ||
| 229 | +] | ||
| 230 | +"test/jit/**" = [ | ||
| 231 | + "PLR0133", # tests require this for JIT | ||
| 232 | + "PYI", | ||
| 233 | + "RUF015", | ||
| 234 | + "UP", # We don't want to modify the jit test as they test specify syntax | ||
| 235 | +] | ||
| 236 | +"test/test_jit.py" = [ | ||
| 237 | + "PLR0133", # tests require this for JIT | ||
| 238 | + "PYI", | ||
| 239 | + "RUF015", | ||
| 240 | + "UP", # We don't want to modify the jit test as they test specify syntax | ||
| 241 | +] | ||
| 242 | +"test/inductor/s429861_repro.py" = [ | ||
| 243 | + "PGH004", | ||
| 244 | +] | ||
| 245 | +"test/inductor/test_torchinductor.py" = [ | ||
| 246 | + "UP037", | ||
| 247 | +] | ||
| 248 | +# autogenerated #TODO figure out why file level noqa is ignored | ||
| 249 | +"torch/_appdirs.py" = ["PGH004"] | ||
| 250 | +"torch/jit/_shape_functions.py" = ["PGH004"] | ||
| 251 | +"torch/_inductor/fx_passes/serialized_patterns/**" = ["F401", "F501"] | ||
| 252 | +"torch/_inductor/autoheuristic/artifacts/**" = ["F401", "F501"] | ||
| 253 | +"torch/_inductor/codegen/**" = [ | ||
| 254 | + "PGH004" | ||
| 255 | +] | ||
| 256 | +"torchgen/api/types/__init__.py" = [ | ||
| 257 | + "F401", | ||
| 258 | + "F403", | ||
| 259 | +] | ||
| 260 | +"torch/utils/collect_env.py" = [ | ||
| 261 | + "UP", # collect_env.py needs to work with older versions of Python | ||
| 262 | +] | ||
| 263 | +"torch/_vendor/**" = [ | ||
| 264 | + "UP", # No need to mess with _vendor | ||
| 265 | +] | ||
| 266 | +"tools/linter/**" = [ | ||
| 267 | + "LOG015" # please fix | ||
| 268 | +] | ||
| 269 | + | ||
| 270 | +# torch/ folders still needing S101 migration | ||
| 271 | +"torch/_dynamo/**" = ["S101"] | ||
| 272 | +"torch/_inductor/**" = ["S101"] | ||
| 273 | + | ||
| 274 | +[tool.codespell] | ||
| 275 | +ignore-words = "tools/linter/dictionary.txt" | ||
| 276 | + | ||
| 277 | +[tool.spin] | ||
| 278 | +package = 'torch' | ||
| 279 | + | ||
| 280 | +[tool.spin.commands] | ||
| 281 | +"Build" = [ | ||
| 282 | + ".spin/cmds.py:lint", | ||
| 283 | + ".spin/cmds.py:fixlint", | ||
| 284 | + ".spin/cmds.py:quicklint", | ||
| 285 | + ".spin/cmds.py:quickfix", | ||
| 286 | +] | ||
| 287 | +"Regenerate" = [ | ||
| 288 | + ".spin/cmds.py:regenerate_version", | ||
| 289 | + ".spin/cmds.py:regenerate_type_stubs", | ||
| 290 | + ".spin/cmds.py:regenerate_clangtidy_files", | ||
| 291 | + ".spin/cmds.py:regenerate_github_workflows", | ||
| 292 | +] | ||
| @@ -0,0 +1,160 @@ | |||
| 1 | +# A Pyrefly configuration for PyTorch | ||
| 2 | +# Based on https://github.com/pytorch/pytorch/blob/main/mypy.ini | ||
| 3 | +python-version = "3.12" | ||
| 4 | + | ||
| 5 | +project-includes = [ | ||
| 6 | + "torch", | ||
| 7 | + "caffe2", | ||
| 8 | + "tools", | ||
| 9 | + "test/test_bundled_images.py", | ||
| 10 | + "test/test_bundled_inputs.py", | ||
| 11 | + "test/test_complex.py", | ||
| 12 | + "test/test_datapipe.py", | ||
| 13 | + "test/test_futures.py", | ||
| 14 | + "test/test_numpy_interop.py", | ||
| 15 | + # We exclude test_torch.py because it is full of errors, but most functions lack type signatures, | ||
| 16 | + # and mypy.ini specifies `check_untyped_defs = False` for this file. | ||
| 17 | + # If you check even the unannotated stuff mypy produces 322 errors. | ||
| 18 | + # "test/test_torch.py", | ||
| 19 | + "test/test_type_hints.py", | ||
| 20 | + "test/test_type_info.py", | ||
| 21 | + "test/test_utils.py", | ||
| 22 | +] | ||
| 23 | +project-excludes = [ | ||
| 24 | + "torch/_inductor/runtime/triton_heuristics.py", | ||
| 25 | + "torch/_inductor/runtime/triton_helpers.py", | ||
| 26 | + "torch/_inductor/runtime/halide_helpers.py", | ||
| 27 | + "torch/utils/data/typing.ipynb", | ||
| 28 | + "torch/utils/data/dataframes_pipes.ipynb", | ||
| 29 | + "torch/utils/data/standard_pipes.ipynb", | ||
| 30 | + "torch/onnx/_internal/exporter/_torchlib/ops/nn.py", | ||
| 31 | + "torch/include/**", | ||
| 32 | + "torch/csrc/**", | ||
| 33 | + "torch/distributed/elastic/agent/server/api.py", | ||
| 34 | + "torch/testing/_internal/**", | ||
| 35 | + "torch/distributed/fsdp/fully_sharded_data_parallel.py", | ||
| 36 | + "torch/ao/quantization/pt2e/_affine_quantization.py", | ||
| 37 | + "torch/nn/modules/pooling.py", | ||
| 38 | + "torch/nn/parallel/_functions.py", | ||
| 39 | + "torch/_appdirs.py", | ||
| 40 | + "torch/multiprocessing/pool.py", | ||
| 41 | + "torch/overrides.py", | ||
| 42 | + "*/__pycache__/**", | ||
| 43 | + "*/.*", | ||
| 44 | + "torch/_inductor/kernel/vendored_templates/cutedsl/kernels/**", | ||
| 45 | + "torch/_inductor/kernel/vendored_templates/cutedsl/dense_blockscaled_gemm_persistent.py", | ||
| 46 | +] | ||
| 47 | +ignore-missing-imports = [ | ||
| 48 | + # XPU memory symbols not present for builds without XPU support | ||
| 49 | + "torch._C._xpu_beginAllocateCurrentThreadToPool", | ||
| 50 | + "torch._C._xpu_endAllocateToPool", | ||
| 51 | + "torch._C._xpu_releasePool", | ||
| 52 | + "torch._C._xpu_XPUAllocator", | ||
| 53 | + "torch._C._XPUMemPool", | ||
| 54 | + "torch._C._StaticXpuLauncher", | ||
| 55 | + "torch._C._jit_tree_views.*", | ||
| 56 | + "torch.for_onnx.onnx.*", | ||
| 57 | + "torch.ao.quantization.experimental.apot_utils.*", | ||
| 58 | + "torch.ao.quantization.experimental.quantizer.*", | ||
| 59 | + "torch.ao.quantization.experimental.observer.*", | ||
| 60 | + "torch.ao.quantization.experimental.APoT_tensor.*", | ||
| 61 | + "torch.ao.quantization.experimental.fake_quantize_function.*", | ||
| 62 | + "torch.ao.quantization.experimental.fake_quantize.*", | ||
| 63 | + "triton.*", | ||
| 64 | + "tensorflow.*", | ||
| 65 | + "tensorboard.*", | ||
| 66 | + "matplotlib.*", | ||
| 67 | + "numpy.*", | ||
| 68 | + "sympy.*", | ||
| 69 | + "hypothesis.*", | ||
| 70 | + "tqdm.*", | ||
| 71 | + "multiprocessing.*", | ||
| 72 | + "setuptools.*", | ||
| 73 | + "distutils.*", | ||
| 74 | + "nvd3.*", | ||
| 75 | + "future.utils.*", | ||
| 76 | + "past.builtins.*", | ||
| 77 | + "numba.*", | ||
| 78 | + "nvMatmulHeuristics", | ||
| 79 | + "PIL.*", | ||
| 80 | + "moviepy.*", | ||
| 81 | + "cv2.*", | ||
| 82 | + "torchvision.*", | ||
| 83 | + "pycuda.*", | ||
| 84 | + "tensorrt.*", | ||
| 85 | + "tornado.*", | ||
| 86 | + "pydot.*", | ||
| 87 | + "networkx.*", | ||
| 88 | + "scipy.*", | ||
| 89 | + "IPython.*", | ||
| 90 | + "google.protobuf.textformat.*", | ||
| 91 | + "lmdb.*", | ||
| 92 | + "mpi4py.*", | ||
| 93 | + "skimage.*", | ||
| 94 | + "librosa.*", | ||
| 95 | + "mypy.*", | ||
| 96 | + "xml.*", | ||
| 97 | + "boto3.*", | ||
| 98 | + "dill.*", | ||
| 99 | + "usort.*", | ||
| 100 | + "cutlass.*", | ||
| 101 | + "cutlass_library.*", | ||
| 102 | + "cutlass_api.*", | ||
| 103 | + "deeplearning.*", | ||
| 104 | + "einops.*", | ||
| 105 | + "libfb.*", | ||
| 106 | + "torch.fb.*", | ||
| 107 | + "torch.*.fb.*", | ||
| 108 | + "torch_xla.*", | ||
| 109 | + "onnx.*", | ||
| 110 | + "onnxruntime.*", | ||
| 111 | + "onnxscript.*", | ||
| 112 | + "redis.*", | ||
| 113 | +] | ||
| 114 | +# By default, mypy does not check untyped definitions. | ||
| 115 | +# However, mypy has a configuration called check_untyped_defs which is used | ||
| 116 | +# to typecheck the interior of untyped functions. | ||
| 117 | +untyped-def-behavior = "check-and-infer-return-any" | ||
| 118 | +# In lots of places they define their attributes in `_init` or similar. | ||
| 119 | +# https://github.com/pytorch/pytorch/blob/75f3e5a88df60caef27fd9c9df3fd51161378fcc/torch/fx/experimental/symbolic_shapes.py#L3632C1-L3633C1 | ||
| 120 | +errors.implicitly-defined-attribute = false | ||
| 121 | +# In many methods that are overridden, parameters are renamed. | ||
| 122 | +# We can come up with a codemod for this in the future | ||
| 123 | +errors.bad-param-name-override = false | ||
| 124 | +# Mypy doesn't require that imports are explicitly imported, so be compatible with that. | ||
| 125 | +# Might be a good idea to turn this on in future. | ||
| 126 | +errors.implicit-import = false | ||
| 127 | +errors.deprecated = false # re-enable after we've fix import formatting | ||
| 128 | + | ||
| 129 | +permissive-ignores = true | ||
| 130 | +replace-imports-with-any = ["!sympy.printing.*", "sympy.*", "onnxscript.onnx_opset.*", "networkx.*"] | ||
| 131 | +search-path = ["tools/experimental"] | ||
| 132 | + | ||
| 133 | +# Dynamo sub-config - note, we experiment with stricter typing here | ||
| 134 | +[[sub-config]] | ||
| 135 | +matches = "torch/_dynamo/**" | ||
| 136 | +[sub-config.errors] | ||
| 137 | +implicit-import = false | ||
| 138 | +implicit-any = true | ||
| 139 | +# TODO: Turn on in later version | ||
| 140 | +# unannotated-attribute=true | ||
| 141 | +# unannotated-parameter=true | ||
| 142 | +# unannotated-return=true | ||
| 143 | + | ||
| 144 | +[[sub-config]] | ||
| 145 | +matches = "torch/_dispatch/**" | ||
| 146 | +[sub-config.errors] | ||
| 147 | +implicit-import = false | ||
| 148 | +implicit-any = true | ||
| 149 | + | ||
| 150 | +[[sub-config]] | ||
| 151 | +matches = "torch/_subclasses/**" | ||
| 152 | +[sub-config.errors] | ||
| 153 | +implicit-import = false | ||
| 154 | +implicit-any = true | ||
| 155 | + | ||
| 156 | +[[sub-config]] | ||
| 157 | +matches = "torch/_functorch/**" | ||
| 158 | +[sub-config.errors] | ||
| 159 | +implicit-import = false | ||
| 160 | +implicit-any = true | ||
| @@ -0,0 +1,10 @@ | |||
| 1 | +# lintrunner adapters | ||
| 2 | + | ||
| 3 | +These files adapt our various linters to work with `lintrunner`. | ||
| 4 | + | ||
| 5 | +## Adding a new linter | ||
| 6 | +1. init and linter | ||
| 7 | +2. {{DRYRUN}} and {{PATHSFILE}} | ||
| 8 | +3. never exit uncleanly | ||
| 9 | +4. Communication protocol | ||
| 10 | +5. Self-contained | ||
| @@ -0,0 +1,49 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import token | ||
| 4 | +from pathlib import Path | ||
| 5 | +from typing import Any, TYPE_CHECKING | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +if TYPE_CHECKING: | ||
| 9 | + from collections.abc import Sequence | ||
| 10 | + from tokenize import TokenInfo | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +__all__ = [ | ||
| 14 | + "Block", | ||
| 15 | + "FileLinter", | ||
| 16 | + "is_empty", | ||
| 17 | + # pyrefly: ignore [bad-dunder-all] | ||
| 18 | + "LineWithSets", | ||
| 19 | + "LintResult", | ||
| 20 | + "ParseError", | ||
| 21 | + "PythonFile", | ||
| 22 | + "ROOT", | ||
| 23 | +] | ||
| 24 | + | ||
| 25 | +NO_TOKEN = -1 | ||
| 26 | + | ||
| 27 | +# Python 3.12 and up have two new token types, FSTRING_START and FSTRING_END | ||
| 28 | +_START_OF_LINE_TOKENS = token.DEDENT, token.INDENT, token.NEWLINE | ||
| 29 | +_IGNORED_TOKENS = token.COMMENT, token.ENDMARKER, token.ENCODING, token.NL | ||
| 30 | +_EMPTY_TOKENS = dict.fromkeys(_START_OF_LINE_TOKENS + _IGNORED_TOKENS) | ||
| 31 | + | ||
| 32 | +_LINTER = Path(__file__).absolute().parents[0] | ||
| 33 | +ROOT = _LINTER.parents[3] | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +class ParseError(ValueError): | ||
| 37 | + def __init__(self, token: TokenInfo, *args: str) -> None: | ||
| 38 | + super().__init__(*args) | ||
| 39 | + self.token = token | ||
| 40 | + | ||
| 41 | + | ||
| 42 | +def is_empty(t: TokenInfo) -> bool: | ||
| 43 | + return t.type in _EMPTY_TOKENS | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +from .block import Block | ||
| 47 | +from .file_linter import FileLinter | ||
| 48 | +from .messages import LintResult | ||
| 49 | +from .python_file import PythonFile | ||
| @@ -0,0 +1,50 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import sys | ||
| 5 | +from typing import Any, TYPE_CHECKING | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +if TYPE_CHECKING: | ||
| 9 | + from typing_extensions import Never | ||
| 10 | + | ||
| 11 | + | ||
| 12 | +class ArgumentParser(argparse.ArgumentParser): | ||
| 13 | + """ | ||
| 14 | + Adds better help formatting and default arguments to argparse.ArgumentParser | ||
| 15 | + """ | ||
| 16 | + | ||
| 17 | + def __init__( | ||
| 18 | + self, | ||
| 19 | + prog: str | None = None, | ||
| 20 | + usage: str | None = None, | ||
| 21 | + description: str | None = None, | ||
| 22 | + epilog: str | None = None, | ||
| 23 | + is_fixer: bool = False, | ||
| 24 | + **kwargs: Any, | ||
| 25 | + ) -> None: | ||
| 26 | + super().__init__(prog, usage, description, None, **kwargs) | ||
| 27 | + self._epilog = epilog | ||
| 28 | + | ||
| 29 | + help = "A list of files or directories to lint" | ||
| 30 | + self.add_argument("files", nargs="*", help=help) | ||
| 31 | + # TODO(rec): get fromfile_prefix_chars="@", type=argparse.FileType to work | ||
| 32 | + | ||
| 33 | + help = "Fix lint errors if possible" if is_fixer else argparse.SUPPRESS | ||
| 34 | + self.add_argument("-f", "--fix", action="store_true", help=help) | ||
| 35 | + | ||
| 36 | + help = "Run for lintrunner and print LintMessages which aren't edits" | ||
| 37 | + self.add_argument("-l", "--lintrunner", action="store_true", help=help) | ||
| 38 | + | ||
| 39 | + help = "Print more debug info" | ||
| 40 | + self.add_argument("-v", "--verbose", action="store_true", help=help) | ||
| 41 | + | ||
| 42 | + def exit(self, status: int = 0, message: str | None = None) -> Never: | ||
| 43 | + """ | ||
| 44 | + Overriding this method is a workaround for argparse throwing away all | ||
| 45 | + line breaks when printing the `epilog` section of the help message. | ||
| 46 | + """ | ||
| 47 | + argv = sys.argv[1:] | ||
| 48 | + if self._epilog and not status and "-h" in argv or "--help" in argv: | ||
| 49 | + print(self._epilog) | ||
| 50 | + super().exit(status, message) | ||
| @@ -0,0 +1,174 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import dataclasses as dc | ||
| 4 | +import itertools | ||
| 5 | +import token | ||
| 6 | +from enum import Enum | ||
| 7 | +from functools import cached_property, total_ordering | ||
| 8 | +from typing import Any, TYPE_CHECKING | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +if TYPE_CHECKING: | ||
| 12 | + from collections.abc import Iterator, Sequence | ||
| 13 | + from tokenize import TokenInfo | ||
| 14 | + from typing_extensions import Self | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +_OVERRIDES = {"@override", "@typing_extensions.override", "@typing.override"} | ||
| 18 | + | ||
| 19 | + | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +class Block: | ||
| 23 | + """A block of Python code starting with either `def` or `class`""" | ||
| 24 | + | ||
| 25 | + class Category(str, Enum): | ||
| 26 | + CLASS = "class" | ||
| 27 | + DEF = "def" | ||
| 28 | + | ||
| 29 | + category: Category | ||
| 30 | + | ||
| 31 | + # The sequence of tokens that contains this Block. | ||
| 32 | + # Tokens are represented in `Block` as indexes into `self.tokens` | ||
| 33 | + tokens: Sequence[TokenInfo] = dc.field(repr=False) | ||
| 34 | + | ||
| 35 | + # The name of the function or class being defined | ||
| 36 | + name: str | ||
| 37 | + | ||
| 38 | + # The index of the very first token in the block (the "class" or "def" keyword) | ||
| 39 | + begin: int | ||
| 40 | + | ||
| 41 | + # The index of the last token for this block | ||
| 42 | + end: int | ||
| 43 | + | ||
| 44 | + # The docstring for the block | ||
| 45 | + docstring: str | ||
| 46 | + | ||
| 47 | + # These next members only get filled in after all blocks have been constructed | ||
| 48 | + # and figure out family ties | ||
| 49 | + | ||
| 50 | + # The full qualified name of the block within the file. | ||
| 51 | + # This is the name of this block and all its parents, joined with `.`. | ||
| 52 | + full_name: str = "" | ||
| 53 | + | ||
| 54 | + # The index of this block within the full list of blocks in the file | ||
| 55 | + index: int = 0 | ||
| 56 | + | ||
| 57 | + # Is this block contained within a function definition? | ||
| 58 | + is_local: bool = dc.field(default=False, repr=False) | ||
| 59 | + | ||
| 60 | + # Is this block a function definition in a class definition? | ||
| 61 | + is_method: bool = dc.field(default=False, repr=False) | ||
| 62 | + | ||
| 63 | + # A block index to the parent of this block, or None for a top-level block. | ||
| 64 | + parent: int | None = None | ||
| 65 | + | ||
| 66 | + # A list of block indexes for the children | ||
| 67 | + children: list[int] = dc.field(default_factory=list) | ||
| 68 | + | ||
| 69 | + | ||
| 70 | + def start_line(self) -> int: | ||
| 71 | + """The line number for the def or class statement""" | ||
| 72 | + return self.tokens[self.begin].start[0] | ||
| 73 | + | ||
| 74 | + | ||
| 75 | + def end_line(self) -> int: | ||
| 76 | + return self.tokens[self.end].start[0] | ||
| 77 | + | ||
| 78 | + | ||
| 79 | + def line_count(self) -> int: | ||
| 80 | + return self.end_line - self.start_line + 1 | ||
| 81 | + | ||
| 82 | + | ||
| 83 | + def line_range(self) -> range: | ||
| 84 | + return range(self.start_line, self.end_line + 1) | ||
| 85 | + | ||
| 86 | + | ||
| 87 | + def is_class(self) -> bool: | ||
| 88 | + return self.category == Block.Category.CLASS | ||
| 89 | + | ||
| 90 | + | ||
| 91 | + def display_name(self) -> str: | ||
| 92 | + """A user-friendly name like 'class One' or 'def One.method()'""" | ||
| 93 | + ending = "" if self.is_class else "()" | ||
| 94 | + return f"{self.category.value} {self.full_name}{ending}" | ||
| 95 | + | ||
| 96 | + | ||
| 97 | + def decorators(self) -> list[str]: | ||
| 98 | + """A list of decorators for this function or method. | ||
| 99 | + | ||
| 100 | + Each decorator both the @ symbol and any arguments to the decorator | ||
| 101 | + but no extra whitespace. | ||
| 102 | + """ | ||
| 103 | + return _get_decorators(self.tokens, self.begin) | ||
| 104 | + | ||
| 105 | + | ||
| 106 | + def is_override(self) -> bool: | ||
| 107 | + return not self.is_class and bool(_OVERRIDES.intersection(self.decorators)) | ||
| 108 | + | ||
| 109 | + DATA_FIELDS = ( | ||
| 110 | + "category", | ||
| 111 | + "children", | ||
| 112 | + "decorators", | ||
| 113 | + "display_name", | ||
| 114 | + "docstring", | ||
| 115 | + "full_name", | ||
| 116 | + "index", | ||
| 117 | + "is_local", | ||
| 118 | + "is_method", | ||
| 119 | + "line_count", | ||
| 120 | + "parent", | ||
| 121 | + "start_line", | ||
| 122 | + ) | ||
| 123 | + | ||
| 124 | + def as_data(self) -> dict[str, Any]: | ||
| 125 | + d = {i: getattr(self, i) for i in self.DATA_FIELDS} | ||
| 126 | + d["category"] = d["category"].value | ||
| 127 | + return d | ||
| 128 | + | ||
| 129 | + | ||
| 130 | + def is_init(self) -> bool: | ||
| 131 | + return not self.is_class and self.name == "__init__" | ||
| 132 | + | ||
| 133 | + def contains(self, b: Block) -> bool: | ||
| 134 | + return self.start_line < b.start_line and self.end_line >= b.end_line | ||
| 135 | + | ||
| 136 | + def __eq__(self, o: object) -> bool: | ||
| 137 | + if not isinstance(o, Block): | ||
| 138 | + raise AssertionError(f"Expected Block, got {type(o)}") | ||
| 139 | + return o.tokens is self.tokens and o.index == self.index | ||
| 140 | + | ||
| 141 | + def __hash__(self) -> int: | ||
| 142 | + return super().__hash__() | ||
| 143 | + | ||
| 144 | + def __lt__(self, o: Self) -> bool: | ||
| 145 | + if not (isinstance(o, Block) and o.tokens is self.tokens): | ||
| 146 | + raise AssertionError("Expected Block with same tokens") | ||
| 147 | + return o.index < self.index | ||
| 148 | + | ||
| 149 | + | ||
| 150 | +_IGNORE = {token.COMMENT, token.DEDENT, token.INDENT, token.NL} | ||
| 151 | + | ||
| 152 | + | ||
| 153 | +def _get_decorators(tokens: Sequence[TokenInfo], block_start: int) -> list[str]: | ||
| 154 | + def decorators() -> Iterator[str]: | ||
| 155 | + rev = reversed(range(block_start)) | ||
| 156 | + newlines = (i for i in rev if tokens[i].type == token.NEWLINE) | ||
| 157 | + it = iter(itertools.chain(newlines, [-1])) | ||
| 158 | + # The -1 accounts for the very first line in the file | ||
| 159 | + | ||
| 160 | + end = next(it, -1) # Like itertools.pairwise in Python 3.10 | ||
| 161 | + for begin in it: | ||
| 162 | + for i in range(begin + 1, end): | ||
| 163 | + t = tokens[i] | ||
| 164 | + if t.type == token.OP and t.string == "@": | ||
| 165 | + useful = (t for t in tokens[i:end] if t.type not in _IGNORE) | ||
| 166 | + yield "".join(s.string.strip("\n") for s in useful) | ||
| 167 | + break | ||
| 168 | + elif t.type not in _IGNORE: | ||
| 169 | + return # A statement means no more decorators | ||
| 170 | + end = begin | ||
| 171 | + | ||
| 172 | + out = list(decorators()) | ||
| 173 | + out.reverse() | ||
| 174 | + return out | ||
| @@ -0,0 +1,91 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import token | ||
| 4 | +from typing import TYPE_CHECKING | ||
| 5 | + | ||
| 6 | +from . import is_empty | ||
| 7 | +from .block import Block | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +if TYPE_CHECKING: | ||
| 11 | + from collections.abc import Sequence | ||
| 12 | + | ||
| 13 | + from .python_file import PythonFile | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +def blocks(pf: PythonFile) -> list[Block]: | ||
| 17 | + blocks: list[Block] = [] | ||
| 18 | + | ||
| 19 | + it = (i for i, t in enumerate(pf.tokens) if t.string in ("class", "def")) | ||
| 20 | + blocks = [_make_block(pf, i) for i in it] | ||
| 21 | + | ||
| 22 | + for i, parent in enumerate(blocks): | ||
| 23 | + for j in range(i + 1, len(blocks)): | ||
| 24 | + if parent.contains(child := blocks[j]): | ||
| 25 | + child.parent = i | ||
| 26 | + parent.children.append(j) | ||
| 27 | + else: | ||
| 28 | + break | ||
| 29 | + | ||
| 30 | + for i, b in enumerate(blocks): | ||
| 31 | + b.index = i | ||
| 32 | + parents = [b] | ||
| 33 | + while (p := parents[-1].parent) is not None: | ||
| 34 | + parents.append(blocks[p]) | ||
| 35 | + parents = parents[1:] | ||
| 36 | + | ||
| 37 | + b.is_local = not all(p.is_class for p in parents) | ||
| 38 | + b.is_method = not b.is_class and bool(parents) and parents[0].is_class | ||
| 39 | + | ||
| 40 | + _add_full_names(blocks, [b for b in blocks if b.parent is None]) | ||
| 41 | + return blocks | ||
| 42 | + | ||
| 43 | + | ||
| 44 | +def _add_full_names( | ||
| 45 | + blocks: Sequence[Block], children: Sequence[Block], prefix: str = "" | ||
| 46 | +) -> None: | ||
| 47 | + # Would be trivial except that there can be duplicate names at any level | ||
| 48 | + dupes: dict[str, list[Block]] = {} | ||
| 49 | + for b in children: | ||
| 50 | + dupes.setdefault(b.name, []).append(b) | ||
| 51 | + | ||
| 52 | + for dl in dupes.values(): | ||
| 53 | + for i, b in enumerate(dl): | ||
| 54 | + suffix = f"[{i + 1}]" if len(dl) > 1 else "" | ||
| 55 | + b.full_name = prefix + b.name + suffix | ||
| 56 | + | ||
| 57 | + for b in children: | ||
| 58 | + if kids := [blocks[i] for i in b.children]: | ||
| 59 | + _add_full_names(blocks, kids, b.full_name + ".") | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def _make_block(pf: PythonFile, begin: int) -> Block: | ||
| 63 | + name = docstring = "" | ||
| 64 | + end = 0 | ||
| 65 | + | ||
| 66 | + for i in range(begin + 1, len(pf.tokens)): | ||
| 67 | + t = pf.tokens[i] | ||
| 68 | + if not name and t.type == token.NAME: | ||
| 69 | + name = t.string | ||
| 70 | + elif not end: | ||
| 71 | + if t.type == token.INDENT: | ||
| 72 | + end = pf.indent_to_dedent[i] | ||
| 73 | + while is_empty(pf.tokens[end := end - 1]): | ||
| 74 | + pass | ||
| 75 | + elif t.string == "...": | ||
| 76 | + end = i | ||
| 77 | + elif t.type == token.STRING: | ||
| 78 | + docstring = t.string | ||
| 79 | + break | ||
| 80 | + elif not is_empty(t): | ||
| 81 | + break | ||
| 82 | + | ||
| 83 | + category = Block.Category[pf.tokens[begin].string.upper()] | ||
| 84 | + return Block( | ||
| 85 | + begin=begin, | ||
| 86 | + category=category, | ||
| 87 | + docstring=docstring, | ||
| 88 | + end=end, | ||
| 89 | + name=name, | ||
| 90 | + tokens=pf.tokens, | ||
| 91 | + ) | ||
| @@ -0,0 +1,45 @@ | |||
| 1 | +import token | ||
| 2 | +from collections.abc import Sequence | ||
| 3 | +from tokenize import TokenInfo | ||
| 4 | + | ||
| 5 | +from . import NO_TOKEN, ParseError | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +FSTRING_START: int = getattr(token, "FSTRING_START", NO_TOKEN) | ||
| 9 | +FSTRING_END: int = getattr(token, "FSTRING_END", NO_TOKEN) | ||
| 10 | + | ||
| 11 | +BRACKETS = {"{": "}", "(": ")", "[": "]"} | ||
| 12 | +BRACKETS_INV = {j: i for i, j in BRACKETS.items()} | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +def bracket_pairs(tokens: Sequence[TokenInfo]) -> dict[int, int]: | ||
| 16 | + """Returns a dictionary mapping opening to closing brackets""" | ||
| 17 | + braces: dict[int, int] = {} | ||
| 18 | + stack: list[int] = [] | ||
| 19 | + in_fstring = False | ||
| 20 | + | ||
| 21 | + for i, t in enumerate(tokens): | ||
| 22 | + if t.type == token.OP and not in_fstring: | ||
| 23 | + if t.string in BRACKETS: | ||
| 24 | + stack.append(i) | ||
| 25 | + elif inv := BRACKETS_INV.get(t.string): | ||
| 26 | + if not stack: | ||
| 27 | + raise ParseError(t, "Never opened") | ||
| 28 | + begin = stack.pop() | ||
| 29 | + | ||
| 30 | + if not (stack and stack[-1] == FSTRING_START): | ||
| 31 | + braces[begin] = i | ||
| 32 | + | ||
| 33 | + b = tokens[begin].string | ||
| 34 | + if b != inv: | ||
| 35 | + raise ParseError(t, f"Mismatched braces '{b}' at {begin}") | ||
| 36 | + elif t.type == FSTRING_START: | ||
| 37 | + stack.append(FSTRING_START) | ||
| 38 | + in_fstring = True | ||
| 39 | + elif t.type == FSTRING_END: | ||
| 40 | + if stack.pop() != FSTRING_START: | ||
| 41 | + raise ParseError(t, "Mismatched FSTRING_START/FSTRING_END") | ||
| 42 | + in_fstring = False | ||
| 43 | + if stack: | ||
| 44 | + raise ParseError(t, "Left open") | ||
| 45 | + return braces | ||
| @@ -0,0 +1,190 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import json | ||
| 4 | +import sys | ||
| 5 | +from abc import abstractmethod | ||
| 6 | +from functools import cached_property | ||
| 7 | +from pathlib import Path | ||
| 8 | +from typing import TYPE_CHECKING | ||
| 9 | + | ||
| 10 | +from . import ParseError | ||
| 11 | +from .argument_parser import ArgumentParser | ||
| 12 | +from .messages import LintResult | ||
| 13 | +from .python_file import PythonFile | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +if TYPE_CHECKING: | ||
| 17 | + from argparse import Namespace | ||
| 18 | + from collections.abc import Iterator, Sequence | ||
| 19 | + from typing_extensions import Never | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +class ErrorLines: | ||
| 23 | + """How many lines to display before and after an error""" | ||
| 24 | + | ||
| 25 | + WINDOW = 5 | ||
| 26 | + BEFORE = 2 | ||
| 27 | + AFTER = WINDOW - BEFORE - 1 | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +class FileLinter: | ||
| 31 | + """The base class that all token-based linters inherit from""" | ||
| 32 | + | ||
| 33 | + description: str | ||
| 34 | + linter_name: str | ||
| 35 | + | ||
| 36 | + epilog: str | None = None | ||
| 37 | + is_fixer: bool = True | ||
| 38 | + report_column_numbers: bool = False | ||
| 39 | + | ||
| 40 | + | ||
| 41 | + def _lint(self, python_file: PythonFile) -> Iterator[LintResult]: | ||
| 42 | + raise NotImplementedError | ||
| 43 | + | ||
| 44 | + def __init__(self, argv: Sequence[str] | None = None) -> None: | ||
| 45 | + self.argv = argv | ||
| 46 | + self.parser = ArgumentParser( | ||
| 47 | + is_fixer=self.is_fixer, | ||
| 48 | + description=self.description, | ||
| 49 | + epilog=self.epilog, | ||
| 50 | + ) | ||
| 51 | + self.result_shown = False | ||
| 52 | + | ||
| 53 | + | ||
| 54 | + def run(cls) -> Never: | ||
| 55 | + linter = cls() | ||
| 56 | + sys.exit(not (linter.lint_all() or linter.args.lintrunner)) | ||
| 57 | + | ||
| 58 | + def lint_all(self) -> bool: | ||
| 59 | + success = True | ||
| 60 | + for p in self.paths: | ||
| 61 | + success = self._lint_file(p) and success | ||
| 62 | + return success | ||
| 63 | + | ||
| 64 | + | ||
| 65 | + def make_file(cls, pc: Path | str | None = None) -> PythonFile: | ||
| 66 | + return PythonFile.make(cls.linter_name, pc) | ||
| 67 | + | ||
| 68 | + | ||
| 69 | + def args(self) -> Namespace: | ||
| 70 | + args = self.parser.parse_args(self.argv) | ||
| 71 | + | ||
| 72 | + if args.fix and args.lintrunner: | ||
| 73 | + raise ValueError("--fix and --lintrunner are incompatible") | ||
| 74 | + return args | ||
| 75 | + | ||
| 76 | + | ||
| 77 | + def code(self) -> str: | ||
| 78 | + return self.linter_name.upper() | ||
| 79 | + | ||
| 80 | + | ||
| 81 | + def paths(self) -> list[Path]: | ||
| 82 | + files = [] | ||
| 83 | + file_parts = (f for fp in self.args.files for f in fp.split(":")) | ||
| 84 | + for f in file_parts: | ||
| 85 | + if f.startswith("@"): | ||
| 86 | + files.extend(Path(f[1:]).read_text().splitlines()) | ||
| 87 | + elif f != "--": | ||
| 88 | + files.append(f) | ||
| 89 | + return sorted(Path(f) for f in files) | ||
| 90 | + | ||
| 91 | + def _lint_file(self, p: Path) -> bool: | ||
| 92 | + if self.args.verbose: | ||
| 93 | + print(p, "Reading", file=sys.stderr) | ||
| 94 | + | ||
| 95 | + pf = self.make_file(p) | ||
| 96 | + replacement, results = self._replace(pf) | ||
| 97 | + | ||
| 98 | + if display := list(self._display(pf, results)): | ||
| 99 | + print(*display, sep="\n") | ||
| 100 | + if results and self.args.fix and pf.path and pf.contents != replacement: | ||
| 101 | + pf.path.write_text(replacement) | ||
| 102 | + | ||
| 103 | + return not results or self.args.fix and all(r.is_edit for r in results) | ||
| 104 | + | ||
| 105 | + def _error(self, pf: PythonFile, result: LintResult) -> None: | ||
| 106 | + """Called on files that are unparsable""" | ||
| 107 | + | ||
| 108 | + def _replace(self, pf: PythonFile) -> tuple[str, list[LintResult]]: | ||
| 109 | + # Because of recursive replacements, we need to repeat replacing and reparsing | ||
| 110 | + # from the inside out until all possible replacements are complete | ||
| 111 | + previous_result_count = float("inf") | ||
| 112 | + first_results: list[LintResult] = [] | ||
| 113 | + original = replacement = pf.contents | ||
| 114 | + results: list[LintResult] = [] | ||
| 115 | + | ||
| 116 | + while True: | ||
| 117 | + try: | ||
| 118 | + results = sorted(self._lint(pf), key=LintResult.sort_key) | ||
| 119 | + except IndentationError as e: | ||
| 120 | + error, (_name, lineno, column, _line) = e.args | ||
| 121 | + | ||
| 122 | + results = [LintResult(error, lineno, column)] | ||
| 123 | + self._error(pf, *results) | ||
| 124 | + | ||
| 125 | + except ParseError as e: | ||
| 126 | + results = [LintResult(str(e), *e.token.start)] | ||
| 127 | + self._error(pf, *results) | ||
| 128 | + | ||
| 129 | + for i, ri in enumerate(results): | ||
| 130 | + if not ri.is_recursive: | ||
| 131 | + for rj in results[i + 1 :]: | ||
| 132 | + if ri.contains(rj): | ||
| 133 | + rj.is_recursive = True | ||
| 134 | + else: | ||
| 135 | + break | ||
| 136 | + | ||
| 137 | + first_results = first_results or results | ||
| 138 | + if not results or len(results) >= previous_result_count: | ||
| 139 | + break | ||
| 140 | + previous_result_count = len(results) | ||
| 141 | + | ||
| 142 | + lines = pf.lines[:] | ||
| 143 | + for r in reversed(results): | ||
| 144 | + r.apply(lines) | ||
| 145 | + replacement = "".join(lines) | ||
| 146 | + | ||
| 147 | + if not any(r.is_recursive for r in results): | ||
| 148 | + break | ||
| 149 | + pf = pf.with_contents(replacement) | ||
| 150 | + | ||
| 151 | + if first_results and self.args.lintrunner: | ||
| 152 | + name = f"Suggested fixes for {self.linter_name}" | ||
| 153 | + msg = LintResult(name=name, original=original, replacement=replacement) | ||
| 154 | + first_results.append(msg) | ||
| 155 | + | ||
| 156 | + return replacement, first_results | ||
| 157 | + | ||
| 158 | + def _display(self, pf: PythonFile, results: list[LintResult]) -> Iterator[str]: | ||
| 159 | + """Emit a series of human-readable strings representing the results""" | ||
| 160 | + for r in results: | ||
| 161 | + if self.args.lintrunner: | ||
| 162 | + msg = r.as_message(code=self.code, path=str(pf.path)) | ||
| 163 | + yield json.dumps(msg.asdict(), sort_keys=True) | ||
| 164 | + else: | ||
| 165 | + if self.result_shown: | ||
| 166 | + yield "" | ||
| 167 | + else: | ||
| 168 | + self.result_shown = True | ||
| 169 | + if r.line is None: | ||
| 170 | + yield f"{pf.path}: {r.name}" | ||
| 171 | + else: | ||
| 172 | + yield from (i.rstrip() for i in self._display_window(pf, r)) | ||
| 173 | + | ||
| 174 | + def _display_window(self, pf: PythonFile, r: LintResult) -> Iterator[str]: | ||
| 175 | + """Display a window onto the code with an error""" | ||
| 176 | + if r.char is None or not self.report_column_numbers: | ||
| 177 | + yield f"{pf.path}:{r.line}: {r.name}" | ||
| 178 | + else: | ||
| 179 | + yield f"{pf.path}:{r.line}:{r.char + 1}: {r.name}" | ||
| 180 | + | ||
| 181 | + begin = max((r.line or 0) - ErrorLines.BEFORE, 1) | ||
| 182 | + end = min(begin + ErrorLines.WINDOW, 1 + len(pf.lines)) | ||
| 183 | + | ||
| 184 | + for lineno in range(begin, end): | ||
| 185 | + source_line = pf.lines[lineno - 1].rstrip() | ||
| 186 | + yield f"{lineno:5} | {source_line}" | ||
| 187 | + if lineno == r.line: | ||
| 188 | + spaces = 8 + (r.char or 0) | ||
| 189 | + carets = len(source_line) if r.char is None else (r.length or 1) | ||
| 190 | + yield spaces * " " + carets * "^" | ||
| @@ -0,0 +1,110 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import dataclasses as dc | ||
| 4 | +from enum import Enum | ||
| 5 | + | ||
| 6 | + | ||
| 7 | +class LintSeverity(str, Enum): | ||
| 8 | + ERROR = "error" | ||
| 9 | + WARNING = "warning" | ||
| 10 | + ADVICE = "advice" | ||
| 11 | + DISABLED = "disabled" | ||
| 12 | + | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +class LintMessage: | ||
| 16 | + """This is a datatype representation of the JSON that gets sent to lintrunner | ||
| 17 | + as described here: | ||
| 18 | + https://docs.rs/lintrunner/latest/lintrunner/lint_message/struct.LintMessage.html | ||
| 19 | + """ | ||
| 20 | + | ||
| 21 | + code: str | ||
| 22 | + name: str | ||
| 23 | + severity: LintSeverity | ||
| 24 | + | ||
| 25 | + char: int | None = None | ||
| 26 | + description: str | None = None | ||
| 27 | + line: int | None = None | ||
| 28 | + original: str | None = None | ||
| 29 | + path: str | None = None | ||
| 30 | + replacement: str | None = None | ||
| 31 | + | ||
| 32 | + asdict = dc.asdict | ||
| 33 | + | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +class LintResult: | ||
| 37 | + """LintResult is a single result from a linter. | ||
| 38 | + | ||
| 39 | + Like LintMessage but the .length member allows you to make specific edits to | ||
| 40 | + one location within a file, not just replace the whole file. | ||
| 41 | + | ||
| 42 | + Linters can generate recursive results - results that contain other results. | ||
| 43 | + | ||
| 44 | + For example, the annotation linter would find two results in this code sample: | ||
| 45 | + | ||
| 46 | + index = Union[Optional[str], int] | ||
| 47 | + | ||
| 48 | + And the first result, `Union[Optional[str], int]`, contains the second one, | ||
| 49 | + `Optional[str]`, so the first result is recursive but the second is not. | ||
| 50 | + | ||
| 51 | + If --fix is selected, the linter does a cycle of tokenizing and fixing all | ||
| 52 | + the non-recursive edits until no edits remain. | ||
| 53 | + """ | ||
| 54 | + | ||
| 55 | + name: str | ||
| 56 | + | ||
| 57 | + line: int | None = None | ||
| 58 | + char: int | None = None | ||
| 59 | + replacement: str | None = None | ||
| 60 | + length: int | None = None # Not in LintMessage | ||
| 61 | + description: str | None = None | ||
| 62 | + original: str | None = None | ||
| 63 | + | ||
| 64 | + is_recursive: bool = False # Not in LintMessage | ||
| 65 | + | ||
| 66 | + | ||
| 67 | + def is_edit(self) -> bool: | ||
| 68 | + return None not in (self.char, self.length, self.line, self.replacement) | ||
| 69 | + | ||
| 70 | + def apply(self, lines: list[str]) -> None: | ||
| 71 | + if not ( | ||
| 72 | + self.char is None | ||
| 73 | + or self.length is None | ||
| 74 | + or self.line is None | ||
| 75 | + or self.replacement is None | ||
| 76 | + or self.is_recursive | ||
| 77 | + ): | ||
| 78 | + line = lines[self.line - 1] | ||
| 79 | + before = line[: self.char] | ||
| 80 | + after = line[self.char + self.length :] | ||
| 81 | + lines[self.line - 1] = f"{before}{self.replacement}{after}" | ||
| 82 | + | ||
| 83 | + def contains(self, r: LintResult) -> bool: | ||
| 84 | + if self.char is None or self.line is None: | ||
| 85 | + raise AssertionError("self.char and self.line must not be None") | ||
| 86 | + if r.char is None or r.line is None: | ||
| 87 | + raise AssertionError("r.char and r.line must not be None") | ||
| 88 | + return self.line == r.line and self.char <= r.char and self.end >= r.end | ||
| 89 | + | ||
| 90 | + | ||
| 91 | + def end(self) -> int: | ||
| 92 | + if self.char is None or self.length is None: | ||
| 93 | + raise AssertionError("self.char and self.length must not be None") | ||
| 94 | + return self.char + self.length | ||
| 95 | + | ||
| 96 | + def as_message(self, code: str, path: str) -> LintMessage: | ||
| 97 | + d = dc.asdict(self) | ||
| 98 | + d.pop("is_recursive") | ||
| 99 | + d.pop("length") | ||
| 100 | + if self.is_edit: | ||
| 101 | + # This is one of our , which we don't want to | ||
| 102 | + # send to lintrunner as a replacement | ||
| 103 | + d["replacement"] = None | ||
| 104 | + | ||
| 105 | + return LintMessage(code=code, path=path, severity=LintSeverity.ERROR, **d) | ||
| 106 | + | ||
| 107 | + def sort_key(self) -> tuple[int, int, str]: | ||
| 108 | + line = -1 if self.line is None else self.line | ||
| 109 | + char = -1 if self.char is None else self.char | ||
| 110 | + return line, char, self.name | ||
| @@ -0,0 +1,185 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import token | ||
| 4 | +from functools import cached_property | ||
| 5 | +from pathlib import Path | ||
| 6 | +from tokenize import generate_tokens, TokenInfo | ||
| 7 | +from typing import TYPE_CHECKING | ||
| 8 | + | ||
| 9 | +from . import is_empty, NO_TOKEN, ParseError, ROOT | ||
| 10 | +from .sets import LineWithSets | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +if TYPE_CHECKING: | ||
| 14 | + from collections.abc import Sequence | ||
| 15 | + from typing_extensions import Self | ||
| 16 | + | ||
| 17 | + from .block import Block | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +class PythonFile: | ||
| 21 | + path: Path | None | ||
| 22 | + linter_name: str | ||
| 23 | + | ||
| 24 | + def __init__( | ||
| 25 | + self, | ||
| 26 | + linter_name: str, | ||
| 27 | + *, | ||
| 28 | + contents: str | None = None, | ||
| 29 | + path: Path | None = None, | ||
| 30 | + ) -> None: | ||
| 31 | + self.linter_name = linter_name | ||
| 32 | + self._contents = contents | ||
| 33 | + self.path = path.relative_to(ROOT) if path and path.is_absolute() else path | ||
| 34 | + | ||
| 35 | + | ||
| 36 | + def contents(self) -> str: | ||
| 37 | + if self._contents is not None: | ||
| 38 | + return self._contents | ||
| 39 | + return self.path.read_text() if self.path else "" | ||
| 40 | + | ||
| 41 | + | ||
| 42 | + def lines(self) -> list[str]: | ||
| 43 | + return self.contents.splitlines(keepends=True) | ||
| 44 | + | ||
| 45 | + | ||
| 46 | + def make(cls, linter_name: str, pc: Path | str | None = None) -> Self: | ||
| 47 | + if isinstance(pc, Path): | ||
| 48 | + return cls(linter_name, path=pc) | ||
| 49 | + else: | ||
| 50 | + return cls(linter_name, contents=pc) | ||
| 51 | + | ||
| 52 | + def with_contents(self, contents: str) -> Self: | ||
| 53 | + return self.__class__(self.linter_name, contents=contents, path=self.path) | ||
| 54 | + | ||
| 55 | + | ||
| 56 | + def omitted(self) -> OmittedLines: | ||
| 57 | + if self.linter_name is None: | ||
| 58 | + raise AssertionError("linter_name is None") | ||
| 59 | + return OmittedLines(self.lines, self.linter_name) | ||
| 60 | + | ||
| 61 | + | ||
| 62 | + def tokens(self) -> list[TokenInfo]: | ||
| 63 | + """This file, tokenized. Raises IndentationError on badly indented code.""" | ||
| 64 | + return list(generate_tokens(iter(self.lines).__next__)) | ||
| 65 | + | ||
| 66 | + | ||
| 67 | + def token_lines(self) -> list[list[TokenInfo]]: | ||
| 68 | + """Returns lists of TokenInfo segmented by token.NEWLINE""" | ||
| 69 | + token_lines: list[list[TokenInfo]] = [[]] | ||
| 70 | + | ||
| 71 | + for t in self.tokens: | ||
| 72 | + if t.type not in (token.COMMENT, token.ENDMARKER, token.NL): | ||
| 73 | + token_lines[-1].append(t) | ||
| 74 | + if t.type == token.NEWLINE: | ||
| 75 | + token_lines.append([]) | ||
| 76 | + if token_lines and not token_lines[-1]: | ||
| 77 | + token_lines.pop() | ||
| 78 | + return token_lines | ||
| 79 | + | ||
| 80 | + | ||
| 81 | + def import_lines(self) -> list[list[int]]: | ||
| 82 | + froms, imports = [], [] | ||
| 83 | + for i, (t, *_) in enumerate(self.token_lines): | ||
| 84 | + if t.type == token.INDENT: | ||
| 85 | + break | ||
| 86 | + if t.type == token.NAME: | ||
| 87 | + if t.string == "from": | ||
| 88 | + froms.append(i) | ||
| 89 | + elif t.string == "import": | ||
| 90 | + imports.append(i) | ||
| 91 | + | ||
| 92 | + return [froms, imports] | ||
| 93 | + | ||
| 94 | + | ||
| 95 | + def opening_comment_lines(self) -> int: | ||
| 96 | + """The number of comments at the very top of the file.""" | ||
| 97 | + it = (i for i, s in enumerate(self.lines) if not s.startswith("#")) | ||
| 98 | + return next(it, 0) | ||
| 99 | + | ||
| 100 | + def __getitem__(self, i: int | slice) -> TokenInfo | Sequence[TokenInfo]: | ||
| 101 | + return self.tokens[i] | ||
| 102 | + | ||
| 103 | + def next_token(self, start: int, token_type: int, error: str) -> int: | ||
| 104 | + for i in range(start, len(self.tokens)): | ||
| 105 | + if self.tokens[i].type == token_type: | ||
| 106 | + return i | ||
| 107 | + raise ParseError(self.tokens[-1], error) | ||
| 108 | + | ||
| 109 | + def docstring(self, start: int) -> str: | ||
| 110 | + for i in range(start + 1, len(self.tokens)): | ||
| 111 | + tk = self.tokens[i] | ||
| 112 | + if tk.type == token.STRING: | ||
| 113 | + return tk.string | ||
| 114 | + if is_empty(tk): | ||
| 115 | + return "" | ||
| 116 | + return "" | ||
| 117 | + | ||
| 118 | + | ||
| 119 | + def indent_to_dedent(self) -> dict[int, int]: | ||
| 120 | + dedents = dict[int, int]() | ||
| 121 | + stack = list[int]() | ||
| 122 | + | ||
| 123 | + for i, t in enumerate(self.tokens): | ||
| 124 | + if t.type == token.INDENT: | ||
| 125 | + stack.append(i) | ||
| 126 | + elif t.type == token.DEDENT: | ||
| 127 | + dedents[stack.pop()] = i | ||
| 128 | + | ||
| 129 | + return dedents | ||
| 130 | + | ||
| 131 | + | ||
| 132 | + def braced_sets(self) -> list[Sequence[TokenInfo]]: | ||
| 133 | + lines = [t for tl in self._lines_with_sets for t in tl.braced_sets] | ||
| 134 | + return [s for s in lines if not self.omitted(s)] | ||
| 135 | + | ||
| 136 | + | ||
| 137 | + def sets(self) -> list[TokenInfo]: | ||
| 138 | + tokens = [t for tl in self._lines_with_sets for t in tl.sets] | ||
| 139 | + return [t for t in tokens if not self.omitted([t])] | ||
| 140 | + | ||
| 141 | + | ||
| 142 | + def insert_import_line(self) -> int | None: | ||
| 143 | + froms, imports = self.import_lines | ||
| 144 | + for i in froms + imports: | ||
| 145 | + tl = self.token_lines[i] | ||
| 146 | + if any(i.type == token.NAME and i.string == "OrderedSet" for i in tl): | ||
| 147 | + return None | ||
| 148 | + if section := froms or imports: | ||
| 149 | + return self._lines_with_sets[section[-1]].tokens[-1].start[0] + 1 | ||
| 150 | + return self.opening_comment_lines + 1 | ||
| 151 | + | ||
| 152 | + | ||
| 153 | + def _lines_with_sets(self) -> list[LineWithSets]: | ||
| 154 | + return [LineWithSets(tl) for tl in self.token_lines] | ||
| 155 | + | ||
| 156 | + | ||
| 157 | + def blocks(self) -> list[Block]: | ||
| 158 | + from .blocks import blocks | ||
| 159 | + | ||
| 160 | + return blocks(self) | ||
| 161 | + | ||
| 162 | + | ||
| 163 | +class OmittedLines: | ||
| 164 | + """Read lines textually and find comment lines that end in 'noqa {linter_name}'""" | ||
| 165 | + | ||
| 166 | + omitted: set[int] | ||
| 167 | + | ||
| 168 | + def __init__(self, lines: Sequence[str], linter_name: str) -> None: | ||
| 169 | + self.lines = lines | ||
| 170 | + suffix = f"# noqa: {linter_name}" | ||
| 171 | + omitted = ((i, s.rstrip()) for i, s in enumerate(lines)) | ||
| 172 | + self.omitted = {i + 1 for i, s in omitted if s.endswith(suffix)} | ||
| 173 | + | ||
| 174 | + def __call__( | ||
| 175 | + self, tokens: Sequence[TokenInfo], begin: int = 0, end: int = NO_TOKEN | ||
| 176 | + ) -> bool: | ||
| 177 | + if end == NO_TOKEN: | ||
| 178 | + end = len(tokens) | ||
| 179 | + # A token_line might span multiple physical lines | ||
| 180 | + start = min((tokens[i].start[0] for i in range(begin, end)), default=0) | ||
| 181 | + end = max((tokens[i].end[0] for i in range(begin, end)), default=-1) | ||
| 182 | + return self.contains_lines(start, end) | ||
| 183 | + | ||
| 184 | + def contains_lines(self, begin: int, end: int) -> bool: | ||
| 185 | + return bool(self.omitted.intersection(range(begin, end + 1))) | ||
| @@ -0,0 +1,73 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import dataclasses as dc | ||
| 4 | +import token | ||
| 5 | +from functools import cached_property | ||
| 6 | +from typing import TYPE_CHECKING | ||
| 7 | + | ||
| 8 | +from . import is_empty | ||
| 9 | +from .bracket_pairs import bracket_pairs | ||
| 10 | + | ||
| 11 | + | ||
| 12 | +if TYPE_CHECKING: | ||
| 13 | + from tokenize import TokenInfo | ||
| 14 | + | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +class LineWithSets: | ||
| 18 | + """A logical line of Python tokens, terminated by a NEWLINE or the end of file""" | ||
| 19 | + | ||
| 20 | + tokens: list[TokenInfo] | ||
| 21 | + | ||
| 22 | + | ||
| 23 | + def sets(self) -> list[TokenInfo]: | ||
| 24 | + """A list of tokens which use the built-in set symbol""" | ||
| 25 | + return [t for i, t in enumerate(self.tokens) if self.is_set(i)] | ||
| 26 | + | ||
| 27 | + | ||
| 28 | + def braced_sets(self) -> list[list[TokenInfo]]: | ||
| 29 | + """A list of lists of tokens, each representing a braced set, like {1}""" | ||
| 30 | + return [ | ||
| 31 | + self.tokens[b : e + 1] | ||
| 32 | + for b, e in self.bracket_pairs.items() | ||
| 33 | + if self.is_braced_set(b, e) | ||
| 34 | + ] | ||
| 35 | + | ||
| 36 | + | ||
| 37 | + def bracket_pairs(self) -> dict[int, int]: | ||
| 38 | + return bracket_pairs(self.tokens) | ||
| 39 | + | ||
| 40 | + def is_set(self, i: int) -> bool: | ||
| 41 | + t = self.tokens[i] | ||
| 42 | + after = i < len(self.tokens) - 1 and self.tokens[i + 1] | ||
| 43 | + if t.string == "Set" and t.type == token.NAME: | ||
| 44 | + # pyrefly: ignore [bad-return] | ||
| 45 | + return after and after.string == "[" and after.type == token.OP | ||
| 46 | + return ( | ||
| 47 | + (t.string == "set" and t.type == token.NAME) | ||
| 48 | + and not (i and self.tokens[i - 1].string in ("def", ".")) | ||
| 49 | + and not (after and after.string == "=" and after.type == token.OP) | ||
| 50 | + ) | ||
| 51 | + | ||
| 52 | + def is_braced_set(self, begin: int, end: int) -> bool: | ||
| 53 | + if ( | ||
| 54 | + begin + 1 == end | ||
| 55 | + or self.tokens[begin].string != "{" | ||
| 56 | + or begin | ||
| 57 | + and self.tokens[begin - 1].string == "in" # skip `x in {1, 2, 3}` | ||
| 58 | + ): | ||
| 59 | + return False | ||
| 60 | + | ||
| 61 | + i = begin + 1 | ||
| 62 | + empty = True | ||
| 63 | + while i < end: | ||
| 64 | + t = self.tokens[i] | ||
| 65 | + if t.type == token.OP and t.string in (":", "**"): | ||
| 66 | + return False | ||
| 67 | + if brace_end := self.bracket_pairs.get(i): | ||
| 68 | + # Skip to the end of a subexpression | ||
| 69 | + i = brace_end | ||
| 70 | + elif not is_empty(t): | ||
| 71 | + empty = False | ||
| 72 | + i += 1 | ||
| 73 | + return not empty | ||
| @@ -0,0 +1,170 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import concurrent.futures | ||
| 5 | +import json | ||
| 6 | +import logging | ||
| 7 | +import os | ||
| 8 | +import re | ||
| 9 | +import subprocess | ||
| 10 | +import sys | ||
| 11 | +import time | ||
| 12 | +from enum import Enum | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +LINTER_CODE = "ACTIONLINT" | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +class LintSeverity(str, Enum): | ||
| 20 | + ERROR = "error" | ||
| 21 | + WARNING = "warning" | ||
| 22 | + ADVICE = "advice" | ||
| 23 | + DISABLED = "disabled" | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +class LintMessage(NamedTuple): | ||
| 27 | + path: str | None | ||
| 28 | + line: int | None | ||
| 29 | + char: int | None | ||
| 30 | + code: str | ||
| 31 | + severity: LintSeverity | ||
| 32 | + name: str | ||
| 33 | + original: str | None | ||
| 34 | + replacement: str | None | ||
| 35 | + description: str | None | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 39 | + r"""(?mx) | ||
| 40 | + ^ | ||
| 41 | + (?P<file>.*?): | ||
| 42 | + (?P<line>\d+): | ||
| 43 | + (?P<char>\d+): | ||
| 44 | + \s(?P<message>.*) | ||
| 45 | + \s(?P<code>\[.*\]) | ||
| 46 | + $ | ||
| 47 | + """ | ||
| 48 | +) | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def run_command( | ||
| 52 | + args: list[str], | ||
| 53 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 54 | + logging.debug("$ %s", " ".join(args)) | ||
| 55 | + start_time = time.monotonic() | ||
| 56 | + try: | ||
| 57 | + return subprocess.run( | ||
| 58 | + args, | ||
| 59 | + capture_output=True, | ||
| 60 | + ) | ||
| 61 | + finally: | ||
| 62 | + end_time = time.monotonic() | ||
| 63 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +def check_file( | ||
| 67 | + binary: str, | ||
| 68 | + file: str, | ||
| 69 | +) -> list[LintMessage]: | ||
| 70 | + try: | ||
| 71 | + proc = run_command( | ||
| 72 | + [ | ||
| 73 | + binary, | ||
| 74 | + "-ignore", | ||
| 75 | + '"runs-on" section must be sequence node but got mapping node with "!!map" tag', | ||
| 76 | + "-ignore", | ||
| 77 | + 'input "freethreaded" is not defined in action "actions/setup-python@v', | ||
| 78 | + # GitHub increased workflow_dispatch limit to 25 inputs (Dec 2025). | ||
| 79 | + # actionlint fixed this in v1.7.10; remove after upgrading from v1.7.7. | ||
| 80 | + "-ignore", | ||
| 81 | + 'maximum number of inputs for "workflow_dispatch" event is 10 but', | ||
| 82 | + file, | ||
| 83 | + ] | ||
| 84 | + ) | ||
| 85 | + except OSError as err: | ||
| 86 | + return [ | ||
| 87 | + LintMessage( | ||
| 88 | + path=None, | ||
| 89 | + line=None, | ||
| 90 | + char=None, | ||
| 91 | + code=LINTER_CODE, | ||
| 92 | + severity=LintSeverity.ERROR, | ||
| 93 | + name="command-failed", | ||
| 94 | + original=None, | ||
| 95 | + replacement=None, | ||
| 96 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 97 | + ) | ||
| 98 | + ] | ||
| 99 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 100 | + return [ | ||
| 101 | + LintMessage( | ||
| 102 | + path=match["file"], | ||
| 103 | + name=match["code"], | ||
| 104 | + description=match["message"], | ||
| 105 | + line=int(match["line"]), | ||
| 106 | + char=int(match["char"]), | ||
| 107 | + code=LINTER_CODE, | ||
| 108 | + severity=LintSeverity.ERROR, | ||
| 109 | + original=None, | ||
| 110 | + replacement=None, | ||
| 111 | + ) | ||
| 112 | + for match in RESULTS_RE.finditer(stdout) | ||
| 113 | + ] | ||
| 114 | + | ||
| 115 | + | ||
| 116 | +if __name__ == "__main__": | ||
| 117 | + parser = argparse.ArgumentParser( | ||
| 118 | + description="actionlint runner", | ||
| 119 | + fromfile_prefix_chars="@", | ||
| 120 | + ) | ||
| 121 | + parser.add_argument( | ||
| 122 | + "--binary", | ||
| 123 | + required=True, | ||
| 124 | + help="actionlint binary path", | ||
| 125 | + ) | ||
| 126 | + parser.add_argument( | ||
| 127 | + "filenames", | ||
| 128 | + nargs="+", | ||
| 129 | + help="paths to lint", | ||
| 130 | + ) | ||
| 131 | + | ||
| 132 | + args = parser.parse_args() | ||
| 133 | + | ||
| 134 | + if not os.path.exists(args.binary): | ||
| 135 | + err_msg = LintMessage( | ||
| 136 | + path="<none>", | ||
| 137 | + line=None, | ||
| 138 | + char=None, | ||
| 139 | + code=LINTER_CODE, | ||
| 140 | + severity=LintSeverity.ERROR, | ||
| 141 | + name="command-failed", | ||
| 142 | + original=None, | ||
| 143 | + replacement=None, | ||
| 144 | + description=( | ||
| 145 | + f"Could not find actionlint binary at {args.binary}," | ||
| 146 | + " you may need to run `lintrunner init`." | ||
| 147 | + ), | ||
| 148 | + ) | ||
| 149 | + print(json.dumps(err_msg._asdict()), flush=True) | ||
| 150 | + sys.exit(0) | ||
| 151 | + | ||
| 152 | + with concurrent.futures.ThreadPoolExecutor( | ||
| 153 | + max_workers=os.cpu_count(), | ||
| 154 | + thread_name_prefix="Thread", | ||
| 155 | + ) as executor: | ||
| 156 | + futures = { | ||
| 157 | + executor.submit( | ||
| 158 | + check_file, | ||
| 159 | + args.binary, | ||
| 160 | + filename, | ||
| 161 | + ): filename | ||
| 162 | + for filename in args.filenames | ||
| 163 | + } | ||
| 164 | + for future in concurrent.futures.as_completed(futures): | ||
| 165 | + try: | ||
| 166 | + for lint_message in future.result(): | ||
| 167 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 168 | + except Exception: | ||
| 169 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 170 | + raise | ||
| @@ -0,0 +1,197 @@ | |||
| 1 | +""" | ||
| 2 | +This linter ensures that users don't set a SHA hash checksum in Bazel for the http_archive. | ||
| 3 | +Although the security practice of setting the checksum is good, it doesn't work when the | ||
| 4 | +archive is downloaded from some sites like GitHub because it can change. Specifically, | ||
| 5 | +GitHub gives no guarantee to keep the same value forever. Check for more details at | ||
| 6 | +https://github.com/community/community/discussions/46034. | ||
| 7 | +""" | ||
| 8 | + | ||
| 9 | +from __future__ import annotations | ||
| 10 | + | ||
| 11 | +import argparse | ||
| 12 | +import json | ||
| 13 | +import re | ||
| 14 | +import shlex | ||
| 15 | +import subprocess | ||
| 16 | +import sys | ||
| 17 | +import xml.etree.ElementTree as ET | ||
| 18 | +from enum import Enum | ||
| 19 | +from typing import NamedTuple | ||
| 20 | +from urllib.parse import urlparse | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +LINTER_CODE = "BAZEL_LINTER" | ||
| 24 | +SHA256_REGEX = re.compile(r"\s*sha256\s*=\s*['\"](?P<sha256>[a-zA-Z0-9]{64})['\"]\s*,") | ||
| 25 | +DOMAINS_WITH_UNSTABLE_CHECKSUM = {"github.com"} | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +class LintSeverity(str, Enum): | ||
| 29 | + ERROR = "error" | ||
| 30 | + WARNING = "warning" | ||
| 31 | + ADVICE = "advice" | ||
| 32 | + DISABLED = "disabled" | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +class LintMessage(NamedTuple): | ||
| 36 | + path: str | None | ||
| 37 | + line: int | None | ||
| 38 | + char: int | None | ||
| 39 | + code: str | ||
| 40 | + severity: LintSeverity | ||
| 41 | + name: str | ||
| 42 | + original: str | None | ||
| 43 | + replacement: str | None | ||
| 44 | + description: str | None | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +def is_required_checksum(urls: list[str | None]) -> bool: | ||
| 48 | + if not urls: | ||
| 49 | + return False | ||
| 50 | + | ||
| 51 | + for url in urls: | ||
| 52 | + if not url: | ||
| 53 | + continue | ||
| 54 | + | ||
| 55 | + parsed_url = urlparse(url) | ||
| 56 | + if parsed_url.hostname in DOMAINS_WITH_UNSTABLE_CHECKSUM: | ||
| 57 | + return False | ||
| 58 | + | ||
| 59 | + return True | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def get_disallowed_checksums( | ||
| 63 | + binary: str, | ||
| 64 | +) -> set[str]: | ||
| 65 | + """ | ||
| 66 | + Return the set of disallowed checksums from all http_archive rules | ||
| 67 | + """ | ||
| 68 | + # Use bazel to get the list of external dependencies in XML format | ||
| 69 | + proc = subprocess.run( | ||
| 70 | + [binary, "query", "kind(http_archive, //external:*)", "--output=xml"], | ||
| 71 | + capture_output=True, | ||
| 72 | + check=True, | ||
| 73 | + text=True, | ||
| 74 | + ) | ||
| 75 | + | ||
| 76 | + root = ET.fromstring(proc.stdout) | ||
| 77 | + | ||
| 78 | + disallowed_checksums = set() | ||
| 79 | + # Parse all the http_archive rules in the XML output | ||
| 80 | + for rule in root.findall('.//rule[@class="http_archive"]'): | ||
| 81 | + urls_node = rule.find('.//list[@name="urls"]') | ||
| 82 | + if urls_node is None: | ||
| 83 | + continue | ||
| 84 | + urls = [n.get("value") for n in urls_node.findall(".//string")] | ||
| 85 | + | ||
| 86 | + checksum_node = rule.find('.//string[@name="sha256"]') | ||
| 87 | + if checksum_node is None: | ||
| 88 | + continue | ||
| 89 | + checksum = checksum_node.get("value") | ||
| 90 | + | ||
| 91 | + if not checksum: | ||
| 92 | + continue | ||
| 93 | + | ||
| 94 | + if not is_required_checksum(urls): | ||
| 95 | + disallowed_checksums.add(checksum) | ||
| 96 | + | ||
| 97 | + return disallowed_checksums | ||
| 98 | + | ||
| 99 | + | ||
| 100 | +def check_bazel( | ||
| 101 | + filename: str, | ||
| 102 | + disallowed_checksums: set[str], | ||
| 103 | +) -> list[LintMessage]: | ||
| 104 | + original = "" | ||
| 105 | + replacement = "" | ||
| 106 | + | ||
| 107 | + with open(filename) as f: | ||
| 108 | + for line in f: | ||
| 109 | + original += f"{line}" | ||
| 110 | + | ||
| 111 | + m = SHA256_REGEX.match(line) | ||
| 112 | + if m: | ||
| 113 | + sha256 = m.group("sha256") | ||
| 114 | + | ||
| 115 | + if sha256 in disallowed_checksums: | ||
| 116 | + continue | ||
| 117 | + | ||
| 118 | + replacement += f"{line}" | ||
| 119 | + | ||
| 120 | + if original == replacement: | ||
| 121 | + return [] | ||
| 122 | + | ||
| 123 | + return [ | ||
| 124 | + LintMessage( | ||
| 125 | + path=filename, | ||
| 126 | + line=None, | ||
| 127 | + char=None, | ||
| 128 | + code=LINTER_CODE, | ||
| 129 | + severity=LintSeverity.ADVICE, | ||
| 130 | + name="format", | ||
| 131 | + original=original, | ||
| 132 | + replacement=replacement, | ||
| 133 | + description="Found redundant SHA checksums. Run `lintrunner -a` to apply this patch.", | ||
| 134 | + ) | ||
| 135 | + ] | ||
| 136 | + | ||
| 137 | + | ||
| 138 | +def main() -> None: | ||
| 139 | + parser = argparse.ArgumentParser( | ||
| 140 | + description="A custom linter to detect redundant SHA checksums in Bazel", | ||
| 141 | + fromfile_prefix_chars="@", | ||
| 142 | + ) | ||
| 143 | + parser.add_argument( | ||
| 144 | + "--binary", | ||
| 145 | + required=True, | ||
| 146 | + help="bazel binary path", | ||
| 147 | + ) | ||
| 148 | + parser.add_argument( | ||
| 149 | + "filenames", | ||
| 150 | + nargs="+", | ||
| 151 | + help="paths to lint", | ||
| 152 | + ) | ||
| 153 | + args = parser.parse_args() | ||
| 154 | + | ||
| 155 | + try: | ||
| 156 | + disallowed_checksums = get_disallowed_checksums(args.binary) | ||
| 157 | + except subprocess.CalledProcessError as err: | ||
| 158 | + err_msg = LintMessage( | ||
| 159 | + path=None, | ||
| 160 | + line=None, | ||
| 161 | + char=None, | ||
| 162 | + code=__file__, | ||
| 163 | + severity=LintSeverity.ADVICE, | ||
| 164 | + name="command-failed", | ||
| 165 | + original=None, | ||
| 166 | + replacement=None, | ||
| 167 | + description=( | ||
| 168 | + f"COMMAND (exit code {err.returncode})\n" | ||
| 169 | + f"{shlex.join(err.cmd)}\n\n" | ||
| 170 | + f"STDERR\n{err.stderr or '(empty)'}\n\n" | ||
| 171 | + f"STDOUT\n{err.stdout or '(empty)'}" | ||
| 172 | + ), | ||
| 173 | + ) | ||
| 174 | + print(json.dumps(err_msg._asdict())) | ||
| 175 | + return | ||
| 176 | + except Exception as e: | ||
| 177 | + err_msg = LintMessage( | ||
| 178 | + path=None, | ||
| 179 | + line=None, | ||
| 180 | + char=None, | ||
| 181 | + code=LINTER_CODE, | ||
| 182 | + severity=LintSeverity.ERROR, | ||
| 183 | + name="command-failed", | ||
| 184 | + original=None, | ||
| 185 | + replacement=None, | ||
| 186 | + description=(f"Failed due to {e.__class__.__name__}:\n{e}"), | ||
| 187 | + ) | ||
| 188 | + print(json.dumps(err_msg._asdict()), flush=True) | ||
| 189 | + sys.exit(0) | ||
| 190 | + | ||
| 191 | + for filename in args.filenames: | ||
| 192 | + for lint_message in check_bazel(filename, disallowed_checksums): | ||
| 193 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 194 | + | ||
| 195 | + | ||
| 196 | +if __name__ == "__main__": | ||
| 197 | + main() | ||
| @@ -0,0 +1,246 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import concurrent.futures | ||
| 5 | +import json | ||
| 6 | +import logging | ||
| 7 | +import os | ||
| 8 | +import subprocess | ||
| 9 | +import sys | ||
| 10 | +import time | ||
| 11 | +from enum import Enum | ||
| 12 | +from pathlib import Path | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +IS_WINDOWS: bool = os.name == "nt" | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +class LintSeverity(str, Enum): | ||
| 20 | + ERROR = "error" | ||
| 21 | + WARNING = "warning" | ||
| 22 | + ADVICE = "advice" | ||
| 23 | + DISABLED = "disabled" | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +class LintMessage(NamedTuple): | ||
| 27 | + path: str | None | ||
| 28 | + line: int | None | ||
| 29 | + char: int | None | ||
| 30 | + code: str | ||
| 31 | + severity: LintSeverity | ||
| 32 | + name: str | ||
| 33 | + original: str | None | ||
| 34 | + replacement: str | None | ||
| 35 | + description: str | None | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +def as_posix(name: str) -> str: | ||
| 39 | + return name.replace("\\", "/") if IS_WINDOWS else name | ||
| 40 | + | ||
| 41 | + | ||
| 42 | +def _run_command( | ||
| 43 | + args: list[str], | ||
| 44 | + *, | ||
| 45 | + timeout: int, | ||
| 46 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 47 | + logging.debug("$ %s", " ".join(args)) | ||
| 48 | + start_time = time.monotonic() | ||
| 49 | + try: | ||
| 50 | + return subprocess.run( | ||
| 51 | + args, | ||
| 52 | + capture_output=True, | ||
| 53 | + shell=IS_WINDOWS, # So batch scripts are found. | ||
| 54 | + timeout=timeout, | ||
| 55 | + check=True, | ||
| 56 | + ) | ||
| 57 | + finally: | ||
| 58 | + end_time = time.monotonic() | ||
| 59 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def run_command( | ||
| 63 | + args: list[str], | ||
| 64 | + *, | ||
| 65 | + retries: int, | ||
| 66 | + timeout: int, | ||
| 67 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 68 | + remaining_retries = retries | ||
| 69 | + while True: | ||
| 70 | + try: | ||
| 71 | + return _run_command(args, timeout=timeout) | ||
| 72 | + except subprocess.TimeoutExpired as err: | ||
| 73 | + if remaining_retries == 0: | ||
| 74 | + raise err | ||
| 75 | + remaining_retries -= 1 | ||
| 76 | + logging.warning( # noqa: G200 | ||
| 77 | + "(%s/%s) Retrying because command failed with: %r", | ||
| 78 | + retries - remaining_retries, | ||
| 79 | + retries, | ||
| 80 | + err, | ||
| 81 | + ) | ||
| 82 | + time.sleep(1) | ||
| 83 | + | ||
| 84 | + | ||
| 85 | +def check_file( | ||
| 86 | + filename: str, | ||
| 87 | + binary: str, | ||
| 88 | + retries: int, | ||
| 89 | + timeout: int, | ||
| 90 | +) -> list[LintMessage]: | ||
| 91 | + try: | ||
| 92 | + with open(filename, "rb") as f: | ||
| 93 | + original = f.read() | ||
| 94 | + proc = run_command( | ||
| 95 | + [binary, filename], | ||
| 96 | + retries=retries, | ||
| 97 | + timeout=timeout, | ||
| 98 | + ) | ||
| 99 | + except subprocess.TimeoutExpired: | ||
| 100 | + return [ | ||
| 101 | + LintMessage( | ||
| 102 | + path=filename, | ||
| 103 | + line=None, | ||
| 104 | + char=None, | ||
| 105 | + code="CLANGFORMAT", | ||
| 106 | + severity=LintSeverity.ERROR, | ||
| 107 | + name="timeout", | ||
| 108 | + original=None, | ||
| 109 | + replacement=None, | ||
| 110 | + description=( | ||
| 111 | + "clang-format timed out while trying to process a file. " | ||
| 112 | + "Please report an issue in pytorch/pytorch with the " | ||
| 113 | + "label 'module: lint'" | ||
| 114 | + ), | ||
| 115 | + ) | ||
| 116 | + ] | ||
| 117 | + except (OSError, subprocess.CalledProcessError) as err: | ||
| 118 | + return [ | ||
| 119 | + LintMessage( | ||
| 120 | + path=filename, | ||
| 121 | + line=None, | ||
| 122 | + char=None, | ||
| 123 | + code="CLANGFORMAT", | ||
| 124 | + severity=LintSeverity.ADVICE, | ||
| 125 | + name="command-failed", | ||
| 126 | + original=None, | ||
| 127 | + replacement=None, | ||
| 128 | + description=( | ||
| 129 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 130 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 131 | + else ( | ||
| 132 | + "COMMAND (exit code {returncode})\n" | ||
| 133 | + "{command}\n\n" | ||
| 134 | + "STDERR\n{stderr}\n\n" | ||
| 135 | + "STDOUT\n{stdout}" | ||
| 136 | + ).format( | ||
| 137 | + returncode=err.returncode, | ||
| 138 | + command=" ".join(as_posix(x) for x in err.cmd), | ||
| 139 | + stderr=err.stderr.decode("utf-8").strip() or "(empty)", | ||
| 140 | + stdout=err.stdout.decode("utf-8").strip() or "(empty)", | ||
| 141 | + ) | ||
| 142 | + ), | ||
| 143 | + ) | ||
| 144 | + ] | ||
| 145 | + | ||
| 146 | + replacement = proc.stdout | ||
| 147 | + if original == replacement: | ||
| 148 | + return [] | ||
| 149 | + | ||
| 150 | + return [ | ||
| 151 | + LintMessage( | ||
| 152 | + path=filename, | ||
| 153 | + line=None, | ||
| 154 | + char=None, | ||
| 155 | + code="CLANGFORMAT", | ||
| 156 | + severity=LintSeverity.WARNING, | ||
| 157 | + name="format", | ||
| 158 | + original=original.decode("utf-8"), | ||
| 159 | + replacement=replacement.decode("utf-8"), | ||
| 160 | + description="See https://clang.llvm.org/docs/ClangFormat.html.\nRun `lintrunner -a` to apply this patch.", | ||
| 161 | + ) | ||
| 162 | + ] | ||
| 163 | + | ||
| 164 | + | ||
| 165 | +def main() -> None: | ||
| 166 | + parser = argparse.ArgumentParser( | ||
| 167 | + description="Format files with clang-format.", | ||
| 168 | + fromfile_prefix_chars="@", | ||
| 169 | + ) | ||
| 170 | + parser.add_argument( | ||
| 171 | + "--binary", | ||
| 172 | + required=True, | ||
| 173 | + help="clang-format binary path", | ||
| 174 | + ) | ||
| 175 | + parser.add_argument( | ||
| 176 | + "--retries", | ||
| 177 | + default=3, | ||
| 178 | + type=int, | ||
| 179 | + help="times to retry timed out clang-format", | ||
| 180 | + ) | ||
| 181 | + parser.add_argument( | ||
| 182 | + "--timeout", | ||
| 183 | + default=90, | ||
| 184 | + type=int, | ||
| 185 | + help="seconds to wait for clang-format", | ||
| 186 | + ) | ||
| 187 | + parser.add_argument( | ||
| 188 | + "--verbose", | ||
| 189 | + action="store_true", | ||
| 190 | + help="verbose logging", | ||
| 191 | + ) | ||
| 192 | + parser.add_argument( | ||
| 193 | + "filenames", | ||
| 194 | + nargs="+", | ||
| 195 | + help="paths to lint", | ||
| 196 | + ) | ||
| 197 | + args = parser.parse_args() | ||
| 198 | + | ||
| 199 | + logging.basicConfig( | ||
| 200 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 201 | + level=logging.NOTSET | ||
| 202 | + if args.verbose | ||
| 203 | + else logging.DEBUG | ||
| 204 | + if len(args.filenames) < 1000 | ||
| 205 | + else logging.INFO, | ||
| 206 | + stream=sys.stderr, | ||
| 207 | + ) | ||
| 208 | + | ||
| 209 | + binary = os.path.normpath(args.binary) if IS_WINDOWS else args.binary | ||
| 210 | + if not Path(binary).exists(): | ||
| 211 | + lint_message = LintMessage( | ||
| 212 | + path=None, | ||
| 213 | + line=None, | ||
| 214 | + char=None, | ||
| 215 | + code="CLANGFORMAT", | ||
| 216 | + severity=LintSeverity.ERROR, | ||
| 217 | + name="init-error", | ||
| 218 | + original=None, | ||
| 219 | + replacement=None, | ||
| 220 | + description=( | ||
| 221 | + f"Could not find clang-format binary at {binary}, " | ||
| 222 | + "did you forget to run `lintrunner init`?" | ||
| 223 | + ), | ||
| 224 | + ) | ||
| 225 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 226 | + sys.exit(0) | ||
| 227 | + | ||
| 228 | + with concurrent.futures.ThreadPoolExecutor( | ||
| 229 | + max_workers=os.cpu_count(), | ||
| 230 | + thread_name_prefix="Thread", | ||
| 231 | + ) as executor: | ||
| 232 | + futures = { | ||
| 233 | + executor.submit(check_file, x, binary, args.retries, args.timeout): x | ||
| 234 | + for x in args.filenames | ||
| 235 | + } | ||
| 236 | + for future in concurrent.futures.as_completed(futures): | ||
| 237 | + try: | ||
| 238 | + for lint_message in future.result(): | ||
| 239 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 240 | + except Exception: | ||
| 241 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 242 | + raise | ||
| 243 | + | ||
| 244 | + | ||
| 245 | +if __name__ == "__main__": | ||
| 246 | + main() | ||
| @@ -0,0 +1,314 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import concurrent.futures | ||
| 5 | +import json | ||
| 6 | +import logging | ||
| 7 | +import os | ||
| 8 | +import re | ||
| 9 | +import shutil | ||
| 10 | +import subprocess | ||
| 11 | +import sys | ||
| 12 | +import time | ||
| 13 | +from enum import Enum | ||
| 14 | +from pathlib import Path | ||
| 15 | +from sysconfig import get_paths as gp | ||
| 16 | +from typing import NamedTuple | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +# PyTorch directory root | ||
| 20 | +def scm_root() -> str: | ||
| 21 | + path = os.path.abspath(os.getcwd()) | ||
| 22 | + # pyrefly: ignore [bad-assignment] | ||
| 23 | + while True: | ||
| 24 | + if os.path.exists(os.path.join(path, ".git")): | ||
| 25 | + return path | ||
| 26 | + if os.path.isdir(os.path.join(path, ".hg")): | ||
| 27 | + return path | ||
| 28 | + # pyrefly: ignore [bad-argument-type] | ||
| 29 | + n = len(path) | ||
| 30 | + path = os.path.dirname(path) | ||
| 31 | + if len(path) == n: | ||
| 32 | + raise RuntimeError("Unable to find SCM root") | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +PYTORCH_ROOT = scm_root() | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +# Returns '/usr/local/include/python<version number>' | ||
| 39 | +def get_python_include_dir() -> str: | ||
| 40 | + return gp()["include"] | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +class LintSeverity(str, Enum): | ||
| 44 | + ERROR = "error" | ||
| 45 | + WARNING = "warning" | ||
| 46 | + ADVICE = "advice" | ||
| 47 | + DISABLED = "disabled" | ||
| 48 | + | ||
| 49 | + | ||
| 50 | +class LintMessage(NamedTuple): | ||
| 51 | + path: str | None | ||
| 52 | + line: int | None | ||
| 53 | + char: int | None | ||
| 54 | + code: str | ||
| 55 | + severity: LintSeverity | ||
| 56 | + name: str | ||
| 57 | + original: str | None | ||
| 58 | + replacement: str | None | ||
| 59 | + description: str | None | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +# c10/core/DispatchKey.cpp:281:26: error: 'k' used after it was moved [bugprone-use-after-move] | ||
| 63 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 64 | + r"""(?mx) | ||
| 65 | + ^ | ||
| 66 | + (?P<file>.*?): | ||
| 67 | + (?P<line>\d+): | ||
| 68 | + (?:(?P<column>-?\d+):)? | ||
| 69 | + \s(?P<severity>\S+?):? | ||
| 70 | + \s(?P<message>.*) | ||
| 71 | + \s(?P<code>\[.*\]) | ||
| 72 | + $ | ||
| 73 | + """ | ||
| 74 | +) | ||
| 75 | + | ||
| 76 | + | ||
| 77 | +def run_command( | ||
| 78 | + args: list[str], | ||
| 79 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 80 | + logging.debug("$ %s", " ".join(args)) | ||
| 81 | + start_time = time.monotonic() | ||
| 82 | + try: | ||
| 83 | + return subprocess.run( | ||
| 84 | + args, | ||
| 85 | + capture_output=True, | ||
| 86 | + check=False, | ||
| 87 | + ) | ||
| 88 | + finally: | ||
| 89 | + end_time = time.monotonic() | ||
| 90 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 91 | + | ||
| 92 | + | ||
| 93 | +# Severity is either "error" or "note": | ||
| 94 | +# https://github.com/python/mypy/blob/8b47a032e1317fb8e3f9a818005a6b63e9bf0311/mypy/errors.py#L46-L47 | ||
| 95 | +severities = { | ||
| 96 | + "error": LintSeverity.ERROR, | ||
| 97 | + "warning": LintSeverity.WARNING, | ||
| 98 | +} | ||
| 99 | + | ||
| 100 | + | ||
| 101 | +def clang_search_dirs() -> list[str]: | ||
| 102 | + # Compilers are ordered based on fallback preference | ||
| 103 | + # We pick the first one that is available on the system | ||
| 104 | + compilers = ["clang", "gcc", "cpp", "cc"] | ||
| 105 | + compilers = [c for c in compilers if shutil.which(c) is not None] | ||
| 106 | + if len(compilers) == 0: | ||
| 107 | + raise RuntimeError(f"None of {compilers} were found") | ||
| 108 | + compiler = compilers[0] | ||
| 109 | + | ||
| 110 | + result = subprocess.run( | ||
| 111 | + [compiler, "-E", "-x", "c++", "-", "-v"], | ||
| 112 | + stdin=subprocess.DEVNULL, | ||
| 113 | + capture_output=True, | ||
| 114 | + check=True, | ||
| 115 | + ) | ||
| 116 | + stderr = result.stderr.decode().strip().split("\n") | ||
| 117 | + search_start = r"#include.*search starts here:" | ||
| 118 | + search_end = r"End of search list." | ||
| 119 | + | ||
| 120 | + append_path = False | ||
| 121 | + search_paths = [] | ||
| 122 | + for line in stderr: | ||
| 123 | + if re.match(search_start, line): | ||
| 124 | + if append_path: | ||
| 125 | + continue | ||
| 126 | + else: | ||
| 127 | + append_path = True | ||
| 128 | + elif re.match(search_end, line): | ||
| 129 | + break | ||
| 130 | + elif append_path: | ||
| 131 | + search_paths.append(line.strip()) | ||
| 132 | + | ||
| 133 | + return search_paths | ||
| 134 | + | ||
| 135 | + | ||
| 136 | +include_args = [] | ||
| 137 | +include_dir = [ | ||
| 138 | + "/usr/lib/llvm-11/include/openmp", | ||
| 139 | + get_python_include_dir(), | ||
| 140 | + os.path.join(PYTORCH_ROOT, "third_party/pybind11/include"), | ||
| 141 | + PYTORCH_ROOT, | ||
| 142 | +] + clang_search_dirs() | ||
| 143 | +for dir in include_dir: | ||
| 144 | + include_args += ["--extra-arg", f"-I{dir}"] | ||
| 145 | + | ||
| 146 | + | ||
| 147 | +def check_file( | ||
| 148 | + filename: str, | ||
| 149 | + binary: str, | ||
| 150 | + build_dir: Path, | ||
| 151 | + std: str | None, | ||
| 152 | +) -> list[LintMessage]: | ||
| 153 | + # Explicitly pass include path for linters that only check headers. | ||
| 154 | + build_include_args = include_args + ["--extra-arg", f"-I{build_dir}"] | ||
| 155 | + cmd = [ | ||
| 156 | + binary, | ||
| 157 | + f"-p={build_dir}", | ||
| 158 | + *build_include_args, | ||
| 159 | + filename, | ||
| 160 | + ] | ||
| 161 | + # Only add -- and -std flag if std is explicitly specified | ||
| 162 | + if std is not None: | ||
| 163 | + cmd.extend(["--", f"-std={std}"]) | ||
| 164 | + | ||
| 165 | + try: | ||
| 166 | + proc = run_command(cmd) | ||
| 167 | + except OSError as err: | ||
| 168 | + return [ | ||
| 169 | + LintMessage( | ||
| 170 | + path=filename, | ||
| 171 | + line=None, | ||
| 172 | + char=None, | ||
| 173 | + code="CLANGTIDY", | ||
| 174 | + severity=LintSeverity.ERROR, | ||
| 175 | + name="command-failed", | ||
| 176 | + original=None, | ||
| 177 | + replacement=None, | ||
| 178 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 179 | + ) | ||
| 180 | + ] | ||
| 181 | + lint_messages = [] | ||
| 182 | + try: | ||
| 183 | + # Change the current working directory to the build directory, since | ||
| 184 | + # clang-tidy will report files relative to the build directory. | ||
| 185 | + saved_cwd = os.getcwd() | ||
| 186 | + os.chdir(build_dir) | ||
| 187 | + | ||
| 188 | + for match in RESULTS_RE.finditer(proc.stdout.decode()): | ||
| 189 | + # Convert the reported path to an absolute path. | ||
| 190 | + abs_path = str(Path(match["file"]).resolve()) | ||
| 191 | + if not abs_path.startswith(PYTORCH_ROOT): | ||
| 192 | + continue | ||
| 193 | + message = LintMessage( | ||
| 194 | + path=abs_path, | ||
| 195 | + name=match["code"], | ||
| 196 | + description=match["message"], | ||
| 197 | + line=int(match["line"]), | ||
| 198 | + char=int(match["column"]) | ||
| 199 | + if match["column"] is not None and not match["column"].startswith("-") | ||
| 200 | + else None, | ||
| 201 | + code="CLANGTIDY", | ||
| 202 | + severity=severities.get(match["severity"], LintSeverity.ERROR), | ||
| 203 | + original=None, | ||
| 204 | + replacement=None, | ||
| 205 | + ) | ||
| 206 | + lint_messages.append(message) | ||
| 207 | + finally: | ||
| 208 | + os.chdir(saved_cwd) | ||
| 209 | + | ||
| 210 | + return lint_messages | ||
| 211 | + | ||
| 212 | + | ||
| 213 | +def main() -> None: | ||
| 214 | + parser = argparse.ArgumentParser( | ||
| 215 | + description="clang-tidy wrapper linter.", | ||
| 216 | + fromfile_prefix_chars="@", | ||
| 217 | + ) | ||
| 218 | + parser.add_argument( | ||
| 219 | + "--binary", | ||
| 220 | + required=True, | ||
| 221 | + help="clang-tidy binary path", | ||
| 222 | + ) | ||
| 223 | + parser.add_argument( | ||
| 224 | + "--build-dir", | ||
| 225 | + "--build_dir", | ||
| 226 | + required=True, | ||
| 227 | + help=( | ||
| 228 | + "Where the compile_commands.json file is located. " | ||
| 229 | + "Gets passed to clang-tidy -p" | ||
| 230 | + ), | ||
| 231 | + ) | ||
| 232 | + parser.add_argument( | ||
| 233 | + "--std", | ||
| 234 | + default=None, | ||
| 235 | + help=( | ||
| 236 | + "C++ standard to use for compilation (e.g., c++17, c++20). " | ||
| 237 | + "If not specified, uses the standard from compile_commands.json." | ||
| 238 | + ), | ||
| 239 | + ) | ||
| 240 | + parser.add_argument( | ||
| 241 | + "--verbose", | ||
| 242 | + action="store_true", | ||
| 243 | + help="verbose logging", | ||
| 244 | + ) | ||
| 245 | + parser.add_argument( | ||
| 246 | + "filenames", | ||
| 247 | + nargs="+", | ||
| 248 | + help="paths to lint", | ||
| 249 | + ) | ||
| 250 | + args = parser.parse_args() | ||
| 251 | + | ||
| 252 | + logging.basicConfig( | ||
| 253 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 254 | + level=logging.NOTSET | ||
| 255 | + if args.verbose | ||
| 256 | + else logging.DEBUG | ||
| 257 | + if len(args.filenames) < 1000 | ||
| 258 | + else logging.INFO, | ||
| 259 | + stream=sys.stderr, | ||
| 260 | + ) | ||
| 261 | + | ||
| 262 | + if not os.path.exists(args.binary): | ||
| 263 | + err_msg = LintMessage( | ||
| 264 | + path="<none>", | ||
| 265 | + line=None, | ||
| 266 | + char=None, | ||
| 267 | + code="CLANGTIDY", | ||
| 268 | + severity=LintSeverity.ERROR, | ||
| 269 | + name="command-failed", | ||
| 270 | + original=None, | ||
| 271 | + replacement=None, | ||
| 272 | + description=( | ||
| 273 | + f"Could not find clang-tidy binary at {args.binary}," | ||
| 274 | + " you may need to run `lintrunner init`." | ||
| 275 | + ), | ||
| 276 | + ) | ||
| 277 | + print(json.dumps(err_msg._asdict()), flush=True) | ||
| 278 | + sys.exit(0) | ||
| 279 | + | ||
| 280 | + abs_build_dir = Path(args.build_dir).resolve() | ||
| 281 | + | ||
| 282 | + # Get the absolute path to clang-tidy and use this instead of the relative | ||
| 283 | + # path such as .lintbin/clang-tidy. The problem here is that os.chdir is | ||
| 284 | + # per process, and the linter uses it to move between the current directory | ||
| 285 | + # and the build folder. And there is no .lintbin directory in the latter. | ||
| 286 | + # When it happens in a race condition, the linter command will fails with | ||
| 287 | + # the following no such file or directory error: '.lintbin/clang-tidy' | ||
| 288 | + binary_path = os.path.abspath(args.binary) | ||
| 289 | + | ||
| 290 | + with concurrent.futures.ThreadPoolExecutor( | ||
| 291 | + max_workers=os.cpu_count(), | ||
| 292 | + thread_name_prefix="Thread", | ||
| 293 | + ) as executor: | ||
| 294 | + futures = { | ||
| 295 | + executor.submit( | ||
| 296 | + check_file, | ||
| 297 | + filename, | ||
| 298 | + binary_path, | ||
| 299 | + abs_build_dir, | ||
| 300 | + args.std, | ||
| 301 | + ): filename | ||
| 302 | + for filename in args.filenames | ||
| 303 | + } | ||
| 304 | + for future in concurrent.futures.as_completed(futures): | ||
| 305 | + try: | ||
| 306 | + for lint_message in future.result(): | ||
| 307 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 308 | + except Exception: | ||
| 309 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 310 | + raise | ||
| 311 | + | ||
| 312 | + | ||
| 313 | +if __name__ == "__main__": | ||
| 314 | + main() | ||
| @@ -0,0 +1,146 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "cmakelint==1.4.1", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +from __future__ import annotations | ||
| 8 | + | ||
| 9 | +import argparse | ||
| 10 | +import concurrent.futures | ||
| 11 | +import json | ||
| 12 | +import logging | ||
| 13 | +import os | ||
| 14 | +import re | ||
| 15 | +import subprocess | ||
| 16 | +import time | ||
| 17 | +from enum import Enum | ||
| 18 | +from typing import NamedTuple | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +LINTER_CODE = "CMAKE" | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +class LintSeverity(str, Enum): | ||
| 25 | + ERROR = "error" | ||
| 26 | + WARNING = "warning" | ||
| 27 | + ADVICE = "advice" | ||
| 28 | + DISABLED = "disabled" | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +class LintMessage(NamedTuple): | ||
| 32 | + path: str | None | ||
| 33 | + line: int | None | ||
| 34 | + char: int | None | ||
| 35 | + code: str | ||
| 36 | + severity: LintSeverity | ||
| 37 | + name: str | ||
| 38 | + original: str | None | ||
| 39 | + replacement: str | None | ||
| 40 | + description: str | None | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +# CMakeLists.txt:901: Lines should be <= 80 characters long [linelength] | ||
| 44 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 45 | + r"""(?mx) | ||
| 46 | + ^ | ||
| 47 | + (?P<file>.*?): | ||
| 48 | + (?P<line>\d+): | ||
| 49 | + \s(?P<message>.*) | ||
| 50 | + \s(?P<code>\[.*\]) | ||
| 51 | + $ | ||
| 52 | + """ | ||
| 53 | +) | ||
| 54 | + | ||
| 55 | + | ||
| 56 | +def run_command( | ||
| 57 | + args: list[str], | ||
| 58 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 59 | + logging.debug("$ %s", " ".join(args)) | ||
| 60 | + start_time = time.monotonic() | ||
| 61 | + try: | ||
| 62 | + return subprocess.run( | ||
| 63 | + args, | ||
| 64 | + capture_output=True, | ||
| 65 | + ) | ||
| 66 | + finally: | ||
| 67 | + end_time = time.monotonic() | ||
| 68 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 69 | + | ||
| 70 | + | ||
| 71 | +def check_file( | ||
| 72 | + filename: str, | ||
| 73 | + config: str, | ||
| 74 | +) -> list[LintMessage]: | ||
| 75 | + try: | ||
| 76 | + proc = run_command( | ||
| 77 | + ["cmakelint", f"--config={config}", filename], | ||
| 78 | + ) | ||
| 79 | + except OSError as err: | ||
| 80 | + return [ | ||
| 81 | + LintMessage( | ||
| 82 | + path=None, | ||
| 83 | + line=None, | ||
| 84 | + char=None, | ||
| 85 | + code=LINTER_CODE, | ||
| 86 | + severity=LintSeverity.ERROR, | ||
| 87 | + name="command-failed", | ||
| 88 | + original=None, | ||
| 89 | + replacement=None, | ||
| 90 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 91 | + ) | ||
| 92 | + ] | ||
| 93 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 94 | + return [ | ||
| 95 | + LintMessage( | ||
| 96 | + path=match["file"], | ||
| 97 | + name=match["code"], | ||
| 98 | + description=match["message"], | ||
| 99 | + line=int(match["line"]), | ||
| 100 | + char=None, | ||
| 101 | + code=LINTER_CODE, | ||
| 102 | + severity=LintSeverity.ERROR, | ||
| 103 | + original=None, | ||
| 104 | + replacement=None, | ||
| 105 | + ) | ||
| 106 | + for match in RESULTS_RE.finditer(stdout) | ||
| 107 | + ] | ||
| 108 | + | ||
| 109 | + | ||
| 110 | +if __name__ == "__main__": | ||
| 111 | + parser = argparse.ArgumentParser( | ||
| 112 | + description="cmakelint runner", | ||
| 113 | + fromfile_prefix_chars="@", | ||
| 114 | + ) | ||
| 115 | + parser.add_argument( | ||
| 116 | + "--config", | ||
| 117 | + required=True, | ||
| 118 | + help="location of cmakelint config", | ||
| 119 | + ) | ||
| 120 | + parser.add_argument( | ||
| 121 | + "filenames", | ||
| 122 | + nargs="+", | ||
| 123 | + help="paths to lint", | ||
| 124 | + ) | ||
| 125 | + | ||
| 126 | + args = parser.parse_args() | ||
| 127 | + | ||
| 128 | + with concurrent.futures.ThreadPoolExecutor( | ||
| 129 | + max_workers=os.cpu_count(), | ||
| 130 | + thread_name_prefix="Thread", | ||
| 131 | + ) as executor: | ||
| 132 | + futures = { | ||
| 133 | + executor.submit( | ||
| 134 | + check_file, | ||
| 135 | + filename, | ||
| 136 | + args.config, | ||
| 137 | + ): filename | ||
| 138 | + for filename in args.filenames | ||
| 139 | + } | ||
| 140 | + for future in concurrent.futures.as_completed(futures): | ||
| 141 | + try: | ||
| 142 | + for lint_message in future.result(): | ||
| 143 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 144 | + except Exception: | ||
| 145 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 146 | + raise | ||
| @@ -0,0 +1,252 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "packaging==25.0", | ||
| 5 | +# "tomli==2.2.1 ; python_version < '3.11'", | ||
| 6 | +# ] | ||
| 7 | +# /// | ||
| 8 | +from __future__ import annotations | ||
| 9 | + | ||
| 10 | +import argparse | ||
| 11 | +import concurrent.futures | ||
| 12 | +import fnmatch | ||
| 13 | +import json | ||
| 14 | +import logging | ||
| 15 | +import os | ||
| 16 | +import re | ||
| 17 | +import sys | ||
| 18 | +from enum import Enum | ||
| 19 | +from pathlib import Path | ||
| 20 | +from typing import NamedTuple | ||
| 21 | + | ||
| 22 | +from packaging.requirements import Requirement | ||
| 23 | +from packaging.version import Version | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +if sys.version_info >= (3, 11): | ||
| 27 | + import tomllib | ||
| 28 | +else: | ||
| 29 | + import tomli as tomllib # type: ignore[import-not-found] | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +REPO_ROOT = Path(__file__).absolute().parents[3] | ||
| 33 | +sys.path.insert(0, str(REPO_ROOT)) | ||
| 34 | + | ||
| 35 | +from tools.setup_helpers.env import CMAKE_MINIMUM_VERSION_STRING | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +sys.path.remove(str(REPO_ROOT)) | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +LINTER_CODE = "CMAKE_MINIMUM_REQUIRED" | ||
| 42 | +CMAKE_MINIMUM_VERSION = Version(CMAKE_MINIMUM_VERSION_STRING) | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +class LintSeverity(str, Enum): | ||
| 46 | + ERROR = "error" | ||
| 47 | + WARNING = "warning" | ||
| 48 | + ADVICE = "advice" | ||
| 49 | + DISABLED = "disabled" | ||
| 50 | + | ||
| 51 | + | ||
| 52 | +class LintMessage(NamedTuple): | ||
| 53 | + path: str | None | ||
| 54 | + line: int | None | ||
| 55 | + char: int | None | ||
| 56 | + code: str | ||
| 57 | + severity: LintSeverity | ||
| 58 | + name: str | ||
| 59 | + original: str | None | ||
| 60 | + replacement: str | None | ||
| 61 | + description: str | None | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def format_error_message( | ||
| 65 | + filename: str, | ||
| 66 | + error: Exception | None = None, | ||
| 67 | + *, | ||
| 68 | + line: int | None = None, | ||
| 69 | + message: str | None = None, | ||
| 70 | +) -> LintMessage: | ||
| 71 | + if message is None and error is not None: | ||
| 72 | + message = f"Failed due to {error.__class__.__name__}:\n{error}" | ||
| 73 | + return LintMessage( | ||
| 74 | + path=filename, | ||
| 75 | + line=line, | ||
| 76 | + char=None, | ||
| 77 | + code=LINTER_CODE, | ||
| 78 | + severity=LintSeverity.ERROR, | ||
| 79 | + name="CMake minimum version", | ||
| 80 | + original=None, | ||
| 81 | + replacement=None, | ||
| 82 | + description=message, | ||
| 83 | + ) | ||
| 84 | + | ||
| 85 | + | ||
| 86 | +CMAKE_MINIMUM_REQUIRED_PATTERN = re.compile( | ||
| 87 | + r"cmake_minimum_required\(VERSION\s+(?P<version>\d+\.\d+(\.\d+)?)\b.*\)", | ||
| 88 | + flags=re.IGNORECASE, | ||
| 89 | +) | ||
| 90 | + | ||
| 91 | + | ||
| 92 | +def check_cmake(path: Path) -> list[LintMessage]: | ||
| 93 | + with path.open(encoding="utf-8") as f: | ||
| 94 | + for i, line in enumerate(f, start=1): | ||
| 95 | + if match := CMAKE_MINIMUM_REQUIRED_PATTERN.search(line): | ||
| 96 | + version = match.group("version") | ||
| 97 | + if path.samefile(REPO_ROOT / "CMakeLists.txt"): | ||
| 98 | + if Version(version) != CMAKE_MINIMUM_VERSION: | ||
| 99 | + return [ | ||
| 100 | + format_error_message( | ||
| 101 | + str(path), | ||
| 102 | + line=i, | ||
| 103 | + message=( | ||
| 104 | + f"CMake minimum version must be {CMAKE_MINIMUM_VERSION}, " | ||
| 105 | + f"but found {version}." | ||
| 106 | + ), | ||
| 107 | + ) | ||
| 108 | + ] | ||
| 109 | + elif Version(version) > CMAKE_MINIMUM_VERSION: | ||
| 110 | + return [ | ||
| 111 | + format_error_message( | ||
| 112 | + str(path), | ||
| 113 | + line=i, | ||
| 114 | + message=( | ||
| 115 | + f"The environment can only provide CMake {CMAKE_MINIMUM_VERSION}, " | ||
| 116 | + f"but found requiring {version}." | ||
| 117 | + ), | ||
| 118 | + ) | ||
| 119 | + ] | ||
| 120 | + return [] | ||
| 121 | + | ||
| 122 | + | ||
| 123 | +def check_requirement( | ||
| 124 | + requirement: Requirement, | ||
| 125 | + path: Path, | ||
| 126 | + *, | ||
| 127 | + line: int | None = None, | ||
| 128 | +) -> LintMessage | None: | ||
| 129 | + if requirement.name.lower() != "cmake": | ||
| 130 | + return None | ||
| 131 | + | ||
| 132 | + for spec in requirement.specifier: | ||
| 133 | + if ( | ||
| 134 | + spec.operator in ("==", ">=") | ||
| 135 | + and Version(spec.version.removesuffix(".*")) < CMAKE_MINIMUM_VERSION | ||
| 136 | + ): | ||
| 137 | + return format_error_message( | ||
| 138 | + str(path), | ||
| 139 | + line=line, | ||
| 140 | + message=( | ||
| 141 | + f"CMake minimum version must be at least {CMAKE_MINIMUM_VERSION}, " | ||
| 142 | + f"but found {spec}." | ||
| 143 | + ), | ||
| 144 | + ) | ||
| 145 | + | ||
| 146 | + return None | ||
| 147 | + | ||
| 148 | + | ||
| 149 | +def check_pyproject(path: Path) -> list[LintMessage]: | ||
| 150 | + try: | ||
| 151 | + pyproject = tomllib.loads(path.read_text(encoding="utf-8")) | ||
| 152 | + except (tomllib.TOMLDecodeError, OSError) as err: | ||
| 153 | + return [format_error_message(str(path), err)] | ||
| 154 | + | ||
| 155 | + if not isinstance(pyproject, dict): | ||
| 156 | + return [] | ||
| 157 | + if not isinstance(pyproject.get("build-system"), dict): | ||
| 158 | + return [] | ||
| 159 | + | ||
| 160 | + build_system = pyproject["build-system"] | ||
| 161 | + requires = build_system.get("requires") | ||
| 162 | + if not isinstance(requires, list): | ||
| 163 | + return [] | ||
| 164 | + return list( | ||
| 165 | + filter( | ||
| 166 | + None, | ||
| 167 | + (check_requirement(Requirement(req), path=path) for req in requires), | ||
| 168 | + ) | ||
| 169 | + ) | ||
| 170 | + | ||
| 171 | + | ||
| 172 | +def check_requirements(path: Path) -> list[LintMessage]: | ||
| 173 | + try: | ||
| 174 | + with path.open(encoding="utf-8") as f: | ||
| 175 | + lines = f.readlines() | ||
| 176 | + except OSError as err: | ||
| 177 | + return [format_error_message(str(path), err)] | ||
| 178 | + | ||
| 179 | + lint_messages = [] | ||
| 180 | + for i, line in enumerate(lines, start=1): | ||
| 181 | + line = line.strip() | ||
| 182 | + if not line or line.startswith(("#", "-")): | ||
| 183 | + continue | ||
| 184 | + try: | ||
| 185 | + requirement = Requirement(line) | ||
| 186 | + except Exception: | ||
| 187 | + continue | ||
| 188 | + lint_message = check_requirement(requirement, path=path, line=i) | ||
| 189 | + if lint_message is not None: | ||
| 190 | + lint_messages.append(lint_message) | ||
| 191 | + | ||
| 192 | + return lint_messages | ||
| 193 | + | ||
| 194 | + | ||
| 195 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 196 | + path = Path(filename).absolute() | ||
| 197 | + basename = path.name.lower() | ||
| 198 | + if basename in ("cmakelists.txt", "cmakelists.txt.in") or basename.endswith( | ||
| 199 | + (".cmake", ".cmake.in") | ||
| 200 | + ): | ||
| 201 | + return check_cmake(path) | ||
| 202 | + if basename == "pyproject.toml": | ||
| 203 | + return check_pyproject(path) | ||
| 204 | + if fnmatch.fnmatch(basename, "*requirements*.txt") or fnmatch.fnmatch( | ||
| 205 | + basename, "*requirements*.in" | ||
| 206 | + ): | ||
| 207 | + return check_requirements(path) | ||
| 208 | + return [] | ||
| 209 | + | ||
| 210 | + | ||
| 211 | +def main() -> None: | ||
| 212 | + parser = argparse.ArgumentParser( | ||
| 213 | + description="Check consistency of cmake minimum version in requirement files.", | ||
| 214 | + fromfile_prefix_chars="@", | ||
| 215 | + ) | ||
| 216 | + parser.add_argument( | ||
| 217 | + "--verbose", | ||
| 218 | + action="store_true", | ||
| 219 | + help="verbose logging", | ||
| 220 | + ) | ||
| 221 | + parser.add_argument( | ||
| 222 | + "filenames", | ||
| 223 | + nargs="+", | ||
| 224 | + help="paths to lint", | ||
| 225 | + ) | ||
| 226 | + args = parser.parse_args() | ||
| 227 | + | ||
| 228 | + logging.basicConfig( | ||
| 229 | + format="<%(processName)s:%(levelname)s> %(message)s", | ||
| 230 | + level=logging.NOTSET | ||
| 231 | + if args.verbose | ||
| 232 | + else logging.DEBUG | ||
| 233 | + if len(args.filenames) < 1000 | ||
| 234 | + else logging.INFO, | ||
| 235 | + stream=sys.stderr, | ||
| 236 | + ) | ||
| 237 | + | ||
| 238 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 239 | + max_workers=os.cpu_count(), | ||
| 240 | + ) as executor: | ||
| 241 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 242 | + for future in concurrent.futures.as_completed(futures): | ||
| 243 | + try: | ||
| 244 | + for lint_message in future.result(): | ||
| 245 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 246 | + except Exception: | ||
| 247 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 248 | + raise | ||
| 249 | + | ||
| 250 | + | ||
| 251 | +if __name__ == "__main__": | ||
| 252 | + main() | ||
| @@ -0,0 +1,206 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "codespell[toml]==2.4.1", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +from __future__ import annotations | ||
| 8 | + | ||
| 9 | +import argparse | ||
| 10 | +import concurrent.futures | ||
| 11 | +import json | ||
| 12 | +import logging | ||
| 13 | +import os | ||
| 14 | +import subprocess | ||
| 15 | +import sys | ||
| 16 | +from enum import Enum | ||
| 17 | +from pathlib import Path | ||
| 18 | +from typing import NamedTuple | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +REPO_ROOT = Path(__file__).absolute().parents[3] | ||
| 22 | +PYPROJECT = REPO_ROOT / "pyproject.toml" | ||
| 23 | +DICTIONARY = REPO_ROOT / "tools" / "linter" / "dictionary.txt" | ||
| 24 | + | ||
| 25 | +FORBIDDEN_WORDS = { | ||
| 26 | + "multipy", # project pytorch/multipy is dead # codespell:ignore multipy | ||
| 27 | +} | ||
| 28 | + | ||
| 29 | +MAX_FILE_SIZE: int = 1024 * 1024 * 1024 # 1GB in bytes | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +class LintSeverity(str, Enum): | ||
| 33 | + ERROR = "error" | ||
| 34 | + WARNING = "warning" | ||
| 35 | + ADVICE = "advice" | ||
| 36 | + DISABLED = "disabled" | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +class LintMessage(NamedTuple): | ||
| 40 | + path: str | None | ||
| 41 | + line: int | None | ||
| 42 | + char: int | None | ||
| 43 | + code: str | ||
| 44 | + severity: LintSeverity | ||
| 45 | + name: str | ||
| 46 | + original: str | None | ||
| 47 | + replacement: str | None | ||
| 48 | + description: str | None | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def format_error_message( | ||
| 52 | + filename: str, | ||
| 53 | + error: Exception | None = None, | ||
| 54 | + *, | ||
| 55 | + message: str | None = None, | ||
| 56 | +) -> LintMessage: | ||
| 57 | + if message is None and error is not None: | ||
| 58 | + message = ( | ||
| 59 | + f"Failed due to {error.__class__.__name__}:\n{error}\n" | ||
| 60 | + "Please either fix the error or add the word(s) to the dictionary file.\n" | ||
| 61 | + "HINT: all-lowercase words in the dictionary can cover all case variations." | ||
| 62 | + ) | ||
| 63 | + return LintMessage( | ||
| 64 | + path=filename, | ||
| 65 | + line=None, | ||
| 66 | + char=None, | ||
| 67 | + code="CODESPELL", | ||
| 68 | + severity=LintSeverity.ERROR, | ||
| 69 | + name="spelling error", | ||
| 70 | + original=None, | ||
| 71 | + replacement=None, | ||
| 72 | + description=message, | ||
| 73 | + ) | ||
| 74 | + | ||
| 75 | + | ||
| 76 | +def run_codespell(path: Path) -> str: | ||
| 77 | + try: | ||
| 78 | + return subprocess.check_output( | ||
| 79 | + [ | ||
| 80 | + sys.executable, | ||
| 81 | + "-m", | ||
| 82 | + "codespell_lib", | ||
| 83 | + "--toml", | ||
| 84 | + str(PYPROJECT), | ||
| 85 | + str(path), | ||
| 86 | + ], | ||
| 87 | + stderr=subprocess.STDOUT, | ||
| 88 | + text=True, | ||
| 89 | + encoding="utf-8", | ||
| 90 | + ) | ||
| 91 | + except subprocess.CalledProcessError as exc: | ||
| 92 | + raise ValueError(exc.output) from exc | ||
| 93 | + | ||
| 94 | + | ||
| 95 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 96 | + path = Path(filename).absolute() | ||
| 97 | + | ||
| 98 | + # Check if file is too large | ||
| 99 | + try: | ||
| 100 | + file_size = os.path.getsize(path) | ||
| 101 | + if file_size > MAX_FILE_SIZE: | ||
| 102 | + return [ | ||
| 103 | + LintMessage( | ||
| 104 | + path=filename, | ||
| 105 | + line=None, | ||
| 106 | + char=None, | ||
| 107 | + code="CODESPELL", | ||
| 108 | + severity=LintSeverity.WARNING, | ||
| 109 | + name="file-too-large", | ||
| 110 | + original=None, | ||
| 111 | + replacement=None, | ||
| 112 | + description=f"File size ({file_size} bytes) exceeds {MAX_FILE_SIZE} bytes limit, skipping", | ||
| 113 | + ) | ||
| 114 | + ] | ||
| 115 | + except OSError as err: | ||
| 116 | + return [ | ||
| 117 | + LintMessage( | ||
| 118 | + path=filename, | ||
| 119 | + line=None, | ||
| 120 | + char=None, | ||
| 121 | + code="CODESPELL", | ||
| 122 | + severity=LintSeverity.ERROR, | ||
| 123 | + name="file-access-error", | ||
| 124 | + original=None, | ||
| 125 | + replacement=None, | ||
| 126 | + description=f"Failed to get file size: {err}", | ||
| 127 | + ) | ||
| 128 | + ] | ||
| 129 | + | ||
| 130 | + try: | ||
| 131 | + run_codespell(path) | ||
| 132 | + except Exception as err: | ||
| 133 | + return [format_error_message(filename, err)] | ||
| 134 | + return [] | ||
| 135 | + | ||
| 136 | + | ||
| 137 | +def check_dictionary(filename: str) -> list[LintMessage]: | ||
| 138 | + """Check the dictionary file for duplicates.""" | ||
| 139 | + path = Path(filename).absolute() | ||
| 140 | + try: | ||
| 141 | + words = path.read_text(encoding="utf-8").splitlines() | ||
| 142 | + words_set = set(words) | ||
| 143 | + if len(words) != len(words_set): | ||
| 144 | + raise ValueError("The dictionary file contains duplicate entries.") | ||
| 145 | + # pyrefly: ignore [no-matching-overload] | ||
| 146 | + uncased_words = list(map(str.lower, words)) | ||
| 147 | + if uncased_words != sorted(uncased_words): | ||
| 148 | + raise ValueError( | ||
| 149 | + "The dictionary file is not sorted alphabetically (case-insensitive)." | ||
| 150 | + ) | ||
| 151 | + for forbidden_word in sorted( | ||
| 152 | + FORBIDDEN_WORDS & (words_set | set(uncased_words)) | ||
| 153 | + ): | ||
| 154 | + raise ValueError( | ||
| 155 | + f"The dictionary file contains a forbidden word: {forbidden_word!r}. " | ||
| 156 | + "Please remove it from the dictionary file and use 'codespell:ignore' " | ||
| 157 | + "inline comment instead." | ||
| 158 | + ) | ||
| 159 | + except Exception as err: | ||
| 160 | + return [format_error_message(str(filename), err)] | ||
| 161 | + return [] | ||
| 162 | + | ||
| 163 | + | ||
| 164 | +def main() -> None: | ||
| 165 | + parser = argparse.ArgumentParser( | ||
| 166 | + description="Check files for spelling mistakes using codespell.", | ||
| 167 | + fromfile_prefix_chars="@", | ||
| 168 | + ) | ||
| 169 | + parser.add_argument( | ||
| 170 | + "--verbose", | ||
| 171 | + action="store_true", | ||
| 172 | + help="verbose logging", | ||
| 173 | + ) | ||
| 174 | + parser.add_argument( | ||
| 175 | + "filenames", | ||
| 176 | + nargs="+", | ||
| 177 | + help="paths to lint", | ||
| 178 | + ) | ||
| 179 | + args = parser.parse_args() | ||
| 180 | + | ||
| 181 | + logging.basicConfig( | ||
| 182 | + format="<%(processName)s:%(levelname)s> %(message)s", | ||
| 183 | + level=logging.NOTSET | ||
| 184 | + if args.verbose | ||
| 185 | + else logging.DEBUG | ||
| 186 | + if len(args.filenames) < 1000 | ||
| 187 | + else logging.INFO, | ||
| 188 | + stream=sys.stderr, | ||
| 189 | + ) | ||
| 190 | + | ||
| 191 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 192 | + max_workers=os.cpu_count(), | ||
| 193 | + ) as executor: | ||
| 194 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 195 | + futures[executor.submit(check_dictionary, str(DICTIONARY))] = str(DICTIONARY) | ||
| 196 | + for future in concurrent.futures.as_completed(futures): | ||
| 197 | + try: | ||
| 198 | + for lint_message in future.result(): | ||
| 199 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 200 | + except Exception: | ||
| 201 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 202 | + raise | ||
| 203 | + | ||
| 204 | + | ||
| 205 | +if __name__ == "__main__": | ||
| 206 | + main() | ||
| @@ -0,0 +1,226 @@ | |||
| 1 | +{ | ||
| 2 | + "torch/_inductor/autoheuristic/artifacts/_MMRankingA100.py": { | ||
| 3 | + "class MMRankingA100": 280, | ||
| 4 | + "def MMRankingA100.fill_choices()": 199 | ||
| 5 | + }, | ||
| 6 | + "torch/_inductor/autoheuristic/artifacts/_MMRankingH100.py": { | ||
| 7 | + "class MMRankingH100": 305, | ||
| 8 | + "def MMRankingH100.fill_choices()": 203 | ||
| 9 | + }, | ||
| 10 | + "torch/_inductor/autoheuristic/artifacts/_MixedMMA100.py": { | ||
| 11 | + "class MixedMMA100": 134, | ||
| 12 | + "def MixedMMA100.get_best_choices()": 86 | ||
| 13 | + }, | ||
| 14 | + "torch/_inductor/autoheuristic/artifacts/_MixedMMH100.py": { | ||
| 15 | + "class MixedMMH100": 133, | ||
| 16 | + "def MixedMMH100.get_best_choices()": 86 | ||
| 17 | + }, | ||
| 18 | + "torch/_inductor/bounds.py": { | ||
| 19 | + "class ValueRangeAnalysis": 108 | ||
| 20 | + }, | ||
| 21 | + "torch/_inductor/codecache.py": { | ||
| 22 | + "class CppPythonBindingsCodeCache": 191, | ||
| 23 | + "class HalideCodeCache": 358, | ||
| 24 | + "class PyCodeCache": 101 | ||
| 25 | + }, | ||
| 26 | + "torch/_inductor/codegen/common.py": { | ||
| 27 | + "class CSE": 173, | ||
| 28 | + "class Kernel": 305, | ||
| 29 | + "class KernelArgs": 345, | ||
| 30 | + "class OpOverrides": 208 | ||
| 31 | + }, | ||
| 32 | + "torch/_inductor/codegen/cpp.py": { | ||
| 33 | + "class CppKernelProxy": 616, | ||
| 34 | + "class CppOverrides": 437, | ||
| 35 | + "class CppScheduling": 819, | ||
| 36 | + "class CppVecKernel": 867, | ||
| 37 | + "class OuterLoopFusedSchedulerNode": 159, | ||
| 38 | + "def CppKernel.codegen_loops_impl()": 144, | ||
| 39 | + "def CppKernelProxy.codegen_functions()": 179, | ||
| 40 | + "def CppKernelProxy.legalize_lowp_fp_dtype_loopbody()": 228, | ||
| 41 | + "def CppScheduling.fuse()": 81, | ||
| 42 | + "def CppVecKernel.reduction_combine_vec()": 103, | ||
| 43 | + "def OuterLoopFusedSchedulerNode.check_outer_fusion_loop_level_attr()": 85, | ||
| 44 | + "def TilingSelect.select_tiling()": 170 | ||
| 45 | + }, | ||
| 46 | + "torch/_inductor/codegen/cpp_flex_attention_template.py": { | ||
| 47 | + "class CppFlexAttentionTemplate": 404, | ||
| 48 | + "def CppFlexAttentionTemplate.modification()": 102 | ||
| 49 | + }, | ||
| 50 | + "torch/_inductor/codegen/cpp_gemm_template.py": { | ||
| 51 | + "def CppGemmTemplate.get_options()": 255 | ||
| 52 | + }, | ||
| 53 | + "torch/_inductor/codegen/cpp_grouped_gemm_template.py": { | ||
| 54 | + "def CppGroupedGemmTemplate.render()": 157 | ||
| 55 | + }, | ||
| 56 | + "torch/_inductor/codegen/cpp_template.py": { | ||
| 57 | + "class CppTemplate": 117 | ||
| 58 | + }, | ||
| 59 | + "torch/_inductor/codegen/cpp_template_kernel.py": { | ||
| 60 | + "class CppTemplateKernel": 504 | ||
| 61 | + }, | ||
| 62 | + "torch/_inductor/codegen/cpp_utils.py": { | ||
| 63 | + "def create_epilogue_with_attr()": 164 | ||
| 64 | + }, | ||
| 65 | + "torch/_inductor/codegen/cpp_wrapper_cpu.py": { | ||
| 66 | + "def CppWrapperCpu.generate_py_arg()": 96, | ||
| 67 | + "def CppWrapperCpu.val_to_arg_str()": 88, | ||
| 68 | + "def CppWrapperCpu.write_wrapper_decl()": 142 | ||
| 69 | + }, | ||
| 70 | + "torch/_inductor/codegen/cpp_wrapper_cpu_array_ref.py": { | ||
| 71 | + "def CppWrapperCpuArrayRef.generate_return()": 128, | ||
| 72 | + "def CppWrapperCpuArrayRef.write_wrapper_decl()": 208 | ||
| 73 | + }, | ||
| 74 | + "torch/_inductor/codegen/cutlass/gemm_template.py": { | ||
| 75 | + "class CUTLASS2xGemmTemplate": 268 | ||
| 76 | + }, | ||
| 77 | + "torch/_inductor/codegen/debug_utils.py": { | ||
| 78 | + "class DebugPrinterManager": 232 | ||
| 79 | + }, | ||
| 80 | + "torch/_inductor/codegen/halide.py": { | ||
| 81 | + "class HalideKernel": 999, | ||
| 82 | + "class HalideOverrides": 339, | ||
| 83 | + "class HalidePrinter": 128, | ||
| 84 | + "def HalideKernel.halide_kernel_meta()": 82 | ||
| 85 | + }, | ||
| 86 | + "torch/_inductor/codegen/rocm/ck_conv_template.py": { | ||
| 87 | + "class CKGroupedConvFwdTemplate": 547, | ||
| 88 | + "def CKGroupedConvFwdTemplate.globals()": 145 | ||
| 89 | + }, | ||
| 90 | + "torch/_inductor/codegen/rocm/ck_universal_gemm_template.py": { | ||
| 91 | + "class CKGemmTemplate": 954 | ||
| 92 | + }, | ||
| 93 | + "torch/_inductor/codegen/rocm/rocm_benchmark_request.py": { | ||
| 94 | + "class ROCmBenchmarkRequest": 118 | ||
| 95 | + }, | ||
| 96 | + "torch/_inductor/codegen/simd.py": { | ||
| 97 | + "def SIMDScheduling.candidate_tilings()": 126, | ||
| 98 | + "def SIMDScheduling.generate_node_schedule()": 95 | ||
| 99 | + }, | ||
| 100 | + "torch/_inductor/codegen/triton.py": { | ||
| 101 | + "class TritonPrinter": 179, | ||
| 102 | + "class TritonScheduling": 413, | ||
| 103 | + "def TritonScheduling.benchmark_codegened_module()": 88, | ||
| 104 | + "def TritonScheduling.benchmark_combo_kernel()": 93 | ||
| 105 | + }, | ||
| 106 | + "torch/_inductor/codegen/triton_combo_kernel.py": { | ||
| 107 | + "class ComboKernel": 871 | ||
| 108 | + }, | ||
| 109 | + "torch/_inductor/codegen/wrapper.py": { | ||
| 110 | + "def PythonWrapperCodegen.benchmark_compiled_module()": 96, | ||
| 111 | + "def PythonWrapperCodegen.define_user_defined_triton_kernel()": 269, | ||
| 112 | + "def PythonWrapperCodegen.generate_example_arg_value()": 84, | ||
| 113 | + "def user_defined_kernel_grid_fn_code()": 108 | ||
| 114 | + }, | ||
| 115 | + "torch/_inductor/comms.py": { | ||
| 116 | + "def enforce_comm_ordering_for_fsdp()": 175, | ||
| 117 | + "def reinplace_fsdp_all_gather()": 107 | ||
| 118 | + }, | ||
| 119 | + "torch/_inductor/constant_folding.py": { | ||
| 120 | + "class ConstantFolder": 225, | ||
| 121 | + "def ConstantFolder.run_node()": 94 | ||
| 122 | + }, | ||
| 123 | + "torch/_inductor/cpu_vec_isa.py": { | ||
| 124 | + "class VecISA": 119 | ||
| 125 | + }, | ||
| 126 | + "torch/_inductor/debug.py": { | ||
| 127 | + "class DebugContext": 154, | ||
| 128 | + "class DebugFormatter": 171, | ||
| 129 | + "def DebugFormatter.log_autotuning_results()": 89 | ||
| 130 | + }, | ||
| 131 | + "torch/_inductor/dependencies.py": { | ||
| 132 | + "class MemoryDep": 241 | ||
| 133 | + }, | ||
| 134 | + "torch/_inductor/fx_passes/b2b_gemm.py": { | ||
| 135 | + "def b2b_gemm_handler()": 192 | ||
| 136 | + }, | ||
| 137 | + "torch/_inductor/fx_passes/binary_folding.py": { | ||
| 138 | + "def binary_folding_init()": 417 | ||
| 139 | + }, | ||
| 140 | + "torch/_inductor/fx_passes/group_batch_fusion.py": { | ||
| 141 | + "def BatchLayernormFusion.fuse()": 136, | ||
| 142 | + "def PostGradBatchLinearFusion.fuse()": 82, | ||
| 143 | + "def PreGradBatchLinearFusion.fuse()": 86 | ||
| 144 | + }, | ||
| 145 | + "torch/_inductor/fx_passes/joint_graph.py": { | ||
| 146 | + "def constant_fold_uniform_value()": 108, | ||
| 147 | + "def remove_no_ops()": 96 | ||
| 148 | + }, | ||
| 149 | + "torch/_inductor/fx_passes/micro_pipeline_tp.py": { | ||
| 150 | + "def find_all_gather_patterns()": 115, | ||
| 151 | + "def find_reduce_scatter_patterns()": 124 | ||
| 152 | + }, | ||
| 153 | + "torch/_inductor/fx_passes/split_cat.py": { | ||
| 154 | + "def SplitCatSimplifier.replace_cat()": 160, | ||
| 155 | + "def merge_split_cat_aten()": 90, | ||
| 156 | + "def move_reshape_out_of_split_stack()": 109 | ||
| 157 | + }, | ||
| 158 | + "torch/_inductor/graph.py": { | ||
| 159 | + "class GraphLowering": 2224, | ||
| 160 | + "def GraphLowering.call_function()": 119, | ||
| 161 | + "def GraphLowering.create_deferred_runtime_asserts()": 84, | ||
| 162 | + "def GraphLowering.extract_autotune_inputs()": 94, | ||
| 163 | + "def GraphLowering.output()": 92, | ||
| 164 | + "def GraphLowering.placeholder()": 103, | ||
| 165 | + "def GraphLowering.run_node()": 364 | ||
| 166 | + }, | ||
| 167 | + "torch/_inductor/ir.py": { | ||
| 168 | + "class Buffer": 134, | ||
| 169 | + "class Loops": 125, | ||
| 170 | + "class Reduction": 803, | ||
| 171 | + "class Scan": 199, | ||
| 172 | + "class Sort": 151, | ||
| 173 | + "class UserDefinedTritonKernel": 204, | ||
| 174 | + "class View": 180, | ||
| 175 | + "class WelfordReduction": 220, | ||
| 176 | + "def ExternKernel.process_kernel()": 125, | ||
| 177 | + "def ExternKernel.require_strides()": 162, | ||
| 178 | + "def Reduction.num_splits()": 168 | ||
| 179 | + }, | ||
| 180 | + "torch/_inductor/jagged_lowerings.py": { | ||
| 181 | + "def register_jagged_ops()": 161 | ||
| 182 | + }, | ||
| 183 | + "torch/_inductor/kernel/conv.py": { | ||
| 184 | + "def convolution()": 239 | ||
| 185 | + }, | ||
| 186 | + "torch/_inductor/kernel/vendored_templates/cutedsl_grouped_gemm.py": { | ||
| 187 | + "def create_tensors_for_all_groups()": 95 | ||
| 188 | + }, | ||
| 189 | + "torch/_inductor/loop_body.py": { | ||
| 190 | + "class CaptureIndexing": 175 | ||
| 191 | + }, | ||
| 192 | + "torch/_inductor/lowering.py": { | ||
| 193 | + "def avg_pool2d_backward()": 163, | ||
| 194 | + "def avg_pool3d_backward()": 197, | ||
| 195 | + "def cat()": 122, | ||
| 196 | + "def index_put_impl_()": 117, | ||
| 197 | + "def make_pointwise()": 88, | ||
| 198 | + "def max_pool2d_with_indices_backward()": 143, | ||
| 199 | + "def scatter_reduce_()": 114, | ||
| 200 | + "def sdpa_constraint()": 135, | ||
| 201 | + "def searchsorted()": 95 | ||
| 202 | + }, | ||
| 203 | + "torch/_inductor/mkldnn_ir.py": { | ||
| 204 | + "class MkldnnRnnLayer": 121, | ||
| 205 | + "def MkldnnRnnLayer.create()": 101 | ||
| 206 | + }, | ||
| 207 | + "torch/_inductor/mkldnn_lowerings.py": { | ||
| 208 | + "def register_onednn_fusion_ops()": 1207 | ||
| 209 | + }, | ||
| 210 | + "torch/_inductor/mock_cache.py": { | ||
| 211 | + "class PatchCaches": 109 | ||
| 212 | + }, | ||
| 213 | + "torch/_inductor/quantized_lowerings.py": { | ||
| 214 | + "def register_woq_mm_ops()": 118 | ||
| 215 | + }, | ||
| 216 | + "torch/_inductor/runtime/autotune_cache.py": { | ||
| 217 | + "class AutotuneCache": 199 | ||
| 218 | + }, | ||
| 219 | + "torch/_inductor/scheduler.py": { | ||
| 220 | + "class BaseSchedulerNode": 754, | ||
| 221 | + "class SchedulerBuffer": 105 | ||
| 222 | + }, | ||
| 223 | + "torch/_inductor/utils.py": { | ||
| 224 | + "class IndentedBuffer": 148 | ||
| 225 | + } | ||
| 226 | +} | ||
| @@ -0,0 +1,297 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import itertools | ||
| 4 | +import json | ||
| 5 | +import sys | ||
| 6 | +from functools import cached_property | ||
| 7 | +from pathlib import Path | ||
| 8 | +from typing import Any, TYPE_CHECKING | ||
| 9 | + | ||
| 10 | + | ||
| 11 | +_FILE = Path(__file__).absolute() | ||
| 12 | +_PATH = [Path(p).absolute() for p in sys.path] | ||
| 13 | +_OVERRIDES = {"@override", "@typing_extensions.override", "@typing.override"} | ||
| 14 | + | ||
| 15 | +if TYPE_CHECKING or _FILE.parent not in _PATH: | ||
| 16 | + from . import _linter | ||
| 17 | +else: | ||
| 18 | + import _linter | ||
| 19 | + | ||
| 20 | +if TYPE_CHECKING: | ||
| 21 | + from collections.abc import Callable, Iterator, Sequence | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +GRANDFATHER_LIST = _FILE.parent / "docstring_linter-grandfather.json" | ||
| 25 | + | ||
| 26 | +# We tolerate a 10% increase in block size before demanding a docstring | ||
| 27 | +TOLERANCE_PERCENT = 10 | ||
| 28 | + | ||
| 29 | +MAX_LINES = {"class": 100, "def": 80} | ||
| 30 | + | ||
| 31 | +MIN_DOCSTRING = 50 # docstrings shorter than this are too short | ||
| 32 | + | ||
| 33 | +DESCRIPTION = """ | ||
| 34 | +`docstring_linter` reports on long functions, methods or classes without docstrings | ||
| 35 | +""".strip() | ||
| 36 | + | ||
| 37 | +METHOD_OVERRIDE_HINT = ( | ||
| 38 | + "If the method overrides a method on a parent class, adding the" | ||
| 39 | + " `@typing_extensions.override` decorator will make this error" | ||
| 40 | + " go away." | ||
| 41 | +) | ||
| 42 | + | ||
| 43 | + | ||
| 44 | +class DocstringLinter(_linter.FileLinter): | ||
| 45 | + linter_name = "docstring_linter" | ||
| 46 | + description = DESCRIPTION | ||
| 47 | + is_fixer = False | ||
| 48 | + | ||
| 49 | + path_to_blocks: dict[str, list[dict[str, Any]]] | ||
| 50 | + path_to_errors: dict[str, list[dict[str, Any]]] | ||
| 51 | + | ||
| 52 | + def __init__(self, argv: Sequence[str] | None = None) -> None: | ||
| 53 | + super().__init__(argv) | ||
| 54 | + add_arguments(self.parser.add_argument) | ||
| 55 | + self.path_to_blocks = {} | ||
| 56 | + self.path_to_errors = {} | ||
| 57 | + | ||
| 58 | + def lint_all(self) -> bool: | ||
| 59 | + success = super().lint_all() | ||
| 60 | + self._report() | ||
| 61 | + self._write_grandfather() | ||
| 62 | + return success | ||
| 63 | + | ||
| 64 | + def _lint(self, pf: _linter.PythonFile) -> Iterator[_linter.LintResult]: | ||
| 65 | + if (p := str(pf.path)) in self.path_to_blocks: | ||
| 66 | + print("Repeated file", p, file=sys.stderr) | ||
| 67 | + return | ||
| 68 | + | ||
| 69 | + blocks = pf.blocks | ||
| 70 | + bad = {b for b in blocks if self._is_bad_block(b, pf)} | ||
| 71 | + bad = self._dont_require_constructor_and_class_docs(blocks, bad) | ||
| 72 | + gf = self._grandfathered(pf.path, bad) | ||
| 73 | + | ||
| 74 | + yield from (self._block_result(b, pf) for b in sorted(bad - gf)) | ||
| 75 | + | ||
| 76 | + def as_data(b: _linter.Block) -> dict[str, Any]: | ||
| 77 | + status = "grandfather" if b in gf else "bad" if b in bad else "good" | ||
| 78 | + return {"status": status, **b.as_data()} | ||
| 79 | + | ||
| 80 | + self.path_to_blocks[p] = [as_data(b) for b in blocks] | ||
| 81 | + | ||
| 82 | + def _error(self, pf: _linter.PythonFile, result: _linter.LintResult) -> None: | ||
| 83 | + self.path_to_errors[str(pf.path)] = [{str(result.line): result.name}] | ||
| 84 | + | ||
| 85 | + | ||
| 86 | + def _grandfather(self) -> dict[str, dict[str, Any]]: | ||
| 87 | + try: | ||
| 88 | + with open(self.args.grandfather) as fp: | ||
| 89 | + return json.load(fp) # type: ignore[no-any-return] | ||
| 90 | + except FileNotFoundError: | ||
| 91 | + return {} | ||
| 92 | + except Exception as e: | ||
| 93 | + print("ERROR:", e, "in", GRANDFATHER_LIST, file=sys.stderr) | ||
| 94 | + raise | ||
| 95 | + | ||
| 96 | + | ||
| 97 | + def _max_lines(self) -> dict[str, int]: | ||
| 98 | + return {"class": self.args.max_class, "def": self.args.max_def} | ||
| 99 | + | ||
| 100 | + def _grandfathered( | ||
| 101 | + self, path: Path | None, bad: set[_linter.Block] | ||
| 102 | + ) -> set[_linter.Block]: | ||
| 103 | + if path is None or self.args.no_grandfather or self.args.write_grandfather: | ||
| 104 | + return set() | ||
| 105 | + | ||
| 106 | + grand: dict[str, int] = self._grandfather.get(str(path), {}) | ||
| 107 | + tolerance_ratio = 1 + self.args.grandfather_tolerance / 100.0 | ||
| 108 | + | ||
| 109 | + def grandfathered(b: _linter.Block) -> bool: | ||
| 110 | + lines = int(grand.get(b.display_name, 0) * tolerance_ratio) | ||
| 111 | + return b.line_count <= lines | ||
| 112 | + | ||
| 113 | + return {b for b in bad if grandfathered(b)} | ||
| 114 | + | ||
| 115 | + def _block_result( | ||
| 116 | + self, b: _linter.Block, pf: _linter.PythonFile | ||
| 117 | + ) -> _linter.LintResult: | ||
| 118 | + def_name = "function" if b.category == "def" else "class" | ||
| 119 | + msg = f"docstring found for {def_name} '{b.name}' ({b.line_count} lines)" | ||
| 120 | + if len(b.docstring): | ||
| 121 | + s = "" if len(b.docstring) == 1 else "s" | ||
| 122 | + needed = f"needed {self.args.min_docstring}" | ||
| 123 | + msg = f"{msg} was too short ({len(b.docstring)} character{s}, {needed})" | ||
| 124 | + else: | ||
| 125 | + msg = f"No {msg}" | ||
| 126 | + if b.is_method: | ||
| 127 | + msg = f"{msg}. {METHOD_OVERRIDE_HINT}" | ||
| 128 | + return _linter.LintResult(msg, *pf.tokens[b.begin].start) | ||
| 129 | + | ||
| 130 | + def _display( | ||
| 131 | + self, pf: _linter.PythonFile, results: list[_linter.LintResult] | ||
| 132 | + ) -> Iterator[str]: | ||
| 133 | + if not self.args.report: | ||
| 134 | + yield from super()._display(pf, results) | ||
| 135 | + | ||
| 136 | + def _dont_require_constructor_and_class_docs( | ||
| 137 | + self, blocks: Sequence[_linter.Block], bad: set[_linter.Block] | ||
| 138 | + ) -> set[_linter.Block]: | ||
| 139 | + if self.args.lint_init: | ||
| 140 | + return bad | ||
| 141 | + | ||
| 142 | + good = {b for b in blocks if len(b.docstring) >= self.args.min_docstring} | ||
| 143 | + | ||
| 144 | + def has_class_init_doc(b: _linter.Block) -> bool: | ||
| 145 | + if b.is_class: | ||
| 146 | + # Is it a class whose constructor is documented? | ||
| 147 | + children = (blocks[i] for i in b.children) | ||
| 148 | + return any(b.is_init and b in good for b in children) | ||
| 149 | + | ||
| 150 | + # Is it a constructor whose class is documented? | ||
| 151 | + return b.is_init and b.parent is not None and blocks[b.parent] in good | ||
| 152 | + | ||
| 153 | + return {b for b in bad if not has_class_init_doc(b)} | ||
| 154 | + | ||
| 155 | + def _is_bad_block(self, b: _linter.Block, pf: _linter.PythonFile) -> bool: | ||
| 156 | + max_lines = self._max_lines[b.category] | ||
| 157 | + return ( | ||
| 158 | + not (b.is_override or pf.omitted(pf.tokens, b.begin, b.end + 1)) | ||
| 159 | + and b.line_count > max_lines | ||
| 160 | + and len(b.docstring) < self.args.min_docstring | ||
| 161 | + and (self.args.lint_local or not b.is_local) | ||
| 162 | + and (self.args.lint_protected or not b.name.startswith("_")) | ||
| 163 | + ) | ||
| 164 | + | ||
| 165 | + def _report(self) -> None: | ||
| 166 | + if not self.args.lintrunner and self.path_to_blocks and self.args.report: | ||
| 167 | + report = { | ||
| 168 | + k: s for k, v in self.path_to_blocks.items() if (s := file_summary(v)) | ||
| 169 | + } | self.path_to_errors | ||
| 170 | + print(json.dumps(report, sort_keys=True, indent=2)) | ||
| 171 | + | ||
| 172 | + def _write_grandfather(self) -> None: | ||
| 173 | + if self.args.write_grandfather: | ||
| 174 | + results: dict[str, dict[str, int]] = {} | ||
| 175 | + | ||
| 176 | + for path, blocks in self.path_to_blocks.items(): | ||
| 177 | + for block in blocks: | ||
| 178 | + if block["status"] == "bad": | ||
| 179 | + d = results.setdefault(path, {}) | ||
| 180 | + d[block["display_name"]] = block["line_count"] | ||
| 181 | + | ||
| 182 | + with open(self.args.grandfather, "w") as fp: | ||
| 183 | + json.dump(results, fp, sort_keys=True, indent=2) | ||
| 184 | + | ||
| 185 | + | ||
| 186 | +def make_recursive(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]: | ||
| 187 | + def rec(i: int) -> dict[str, Any]: | ||
| 188 | + d = dict(blocks[i]) | ||
| 189 | + d["children"] = [rec(c) for c in d["children"]] | ||
| 190 | + return d | ||
| 191 | + | ||
| 192 | + return [rec(i) for i, b in enumerate(blocks) if b["parent"] is None] | ||
| 193 | + | ||
| 194 | + | ||
| 195 | +def make_terse( | ||
| 196 | + blocks: Sequence[dict[str, Any]], | ||
| 197 | + index_by_line: bool = True, | ||
| 198 | +) -> dict[str, dict[str, Any]]: | ||
| 199 | + result: dict[str, dict[str, Any]] = {} | ||
| 200 | + | ||
| 201 | + max_line = max(b["start_line"] for b in blocks) if blocks else 0 | ||
| 202 | + line_field_width = len(str(max_line)) | ||
| 203 | + | ||
| 204 | + for b in blocks: | ||
| 205 | + root = f"{b['category']} {b['full_name']}" | ||
| 206 | + for i in itertools.count(): | ||
| 207 | + name = root + bool(i) * f"[{i + 1}]" | ||
| 208 | + if name not in result: | ||
| 209 | + break | ||
| 210 | + | ||
| 211 | + d = { | ||
| 212 | + "docstring_len": len(b["docstring"]), | ||
| 213 | + "lines": b["line_count"], | ||
| 214 | + "status": b.get("status", "good"), | ||
| 215 | + } | ||
| 216 | + | ||
| 217 | + start_line = b["start_line"] | ||
| 218 | + if index_by_line: | ||
| 219 | + d["name"] = name | ||
| 220 | + result[f"{start_line:>{line_field_width}}"] = d | ||
| 221 | + else: | ||
| 222 | + d["line"] = start_line | ||
| 223 | + result[name] = d | ||
| 224 | + | ||
| 225 | + if kids := b["children"]: | ||
| 226 | + if not all(isinstance(k, int) for k in kids): | ||
| 227 | + if not all(isinstance(k, dict) for k in kids): | ||
| 228 | + raise AssertionError("children must be all int or all dict") | ||
| 229 | + d["children"] = make_terse(kids) | ||
| 230 | + | ||
| 231 | + return result | ||
| 232 | + | ||
| 233 | + | ||
| 234 | +def file_summary( | ||
| 235 | + blocks: Sequence[dict[str, Any]], report_all: bool = False | ||
| 236 | +) -> dict[str, str]: | ||
| 237 | + def to_line(v: dict[str, Any]) -> str | None: | ||
| 238 | + if (status := v["status"]) == "good": | ||
| 239 | + if not report_all: | ||
| 240 | + return None | ||
| 241 | + fail = "" | ||
| 242 | + elif status == "grandfather": | ||
| 243 | + fail = ": (grandfathered)" | ||
| 244 | + else: | ||
| 245 | + if status != "bad": | ||
| 246 | + raise AssertionError(f"Expected status 'bad', got '{status}'") | ||
| 247 | + fail = ": FAIL" | ||
| 248 | + name = v["name"] | ||
| 249 | + lines = v["lines"] | ||
| 250 | + docs = v["docstring_len"] | ||
| 251 | + parens = "()" if name.startswith("def ") else "" | ||
| 252 | + return f"{name}{parens}: {lines=}, {docs=}{fail}" | ||
| 253 | + | ||
| 254 | + t = make_terse(blocks) | ||
| 255 | + r = {k: line for k, v in t.items() if (line := to_line(v))} | ||
| 256 | + while r and all(k.startswith(" ") for k in r): | ||
| 257 | + r = {k[1:]: v for k, v in r.items()} | ||
| 258 | + return r | ||
| 259 | + | ||
| 260 | + | ||
| 261 | +def add_arguments(add: Callable[..., Any]) -> None: | ||
| 262 | + h = "Set the grandfather list" | ||
| 263 | + add("--grandfather", "-g", default=str(GRANDFATHER_LIST), type=str, help=h) | ||
| 264 | + | ||
| 265 | + h = "Tolerance for grandfather sizes, in percent" | ||
| 266 | + add("--grandfather-tolerance", "-t", default=TOLERANCE_PERCENT, type=float, help=h) | ||
| 267 | + | ||
| 268 | + h = "Lint __init__ and class separately" | ||
| 269 | + add("--lint-init", "-i", action="store_true", help=h) | ||
| 270 | + | ||
| 271 | + h = "Lint definitions inside other functions" | ||
| 272 | + add("--lint-local", "-o", action="store_true", help=h) | ||
| 273 | + | ||
| 274 | + h = "Lint functions, methods and classes that start with _" | ||
| 275 | + add("--lint-protected", "-p", action="store_true", help=h) | ||
| 276 | + | ||
| 277 | + h = "Maximum number of lines for an undocumented class" | ||
| 278 | + add("--max-class", "-c", default=MAX_LINES["class"], type=int, help=h) | ||
| 279 | + | ||
| 280 | + h = "Maximum number of lines for an undocumented function" | ||
| 281 | + add("--max-def", "-d", default=MAX_LINES["def"], type=int, help=h) | ||
| 282 | + | ||
| 283 | + h = "Minimum number of characters for a docstring" | ||
| 284 | + add("--min-docstring", "-s", default=MIN_DOCSTRING, type=int, help=h) | ||
| 285 | + | ||
| 286 | + h = "Disable the grandfather list" | ||
| 287 | + add("--no-grandfather", "-n", action="store_true", help=h) | ||
| 288 | + | ||
| 289 | + h = "Print a report on all classes and defs" | ||
| 290 | + add("--report", "-r", action="store_true", help=h) | ||
| 291 | + | ||
| 292 | + h = "Rewrite the grandfather list" | ||
| 293 | + add("--write-grandfather", "-w", action="store_true", help=h) | ||
| 294 | + | ||
| 295 | + | ||
| 296 | +if __name__ == "__main__": | ||
| 297 | + DocstringLinter.run() | ||
| @@ -0,0 +1,89 @@ | |||
| 1 | +""" | ||
| 2 | +EXEC: Ensure that source files are not executable. | ||
| 3 | +""" | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import argparse | ||
| 8 | +import json | ||
| 9 | +import logging | ||
| 10 | +import os | ||
| 11 | +import sys | ||
| 12 | +from enum import Enum | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +LINTER_CODE = "EXEC" | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +class LintSeverity(str, Enum): | ||
| 20 | + ERROR = "error" | ||
| 21 | + WARNING = "warning" | ||
| 22 | + ADVICE = "advice" | ||
| 23 | + DISABLED = "disabled" | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +class LintMessage(NamedTuple): | ||
| 27 | + path: str | None | ||
| 28 | + line: int | None | ||
| 29 | + char: int | None | ||
| 30 | + code: str | ||
| 31 | + severity: LintSeverity | ||
| 32 | + name: str | ||
| 33 | + original: str | None | ||
| 34 | + replacement: str | None | ||
| 35 | + description: str | None | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +def check_file(filename: str) -> LintMessage | None: | ||
| 39 | + is_executable = os.access(filename, os.X_OK) | ||
| 40 | + if is_executable: | ||
| 41 | + return LintMessage( | ||
| 42 | + path=filename, | ||
| 43 | + line=None, | ||
| 44 | + char=None, | ||
| 45 | + code=LINTER_CODE, | ||
| 46 | + severity=LintSeverity.ERROR, | ||
| 47 | + name="executable-permissions", | ||
| 48 | + original=None, | ||
| 49 | + replacement=None, | ||
| 50 | + description="This file has executable permission; please remove it by using `chmod -x`.", | ||
| 51 | + ) | ||
| 52 | + return None | ||
| 53 | + | ||
| 54 | + | ||
| 55 | +if __name__ == "__main__": | ||
| 56 | + parser = argparse.ArgumentParser( | ||
| 57 | + description="exec linter", | ||
| 58 | + fromfile_prefix_chars="@", | ||
| 59 | + ) | ||
| 60 | + parser.add_argument( | ||
| 61 | + "--verbose", | ||
| 62 | + action="store_true", | ||
| 63 | + ) | ||
| 64 | + parser.add_argument( | ||
| 65 | + "filenames", | ||
| 66 | + nargs="+", | ||
| 67 | + help="paths to lint", | ||
| 68 | + ) | ||
| 69 | + | ||
| 70 | + args = parser.parse_args() | ||
| 71 | + | ||
| 72 | + logging.basicConfig( | ||
| 73 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 74 | + level=logging.NOTSET | ||
| 75 | + if args.verbose | ||
| 76 | + else logging.DEBUG | ||
| 77 | + if len(args.filenames) < 1000 | ||
| 78 | + else logging.INFO, | ||
| 79 | + stream=sys.stderr, | ||
| 80 | + ) | ||
| 81 | + | ||
| 82 | + lint_messages = [] | ||
| 83 | + for filename in args.filenames: | ||
| 84 | + lint_message = check_file(filename) | ||
| 85 | + if lint_message is not None: | ||
| 86 | + lint_messages.append(lint_message) | ||
| 87 | + | ||
| 88 | + for lint_message in lint_messages: | ||
| 89 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,384 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "flake8==7.3.0", | ||
| 5 | +# "flake8-bugbear==24.12.12", | ||
| 6 | +# "flake8-comprehensions==3.16.0", | ||
| 7 | +# "flake8-executable==2.1.3", | ||
| 8 | +# "flake8-logging-format==2024.24.12", | ||
| 9 | +# "flake8-pyi==25.5.0", | ||
| 10 | +# "flake8-simplify==0.22.0", | ||
| 11 | +# "mccabe==0.7.0", | ||
| 12 | +# "pycodestyle==2.14.0", | ||
| 13 | +# "pyflakes==3.4.0", | ||
| 14 | +# "setuptools<82", | ||
| 15 | +# ] | ||
| 16 | +# /// | ||
| 17 | +from __future__ import annotations | ||
| 18 | + | ||
| 19 | +import argparse | ||
| 20 | +import json | ||
| 21 | +import logging | ||
| 22 | +import os | ||
| 23 | +import re | ||
| 24 | +import subprocess | ||
| 25 | +import sys | ||
| 26 | +import time | ||
| 27 | +from enum import Enum | ||
| 28 | +from typing import NamedTuple | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +IS_WINDOWS: bool = os.name == "nt" | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +class LintSeverity(str, Enum): | ||
| 35 | + ERROR = "error" | ||
| 36 | + WARNING = "warning" | ||
| 37 | + ADVICE = "advice" | ||
| 38 | + DISABLED = "disabled" | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +class LintMessage(NamedTuple): | ||
| 42 | + path: str | None | ||
| 43 | + line: int | None | ||
| 44 | + char: int | None | ||
| 45 | + code: str | ||
| 46 | + severity: LintSeverity | ||
| 47 | + name: str | ||
| 48 | + original: str | None | ||
| 49 | + replacement: str | None | ||
| 50 | + description: str | None | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +def as_posix(name: str) -> str: | ||
| 54 | + return name.replace("\\", "/") if IS_WINDOWS else name | ||
| 55 | + | ||
| 56 | + | ||
| 57 | +# fmt: off | ||
| 58 | +# https://www.flake8rules.com/ | ||
| 59 | +DOCUMENTED_IN_FLAKE8RULES: set[str] = { | ||
| 60 | + "E101", "E111", "E112", "E113", "E114", "E115", "E116", "E117", | ||
| 61 | + "E121", "E122", "E123", "E124", "E125", "E126", "E127", "E128", "E129", | ||
| 62 | + "E131", "E133", | ||
| 63 | + "E201", "E202", "E203", | ||
| 64 | + "E211", | ||
| 65 | + "E221", "E222", "E223", "E224", "E225", "E226", "E227", "E228", | ||
| 66 | + "E231", | ||
| 67 | + "E241", "E242", | ||
| 68 | + "E251", | ||
| 69 | + "E261", "E262", "E265", "E266", | ||
| 70 | + "E271", "E272", "E273", "E274", "E275", | ||
| 71 | + "E301", "E302", "E303", "E304", "E305", "E306", | ||
| 72 | + "E401", "E402", | ||
| 73 | + "E501", "E502", | ||
| 74 | + "E701", "E702", "E703", "E704", | ||
| 75 | + "E711", "E712", "E713", "E714", | ||
| 76 | + "E721", "E722", | ||
| 77 | + "E731", | ||
| 78 | + "E741", "E742", "E743", | ||
| 79 | + "E901", "E902", "E999", | ||
| 80 | + "W191", | ||
| 81 | + "W291", "W292", "W293", | ||
| 82 | + "W391", | ||
| 83 | + "W503", "W504", | ||
| 84 | + "W601", "W602", "W603", "W604", "W605", | ||
| 85 | + "F401", "F402", "F403", "F404", "F405", | ||
| 86 | + "F811", "F812", | ||
| 87 | + "F821", "F822", "F823", | ||
| 88 | + "F831", | ||
| 89 | + "F841", | ||
| 90 | + "F901", | ||
| 91 | + "C901", | ||
| 92 | +} | ||
| 93 | + | ||
| 94 | +# https://pypi.org/project/flake8-comprehensions/#rules | ||
| 95 | +DOCUMENTED_IN_FLAKE8COMPREHENSIONS: set[str] = { | ||
| 96 | + "C400", "C401", "C402", "C403", "C404", "C405", "C406", "C407", "C408", "C409", | ||
| 97 | + "C410", | ||
| 98 | + "C411", "C412", "C413", "C414", "C415", "C416", | ||
| 99 | +} | ||
| 100 | + | ||
| 101 | +# https://github.com/PyCQA/flake8-bugbear#list-of-warnings | ||
| 102 | +DOCUMENTED_IN_BUGBEAR: set[str] = { | ||
| 103 | + "B001", "B002", "B003", "B004", "B005", "B006", "B007", "B008", "B009", "B010", | ||
| 104 | + "B011", "B012", "B013", "B014", "B015", | ||
| 105 | + "B301", "B302", "B303", "B304", "B305", "B306", | ||
| 106 | + "B901", "B902", "B903", "B950", | ||
| 107 | +} | ||
| 108 | +# fmt: on | ||
| 109 | + | ||
| 110 | + | ||
| 111 | +# stdin:2: W802 undefined name 'foo' | ||
| 112 | +# stdin:3:6: T484 Name 'foo' is not defined | ||
| 113 | +# stdin:3:-100: W605 invalid escape sequence '\/' | ||
| 114 | +# stdin:3:1: E302 expected 2 blank lines, found 1 | ||
| 115 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 116 | + r"""(?mx) | ||
| 117 | + ^ | ||
| 118 | + (?P<file>.*?): | ||
| 119 | + (?P<line>\d+): | ||
| 120 | + (?:(?P<column>-?\d+):)? | ||
| 121 | + \s(?P<code>\S+?):? | ||
| 122 | + \s(?P<message>.*) | ||
| 123 | + $ | ||
| 124 | + """ | ||
| 125 | +) | ||
| 126 | + | ||
| 127 | + | ||
| 128 | +def _test_results_re() -> None: | ||
| 129 | + """ | ||
| 130 | + >>> def t(s): | ||
| 131 | + ... return RESULTS_RE.search(s).groupdict() | ||
| 132 | + | ||
| 133 | + >>> t(r"file.py:80:1: E302 expected 2 blank lines, found 1") | ||
| 134 | + ... # doctest: +NORMALIZE_WHITESPACE | ||
| 135 | + {'file': 'file.py', 'line': '80', 'column': '1', 'code': 'E302', | ||
| 136 | + 'message': 'expected 2 blank lines, found 1'} | ||
| 137 | + | ||
| 138 | + >>> t(r"file.py:7:1: P201: Resource `stdout` is acquired but not always released.") | ||
| 139 | + ... # doctest: +NORMALIZE_WHITESPACE | ||
| 140 | + {'file': 'file.py', 'line': '7', 'column': '1', 'code': 'P201', | ||
| 141 | + 'message': 'Resource `stdout` is acquired but not always released.'} | ||
| 142 | + | ||
| 143 | + >>> t(r"file.py:8:-10: W605 invalid escape sequence '/'") | ||
| 144 | + ... # doctest: +NORMALIZE_WHITESPACE | ||
| 145 | + {'file': 'file.py', 'line': '8', 'column': '-10', 'code': 'W605', | ||
| 146 | + 'message': "invalid escape sequence '/'"} | ||
| 147 | + """ | ||
| 148 | + | ||
| 149 | + | ||
| 150 | +def _run_command( | ||
| 151 | + args: list[str], | ||
| 152 | + *, | ||
| 153 | + extra_env: dict[str, str] | None, | ||
| 154 | +) -> subprocess.CompletedProcess[str]: | ||
| 155 | + logging.debug( | ||
| 156 | + "$ %s", | ||
| 157 | + " ".join( | ||
| 158 | + ([f"{k}={v}" for (k, v) in extra_env.items()] if extra_env else []) + args | ||
| 159 | + ), | ||
| 160 | + ) | ||
| 161 | + start_time = time.monotonic() | ||
| 162 | + try: | ||
| 163 | + return subprocess.run( | ||
| 164 | + args, | ||
| 165 | + capture_output=True, | ||
| 166 | + check=True, | ||
| 167 | + encoding="utf-8", | ||
| 168 | + ) | ||
| 169 | + finally: | ||
| 170 | + end_time = time.monotonic() | ||
| 171 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 172 | + | ||
| 173 | + | ||
| 174 | +def run_command( | ||
| 175 | + args: list[str], | ||
| 176 | + *, | ||
| 177 | + extra_env: dict[str, str] | None, | ||
| 178 | + retries: int, | ||
| 179 | +) -> subprocess.CompletedProcess[str]: | ||
| 180 | + remaining_retries = retries | ||
| 181 | + while True: | ||
| 182 | + try: | ||
| 183 | + return _run_command(args, extra_env=extra_env) | ||
| 184 | + except subprocess.CalledProcessError as err: | ||
| 185 | + if remaining_retries == 0 or not re.match( | ||
| 186 | + r"^ERROR:1:1: X000 linting with .+ timed out after \d+ seconds", | ||
| 187 | + err.stdout, | ||
| 188 | + ): | ||
| 189 | + raise err | ||
| 190 | + remaining_retries -= 1 | ||
| 191 | + logging.warning( # noqa: G200 | ||
| 192 | + "(%s/%s) Retrying because command failed with: %r", | ||
| 193 | + retries - remaining_retries, | ||
| 194 | + retries, | ||
| 195 | + err, | ||
| 196 | + ) | ||
| 197 | + time.sleep(1) | ||
| 198 | + | ||
| 199 | + | ||
| 200 | +def get_issue_severity(code: str) -> LintSeverity: | ||
| 201 | + # "B901": `return x` inside a generator | ||
| 202 | + # "B902": Invalid first argument to a method | ||
| 203 | + # "B903": __slots__ efficiency | ||
| 204 | + # "B950": Line too long | ||
| 205 | + # "C4": Flake8 Comprehensions | ||
| 206 | + # "C9": Cyclomatic complexity | ||
| 207 | + # "E2": PEP8 horizontal whitespace "errors" | ||
| 208 | + # "E3": PEP8 blank line "errors" | ||
| 209 | + # "E5": PEP8 line length "errors" | ||
| 210 | + # "F401": Name imported but unused | ||
| 211 | + # "F403": Star imports used | ||
| 212 | + # "F405": Name possibly from star imports | ||
| 213 | + # "T400": type checking Notes | ||
| 214 | + # "T49": internal type checker errors or unmatched messages | ||
| 215 | + if any( | ||
| 216 | + code.startswith(x) | ||
| 217 | + for x in [ | ||
| 218 | + "B9", | ||
| 219 | + "C4", | ||
| 220 | + "C9", | ||
| 221 | + "E2", | ||
| 222 | + "E3", | ||
| 223 | + "E5", | ||
| 224 | + "F401", | ||
| 225 | + "F403", | ||
| 226 | + "F405", | ||
| 227 | + "T400", | ||
| 228 | + "T49", | ||
| 229 | + ] | ||
| 230 | + ): | ||
| 231 | + return LintSeverity.ADVICE | ||
| 232 | + | ||
| 233 | + # "F821": Undefined name | ||
| 234 | + # "E999": syntax error | ||
| 235 | + if any(code.startswith(x) for x in ["F821", "E999"]): | ||
| 236 | + return LintSeverity.ERROR | ||
| 237 | + | ||
| 238 | + # "F": PyFlakes Error | ||
| 239 | + # "B": flake8-bugbear Error | ||
| 240 | + # "E": PEP8 "Error" | ||
| 241 | + # "W": PEP8 Warning | ||
| 242 | + # possibly other plugins... | ||
| 243 | + return LintSeverity.WARNING | ||
| 244 | + | ||
| 245 | + | ||
| 246 | +def get_issue_documentation_url(code: str) -> str: | ||
| 247 | + if code in DOCUMENTED_IN_FLAKE8RULES: | ||
| 248 | + return f"https://www.flake8rules.com/rules/{code}.html" | ||
| 249 | + | ||
| 250 | + if code in DOCUMENTED_IN_FLAKE8COMPREHENSIONS: | ||
| 251 | + return "https://pypi.org/project/flake8-comprehensions/#rules" | ||
| 252 | + | ||
| 253 | + if code in DOCUMENTED_IN_BUGBEAR: | ||
| 254 | + return "https://github.com/PyCQA/flake8-bugbear#list-of-warnings" | ||
| 255 | + | ||
| 256 | + return "" | ||
| 257 | + | ||
| 258 | + | ||
| 259 | +def check_files( | ||
| 260 | + filenames: list[str], | ||
| 261 | + flake8_plugins_path: str | None, | ||
| 262 | + severities: dict[str, LintSeverity], | ||
| 263 | + retries: int, | ||
| 264 | +) -> list[LintMessage]: | ||
| 265 | + try: | ||
| 266 | + proc = run_command( | ||
| 267 | + [sys.executable, "-mflake8", "--exit-zero"] + filenames, | ||
| 268 | + extra_env={"FLAKE8_PLUGINS_PATH": flake8_plugins_path} | ||
| 269 | + if flake8_plugins_path | ||
| 270 | + else None, | ||
| 271 | + retries=retries, | ||
| 272 | + ) | ||
| 273 | + except (OSError, subprocess.CalledProcessError) as err: | ||
| 274 | + return [ | ||
| 275 | + LintMessage( | ||
| 276 | + path=None, | ||
| 277 | + line=None, | ||
| 278 | + char=None, | ||
| 279 | + code="FLAKE8", | ||
| 280 | + severity=LintSeverity.ERROR, | ||
| 281 | + name="command-failed", | ||
| 282 | + original=None, | ||
| 283 | + replacement=None, | ||
| 284 | + description=( | ||
| 285 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 286 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 287 | + else ( | ||
| 288 | + "COMMAND (exit code {returncode})\n" | ||
| 289 | + "{command}\n\n" | ||
| 290 | + "STDERR\n{stderr}\n\n" | ||
| 291 | + "STDOUT\n{stdout}" | ||
| 292 | + ).format( | ||
| 293 | + returncode=err.returncode, | ||
| 294 | + command=" ".join(as_posix(x) for x in err.cmd), | ||
| 295 | + stderr=err.stderr.strip() or "(empty)", | ||
| 296 | + stdout=err.stdout.strip() or "(empty)", | ||
| 297 | + ) | ||
| 298 | + ), | ||
| 299 | + ) | ||
| 300 | + ] | ||
| 301 | + | ||
| 302 | + return [ | ||
| 303 | + LintMessage( | ||
| 304 | + path=match["file"], | ||
| 305 | + name=match["code"], | ||
| 306 | + description=f"{match['message']}\nSee {get_issue_documentation_url(match['code'])}", | ||
| 307 | + line=int(match["line"]), | ||
| 308 | + char=int(match["column"]) | ||
| 309 | + if match["column"] is not None and not match["column"].startswith("-") | ||
| 310 | + else None, | ||
| 311 | + code="FLAKE8", | ||
| 312 | + severity=severities.get(match["code"]) or get_issue_severity(match["code"]), | ||
| 313 | + original=None, | ||
| 314 | + replacement=None, | ||
| 315 | + ) | ||
| 316 | + for match in RESULTS_RE.finditer(proc.stdout) | ||
| 317 | + ] | ||
| 318 | + | ||
| 319 | + | ||
| 320 | +def main() -> None: | ||
| 321 | + parser = argparse.ArgumentParser( | ||
| 322 | + description="Flake8 wrapper linter.", | ||
| 323 | + fromfile_prefix_chars="@", | ||
| 324 | + ) | ||
| 325 | + parser.add_argument( | ||
| 326 | + "--flake8-plugins-path", | ||
| 327 | + help="FLAKE8_PLUGINS_PATH env value", | ||
| 328 | + ) | ||
| 329 | + parser.add_argument( | ||
| 330 | + "--severity", | ||
| 331 | + action="append", | ||
| 332 | + help="map code to severity (e.g. `B950:advice`)", | ||
| 333 | + ) | ||
| 334 | + parser.add_argument( | ||
| 335 | + "--retries", | ||
| 336 | + default=3, | ||
| 337 | + type=int, | ||
| 338 | + help="times to retry timed out flake8", | ||
| 339 | + ) | ||
| 340 | + parser.add_argument( | ||
| 341 | + "--verbose", | ||
| 342 | + action="store_true", | ||
| 343 | + help="verbose logging", | ||
| 344 | + ) | ||
| 345 | + parser.add_argument( | ||
| 346 | + "filenames", | ||
| 347 | + nargs="+", | ||
| 348 | + help="paths to lint", | ||
| 349 | + ) | ||
| 350 | + args = parser.parse_args() | ||
| 351 | + | ||
| 352 | + logging.basicConfig( | ||
| 353 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 354 | + level=logging.NOTSET | ||
| 355 | + if args.verbose | ||
| 356 | + else logging.DEBUG | ||
| 357 | + if len(args.filenames) < 1000 | ||
| 358 | + else logging.INFO, | ||
| 359 | + stream=sys.stderr, | ||
| 360 | + ) | ||
| 361 | + | ||
| 362 | + flake8_plugins_path = ( | ||
| 363 | + None | ||
| 364 | + if args.flake8_plugins_path is None | ||
| 365 | + else os.path.realpath(args.flake8_plugins_path) | ||
| 366 | + ) | ||
| 367 | + | ||
| 368 | + severities: dict[str, LintSeverity] = {} | ||
| 369 | + if args.severity: | ||
| 370 | + for severity in args.severity: | ||
| 371 | + parts = severity.split(":", 1) | ||
| 372 | + if len(parts) != 2: | ||
| 373 | + raise AssertionError(f"invalid severity `{severity}`") | ||
| 374 | + severities[parts[0]] = LintSeverity(parts[1]) | ||
| 375 | + | ||
| 376 | + lint_messages = check_files( | ||
| 377 | + args.filenames, flake8_plugins_path, severities, args.retries | ||
| 378 | + ) | ||
| 379 | + for lint_message in lint_messages: | ||
| 380 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 381 | + | ||
| 382 | + | ||
| 383 | +if __name__ == "__main__": | ||
| 384 | + main() | ||
| @@ -0,0 +1,426 @@ | |||
| 1 | +# mypy: ignore-errors | ||
| 2 | + | ||
| 3 | +from __future__ import annotations | ||
| 4 | + | ||
| 5 | +import argparse | ||
| 6 | +import ast | ||
| 7 | +import functools | ||
| 8 | +import json | ||
| 9 | +import random | ||
| 10 | +import sys | ||
| 11 | +from enum import Enum | ||
| 12 | +from pathlib import Path | ||
| 13 | +from typing import Any, NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +# Patch ast._splitlines_no_ff with caching to avoid O(n²) re-splitting. | ||
| 17 | +# get_source_segment() calls _splitlines_no_ff() for every keyword argument, | ||
| 18 | +# but the same source string is passed repeatedly for the same file. | ||
| 19 | +if hasattr(ast, "_splitlines_no_ff"): | ||
| 20 | + ast._splitlines_no_ff = functools.lru_cache(maxsize=128)(ast._splitlines_no_ff) | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +REPO_ROOT = Path(__file__).resolve().parents[3] | ||
| 24 | +sys.path.insert(0, str(REPO_ROOT)) | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +from tools.dynamo.gb_id_mapping import ( | ||
| 28 | + find_unimplemented_calls, | ||
| 29 | + load_registry, | ||
| 30 | + next_gb_id, | ||
| 31 | +) | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +LINTER_CODE = "GB_REGISTRY" | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +class LintSeverity(str, Enum): | ||
| 38 | + ERROR = "error" | ||
| 39 | + WARNING = "warning" | ||
| 40 | + ADVICE = "advice" | ||
| 41 | + DISABLED = "disabled" | ||
| 42 | + | ||
| 43 | + | ||
| 44 | +class LintMessage(NamedTuple): | ||
| 45 | + path: str | None | ||
| 46 | + line: int | None | ||
| 47 | + char: int | None | ||
| 48 | + code: str | ||
| 49 | + severity: LintSeverity | ||
| 50 | + name: str | ||
| 51 | + original: str | None | ||
| 52 | + replacement: str | None | ||
| 53 | + description: str | None | ||
| 54 | + | ||
| 55 | + | ||
| 56 | +def _is_noqa_suppressed(source_lines: list[str], lineno: int) -> bool: | ||
| 57 | + if lineno <= 0 or lineno > len(source_lines): | ||
| 58 | + return False | ||
| 59 | + if source_lines[lineno - 1].rstrip().endswith(f"# noqa: {LINTER_CODE}"): | ||
| 60 | + return True | ||
| 61 | + if lineno > 1 and source_lines[lineno - 2].strip() == f"# noqa: {LINTER_CODE}": | ||
| 62 | + return True | ||
| 63 | + return False | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +def _is_forbidden_raise(node: ast.Raise) -> bool: | ||
| 67 | + if not isinstance(node.exc, ast.Call): | ||
| 68 | + return False | ||
| 69 | + if isinstance(node.exc.func, ast.Name): | ||
| 70 | + return node.exc.func.id == "Unsupported" | ||
| 71 | + if isinstance(node.exc.func, ast.Attribute): | ||
| 72 | + return node.exc.func.attr == "Unsupported" | ||
| 73 | + return False | ||
| 74 | + | ||
| 75 | + | ||
| 76 | +def _collect_forbidden_unsupported_raises( | ||
| 77 | + dynamo_dir: Path, | ||
| 78 | +) -> list[tuple[Path, int, int]]: | ||
| 79 | + forbidden_raises: list[tuple[Path, int, int]] = [] | ||
| 80 | + | ||
| 81 | + for py_file in dynamo_dir.rglob("*.py"): | ||
| 82 | + source = py_file.read_text(encoding="utf-8") | ||
| 83 | + source_lines = source.splitlines() | ||
| 84 | + try: | ||
| 85 | + tree = ast.parse(source, filename=str(py_file)) | ||
| 86 | + except SyntaxError: | ||
| 87 | + continue | ||
| 88 | + | ||
| 89 | + for node in ast.walk(tree): | ||
| 90 | + if not isinstance(node, ast.Raise) or not _is_forbidden_raise(node): | ||
| 91 | + continue | ||
| 92 | + if _is_noqa_suppressed(source_lines, node.lineno): | ||
| 93 | + continue | ||
| 94 | + forbidden_raises.append((py_file, node.lineno, node.col_offset + 1)) | ||
| 95 | + | ||
| 96 | + return forbidden_raises | ||
| 97 | + | ||
| 98 | + | ||
| 99 | +def _collect_all_calls( | ||
| 100 | + dynamo_dir: Path, | ||
| 101 | +) -> dict[str, list[tuple[dict[str, Any], Path]]]: | ||
| 102 | + """Return mapping *gb_type → list[(call_info, file_path)]* for all occurrences.""" | ||
| 103 | + gb_type_calls: dict[str, list[tuple[dict[str, Any], Path]]] = {} | ||
| 104 | + | ||
| 105 | + for py_file in dynamo_dir.rglob("*.py"): | ||
| 106 | + for call in find_unimplemented_calls(py_file, dynamo_dir): | ||
| 107 | + gb_type = call["gb_type"] | ||
| 108 | + if gb_type not in gb_type_calls: | ||
| 109 | + gb_type_calls[gb_type] = [] | ||
| 110 | + gb_type_calls[gb_type].append((call, py_file)) | ||
| 111 | + | ||
| 112 | + return gb_type_calls | ||
| 113 | + | ||
| 114 | + | ||
| 115 | +def _create_registry_entry( | ||
| 116 | + gb_type: str, context: str, explanation: str, hints: list[str] | ||
| 117 | +) -> dict[str, Any]: | ||
| 118 | + """Create a registry entry with consistent format.""" | ||
| 119 | + return { | ||
| 120 | + "Gb_type": gb_type, | ||
| 121 | + "Context": context, | ||
| 122 | + "Explanation": explanation, | ||
| 123 | + "Hints": hints or [], | ||
| 124 | + } | ||
| 125 | + | ||
| 126 | + | ||
| 127 | +def _update_registry_with_changes( | ||
| 128 | + registry: dict, | ||
| 129 | + calls: dict[str, tuple[dict[str, Any], Path]], | ||
| 130 | + renames: dict[str, str] | None = None, | ||
| 131 | +) -> dict: | ||
| 132 | + """Calculate what the updated registry should look like.""" | ||
| 133 | + renames = renames or {} | ||
| 134 | + updated_registry = dict(registry) | ||
| 135 | + | ||
| 136 | + latest_entry: dict[str, Any] = { | ||
| 137 | + entries[0]["Gb_type"]: entries[0] for entries in registry.values() | ||
| 138 | + } | ||
| 139 | + gb_type_to_key: dict[str, str] = { | ||
| 140 | + entries[0]["Gb_type"]: key for key, entries in registry.items() | ||
| 141 | + } | ||
| 142 | + | ||
| 143 | + # Method for determining add vs. update: | ||
| 144 | + # - If gb_type exists in registry but content differs: UPDATE (append new entry to preserve history) | ||
| 145 | + # - If gb_type is new but content matches existing entry: RENAME (append new entry with new gb_type) | ||
| 146 | + # - If gb_type is completely new: ADD (create new registry entry with a new GBID) | ||
| 147 | + | ||
| 148 | + for old_gb_type, new_gb_type in renames.items(): | ||
| 149 | + registry_key = gb_type_to_key[old_gb_type] | ||
| 150 | + old_entry = updated_registry[registry_key][0] | ||
| 151 | + | ||
| 152 | + new_entry = _create_registry_entry( | ||
| 153 | + new_gb_type, | ||
| 154 | + old_entry["Context"], | ||
| 155 | + old_entry["Explanation"], | ||
| 156 | + old_entry["Hints"], | ||
| 157 | + ) | ||
| 158 | + updated_registry[registry_key] = [new_entry] + updated_registry[registry_key] | ||
| 159 | + | ||
| 160 | + latest_entry[new_gb_type] = new_entry | ||
| 161 | + gb_type_to_key[new_gb_type] = registry_key | ||
| 162 | + del latest_entry[old_gb_type] | ||
| 163 | + del gb_type_to_key[old_gb_type] | ||
| 164 | + | ||
| 165 | + # Collect new entries separately to insert them all at once | ||
| 166 | + new_entries: list[tuple[str, list[dict[str, Any]]]] = [] | ||
| 167 | + | ||
| 168 | + for gb_type, (call, file_path) in calls.items(): | ||
| 169 | + if gb_type in latest_entry: | ||
| 170 | + existing_entry = latest_entry[gb_type] | ||
| 171 | + | ||
| 172 | + if not ( | ||
| 173 | + call["context"] == existing_entry["Context"] | ||
| 174 | + and call["explanation"] == existing_entry["Explanation"] | ||
| 175 | + and sorted(call["hints"]) == sorted(existing_entry["Hints"]) | ||
| 176 | + ): | ||
| 177 | + registry_key = gb_type_to_key[gb_type] | ||
| 178 | + new_entry = _create_registry_entry( | ||
| 179 | + gb_type, call["context"], call["explanation"], call["hints"] | ||
| 180 | + ) | ||
| 181 | + updated_registry[registry_key] = [new_entry] + updated_registry[ | ||
| 182 | + registry_key | ||
| 183 | + ] | ||
| 184 | + else: | ||
| 185 | + # Collect new entries to add later | ||
| 186 | + new_key = next_gb_id(updated_registry) | ||
| 187 | + new_entry = _create_registry_entry( | ||
| 188 | + gb_type, call["context"], call["explanation"], call["hints"] | ||
| 189 | + ) | ||
| 190 | + new_entries.append((new_key, [new_entry])) | ||
| 191 | + # Temporarily add to updated_registry so next_gb_id works correctly | ||
| 192 | + updated_registry[new_key] = [new_entry] | ||
| 193 | + | ||
| 194 | + # Insert all new entries at the same random position to reduce merge conflicts | ||
| 195 | + if new_entries: | ||
| 196 | + # Remove temporarily added entries | ||
| 197 | + for new_key, _ in new_entries: | ||
| 198 | + del updated_registry[new_key] | ||
| 199 | + | ||
| 200 | + registry_items = list(updated_registry.items()) | ||
| 201 | + if registry_items: | ||
| 202 | + # Pick one random position for all new entries | ||
| 203 | + insert_pos = random.randint(0, len(registry_items)) | ||
| 204 | + # Insert all new entries at the same position | ||
| 205 | + for new_key, new_entry in new_entries: | ||
| 206 | + registry_items.insert(insert_pos, (new_key, new_entry)) | ||
| 207 | + insert_pos += 1 # Keep them together | ||
| 208 | + updated_registry = dict(registry_items) | ||
| 209 | + else: | ||
| 210 | + # Empty registry, just add all entries | ||
| 211 | + for new_key, new_entry in new_entries: | ||
| 212 | + updated_registry[new_key] = new_entry | ||
| 213 | + | ||
| 214 | + return updated_registry | ||
| 215 | + | ||
| 216 | + | ||
| 217 | +def check_registry_sync(dynamo_dir: Path, registry_path: Path) -> list[LintMessage]: | ||
| 218 | + """Check registry sync and return lint messages.""" | ||
| 219 | + lint_messages = [] | ||
| 220 | + | ||
| 221 | + forbidden_raises = _collect_forbidden_unsupported_raises(dynamo_dir) | ||
| 222 | + for path, line, char in forbidden_raises: | ||
| 223 | + lint_messages.append( | ||
| 224 | + LintMessage( | ||
| 225 | + path=str(path), | ||
| 226 | + line=line, | ||
| 227 | + char=char, | ||
| 228 | + code=LINTER_CODE, | ||
| 229 | + severity=LintSeverity.ERROR, | ||
| 230 | + name="Direct raise Unsupported", | ||
| 231 | + original=None, | ||
| 232 | + replacement=None, | ||
| 233 | + description=( | ||
| 234 | + "Do not directly `raise Unsupported(...)` in `torch/_dynamo`. " | ||
| 235 | + "Use `unimplemented(...)` for graph breaks, or add `# noqa: GB_REGISTRY` " | ||
| 236 | + "for infra-only exceptions." | ||
| 237 | + ), | ||
| 238 | + ) | ||
| 239 | + ) | ||
| 240 | + | ||
| 241 | + all_calls = _collect_all_calls(dynamo_dir) | ||
| 242 | + | ||
| 243 | + duplicates = [] | ||
| 244 | + for gb_type, call_list in all_calls.items(): | ||
| 245 | + if len(call_list) > 1: | ||
| 246 | + first_call = call_list[0][0] | ||
| 247 | + for call, file_path in call_list[1:]: | ||
| 248 | + if ( | ||
| 249 | + call["context"] != first_call["context"] | ||
| 250 | + or call["explanation"] != first_call["explanation"] | ||
| 251 | + or sorted(call["hints"]) != sorted(first_call["hints"]) | ||
| 252 | + ): | ||
| 253 | + duplicates.append({"gb_type": gb_type, "calls": call_list}) | ||
| 254 | + break | ||
| 255 | + | ||
| 256 | + for dup in duplicates: | ||
| 257 | + gb_type = dup["gb_type"] | ||
| 258 | + calls = dup["calls"] | ||
| 259 | + | ||
| 260 | + description = f"The gb_type '{gb_type}' is used {len(calls)} times with different content. " | ||
| 261 | + description += "Each gb_type must be unique across your entire codebase." | ||
| 262 | + | ||
| 263 | + lint_messages.append( | ||
| 264 | + LintMessage( | ||
| 265 | + path=str(calls[0][1]), | ||
| 266 | + line=None, | ||
| 267 | + char=None, | ||
| 268 | + code=LINTER_CODE, | ||
| 269 | + severity=LintSeverity.ERROR, | ||
| 270 | + name="Duplicate gb_type", | ||
| 271 | + original=None, | ||
| 272 | + replacement=None, | ||
| 273 | + description=description, | ||
| 274 | + ) | ||
| 275 | + ) | ||
| 276 | + | ||
| 277 | + if duplicates: | ||
| 278 | + return lint_messages | ||
| 279 | + | ||
| 280 | + calls = {gb_type: calls[0] for gb_type, calls in all_calls.items()} | ||
| 281 | + | ||
| 282 | + registry = load_registry(registry_path) | ||
| 283 | + | ||
| 284 | + # Check for duplicate gb_types across different GB IDs in the registry | ||
| 285 | + gb_type_to_ids: dict[str, list[str]] = {} | ||
| 286 | + for gb_id, entries in registry.items(): | ||
| 287 | + gb_type = entries[0]["Gb_type"] | ||
| 288 | + if gb_type not in gb_type_to_ids: | ||
| 289 | + gb_type_to_ids[gb_type] = [] | ||
| 290 | + gb_type_to_ids[gb_type].append(gb_id) | ||
| 291 | + | ||
| 292 | + duplicate_gb_types_in_registry = [ | ||
| 293 | + (gb_type, ids) for gb_type, ids in gb_type_to_ids.items() if len(ids) > 1 | ||
| 294 | + ] | ||
| 295 | + | ||
| 296 | + if duplicate_gb_types_in_registry: | ||
| 297 | + for gb_type, ids in duplicate_gb_types_in_registry: | ||
| 298 | + description = ( | ||
| 299 | + f"The gb_type '{gb_type}' appears in multiple GB IDs: {', '.join(sorted(ids))}. " | ||
| 300 | + f"Each gb_type must map to exactly one GB ID. Please manually fix the registry." | ||
| 301 | + ) | ||
| 302 | + lint_messages.append( | ||
| 303 | + LintMessage( | ||
| 304 | + path=str(registry_path), | ||
| 305 | + line=None, | ||
| 306 | + char=None, | ||
| 307 | + code=LINTER_CODE, | ||
| 308 | + severity=LintSeverity.ERROR, | ||
| 309 | + name="Duplicate gb_type in registry", | ||
| 310 | + original=None, | ||
| 311 | + replacement=None, | ||
| 312 | + description=description, | ||
| 313 | + ) | ||
| 314 | + ) | ||
| 315 | + return lint_messages | ||
| 316 | + | ||
| 317 | + latest_entry: dict[str, Any] = { | ||
| 318 | + entries[0]["Gb_type"]: entries[0] for entries in registry.values() | ||
| 319 | + } | ||
| 320 | + | ||
| 321 | + renames: dict[str, str] = {} | ||
| 322 | + remaining_calls = dict(calls) | ||
| 323 | + | ||
| 324 | + for gb_type, (call, file_path) in calls.items(): | ||
| 325 | + if gb_type not in latest_entry: | ||
| 326 | + for existing_gb_type, existing_entry in latest_entry.items(): | ||
| 327 | + if ( | ||
| 328 | + call["context"] == existing_entry["Context"] | ||
| 329 | + and call["explanation"] == existing_entry["Explanation"] | ||
| 330 | + and sorted(call["hints"]) == sorted(existing_entry["Hints"]) | ||
| 331 | + ): | ||
| 332 | + renames[existing_gb_type] = gb_type | ||
| 333 | + del remaining_calls[gb_type] | ||
| 334 | + break | ||
| 335 | + | ||
| 336 | + needs_update = bool(renames) | ||
| 337 | + | ||
| 338 | + for gb_type, (call, file_path) in remaining_calls.items(): | ||
| 339 | + if gb_type in latest_entry: | ||
| 340 | + existing_entry = latest_entry[gb_type] | ||
| 341 | + | ||
| 342 | + if not ( | ||
| 343 | + call["context"] == existing_entry["Context"] | ||
| 344 | + and call["explanation"] == existing_entry["Explanation"] | ||
| 345 | + and sorted(call["hints"] or []) == sorted(existing_entry["Hints"] or []) | ||
| 346 | + ): | ||
| 347 | + needs_update = True | ||
| 348 | + break | ||
| 349 | + else: | ||
| 350 | + needs_update = True | ||
| 351 | + break | ||
| 352 | + | ||
| 353 | + if needs_update: | ||
| 354 | + updated_registry = _update_registry_with_changes( | ||
| 355 | + registry, remaining_calls, renames | ||
| 356 | + ) | ||
| 357 | + | ||
| 358 | + original_content = registry_path.read_text(encoding="utf-8") | ||
| 359 | + | ||
| 360 | + replacement_content = ( | ||
| 361 | + json.dumps(updated_registry, indent=2, ensure_ascii=False) + "\n" | ||
| 362 | + ) | ||
| 363 | + | ||
| 364 | + changes = [] | ||
| 365 | + if renames: | ||
| 366 | + for old, new in renames.items(): | ||
| 367 | + changes.append(f"renamed '{old}' → '{new}'") | ||
| 368 | + if remaining_calls: | ||
| 369 | + new_count = sum( | ||
| 370 | + 1 for gb_type in remaining_calls if gb_type not in latest_entry | ||
| 371 | + ) | ||
| 372 | + if new_count: | ||
| 373 | + changes.append(f"added {new_count} new gb_types") | ||
| 374 | + | ||
| 375 | + description = f"Registry sync needed ({', '.join(changes)}). Run `lintrunner -a` to apply changes." | ||
| 376 | + | ||
| 377 | + lint_messages.append( | ||
| 378 | + LintMessage( | ||
| 379 | + path=str(registry_path), | ||
| 380 | + line=None, | ||
| 381 | + char=None, | ||
| 382 | + code=LINTER_CODE, | ||
| 383 | + severity=LintSeverity.WARNING, | ||
| 384 | + name="Registry sync needed", | ||
| 385 | + original=original_content, | ||
| 386 | + replacement=replacement_content, | ||
| 387 | + description=description, | ||
| 388 | + ) | ||
| 389 | + ) | ||
| 390 | + | ||
| 391 | + return lint_messages | ||
| 392 | + | ||
| 393 | + | ||
| 394 | +if __name__ == "__main__": | ||
| 395 | + script_dir = Path(__file__).resolve() | ||
| 396 | + repo_root = script_dir.parents[3] | ||
| 397 | + default_registry_path = ( | ||
| 398 | + repo_root / "torch" / "_dynamo" / "graph_break_registry.json" | ||
| 399 | + ) | ||
| 400 | + | ||
| 401 | + default_dynamo_dir = repo_root / "torch" / "_dynamo" | ||
| 402 | + | ||
| 403 | + parser = argparse.ArgumentParser( | ||
| 404 | + description="Auto-sync graph break registry with source code" | ||
| 405 | + ) | ||
| 406 | + parser.add_argument( | ||
| 407 | + "--dynamo-dir", | ||
| 408 | + type=Path, | ||
| 409 | + default=default_dynamo_dir, | ||
| 410 | + help=f"Path to the dynamo directory (default: {default_dynamo_dir})", | ||
| 411 | + ) | ||
| 412 | + parser.add_argument( | ||
| 413 | + "--registry-path", | ||
| 414 | + type=Path, | ||
| 415 | + default=default_registry_path, | ||
| 416 | + help=f"Path to the registry file (default: {default_registry_path})", | ||
| 417 | + ) | ||
| 418 | + | ||
| 419 | + args = parser.parse_args() | ||
| 420 | + | ||
| 421 | + lint_messages = check_registry_sync( | ||
| 422 | + dynamo_dir=args.dynamo_dir, registry_path=args.registry_path | ||
| 423 | + ) | ||
| 424 | + | ||
| 425 | + for lint_message in lint_messages: | ||
| 426 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,100 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "ruamel.yaml==0.18.10", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +""" | ||
| 8 | +TODO | ||
| 9 | +""" | ||
| 10 | + | ||
| 11 | +from __future__ import annotations | ||
| 12 | + | ||
| 13 | +import argparse | ||
| 14 | +import json | ||
| 15 | +import os.path | ||
| 16 | +from enum import Enum | ||
| 17 | +from typing import NamedTuple | ||
| 18 | + | ||
| 19 | +import ruamel.yaml # type: ignore[import] | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +class LintSeverity(str, Enum): | ||
| 23 | + ERROR = "error" | ||
| 24 | + WARNING = "warning" | ||
| 25 | + ADVICE = "advice" | ||
| 26 | + DISABLED = "disabled" | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +class LintMessage(NamedTuple): | ||
| 30 | + path: str | None | ||
| 31 | + line: int | None | ||
| 32 | + char: int | None | ||
| 33 | + code: str | ||
| 34 | + severity: LintSeverity | ||
| 35 | + name: str | ||
| 36 | + original: str | None | ||
| 37 | + replacement: str | None | ||
| 38 | + description: str | None | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +if __name__ == "__main__": | ||
| 42 | + parser = argparse.ArgumentParser( | ||
| 43 | + description="github actions linter", | ||
| 44 | + fromfile_prefix_chars="@", | ||
| 45 | + ) | ||
| 46 | + parser.add_argument( | ||
| 47 | + "filenames", | ||
| 48 | + nargs="+", | ||
| 49 | + help="paths to lint", | ||
| 50 | + ) | ||
| 51 | + | ||
| 52 | + args = parser.parse_args() | ||
| 53 | + | ||
| 54 | + for fn in args.filenames: | ||
| 55 | + with open(fn) as f: | ||
| 56 | + contents = f.read() | ||
| 57 | + | ||
| 58 | + yaml = ruamel.yaml.YAML() # type: ignore[attr-defined] | ||
| 59 | + try: | ||
| 60 | + r = yaml.load(contents) | ||
| 61 | + except Exception as err: | ||
| 62 | + msg = LintMessage( | ||
| 63 | + path=None, | ||
| 64 | + line=None, | ||
| 65 | + char=None, | ||
| 66 | + code="GHA", | ||
| 67 | + severity=LintSeverity.ERROR, | ||
| 68 | + name="YAML load failure", | ||
| 69 | + original=None, | ||
| 70 | + replacement=None, | ||
| 71 | + description=f"Failed due to {err.__class__.__name__}:\n{err}", | ||
| 72 | + ) | ||
| 73 | + | ||
| 74 | + print(json.dumps(msg._asdict()), flush=True) | ||
| 75 | + continue | ||
| 76 | + | ||
| 77 | + for job_name, job in r.get("jobs", {}).items(): | ||
| 78 | + # This filter is flexible, the idea is to avoid catching all of | ||
| 79 | + # the random label jobs that don't need secrets | ||
| 80 | + # TODO: binary might be good to have too, but it's a lot and | ||
| 81 | + # they're autogenerated too | ||
| 82 | + uses = os.path.basename(job.get("uses", "")) | ||
| 83 | + if ("build" in uses or "test" in uses) and "binary" not in uses: | ||
| 84 | + if job.get("secrets") != "inherit": | ||
| 85 | + desc = "missing 'secrets: inherit' field" | ||
| 86 | + if job.get("secrets") is not None: | ||
| 87 | + desc = "has 'secrets' field which is not standard form 'secrets: inherit'" | ||
| 88 | + msg = LintMessage( | ||
| 89 | + path=fn, | ||
| 90 | + line=job.lc.line, | ||
| 91 | + char=None, | ||
| 92 | + code="GHA", | ||
| 93 | + severity=LintSeverity.ERROR, | ||
| 94 | + name="missing secrets: inherit", | ||
| 95 | + original=None, | ||
| 96 | + replacement=None, | ||
| 97 | + description=(f"GitHub actions job '{job_name}' {desc}"), | ||
| 98 | + ) | ||
| 99 | + | ||
| 100 | + print(json.dumps(msg._asdict()), flush=True) | ||
| @@ -0,0 +1,414 @@ | |||
| 1 | +""" | ||
| 2 | +Generic linter that greps for a pattern and optionally suggests replacements. | ||
| 3 | +""" | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import argparse | ||
| 8 | +import json | ||
| 9 | +import logging | ||
| 10 | +import os | ||
| 11 | +import subprocess | ||
| 12 | +import sys | ||
| 13 | +import time | ||
| 14 | +from enum import Enum | ||
| 15 | +from typing import NamedTuple | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +IS_WINDOWS: bool = os.name == "nt" | ||
| 19 | +MAX_FILE_SIZE: int = 1024 * 1024 * 1024 # 1GB in bytes | ||
| 20 | +MAX_MATCHES_PER_FILE: int = 100 # Maximum number of matches to report per file | ||
| 21 | +MAX_ORIGINAL_SIZE: int = ( | ||
| 22 | + 512 * 1024 | ||
| 23 | +) # 512KB - don't compute replacement if original is larger | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +class LintSeverity(str, Enum): | ||
| 27 | + ERROR = "error" | ||
| 28 | + WARNING = "warning" | ||
| 29 | + ADVICE = "advice" | ||
| 30 | + DISABLED = "disabled" | ||
| 31 | + | ||
| 32 | + | ||
| 33 | +LINTER_NAME: str = "" | ||
| 34 | +ERROR_DESCRIPTION: str | None = None | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +class LintMessage(NamedTuple): | ||
| 38 | + path: str | None | ||
| 39 | + line: int | None | ||
| 40 | + char: int | None | ||
| 41 | + code: str | ||
| 42 | + severity: LintSeverity | ||
| 43 | + name: str | ||
| 44 | + original: str | None | ||
| 45 | + replacement: str | None | ||
| 46 | + description: str | None | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +def as_posix(name: str) -> str: | ||
| 50 | + return name.replace("\\", "/") if IS_WINDOWS else name | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +def run_command( | ||
| 54 | + args: list[str], | ||
| 55 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 56 | + logging.debug("$ %s", " ".join(args)) | ||
| 57 | + start_time = time.monotonic() | ||
| 58 | + try: | ||
| 59 | + return subprocess.run( | ||
| 60 | + args, | ||
| 61 | + capture_output=True, | ||
| 62 | + ) | ||
| 63 | + finally: | ||
| 64 | + end_time = time.monotonic() | ||
| 65 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 66 | + | ||
| 67 | + | ||
| 68 | +def print_lint_message( | ||
| 69 | + name: str, | ||
| 70 | + severity: LintSeverity = LintSeverity.ERROR, | ||
| 71 | + path: str | None = None, | ||
| 72 | + line: int | None = None, | ||
| 73 | + original: str | None = None, | ||
| 74 | + replacement: str | None = None, | ||
| 75 | + description: str | None = None, | ||
| 76 | +) -> None: | ||
| 77 | + """ | ||
| 78 | + Create a LintMessage and print it as JSON. | ||
| 79 | + | ||
| 80 | + Accepts the same arguments as LintMessage constructor. | ||
| 81 | + """ | ||
| 82 | + char = None | ||
| 83 | + code = LINTER_NAME | ||
| 84 | + description = description or ERROR_DESCRIPTION | ||
| 85 | + lint_message = LintMessage( | ||
| 86 | + path, line, char, code, severity, name, original, replacement, description | ||
| 87 | + ) | ||
| 88 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +def group_lines_by_file(lines: list[str]) -> dict[str, list[str]]: | ||
| 92 | + """ | ||
| 93 | + Group matching lines by filename. | ||
| 94 | + | ||
| 95 | + Args: | ||
| 96 | + lines: List of grep output lines in format "filename:line:content" | ||
| 97 | + | ||
| 98 | + Returns: | ||
| 99 | + Dictionary mapping filename to list of line remainders (without filename prefix) | ||
| 100 | + """ | ||
| 101 | + grouped: dict[str, list[str]] = {} | ||
| 102 | + for line in lines: | ||
| 103 | + if not line: | ||
| 104 | + continue | ||
| 105 | + # Extract filename and remainder from "filename:line:content" format | ||
| 106 | + parts = line.split(":", 1) | ||
| 107 | + filename = parts[0] | ||
| 108 | + remainder = parts[1] if len(parts) > 1 else "" | ||
| 109 | + if filename not in grouped: | ||
| 110 | + grouped[filename] = [] | ||
| 111 | + grouped[filename].append(remainder) | ||
| 112 | + return grouped | ||
| 113 | + | ||
| 114 | + | ||
| 115 | +def check_allowlist( | ||
| 116 | + filename: str, | ||
| 117 | + allowlist_pattern: str, | ||
| 118 | +) -> bool: | ||
| 119 | + """ | ||
| 120 | + Check if a file matches the allowlist pattern. | ||
| 121 | + | ||
| 122 | + Args: | ||
| 123 | + filename: Path to the file to check | ||
| 124 | + allowlist_pattern: Pattern to grep for in the file | ||
| 125 | + | ||
| 126 | + Returns: | ||
| 127 | + True if the file should be skipped (allowlist pattern matched), False otherwise. | ||
| 128 | + Prints error message and returns False if there was an error running grep. | ||
| 129 | + """ | ||
| 130 | + if not allowlist_pattern: | ||
| 131 | + return False | ||
| 132 | + | ||
| 133 | + try: | ||
| 134 | + proc = run_command(["grep", "-nEHI", allowlist_pattern, filename]) | ||
| 135 | + except Exception as err: | ||
| 136 | + print_lint_message( | ||
| 137 | + name="command-failed", | ||
| 138 | + description=( | ||
| 139 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 140 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 141 | + else ( | ||
| 142 | + "COMMAND (exit code {returncode})\n" | ||
| 143 | + "{command}\n\n" | ||
| 144 | + "STDERR\n{stderr}\n\n" | ||
| 145 | + "STDOUT\n{stdout}" | ||
| 146 | + ).format( | ||
| 147 | + returncode=err.returncode, | ||
| 148 | + command=" ".join(as_posix(x) for x in err.cmd), | ||
| 149 | + stderr=err.stderr.decode("utf-8").strip() or "(empty)", | ||
| 150 | + stdout=err.stdout.decode("utf-8").strip() or "(empty)", | ||
| 151 | + ) | ||
| 152 | + ), | ||
| 153 | + ) | ||
| 154 | + return False | ||
| 155 | + | ||
| 156 | + # allowlist pattern was found, abort lint | ||
| 157 | + if proc.returncode == 0: | ||
| 158 | + return True | ||
| 159 | + | ||
| 160 | + return False | ||
| 161 | + | ||
| 162 | + | ||
| 163 | +def lint_file( | ||
| 164 | + filename: str, | ||
| 165 | + line_remainders: list[str], | ||
| 166 | + allowlist_pattern: str, | ||
| 167 | + replace_pattern: str, | ||
| 168 | + error_name: str, | ||
| 169 | +) -> None: | ||
| 170 | + """ | ||
| 171 | + Lint a file with one or more pattern matches, printing LintMessages as they're created. | ||
| 172 | + | ||
| 173 | + Args: | ||
| 174 | + filename: Path to the file being linted | ||
| 175 | + line_remainders: List of line remainders (format: "line:content" without filename prefix) | ||
| 176 | + allowlist_pattern: Pattern to check for allowlisting | ||
| 177 | + replace_pattern: Pattern for sed replacement | ||
| 178 | + error_name: Human-readable error name | ||
| 179 | + """ | ||
| 180 | + if not line_remainders: | ||
| 181 | + return | ||
| 182 | + | ||
| 183 | + should_skip = check_allowlist(filename, allowlist_pattern) | ||
| 184 | + if should_skip: | ||
| 185 | + return | ||
| 186 | + | ||
| 187 | + # Check if file is too large to compute replacement | ||
| 188 | + file_size = os.path.getsize(filename) | ||
| 189 | + compute_replacement = replace_pattern and file_size <= MAX_ORIGINAL_SIZE | ||
| 190 | + | ||
| 191 | + # Apply replacement to entire file if pattern is specified and file is not too large | ||
| 192 | + original = None | ||
| 193 | + replacement = None | ||
| 194 | + if compute_replacement: | ||
| 195 | + # When we have a replacement, report a single message with line=None | ||
| 196 | + try: | ||
| 197 | + with open(filename) as f: | ||
| 198 | + original = f.read() | ||
| 199 | + | ||
| 200 | + proc = run_command(["sed", "-r", replace_pattern, filename]) | ||
| 201 | + replacement = proc.stdout.decode("utf-8") | ||
| 202 | + except Exception as err: | ||
| 203 | + print_lint_message( | ||
| 204 | + name="command-failed", | ||
| 205 | + description=( | ||
| 206 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 207 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 208 | + else ( | ||
| 209 | + "COMMAND (exit code {returncode})\n" | ||
| 210 | + "{command}\n\n" | ||
| 211 | + "STDERR\n{stderr}\n\n" | ||
| 212 | + "STDOUT\n{stdout}" | ||
| 213 | + ).format( | ||
| 214 | + returncode=err.returncode, | ||
| 215 | + command=" ".join(as_posix(x) for x in err.cmd), | ||
| 216 | + stderr=err.stderr.decode("utf-8").strip() or "(empty)", | ||
| 217 | + stdout=err.stdout.decode("utf-8").strip() or "(empty)", | ||
| 218 | + ) | ||
| 219 | + ), | ||
| 220 | + ) | ||
| 221 | + return | ||
| 222 | + | ||
| 223 | + print_lint_message( | ||
| 224 | + path=filename, | ||
| 225 | + name=error_name, | ||
| 226 | + original=original, | ||
| 227 | + replacement=replacement, | ||
| 228 | + ) | ||
| 229 | + else: | ||
| 230 | + # When no replacement, report each matching line (up to MAX_MATCHES_PER_FILE) | ||
| 231 | + total_matches = len(line_remainders) | ||
| 232 | + matches_to_report = min(total_matches, MAX_MATCHES_PER_FILE) | ||
| 233 | + | ||
| 234 | + for line_remainder in line_remainders[:matches_to_report]: | ||
| 235 | + # line_remainder format: "line_number:content" | ||
| 236 | + split = line_remainder.split(":", 1) | ||
| 237 | + line_number = int(split[0]) if split[0] else None | ||
| 238 | + print_lint_message( | ||
| 239 | + path=filename, | ||
| 240 | + line=line_number, | ||
| 241 | + name=error_name, | ||
| 242 | + ) | ||
| 243 | + | ||
| 244 | + # If there are more matches than the limit, print an error | ||
| 245 | + if total_matches > MAX_MATCHES_PER_FILE: | ||
| 246 | + print_lint_message( | ||
| 247 | + path=filename, | ||
| 248 | + name="too-many-matches", | ||
| 249 | + description=f"File has {total_matches} matches, only showing first {MAX_MATCHES_PER_FILE}", | ||
| 250 | + ) | ||
| 251 | + | ||
| 252 | + | ||
| 253 | +def main() -> None: | ||
| 254 | + parser = argparse.ArgumentParser( | ||
| 255 | + description="grep wrapper linter.", | ||
| 256 | + fromfile_prefix_chars="@", | ||
| 257 | + ) | ||
| 258 | + parser.add_argument( | ||
| 259 | + "--pattern", | ||
| 260 | + required=True, | ||
| 261 | + help="pattern to grep for", | ||
| 262 | + ) | ||
| 263 | + parser.add_argument( | ||
| 264 | + "--allowlist-pattern", | ||
| 265 | + help="if this pattern is true in the file, we don't grep for pattern", | ||
| 266 | + ) | ||
| 267 | + parser.add_argument( | ||
| 268 | + "--linter-name", | ||
| 269 | + required=True, | ||
| 270 | + help="name of the linter", | ||
| 271 | + ) | ||
| 272 | + parser.add_argument( | ||
| 273 | + "--match-first-only", | ||
| 274 | + action="store_true", | ||
| 275 | + help="only match the first hit in the file", | ||
| 276 | + ) | ||
| 277 | + parser.add_argument( | ||
| 278 | + "--error-name", | ||
| 279 | + required=True, | ||
| 280 | + help="human-readable description of what the error is", | ||
| 281 | + ) | ||
| 282 | + parser.add_argument( | ||
| 283 | + "--error-description", | ||
| 284 | + required=True, | ||
| 285 | + help="message to display when the pattern is found", | ||
| 286 | + ) | ||
| 287 | + parser.add_argument( | ||
| 288 | + "--replace-pattern", | ||
| 289 | + help=( | ||
| 290 | + "the form of a pattern passed to `sed -r`. " | ||
| 291 | + "If specified, this will become proposed replacement text." | ||
| 292 | + ), | ||
| 293 | + ) | ||
| 294 | + parser.add_argument( | ||
| 295 | + "--verbose", | ||
| 296 | + action="store_true", | ||
| 297 | + help="verbose logging", | ||
| 298 | + ) | ||
| 299 | + parser.add_argument( | ||
| 300 | + "filenames", | ||
| 301 | + nargs="+", | ||
| 302 | + help="paths to lint", | ||
| 303 | + ) | ||
| 304 | + | ||
| 305 | + # Check for duplicate arguments before parsing | ||
| 306 | + seen_args = set() | ||
| 307 | + for arg in sys.argv[1:]: | ||
| 308 | + if arg.startswith("--"): | ||
| 309 | + arg_name = arg.split("=")[0] | ||
| 310 | + if arg_name in seen_args: | ||
| 311 | + parser.error( | ||
| 312 | + f"argument {arg_name}: not allowed to be specified multiple times" | ||
| 313 | + ) | ||
| 314 | + seen_args.add(arg_name) | ||
| 315 | + | ||
| 316 | + args = parser.parse_args() | ||
| 317 | + | ||
| 318 | + global LINTER_NAME, ERROR_DESCRIPTION | ||
| 319 | + LINTER_NAME = args.linter_name | ||
| 320 | + ERROR_DESCRIPTION = args.error_description | ||
| 321 | + | ||
| 322 | + logging.basicConfig( | ||
| 323 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 324 | + level=logging.NOTSET | ||
| 325 | + if args.verbose | ||
| 326 | + else logging.DEBUG | ||
| 327 | + if len(args.filenames) < 1000 | ||
| 328 | + else logging.INFO, | ||
| 329 | + stream=sys.stderr, | ||
| 330 | + ) | ||
| 331 | + | ||
| 332 | + # Filter out files that are too large before running grep | ||
| 333 | + filtered_filenames = [] | ||
| 334 | + for filename in args.filenames: | ||
| 335 | + try: | ||
| 336 | + file_size = os.path.getsize(filename) | ||
| 337 | + if file_size > MAX_FILE_SIZE: | ||
| 338 | + print_lint_message( | ||
| 339 | + path=filename, | ||
| 340 | + severity=LintSeverity.WARNING, | ||
| 341 | + name="file-too-large", | ||
| 342 | + description=f"File size ({file_size} bytes) exceeds {MAX_FILE_SIZE} bytes limit, skipping", | ||
| 343 | + ) | ||
| 344 | + else: | ||
| 345 | + filtered_filenames.append(filename) | ||
| 346 | + except OSError as err: | ||
| 347 | + print_lint_message( | ||
| 348 | + path=filename, | ||
| 349 | + name="file-access-error", | ||
| 350 | + description=f"Failed to get file size: {err}", | ||
| 351 | + ) | ||
| 352 | + | ||
| 353 | + # If all files were filtered out, nothing to do | ||
| 354 | + if not filtered_filenames: | ||
| 355 | + return | ||
| 356 | + | ||
| 357 | + files_with_matches = [] | ||
| 358 | + if args.match_first_only: | ||
| 359 | + files_with_matches = ["--files-with-matches"] | ||
| 360 | + | ||
| 361 | + lines = [] | ||
| 362 | + try: | ||
| 363 | + # Split the grep command into multiple batches to avoid hitting the | ||
| 364 | + # command line length limit of ~1M on my machine | ||
| 365 | + arg_length = sum(len(x) for x in filtered_filenames) | ||
| 366 | + batches = arg_length // 750000 + 1 | ||
| 367 | + batch_size = len(filtered_filenames) // batches | ||
| 368 | + for i in range(0, len(filtered_filenames), batch_size): | ||
| 369 | + proc = run_command( | ||
| 370 | + [ | ||
| 371 | + "grep", | ||
| 372 | + "-nEHI", | ||
| 373 | + *files_with_matches, | ||
| 374 | + args.pattern, | ||
| 375 | + *filtered_filenames[i : i + batch_size], | ||
| 376 | + ] | ||
| 377 | + ) | ||
| 378 | + lines.extend(proc.stdout.decode().splitlines()) | ||
| 379 | + except Exception as err: | ||
| 380 | + print_lint_message( | ||
| 381 | + name="command-failed", | ||
| 382 | + description=( | ||
| 383 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 384 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 385 | + else ( | ||
| 386 | + "COMMAND (exit code {returncode})\n" | ||
| 387 | + "{command}\n\n" | ||
| 388 | + "STDERR\n{stderr}\n\n" | ||
| 389 | + "STDOUT\n{stdout}" | ||
| 390 | + ).format( | ||
| 391 | + returncode=err.returncode, | ||
| 392 | + command=" ".join(as_posix(x) for x in err.cmd), | ||
| 393 | + stderr=err.stderr.decode("utf-8").strip() or "(empty)", | ||
| 394 | + stdout=err.stdout.decode("utf-8").strip() or "(empty)", | ||
| 395 | + ) | ||
| 396 | + ), | ||
| 397 | + ) | ||
| 398 | + sys.exit(0) | ||
| 399 | + | ||
| 400 | + # Group lines by file to call lint_file once per file | ||
| 401 | + grouped_lines = group_lines_by_file(lines) | ||
| 402 | + | ||
| 403 | + for filename, line_remainders in grouped_lines.items(): | ||
| 404 | + lint_file( | ||
| 405 | + filename, | ||
| 406 | + line_remainders, | ||
| 407 | + args.allowlist_pattern, | ||
| 408 | + args.replace_pattern, | ||
| 409 | + args.error_name, | ||
| 410 | + ) | ||
| 411 | + | ||
| 412 | + | ||
| 413 | +if __name__ == "__main__": | ||
| 414 | + main() | ||
| @@ -0,0 +1,140 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +""" | ||
| 3 | +Checks that all symbols in torch/header_only_apis.txt are tested in a .cpp | ||
| 4 | +test file to ensure header-only-ness. The .cpp test file must be built | ||
| 5 | +without linking libtorch. | ||
| 6 | +""" | ||
| 7 | + | ||
| 8 | +import argparse | ||
| 9 | +import json | ||
| 10 | +import re | ||
| 11 | +from enum import Enum | ||
| 12 | +from pathlib import Path | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +LINTER_CODE = "HEADER_ONLY_LINTER" | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +class LintSeverity(str, Enum): | ||
| 20 | + ERROR = "error" | ||
| 21 | + WARNING = "warning" | ||
| 22 | + ADVICE = "advice" | ||
| 23 | + DISABLED = "disabled" | ||
| 24 | + | ||
| 25 | + | ||
| 26 | +class LintMessage(NamedTuple): | ||
| 27 | + path: str | None | ||
| 28 | + line: int | None | ||
| 29 | + char: int | None | ||
| 30 | + code: str | ||
| 31 | + severity: LintSeverity | ||
| 32 | + name: str | ||
| 33 | + original: str | None | ||
| 34 | + replacement: str | None | ||
| 35 | + description: str | None | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +CPP_TEST_GLOBS = [ | ||
| 39 | + "test/cpp/aoti_abi_check/*.cpp", | ||
| 40 | +] | ||
| 41 | + | ||
| 42 | +REPO_ROOT = Path(__file__).parents[3] | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +def find_matched_symbols( | ||
| 46 | + symbols_regex: re.Pattern[str], test_globs: list[str] = CPP_TEST_GLOBS | ||
| 47 | +) -> set[str]: | ||
| 48 | + """ | ||
| 49 | + Goes through all lines not starting with // in the cpp files and | ||
| 50 | + accumulates a list of matches with the symbols_regex. Note that | ||
| 51 | + we expect symbols_regex to be sorted in reverse alphabetical | ||
| 52 | + order to allow superset regexes to get matched. | ||
| 53 | + """ | ||
| 54 | + matched_symbols = set() | ||
| 55 | + # check noncommented out lines of the test files | ||
| 56 | + for cpp_test_glob in test_globs: | ||
| 57 | + for test_file in REPO_ROOT.glob(cpp_test_glob): | ||
| 58 | + with open(test_file) as tf: | ||
| 59 | + for test_file_line in tf: | ||
| 60 | + test_file_line = test_file_line.strip() | ||
| 61 | + if test_file_line.startswith(("//", "#")) or test_file_line == "": | ||
| 62 | + continue | ||
| 63 | + matches = re.findall(symbols_regex, test_file_line) | ||
| 64 | + for m in matches: | ||
| 65 | + if m != "": | ||
| 66 | + matched_symbols.add(m) | ||
| 67 | + return matched_symbols | ||
| 68 | + | ||
| 69 | + | ||
| 70 | +def check_file( | ||
| 71 | + filename: str, test_globs: list[str] = CPP_TEST_GLOBS | ||
| 72 | +) -> list[LintMessage]: | ||
| 73 | + """ | ||
| 74 | + Goes through the header_only_apis.txt file and verifies that all symbols | ||
| 75 | + within the file can be found tested in an appropriately independent .cpp | ||
| 76 | + file. | ||
| 77 | + | ||
| 78 | + Note that we expect CPP_TEST_GLOBS to be passed in as test_globs--the | ||
| 79 | + only reason this is an argument at all is for ease of testing. | ||
| 80 | + """ | ||
| 81 | + lint_messages: list[LintMessage] = [] | ||
| 82 | + | ||
| 83 | + symbols: dict[str, int] = {} # symbol -> lineno | ||
| 84 | + with open(filename) as f: | ||
| 85 | + for idx, line in enumerate(f): | ||
| 86 | + # commented out lines should be skipped | ||
| 87 | + symbol = line.strip() | ||
| 88 | + if not symbol or symbol[0] == "#": | ||
| 89 | + continue | ||
| 90 | + | ||
| 91 | + # symbols can in fact be duplicated and come from different headers. | ||
| 92 | + # we are aware this is a flaw in using simple string matching. | ||
| 93 | + symbols[symbol] = idx + 1 | ||
| 94 | + | ||
| 95 | + # Why reverse the keys? To allow superset regexes to get matched first in | ||
| 96 | + # find_matched_symbols. For example, we want Float8_e5m2fnuz to match | ||
| 97 | + # before Float8_e5m2. Otherwise, both Float8_e5m2fnuz and Float8_e5m2 will | ||
| 98 | + # match Float8_e5m2 | ||
| 99 | + symbols_regex = re.compile("|".join(sorted(symbols.keys(), reverse=True))) | ||
| 100 | + matched_symbols = find_matched_symbols(symbols_regex, test_globs) | ||
| 101 | + | ||
| 102 | + for s, lineno in symbols.items(): | ||
| 103 | + if s not in matched_symbols: | ||
| 104 | + lint_messages.append( | ||
| 105 | + LintMessage( | ||
| 106 | + path=filename, | ||
| 107 | + line=lineno, | ||
| 108 | + char=None, | ||
| 109 | + code=LINTER_CODE, | ||
| 110 | + severity=LintSeverity.ERROR, | ||
| 111 | + name="[untested-symbol]", | ||
| 112 | + original=None, | ||
| 113 | + replacement=None, | ||
| 114 | + description=( | ||
| 115 | + f"{s} has been included as a header-only API " | ||
| 116 | + "but is not tested in any of CPP_TEST_GLOBS, which " | ||
| 117 | + f"contains {CPP_TEST_GLOBS}.\n" | ||
| 118 | + "Please add a .cpp test using the symbol without " | ||
| 119 | + "linking anything to verify that the symbol is in " | ||
| 120 | + "fact header-only. If you already have a test but it's" | ||
| 121 | + " not found, please add the .cpp file to CPP_TEST_GLOBS" | ||
| 122 | + " in tools/linters/adapters/header_only_linter.py." | ||
| 123 | + ), | ||
| 124 | + ) | ||
| 125 | + ) | ||
| 126 | + | ||
| 127 | + return lint_messages | ||
| 128 | + | ||
| 129 | + | ||
| 130 | +if __name__ == "__main__": | ||
| 131 | + parser = argparse.ArgumentParser( | ||
| 132 | + description="header only APIs linter", | ||
| 133 | + fromfile_prefix_chars="@", | ||
| 134 | + ) | ||
| 135 | + args = parser.parse_args() | ||
| 136 | + | ||
| 137 | + for lint_message in check_file( | ||
| 138 | + str(REPO_ROOT) + "/torch/header_only_apis.txt", CPP_TEST_GLOBS | ||
| 139 | + ): | ||
| 140 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,139 @@ | |||
| 1 | +""" | ||
| 2 | +Checks files to make sure there are no imports from disallowed third party | ||
| 3 | +libraries. | ||
| 4 | +""" | ||
| 5 | + | ||
| 6 | +from __future__ import annotations | ||
| 7 | + | ||
| 8 | +import argparse | ||
| 9 | +import json | ||
| 10 | +import os | ||
| 11 | +import sys | ||
| 12 | +import token | ||
| 13 | +from enum import Enum | ||
| 14 | +from pathlib import Path | ||
| 15 | +from typing import NamedTuple, TYPE_CHECKING | ||
| 16 | + | ||
| 17 | + | ||
| 18 | +_PARENT = Path(__file__).parent.absolute() | ||
| 19 | +_PATH = [Path(p).absolute() for p in sys.path] | ||
| 20 | + | ||
| 21 | +if TYPE_CHECKING or _PARENT not in _PATH: | ||
| 22 | + from . import _linter | ||
| 23 | +else: | ||
| 24 | + import _linter | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +class LintSeverity(str, Enum): | ||
| 28 | + ERROR = "error" | ||
| 29 | + WARNING = "warning" | ||
| 30 | + ADVICE = "advice" | ||
| 31 | + DISABLED = "disabled" | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +class LintMessage(NamedTuple): | ||
| 35 | + path: str | None | ||
| 36 | + line: int | None | ||
| 37 | + char: int | None | ||
| 38 | + code: str | ||
| 39 | + severity: LintSeverity | ||
| 40 | + name: str | ||
| 41 | + original: str | None | ||
| 42 | + replacement: str | None | ||
| 43 | + description: str | None | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +LINTER_CODE = "IMPORT_LINTER" | ||
| 47 | +CURRENT_FILE_NAME = os.path.basename(__file__) | ||
| 48 | +_MODULE_NAME_ALLOW_LIST: set[str] = set() | ||
| 49 | + | ||
| 50 | +# Add builtin modules of python. | ||
| 51 | +_MODULE_NAME_ALLOW_LIST.update(sys.stdlib_module_names) | ||
| 52 | + | ||
| 53 | +# Add the allowed third party libraries. Please avoid updating this unless you | ||
| 54 | +# understand the risks -- see `_ERROR_MESSAGE` for why. | ||
| 55 | +_MODULE_NAME_ALLOW_LIST.update( | ||
| 56 | + [ | ||
| 57 | + "sympy", | ||
| 58 | + "einops", | ||
| 59 | + "libfb", | ||
| 60 | + "torch", | ||
| 61 | + "tvm", | ||
| 62 | + "_pytest", | ||
| 63 | + "tabulate", | ||
| 64 | + "optree", | ||
| 65 | + "typing_extensions", | ||
| 66 | + "triton", | ||
| 67 | + "functorch", | ||
| 68 | + "torchrec", | ||
| 69 | + "numpy", | ||
| 70 | + "torch_xla", | ||
| 71 | + "annotationlib", # added in python 3.14 | ||
| 72 | + ] | ||
| 73 | +) | ||
| 74 | + | ||
| 75 | +_ERROR_MESSAGE = """ | ||
| 76 | +Please do not import third-party modules in PyTorch unless they're explicit | ||
| 77 | +requirements of PyTorch. Imports of a third-party library may have side effects | ||
| 78 | +and other unintentional behavior. If you're just checking if a module exists, | ||
| 79 | +use sys.modules.get("torchrec") or the like. | ||
| 80 | +""" | ||
| 81 | + | ||
| 82 | + | ||
| 83 | +def check_file(filepath: str) -> list[LintMessage]: | ||
| 84 | + path = Path(filepath) | ||
| 85 | + file = _linter.PythonFile("import_linter", path=path) | ||
| 86 | + lint_messages = [] | ||
| 87 | + for line_number, line_of_tokens in enumerate(file.token_lines): | ||
| 88 | + # Skip indents | ||
| 89 | + idx = 0 | ||
| 90 | + for tok in line_of_tokens: | ||
| 91 | + if tok.type == token.INDENT: | ||
| 92 | + idx += 1 | ||
| 93 | + else: | ||
| 94 | + break | ||
| 95 | + | ||
| 96 | + # Look for either "import foo..." or "from foo..." | ||
| 97 | + if idx + 1 < len(line_of_tokens): | ||
| 98 | + tok0 = line_of_tokens[idx] | ||
| 99 | + tok1 = line_of_tokens[idx + 1] | ||
| 100 | + if tok0.type == token.NAME and tok0.string in {"import", "from"}: | ||
| 101 | + if tok1.type == token.NAME: | ||
| 102 | + module_name = tok1.string | ||
| 103 | + if module_name not in _MODULE_NAME_ALLOW_LIST: | ||
| 104 | + msg = LintMessage( | ||
| 105 | + path=filepath, | ||
| 106 | + line=line_number, | ||
| 107 | + char=None, | ||
| 108 | + code="IMPORT", | ||
| 109 | + severity=LintSeverity.ERROR, | ||
| 110 | + name="Disallowed import", | ||
| 111 | + original=None, | ||
| 112 | + replacement=None, | ||
| 113 | + description=_ERROR_MESSAGE, | ||
| 114 | + ) | ||
| 115 | + lint_messages.append(msg) | ||
| 116 | + return lint_messages | ||
| 117 | + | ||
| 118 | + | ||
| 119 | +if __name__ == "__main__": | ||
| 120 | + parser = argparse.ArgumentParser( | ||
| 121 | + description="native functions linter", | ||
| 122 | + fromfile_prefix_chars="@", | ||
| 123 | + ) | ||
| 124 | + parser.add_argument( | ||
| 125 | + "filepaths", | ||
| 126 | + nargs="+", | ||
| 127 | + help="paths of files to lint", | ||
| 128 | + ) | ||
| 129 | + args = parser.parse_args() | ||
| 130 | + | ||
| 131 | + # Check all files. | ||
| 132 | + all_lint_messages = [] | ||
| 133 | + for filepath in args.filepaths: | ||
| 134 | + lint_messages = check_file(filepath) | ||
| 135 | + all_lint_messages.extend(lint_messages) | ||
| 136 | + | ||
| 137 | + # Print out lint messages. | ||
| 138 | + for lint_message in all_lint_messages: | ||
| 139 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,82 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import json | ||
| 4 | +import subprocess | ||
| 5 | +import sys | ||
| 6 | +from enum import Enum | ||
| 7 | +from typing import NamedTuple | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +LINTER_CODE = "LINTRUNNER_VERSION" | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +class LintSeverity(str, Enum): | ||
| 14 | + ERROR = "error" | ||
| 15 | + WARNING = "warning" | ||
| 16 | + ADVICE = "advice" | ||
| 17 | + DISABLED = "disabled" | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +class LintMessage(NamedTuple): | ||
| 21 | + path: str | None | ||
| 22 | + line: int | None | ||
| 23 | + char: int | None | ||
| 24 | + code: str | ||
| 25 | + severity: LintSeverity | ||
| 26 | + name: str | ||
| 27 | + original: str | None | ||
| 28 | + replacement: str | None | ||
| 29 | + description: str | None | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +def toVersionString(version_tuple: tuple[int, int, int]) -> str: | ||
| 33 | + return ".".join(str(x) for x in version_tuple) | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +if __name__ == "__main__": | ||
| 37 | + version_str = ( | ||
| 38 | + subprocess.run(["lintrunner", "-V"], stdout=subprocess.PIPE) | ||
| 39 | + .stdout.decode("utf-8") | ||
| 40 | + .strip() | ||
| 41 | + ) | ||
| 42 | + | ||
| 43 | + import re | ||
| 44 | + | ||
| 45 | + version_match = re.compile(r"lintrunner (\d+)\.(\d+)\.(\d+)").match(version_str) | ||
| 46 | + | ||
| 47 | + if not version_match: | ||
| 48 | + err_msg = LintMessage( | ||
| 49 | + path="<none>", | ||
| 50 | + line=None, | ||
| 51 | + char=None, | ||
| 52 | + code=LINTER_CODE, | ||
| 53 | + severity=LintSeverity.ERROR, | ||
| 54 | + name="command-failed", | ||
| 55 | + original=None, | ||
| 56 | + replacement=None, | ||
| 57 | + description="Lintrunner is not installed, did you forget to run `make setup-lint && make lint`?", | ||
| 58 | + ) | ||
| 59 | + sys.exit(0) | ||
| 60 | + | ||
| 61 | + curr_version = int(version_match[1]), int(version_match[2]), int(version_match[3]) | ||
| 62 | + min_version = (0, 10, 7) | ||
| 63 | + | ||
| 64 | + if curr_version < min_version: | ||
| 65 | + err_msg = LintMessage( | ||
| 66 | + path="<none>", | ||
| 67 | + line=None, | ||
| 68 | + char=None, | ||
| 69 | + code=LINTER_CODE, | ||
| 70 | + severity=LintSeverity.ADVICE, | ||
| 71 | + name="command-failed", | ||
| 72 | + original=None, | ||
| 73 | + replacement=None, | ||
| 74 | + description="".join( | ||
| 75 | + ( | ||
| 76 | + f"Lintrunner is out of date (you have v{toVersionString(curr_version)} ", | ||
| 77 | + f"instead of v{toVersionString(min_version)}). ", | ||
| 78 | + "Please run `pip install lintrunner -U` to update it", | ||
| 79 | + ) | ||
| 80 | + ), | ||
| 81 | + ) | ||
| 82 | + print(json.dumps(err_msg._asdict()), flush=True) | ||
| @@ -0,0 +1,263 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import json | ||
| 5 | +import logging | ||
| 6 | +import os | ||
| 7 | +import re | ||
| 8 | +import subprocess | ||
| 9 | +import sys | ||
| 10 | +import time | ||
| 11 | +from enum import Enum | ||
| 12 | +from pathlib import Path | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +class LintSeverity(str, Enum): | ||
| 17 | + ERROR = "error" | ||
| 18 | + WARNING = "warning" | ||
| 19 | + ADVICE = "advice" | ||
| 20 | + DISABLED = "disabled" | ||
| 21 | + | ||
| 22 | + | ||
| 23 | +class LintMessage(NamedTuple): | ||
| 24 | + path: str | None | ||
| 25 | + line: int | None | ||
| 26 | + char: int | None | ||
| 27 | + code: str | ||
| 28 | + severity: LintSeverity | ||
| 29 | + name: str | ||
| 30 | + original: str | None | ||
| 31 | + replacement: str | None | ||
| 32 | + description: str | None | ||
| 33 | + | ||
| 34 | + | ||
| 35 | +# tools/linter/flake8_linter.py:15:13: error: Incompatibl...int") [assignment] | ||
| 36 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 37 | + r"""(?mx) | ||
| 38 | + ^ | ||
| 39 | + (?P<file>.*?): | ||
| 40 | + (?P<line>\d+): | ||
| 41 | + (?:(?P<column>-?\d+):)? | ||
| 42 | + \s(?P<severity>\S+?):? | ||
| 43 | + \s(?P<message>.*) | ||
| 44 | + \s(?P<code>\[.*\]) | ||
| 45 | + $ | ||
| 46 | + """ | ||
| 47 | +) | ||
| 48 | + | ||
| 49 | +# torch/_dynamo/variables/tensor.py:363: error: INTERNAL ERROR | ||
| 50 | +INTERNAL_ERROR_RE: re.Pattern[str] = re.compile( | ||
| 51 | + r"""(?mx) | ||
| 52 | + ^ | ||
| 53 | + (?P<file>.*?): | ||
| 54 | + (?P<line>\d+): | ||
| 55 | + \s(?P<severity>\S+?):? | ||
| 56 | + \s(?P<message>INTERNAL\sERROR.*) | ||
| 57 | + $ | ||
| 58 | + """ | ||
| 59 | +) | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def run_command( | ||
| 63 | + args: list[str], | ||
| 64 | + *, | ||
| 65 | + extra_env: dict[str, str] | None, | ||
| 66 | + retries: int, | ||
| 67 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 68 | + logging.debug("$ %s", " ".join(args)) | ||
| 69 | + start_time = time.monotonic() | ||
| 70 | + try: | ||
| 71 | + return subprocess.run( | ||
| 72 | + args, | ||
| 73 | + capture_output=True, | ||
| 74 | + ) | ||
| 75 | + finally: | ||
| 76 | + end_time = time.monotonic() | ||
| 77 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 78 | + | ||
| 79 | + | ||
| 80 | +# Severity is either "error" or "note": | ||
| 81 | +# https://github.com/python/mypy/blob/8b47a032e1317fb8e3f9a818005a6b63e9bf0311/mypy/errors.py#L46-L47 | ||
| 82 | +severities = { | ||
| 83 | + "error": LintSeverity.ERROR, | ||
| 84 | + "note": LintSeverity.ADVICE, | ||
| 85 | +} | ||
| 86 | + | ||
| 87 | + | ||
| 88 | +def check_mypy_installed(code: str) -> list[LintMessage]: | ||
| 89 | + cmd = [sys.executable, "-mmypy", "-V"] | ||
| 90 | + try: | ||
| 91 | + subprocess.run(cmd, check=True, capture_output=True) | ||
| 92 | + return [] | ||
| 93 | + except subprocess.CalledProcessError as e: | ||
| 94 | + msg = e.stderr.decode(errors="replace") | ||
| 95 | + return [ | ||
| 96 | + LintMessage( | ||
| 97 | + path=None, | ||
| 98 | + line=None, | ||
| 99 | + char=None, | ||
| 100 | + code=code, | ||
| 101 | + severity=LintSeverity.ERROR, | ||
| 102 | + name="command-failed", | ||
| 103 | + original=None, | ||
| 104 | + replacement=None, | ||
| 105 | + description=f"Could not run '{' '.join(cmd)}': {msg}", | ||
| 106 | + ) | ||
| 107 | + ] | ||
| 108 | + | ||
| 109 | + | ||
| 110 | +def in_github_actions() -> bool: | ||
| 111 | + return bool(os.getenv("GITHUB_ACTIONS")) | ||
| 112 | + | ||
| 113 | + | ||
| 114 | +def check_files( | ||
| 115 | + filenames: list[str], | ||
| 116 | + config: str, | ||
| 117 | + retries: int, | ||
| 118 | + code: str, | ||
| 119 | +) -> list[LintMessage]: | ||
| 120 | + # dmypy has a bug where it won't pick up changes if you pass it absolute | ||
| 121 | + # file names, see https://github.com/python/mypy/issues/16768 | ||
| 122 | + filenames = [os.path.relpath(f) for f in filenames] | ||
| 123 | + try: | ||
| 124 | + mypy_commands = ["dmypy", "run", "--"] | ||
| 125 | + if in_github_actions(): | ||
| 126 | + mypy_commands = ["mypy"] | ||
| 127 | + proc = run_command( | ||
| 128 | + [*mypy_commands, f"--config={config}"] + filenames, | ||
| 129 | + extra_env={}, | ||
| 130 | + retries=retries, | ||
| 131 | + ) | ||
| 132 | + except OSError as err: | ||
| 133 | + return [ | ||
| 134 | + LintMessage( | ||
| 135 | + path=None, | ||
| 136 | + line=None, | ||
| 137 | + char=None, | ||
| 138 | + code=code, | ||
| 139 | + severity=LintSeverity.ERROR, | ||
| 140 | + name="command-failed", | ||
| 141 | + original=None, | ||
| 142 | + replacement=None, | ||
| 143 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 144 | + ) | ||
| 145 | + ] | ||
| 146 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 147 | + stderr = str(proc.stderr, "utf-8").strip() | ||
| 148 | + if proc.returncode not in (0, 1): | ||
| 149 | + return [ | ||
| 150 | + LintMessage( | ||
| 151 | + path=None, | ||
| 152 | + line=None, | ||
| 153 | + char=None, | ||
| 154 | + code=code, | ||
| 155 | + severity=LintSeverity.ERROR, | ||
| 156 | + name="command-failed", | ||
| 157 | + original=None, | ||
| 158 | + replacement=None, | ||
| 159 | + description=stderr, | ||
| 160 | + ) | ||
| 161 | + ] | ||
| 162 | + | ||
| 163 | + rc = [ | ||
| 164 | + LintMessage( | ||
| 165 | + path=match["file"], | ||
| 166 | + name=match["code"], | ||
| 167 | + description=match["message"], | ||
| 168 | + line=int(match["line"]), | ||
| 169 | + char=int(match["column"]) | ||
| 170 | + if match["column"] is not None and not match["column"].startswith("-") | ||
| 171 | + else None, | ||
| 172 | + code=code, | ||
| 173 | + severity=severities.get(match["severity"], LintSeverity.ERROR), | ||
| 174 | + original=None, | ||
| 175 | + replacement=None, | ||
| 176 | + ) | ||
| 177 | + for match in RESULTS_RE.finditer(stdout) | ||
| 178 | + ] + [ | ||
| 179 | + LintMessage( | ||
| 180 | + path=match["file"], | ||
| 181 | + name="INTERNAL ERROR", | ||
| 182 | + description=match["message"], | ||
| 183 | + line=int(match["line"]), | ||
| 184 | + char=None, | ||
| 185 | + code=code, | ||
| 186 | + severity=severities.get(match["severity"], LintSeverity.ERROR), | ||
| 187 | + original=None, | ||
| 188 | + replacement=None, | ||
| 189 | + ) | ||
| 190 | + for match in INTERNAL_ERROR_RE.finditer(stderr) | ||
| 191 | + ] | ||
| 192 | + return rc | ||
| 193 | + | ||
| 194 | + | ||
| 195 | +def main() -> None: | ||
| 196 | + parser = argparse.ArgumentParser( | ||
| 197 | + description="mypy wrapper linter.", | ||
| 198 | + fromfile_prefix_chars="@", | ||
| 199 | + ) | ||
| 200 | + parser.add_argument( | ||
| 201 | + "--retries", | ||
| 202 | + default=3, | ||
| 203 | + type=int, | ||
| 204 | + help="times to retry timed out mypy", | ||
| 205 | + ) | ||
| 206 | + parser.add_argument( | ||
| 207 | + "--config", | ||
| 208 | + required=True, | ||
| 209 | + help="path to an mypy .ini config file", | ||
| 210 | + ) | ||
| 211 | + parser.add_argument( | ||
| 212 | + "--code", | ||
| 213 | + default="MYPY", | ||
| 214 | + help="the code this lint should report as", | ||
| 215 | + ) | ||
| 216 | + parser.add_argument( | ||
| 217 | + "--verbose", | ||
| 218 | + action="store_true", | ||
| 219 | + help="verbose logging", | ||
| 220 | + ) | ||
| 221 | + parser.add_argument( | ||
| 222 | + "filenames", | ||
| 223 | + nargs="+", | ||
| 224 | + help="paths to lint", | ||
| 225 | + ) | ||
| 226 | + args = parser.parse_args() | ||
| 227 | + | ||
| 228 | + logging.basicConfig( | ||
| 229 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 230 | + level=logging.NOTSET | ||
| 231 | + if args.verbose | ||
| 232 | + else logging.DEBUG | ||
| 233 | + if len(args.filenames) < 1000 | ||
| 234 | + else logging.INFO, | ||
| 235 | + stream=sys.stderr, | ||
| 236 | + ) | ||
| 237 | + | ||
| 238 | + # Use a dictionary here to preserve order. mypy cares about order, | ||
| 239 | + # tragically, e.g. https://github.com/python/mypy/issues/2015 | ||
| 240 | + filenames: dict[str, bool] = {} | ||
| 241 | + | ||
| 242 | + # If a stub file exists, have mypy check it instead of the original file, in | ||
| 243 | + # accordance with PEP-484 (see https://www.python.org/dev/peps/pep-0484/#stub-files) | ||
| 244 | + for filename in args.filenames: | ||
| 245 | + if filename.endswith(".pyi"): | ||
| 246 | + filenames[filename] = True | ||
| 247 | + continue | ||
| 248 | + | ||
| 249 | + stub_filename = filename.replace(".py", ".pyi") | ||
| 250 | + if Path(stub_filename).exists(): | ||
| 251 | + filenames[stub_filename] = True | ||
| 252 | + else: | ||
| 253 | + filenames[filename] = True | ||
| 254 | + | ||
| 255 | + lint_messages = check_mypy_installed(args.code) + check_files( | ||
| 256 | + list(filenames), args.config, args.retries, args.code | ||
| 257 | + ) | ||
| 258 | + for lint_message in lint_messages: | ||
| 259 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 260 | + | ||
| 261 | + | ||
| 262 | +if __name__ == "__main__": | ||
| 263 | + main() | ||
| @@ -0,0 +1,114 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "ruamel.yaml==0.18.10", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +""" | ||
| 8 | +Verify that it is possible to round-trip native_functions.yaml via ruamel under some | ||
| 9 | +configuration. Keeping native_functions.yaml consistent in this way allows us to | ||
| 10 | +run codemods on the file using ruamel without introducing line noise. Note that we don't | ||
| 11 | +want to normalize the YAML file, as that would to lots of spurious lint failures. Anything | ||
| 12 | +that ruamel understands how to roundtrip, e.g., whitespace and comments, is OK! | ||
| 13 | + | ||
| 14 | +ruamel is a bit picky about inconsistent indentation, so you will have to indent your | ||
| 15 | +file properly. Also, if you are working on changing the syntax of native_functions.yaml, | ||
| 16 | +you may find that you want to use some format that is not what ruamel prefers. If so, | ||
| 17 | +it is OK to modify this script (instead of reformatting native_functions.yaml)--the point | ||
| 18 | +is simply to make sure that there is *some* configuration of ruamel that can round trip | ||
| 19 | +the YAML, not to be prescriptive about it. | ||
| 20 | +""" | ||
| 21 | + | ||
| 22 | +from __future__ import annotations | ||
| 23 | + | ||
| 24 | +import argparse | ||
| 25 | +import json | ||
| 26 | +import sys | ||
| 27 | +from enum import Enum | ||
| 28 | +from io import StringIO | ||
| 29 | +from typing import NamedTuple | ||
| 30 | + | ||
| 31 | +import ruamel.yaml # type: ignore[import] | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +class LintSeverity(str, Enum): | ||
| 35 | + ERROR = "error" | ||
| 36 | + WARNING = "warning" | ||
| 37 | + ADVICE = "advice" | ||
| 38 | + DISABLED = "disabled" | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +class LintMessage(NamedTuple): | ||
| 42 | + path: str | None | ||
| 43 | + line: int | None | ||
| 44 | + char: int | None | ||
| 45 | + code: str | ||
| 46 | + severity: LintSeverity | ||
| 47 | + name: str | ||
| 48 | + original: str | None | ||
| 49 | + replacement: str | None | ||
| 50 | + description: str | None | ||
| 51 | + | ||
| 52 | + | ||
| 53 | +if __name__ == "__main__": | ||
| 54 | + parser = argparse.ArgumentParser( | ||
| 55 | + description="native functions linter", | ||
| 56 | + fromfile_prefix_chars="@", | ||
| 57 | + ) | ||
| 58 | + parser.add_argument( | ||
| 59 | + "--native-functions-yml", | ||
| 60 | + required=True, | ||
| 61 | + help="location of native_functions.yaml", | ||
| 62 | + ) | ||
| 63 | + | ||
| 64 | + args = parser.parse_args() | ||
| 65 | + | ||
| 66 | + with open(args.native_functions_yml) as f: | ||
| 67 | + contents = f.read() | ||
| 68 | + | ||
| 69 | + yaml = ruamel.yaml.YAML() # type: ignore[attr-defined] | ||
| 70 | + yaml.preserve_quotes = True # type: ignore[assignment] | ||
| 71 | + yaml.width = 1000 # type: ignore[assignment] | ||
| 72 | + yaml.boolean_representation = ["False", "True"] # type: ignore[attr-defined] | ||
| 73 | + try: | ||
| 74 | + r = yaml.load(contents) | ||
| 75 | + except Exception as err: | ||
| 76 | + msg = LintMessage( | ||
| 77 | + path=None, | ||
| 78 | + line=None, | ||
| 79 | + char=None, | ||
| 80 | + code="NATIVEFUNCTIONS", | ||
| 81 | + severity=LintSeverity.ERROR, | ||
| 82 | + name="YAML load failure", | ||
| 83 | + original=None, | ||
| 84 | + replacement=None, | ||
| 85 | + description=f"Failed due to {err.__class__.__name__}:\n{err}", | ||
| 86 | + ) | ||
| 87 | + | ||
| 88 | + print(json.dumps(msg._asdict()), flush=True) | ||
| 89 | + sys.exit(0) | ||
| 90 | + | ||
| 91 | + # Cuz ruamel's author intentionally didn't include conversion to string | ||
| 92 | + # https://stackoverflow.com/questions/47614862/best-way-to-use-ruamel-yaml-to-dump-to-string-not-to-stream | ||
| 93 | + string_stream = StringIO() | ||
| 94 | + yaml.dump(r, string_stream) | ||
| 95 | + new_contents = string_stream.getvalue() | ||
| 96 | + string_stream.close() | ||
| 97 | + | ||
| 98 | + if contents != new_contents: | ||
| 99 | + msg = LintMessage( | ||
| 100 | + path=args.native_functions_yml, | ||
| 101 | + line=None, | ||
| 102 | + char=None, | ||
| 103 | + code="NATIVEFUNCTIONS", | ||
| 104 | + severity=LintSeverity.ERROR, | ||
| 105 | + name="roundtrip inconsistency", | ||
| 106 | + original=contents, | ||
| 107 | + replacement=new_contents, | ||
| 108 | + description=( | ||
| 109 | + "YAML roundtrip failed; run `lintrunner --take NATIVEFUNCTIONS -a` to apply the suggested changes. " | ||
| 110 | + "If you think this is in error, please see tools/linter/adapters/nativefunctions_linter.py" | ||
| 111 | + ), | ||
| 112 | + ) | ||
| 113 | + | ||
| 114 | + print(json.dumps(msg._asdict()), flush=True) | ||
| @@ -0,0 +1,197 @@ | |||
| 1 | +""" | ||
| 2 | +NEWLINE: Checks files to make sure there are no trailing newlines. | ||
| 3 | +""" | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import argparse | ||
| 8 | +import json | ||
| 9 | +import logging | ||
| 10 | +import os | ||
| 11 | +import sys | ||
| 12 | +from enum import Enum | ||
| 13 | +from typing import NamedTuple | ||
| 14 | + | ||
| 15 | + | ||
| 16 | +NEWLINE = 10 # ASCII "\n" | ||
| 17 | +CARRIAGE_RETURN = 13 # ASCII "\r" | ||
| 18 | +LINTER_CODE = "NEWLINE" | ||
| 19 | +MAX_FILE_SIZE: int = 1024 * 1024 * 1024 # 1GB in bytes | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +class LintSeverity(str, Enum): | ||
| 23 | + ERROR = "error" | ||
| 24 | + WARNING = "warning" | ||
| 25 | + ADVICE = "advice" | ||
| 26 | + DISABLED = "disabled" | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +class LintMessage(NamedTuple): | ||
| 30 | + path: str | None | ||
| 31 | + line: int | None | ||
| 32 | + char: int | None | ||
| 33 | + code: str | ||
| 34 | + severity: LintSeverity | ||
| 35 | + name: str | ||
| 36 | + original: str | None | ||
| 37 | + replacement: str | None | ||
| 38 | + description: str | None | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +def check_file(filename: str) -> LintMessage | None: | ||
| 42 | + logging.debug("Checking file %s", filename) | ||
| 43 | + | ||
| 44 | + # Check if file is too large | ||
| 45 | + try: | ||
| 46 | + file_size = os.path.getsize(filename) | ||
| 47 | + if file_size > MAX_FILE_SIZE: | ||
| 48 | + return LintMessage( | ||
| 49 | + path=filename, | ||
| 50 | + line=None, | ||
| 51 | + char=None, | ||
| 52 | + code=LINTER_CODE, | ||
| 53 | + severity=LintSeverity.WARNING, | ||
| 54 | + name="file-too-large", | ||
| 55 | + original=None, | ||
| 56 | + replacement=None, | ||
| 57 | + description=f"File size ({file_size} bytes) exceeds {MAX_FILE_SIZE} bytes limit, skipping", | ||
| 58 | + ) | ||
| 59 | + except OSError as err: | ||
| 60 | + return LintMessage( | ||
| 61 | + path=filename, | ||
| 62 | + line=None, | ||
| 63 | + char=None, | ||
| 64 | + code=LINTER_CODE, | ||
| 65 | + severity=LintSeverity.ERROR, | ||
| 66 | + name="file-access-error", | ||
| 67 | + original=None, | ||
| 68 | + replacement=None, | ||
| 69 | + description=f"Failed to get file size: {err}", | ||
| 70 | + ) | ||
| 71 | + | ||
| 72 | + with open(filename, "rb") as f: | ||
| 73 | + lines = f.readlines() | ||
| 74 | + | ||
| 75 | + if len(lines) == 0: | ||
| 76 | + # File is empty, just leave it alone. | ||
| 77 | + return None | ||
| 78 | + | ||
| 79 | + if len(lines) == 1 and len(lines[0]) == 1: | ||
| 80 | + # file is wrong whether or not the only byte is a newline | ||
| 81 | + return LintMessage( | ||
| 82 | + path=filename, | ||
| 83 | + line=None, | ||
| 84 | + char=None, | ||
| 85 | + code=LINTER_CODE, | ||
| 86 | + severity=LintSeverity.ERROR, | ||
| 87 | + name="testestTrailing newline", | ||
| 88 | + original=None, | ||
| 89 | + replacement=None, | ||
| 90 | + description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", | ||
| 91 | + ) | ||
| 92 | + | ||
| 93 | + if len(lines[-1]) == 1 and lines[-1][0] == NEWLINE: | ||
| 94 | + try: | ||
| 95 | + original = b"".join(lines).decode("utf-8") | ||
| 96 | + except Exception as err: | ||
| 97 | + return LintMessage( | ||
| 98 | + path=filename, | ||
| 99 | + line=None, | ||
| 100 | + char=None, | ||
| 101 | + code=LINTER_CODE, | ||
| 102 | + severity=LintSeverity.ERROR, | ||
| 103 | + name="Decoding failure", | ||
| 104 | + original=None, | ||
| 105 | + replacement=None, | ||
| 106 | + description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}", | ||
| 107 | + ) | ||
| 108 | + | ||
| 109 | + return LintMessage( | ||
| 110 | + path=filename, | ||
| 111 | + line=None, | ||
| 112 | + char=None, | ||
| 113 | + code=LINTER_CODE, | ||
| 114 | + severity=LintSeverity.ERROR, | ||
| 115 | + name="Trailing newline", | ||
| 116 | + original=original, | ||
| 117 | + replacement=original.rstrip("\n") + "\n", | ||
| 118 | + description="Trailing newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", | ||
| 119 | + ) | ||
| 120 | + has_changes = False | ||
| 121 | + original_lines: list[bytes] | None = None | ||
| 122 | + for idx, line in enumerate(lines): | ||
| 123 | + if len(line) >= 2 and line[-1] == NEWLINE and line[-2] == CARRIAGE_RETURN: | ||
| 124 | + if not has_changes: | ||
| 125 | + original_lines = list(lines) | ||
| 126 | + has_changes = True | ||
| 127 | + lines[idx] = line[:-2] + b"\n" | ||
| 128 | + | ||
| 129 | + if has_changes: | ||
| 130 | + try: | ||
| 131 | + if original_lines is None: | ||
| 132 | + raise AssertionError("original_lines is None") | ||
| 133 | + original = b"".join(original_lines).decode("utf-8") | ||
| 134 | + replacement = b"".join(lines).decode("utf-8") | ||
| 135 | + except Exception as err: | ||
| 136 | + return LintMessage( | ||
| 137 | + path=filename, | ||
| 138 | + line=None, | ||
| 139 | + char=None, | ||
| 140 | + code=LINTER_CODE, | ||
| 141 | + severity=LintSeverity.ERROR, | ||
| 142 | + name="Decoding failure", | ||
| 143 | + original=None, | ||
| 144 | + replacement=None, | ||
| 145 | + description=f"utf-8 decoding failed due to {err.__class__.__name__}:\n{err}", | ||
| 146 | + ) | ||
| 147 | + return LintMessage( | ||
| 148 | + path=filename, | ||
| 149 | + line=None, | ||
| 150 | + char=None, | ||
| 151 | + code=LINTER_CODE, | ||
| 152 | + severity=LintSeverity.ERROR, | ||
| 153 | + name="DOS newline", | ||
| 154 | + original=original, | ||
| 155 | + replacement=replacement, | ||
| 156 | + description="DOS newline found. Run `lintrunner --take NEWLINE -a` to apply changes.", | ||
| 157 | + ) | ||
| 158 | + | ||
| 159 | + return None | ||
| 160 | + | ||
| 161 | + | ||
| 162 | +if __name__ == "__main__": | ||
| 163 | + parser = argparse.ArgumentParser( | ||
| 164 | + description="native functions linter", | ||
| 165 | + fromfile_prefix_chars="@", | ||
| 166 | + ) | ||
| 167 | + parser.add_argument( | ||
| 168 | + "--verbose", | ||
| 169 | + action="store_true", | ||
| 170 | + help="location of native_functions.yaml", | ||
| 171 | + ) | ||
| 172 | + parser.add_argument( | ||
| 173 | + "filenames", | ||
| 174 | + nargs="+", | ||
| 175 | + help="paths to lint", | ||
| 176 | + ) | ||
| 177 | + | ||
| 178 | + args = parser.parse_args() | ||
| 179 | + | ||
| 180 | + logging.basicConfig( | ||
| 181 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 182 | + level=logging.NOTSET | ||
| 183 | + if args.verbose | ||
| 184 | + else logging.DEBUG | ||
| 185 | + if len(args.filenames) < 1000 | ||
| 186 | + else logging.INFO, | ||
| 187 | + stream=sys.stderr, | ||
| 188 | + ) | ||
| 189 | + | ||
| 190 | + lint_messages = [] | ||
| 191 | + for filename in args.filenames: | ||
| 192 | + lint_message = check_file(filename) | ||
| 193 | + if lint_message is not None: | ||
| 194 | + lint_messages.append(lint_message) | ||
| 195 | + | ||
| 196 | + for lint_message in lint_messages: | ||
| 197 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,103 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import argparse | ||
| 4 | +import concurrent.futures | ||
| 5 | +import json | ||
| 6 | +import logging | ||
| 7 | +import os | ||
| 8 | +import sys | ||
| 9 | +from enum import Enum | ||
| 10 | +from typing import NamedTuple | ||
| 11 | + | ||
| 12 | + | ||
| 13 | +class LintSeverity(str, Enum): | ||
| 14 | + ERROR = "error" | ||
| 15 | + WARNING = "warning" | ||
| 16 | + ADVICE = "advice" | ||
| 17 | + DISABLED = "disabled" | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +class LintMessage(NamedTuple): | ||
| 21 | + path: str | None | ||
| 22 | + line: int | None | ||
| 23 | + char: int | None | ||
| 24 | + code: str | ||
| 25 | + severity: LintSeverity | ||
| 26 | + name: str | ||
| 27 | + original: str | None | ||
| 28 | + replacement: str | None | ||
| 29 | + description: str | None | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 33 | + with open(filename, "rb") as f: | ||
| 34 | + original = f.read().decode("utf-8") | ||
| 35 | + replacement = "" | ||
| 36 | + with open(filename) as f: | ||
| 37 | + lines = f.readlines() | ||
| 38 | + for line in lines: | ||
| 39 | + if len(line.strip()) > 0: | ||
| 40 | + replacement += line | ||
| 41 | + replacement += "\n" * 3 | ||
| 42 | + replacement = replacement[:-3] | ||
| 43 | + | ||
| 44 | + if replacement == original: | ||
| 45 | + return [] | ||
| 46 | + | ||
| 47 | + return [ | ||
| 48 | + LintMessage( | ||
| 49 | + path=filename, | ||
| 50 | + line=None, | ||
| 51 | + char=None, | ||
| 52 | + code="MERGE_CONFLICTLESS_CSV", | ||
| 53 | + severity=LintSeverity.WARNING, | ||
| 54 | + name="format", | ||
| 55 | + original=original, | ||
| 56 | + replacement=replacement, | ||
| 57 | + description="Run `lintrunner -a` to apply this patch.", | ||
| 58 | + ) | ||
| 59 | + ] | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def main() -> None: | ||
| 63 | + parser = argparse.ArgumentParser( | ||
| 64 | + description="Format csv files to have 3 lines of space between each line to prevent merge conflicts.", | ||
| 65 | + fromfile_prefix_chars="@", | ||
| 66 | + ) | ||
| 67 | + parser.add_argument( | ||
| 68 | + "--verbose", | ||
| 69 | + action="store_true", | ||
| 70 | + help="verbose logging", | ||
| 71 | + ) | ||
| 72 | + parser.add_argument( | ||
| 73 | + "filenames", | ||
| 74 | + nargs="+", | ||
| 75 | + help="paths to lint", | ||
| 76 | + ) | ||
| 77 | + args = parser.parse_args() | ||
| 78 | + | ||
| 79 | + logging.basicConfig( | ||
| 80 | + format="<%(processName)s:%(levelname)s> %(message)s", | ||
| 81 | + level=logging.NOTSET | ||
| 82 | + if args.verbose | ||
| 83 | + else logging.DEBUG | ||
| 84 | + if len(args.filenames) < 1000 | ||
| 85 | + else logging.INFO, | ||
| 86 | + stream=sys.stderr, | ||
| 87 | + ) | ||
| 88 | + | ||
| 89 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 90 | + max_workers=os.cpu_count(), | ||
| 91 | + ) as executor: | ||
| 92 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 93 | + for future in concurrent.futures.as_completed(futures): | ||
| 94 | + try: | ||
| 95 | + for lint_message in future.result(): | ||
| 96 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 97 | + except Exception: | ||
| 98 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 99 | + raise | ||
| 100 | + | ||
| 101 | + | ||
| 102 | +if __name__ == "__main__": | ||
| 103 | + main() | ||
| @@ -0,0 +1,244 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "pyyaml==6.0.2", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +""" | ||
| 8 | +This a linter that ensures that jobs that can be triggered by push, | ||
| 9 | +pull_request, or schedule will check if the repository owner is 'pytorch'. This | ||
| 10 | +ensures that forks will not run jobs. | ||
| 11 | + | ||
| 12 | +There are some edge cases that might be caught, and this prevents workflows from | ||
| 13 | +being reused in other organizations, but as of right now, there are no workflows | ||
| 14 | +with both push/pull_request/etc and workflow_call triggers simultaneously, so | ||
| 15 | +this is. | ||
| 16 | + | ||
| 17 | +There is also a setting in Github repos that can disable all workflows for that | ||
| 18 | +repo. | ||
| 19 | +""" | ||
| 20 | + | ||
| 21 | +from __future__ import annotations | ||
| 22 | + | ||
| 23 | +import argparse | ||
| 24 | +import concurrent.futures | ||
| 25 | +import json | ||
| 26 | +import logging | ||
| 27 | +import os | ||
| 28 | +import re | ||
| 29 | +from enum import Enum | ||
| 30 | +from pathlib import Path | ||
| 31 | +from typing import Any, NamedTuple, TYPE_CHECKING | ||
| 32 | + | ||
| 33 | +from yaml import load | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +if TYPE_CHECKING: | ||
| 37 | + from collections.abc import Callable | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +# Safely load fast C Yaml loader/dumper if they are available | ||
| 41 | +try: | ||
| 42 | + from yaml import CSafeLoader as Loader | ||
| 43 | +except ImportError: | ||
| 44 | + from yaml import SafeLoader as Loader # type: ignore[assignment, misc] | ||
| 45 | + | ||
| 46 | + | ||
| 47 | +class LintSeverity(str, Enum): | ||
| 48 | + ERROR = "error" | ||
| 49 | + WARNING = "warning" | ||
| 50 | + ADVICE = "advice" | ||
| 51 | + DISABLED = "disabled" | ||
| 52 | + | ||
| 53 | + | ||
| 54 | +class LintMessage(NamedTuple): | ||
| 55 | + path: str | None | ||
| 56 | + line: int | None | ||
| 57 | + char: int | None | ||
| 58 | + code: str | ||
| 59 | + severity: LintSeverity | ||
| 60 | + name: str | ||
| 61 | + original: str | None | ||
| 62 | + replacement: str | None | ||
| 63 | + description: str | None | ||
| 64 | + | ||
| 65 | + | ||
| 66 | +def load_yaml(path: Path) -> Any: | ||
| 67 | + with open(path) as f: | ||
| 68 | + return load(f, Loader) | ||
| 69 | + | ||
| 70 | + | ||
| 71 | +def gen_lint_message( | ||
| 72 | + filename: str | None = None, | ||
| 73 | + original: str | None = None, | ||
| 74 | + replacement: str | None = None, | ||
| 75 | + description: str | None = None, | ||
| 76 | +) -> LintMessage: | ||
| 77 | + return LintMessage( | ||
| 78 | + path=filename, | ||
| 79 | + line=None, | ||
| 80 | + char=None, | ||
| 81 | + code="NO_WORKFLOWS_ON_FORK", | ||
| 82 | + severity=LintSeverity.ERROR, | ||
| 83 | + name="format", | ||
| 84 | + original=original, | ||
| 85 | + replacement=replacement, | ||
| 86 | + description=description, | ||
| 87 | + ) | ||
| 88 | + | ||
| 89 | + | ||
| 90 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 91 | + logging.debug("Checking file %s", filename) | ||
| 92 | + | ||
| 93 | + workflow = load_yaml(Path(filename)) | ||
| 94 | + bad_jobs: dict[str, str | None] = {} | ||
| 95 | + if type(workflow) is not dict: | ||
| 96 | + return [] | ||
| 97 | + | ||
| 98 | + # yaml parses "on" as True | ||
| 99 | + triggers = workflow.get(True, {}) | ||
| 100 | + triggers_to_check = ["push", "schedule", "pull_request", "pull_request_target"] | ||
| 101 | + if not any(trigger in triggers_to_check for trigger in triggers): | ||
| 102 | + return [] | ||
| 103 | + | ||
| 104 | + jobs = workflow.get("jobs", {}) | ||
| 105 | + for job, definition in jobs.items(): | ||
| 106 | + if definition.get("needs"): | ||
| 107 | + # The parent job will have the if statement | ||
| 108 | + continue | ||
| 109 | + | ||
| 110 | + if_statement = definition.get("if") | ||
| 111 | + | ||
| 112 | + if if_statement is None: | ||
| 113 | + bad_jobs[job] = None | ||
| 114 | + elif type(if_statement) is bool and not if_statement: | ||
| 115 | + # if: false | ||
| 116 | + pass | ||
| 117 | + else: | ||
| 118 | + if_statement = str(if_statement) | ||
| 119 | + valid_checks: list[Callable[[str], bool]] = [ | ||
| 120 | + lambda x: "github.repository == 'pytorch/pytorch'" in x | ||
| 121 | + and "github.event_name != 'schedule' || github.repository == 'pytorch/pytorch'" | ||
| 122 | + not in x, | ||
| 123 | + lambda x: "github.repository_owner == 'pytorch'" in x, | ||
| 124 | + ] | ||
| 125 | + if not any(f(if_statement) for f in valid_checks): | ||
| 126 | + bad_jobs[job] = if_statement | ||
| 127 | + | ||
| 128 | + with open(filename) as f: | ||
| 129 | + lines = f.readlines() | ||
| 130 | + | ||
| 131 | + smart_enough = True | ||
| 132 | + original = "".join(lines) | ||
| 133 | + iterator = iter(range(len(lines))) | ||
| 134 | + replacement = "" | ||
| 135 | + for i in iterator: | ||
| 136 | + line = lines[i] | ||
| 137 | + # Search for job name | ||
| 138 | + re_match = re.match(r"( +)([-_\w]*):", line) | ||
| 139 | + if not re_match or re_match.group(2) not in bad_jobs: | ||
| 140 | + replacement += line | ||
| 141 | + continue | ||
| 142 | + job_name = re_match.group(2) | ||
| 143 | + | ||
| 144 | + failure_type = bad_jobs[job_name] | ||
| 145 | + if failure_type is None: | ||
| 146 | + # Just need to add an if statement | ||
| 147 | + replacement += ( | ||
| 148 | + f"{line}{re_match.group(1)} if: github.repository_owner == 'pytorch'\n" | ||
| 149 | + ) | ||
| 150 | + continue | ||
| 151 | + | ||
| 152 | + # Search for if statement | ||
| 153 | + while re.match(r"^ +if:", line) is None: | ||
| 154 | + replacement += line | ||
| 155 | + i = next(iterator) | ||
| 156 | + line = lines[i] | ||
| 157 | + if i + 1 < len(lines) and not re.match(r"^ +(.*):", lines[i + 1]): | ||
| 158 | + # This is a multi line if statement | ||
| 159 | + smart_enough = False | ||
| 160 | + break | ||
| 161 | + | ||
| 162 | + if_statement_match = re.match(r"^ +if: ([^#]*)(#.*)?$", line) | ||
| 163 | + # Get ... in if: ... # comments | ||
| 164 | + if not if_statement_match: | ||
| 165 | + return [ | ||
| 166 | + gen_lint_message( | ||
| 167 | + description=f"Something went wrong when looking at {job_name}.", | ||
| 168 | + ) | ||
| 169 | + ] | ||
| 170 | + | ||
| 171 | + if_statement = if_statement_match.group(1).strip() | ||
| 172 | + | ||
| 173 | + # Handle comment in if: ... # comments | ||
| 174 | + comments = if_statement_match.group(2) or "" | ||
| 175 | + if comments: | ||
| 176 | + comments = " " + comments | ||
| 177 | + | ||
| 178 | + # Too broad of a check, but should catch everything | ||
| 179 | + needs_parens = "||" in if_statement | ||
| 180 | + | ||
| 181 | + # Handle ${{ ... }} | ||
| 182 | + has_brackets = re.match(r"\$\{\{(.*)\}\}", if_statement) | ||
| 183 | + internal_statement = ( | ||
| 184 | + has_brackets.group(1).strip() if has_brackets else if_statement | ||
| 185 | + ) | ||
| 186 | + | ||
| 187 | + if needs_parens: | ||
| 188 | + internal_statement = f"({internal_statement})" | ||
| 189 | + new_line = f"{internal_statement} && github.repository_owner == 'pytorch'" | ||
| 190 | + | ||
| 191 | + # I don't actually know if we need the ${{ }} but do it just in case | ||
| 192 | + new_line = "${{ " + new_line + " }}" + comments | ||
| 193 | + | ||
| 194 | + replacement += f"{re_match.group(1)} if: {new_line}\n" | ||
| 195 | + | ||
| 196 | + description = ( | ||
| 197 | + "Please add checks for if: github.repository_owner == 'pytorch' in the following jobs in this file: " | ||
| 198 | + + ", ".join(job for job in bad_jobs) | ||
| 199 | + ) | ||
| 200 | + | ||
| 201 | + if not smart_enough: | ||
| 202 | + return [ | ||
| 203 | + gen_lint_message( | ||
| 204 | + filename=filename, | ||
| 205 | + description=description, | ||
| 206 | + ) | ||
| 207 | + ] | ||
| 208 | + | ||
| 209 | + if replacement == original: | ||
| 210 | + return [] | ||
| 211 | + | ||
| 212 | + return [ | ||
| 213 | + gen_lint_message( | ||
| 214 | + filename=filename, | ||
| 215 | + original=original, | ||
| 216 | + replacement=replacement, | ||
| 217 | + description=description, | ||
| 218 | + ) | ||
| 219 | + ] | ||
| 220 | + | ||
| 221 | + | ||
| 222 | +if __name__ == "__main__": | ||
| 223 | + parser = argparse.ArgumentParser( | ||
| 224 | + description="workflow consistency linter.", | ||
| 225 | + fromfile_prefix_chars="@", | ||
| 226 | + ) | ||
| 227 | + parser.add_argument( | ||
| 228 | + "filenames", | ||
| 229 | + nargs="+", | ||
| 230 | + help="paths to lint", | ||
| 231 | + ) | ||
| 232 | + args = parser.parse_args() | ||
| 233 | + | ||
| 234 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 235 | + max_workers=os.cpu_count(), | ||
| 236 | + ) as executor: | ||
| 237 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 238 | + for future in concurrent.futures.as_completed(futures): | ||
| 239 | + try: | ||
| 240 | + for lint_message in future.result(): | ||
| 241 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 242 | + except Exception: | ||
| 243 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 244 | + raise | ||
| @@ -0,0 +1,184 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "usort==1.1.3", | ||
| 5 | +# "isort==6.0.1", | ||
| 6 | +# "ruff==0.14.4", | ||
| 7 | +# ] | ||
| 8 | +# /// | ||
| 9 | +from __future__ import annotations | ||
| 10 | + | ||
| 11 | +import argparse | ||
| 12 | +import concurrent.futures | ||
| 13 | +import json | ||
| 14 | +import logging | ||
| 15 | +import os | ||
| 16 | +import re | ||
| 17 | +import subprocess | ||
| 18 | +import sys | ||
| 19 | +from enum import Enum | ||
| 20 | +from pathlib import Path | ||
| 21 | +from typing import NamedTuple | ||
| 22 | + | ||
| 23 | +# pyrefly: ignore [import-error] | ||
| 24 | +import isort | ||
| 25 | +import usort | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +IS_WINDOWS: bool = os.name == "nt" | ||
| 29 | +REPO_ROOT = Path(__file__).absolute().parents[3] | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +class LintSeverity(str, Enum): | ||
| 33 | + ERROR = "error" | ||
| 34 | + WARNING = "warning" | ||
| 35 | + ADVICE = "advice" | ||
| 36 | + DISABLED = "disabled" | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +class LintMessage(NamedTuple): | ||
| 40 | + path: str | None | ||
| 41 | + line: int | None | ||
| 42 | + char: int | None | ||
| 43 | + code: str | ||
| 44 | + severity: LintSeverity | ||
| 45 | + name: str | ||
| 46 | + original: str | None | ||
| 47 | + replacement: str | None | ||
| 48 | + description: str | None | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def as_posix(name: str) -> str: | ||
| 52 | + return name.replace("\\", "/") if IS_WINDOWS else name | ||
| 53 | + | ||
| 54 | + | ||
| 55 | +def format_error_message(filename: str, err: Exception) -> LintMessage: | ||
| 56 | + return LintMessage( | ||
| 57 | + path=filename, | ||
| 58 | + line=None, | ||
| 59 | + char=None, | ||
| 60 | + code="PYFMT", | ||
| 61 | + severity=LintSeverity.ADVICE, | ||
| 62 | + name="command-failed", | ||
| 63 | + original=None, | ||
| 64 | + replacement=None, | ||
| 65 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 66 | + ) | ||
| 67 | + | ||
| 68 | + | ||
| 69 | +def run_isort(content: str, path: Path) -> str: | ||
| 70 | + isort_config = isort.Config(settings_path=str(REPO_ROOT)) | ||
| 71 | + | ||
| 72 | + is_this_file = path.samefile(__file__) | ||
| 73 | + if not is_this_file: | ||
| 74 | + content = re.sub(r"(#.*\b)usort:\s*skip\b", r"\g<1>isort: split", content) | ||
| 75 | + | ||
| 76 | + content = isort.code(content, config=isort_config, file_path=path) | ||
| 77 | + | ||
| 78 | + if not is_this_file: | ||
| 79 | + content = re.sub(r"(#.*\b)isort: split\b", r"\g<1>usort: skip", content) | ||
| 80 | + | ||
| 81 | + return content | ||
| 82 | + | ||
| 83 | + | ||
| 84 | +def run_usort(content: str, path: Path) -> str: | ||
| 85 | + usort_config = usort.Config.find(path) | ||
| 86 | + | ||
| 87 | + return usort.usort_string(content, path=path, config=usort_config) | ||
| 88 | + | ||
| 89 | + | ||
| 90 | +def run_ruff_format(content: str, path: Path) -> str: | ||
| 91 | + try: | ||
| 92 | + return subprocess.check_output( | ||
| 93 | + [ | ||
| 94 | + sys.executable, | ||
| 95 | + "-m", | ||
| 96 | + "ruff", | ||
| 97 | + "format", | ||
| 98 | + "--config", | ||
| 99 | + str(REPO_ROOT / "pyproject.toml"), | ||
| 100 | + "--stdin-filename", | ||
| 101 | + str(path), | ||
| 102 | + "-", | ||
| 103 | + ], | ||
| 104 | + input=content, | ||
| 105 | + stderr=subprocess.STDOUT, | ||
| 106 | + text=True, | ||
| 107 | + encoding="utf-8", | ||
| 108 | + ) | ||
| 109 | + except subprocess.CalledProcessError as exc: | ||
| 110 | + raise ValueError(exc.output) from exc | ||
| 111 | + | ||
| 112 | + | ||
| 113 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 114 | + path = Path(filename).absolute() | ||
| 115 | + original = replacement = path.read_text(encoding="utf-8") | ||
| 116 | + | ||
| 117 | + try: | ||
| 118 | + # NB: run isort first to enforce style for blank lines | ||
| 119 | + replacement = run_isort(replacement, path=path) | ||
| 120 | + replacement = run_usort(replacement, path=path) | ||
| 121 | + replacement = run_ruff_format(replacement, path=path) | ||
| 122 | + | ||
| 123 | + if original == replacement: | ||
| 124 | + return [] | ||
| 125 | + | ||
| 126 | + return [ | ||
| 127 | + LintMessage( | ||
| 128 | + path=filename, | ||
| 129 | + line=None, | ||
| 130 | + char=None, | ||
| 131 | + code="PYFMT", | ||
| 132 | + severity=LintSeverity.WARNING, | ||
| 133 | + name="format", | ||
| 134 | + original=original, | ||
| 135 | + replacement=replacement, | ||
| 136 | + description="Run `lintrunner -a` to apply this patch.", | ||
| 137 | + ) | ||
| 138 | + ] | ||
| 139 | + except Exception as err: | ||
| 140 | + return [format_error_message(filename, err)] | ||
| 141 | + | ||
| 142 | + | ||
| 143 | +def main() -> None: | ||
| 144 | + parser = argparse.ArgumentParser( | ||
| 145 | + description="Format files with usort + ruff-format.", | ||
| 146 | + fromfile_prefix_chars="@", | ||
| 147 | + ) | ||
| 148 | + parser.add_argument( | ||
| 149 | + "--verbose", | ||
| 150 | + action="store_true", | ||
| 151 | + help="verbose logging", | ||
| 152 | + ) | ||
| 153 | + parser.add_argument( | ||
| 154 | + "filenames", | ||
| 155 | + nargs="+", | ||
| 156 | + help="paths to lint", | ||
| 157 | + ) | ||
| 158 | + args = parser.parse_args() | ||
| 159 | + | ||
| 160 | + logging.basicConfig( | ||
| 161 | + format="<%(processName)s:%(levelname)s> %(message)s", | ||
| 162 | + level=logging.NOTSET | ||
| 163 | + if args.verbose | ||
| 164 | + else logging.DEBUG | ||
| 165 | + if len(args.filenames) < 1000 | ||
| 166 | + else logging.INFO, | ||
| 167 | + stream=sys.stderr, | ||
| 168 | + ) | ||
| 169 | + | ||
| 170 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 171 | + max_workers=os.cpu_count(), | ||
| 172 | + ) as executor: | ||
| 173 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 174 | + for future in concurrent.futures.as_completed(futures): | ||
| 175 | + try: | ||
| 176 | + for lint_message in future.result(): | ||
| 177 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 178 | + except Exception: | ||
| 179 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 180 | + raise | ||
| 181 | + | ||
| 182 | + | ||
| 183 | +if __name__ == "__main__": | ||
| 184 | + main() | ||
| @@ -0,0 +1,256 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "packaging==25.0", | ||
| 5 | +# "tomli==2.2.1 ; python_version < '3.11'", | ||
| 6 | +# ] | ||
| 7 | +# /// | ||
| 8 | +from __future__ import annotations | ||
| 9 | + | ||
| 10 | +import argparse | ||
| 11 | +import concurrent.futures | ||
| 12 | +import json | ||
| 13 | +import logging | ||
| 14 | +import os | ||
| 15 | +import sys | ||
| 16 | +from enum import Enum | ||
| 17 | +from pathlib import Path | ||
| 18 | +from typing import NamedTuple | ||
| 19 | + | ||
| 20 | +from packaging.specifiers import SpecifierSet | ||
| 21 | +from packaging.version import Version | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +if sys.version_info >= (3, 11): | ||
| 25 | + import tomllib | ||
| 26 | +else: | ||
| 27 | + import tomli as tomllib # type: ignore[import-not-found] | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +class LintSeverity(str, Enum): | ||
| 31 | + ERROR = "error" | ||
| 32 | + WARNING = "warning" | ||
| 33 | + ADVICE = "advice" | ||
| 34 | + DISABLED = "disabled" | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +class LintMessage(NamedTuple): | ||
| 38 | + path: str | None | ||
| 39 | + line: int | None | ||
| 40 | + char: int | None | ||
| 41 | + code: str | ||
| 42 | + severity: LintSeverity | ||
| 43 | + name: str | ||
| 44 | + original: str | None | ||
| 45 | + replacement: str | None | ||
| 46 | + description: str | None | ||
| 47 | + | ||
| 48 | + | ||
| 49 | +def format_error_message( | ||
| 50 | + filename: str, | ||
| 51 | + error: Exception | None = None, | ||
| 52 | + *, | ||
| 53 | + message: str | None = None, | ||
| 54 | +) -> LintMessage: | ||
| 55 | + if message is None and error is not None: | ||
| 56 | + message = f"Failed due to {error.__class__.__name__}:\n{error}" | ||
| 57 | + return LintMessage( | ||
| 58 | + path=filename, | ||
| 59 | + line=None, | ||
| 60 | + char=None, | ||
| 61 | + code="PYPROJECT", | ||
| 62 | + severity=LintSeverity.ERROR, | ||
| 63 | + name="pyproject.toml consistency", | ||
| 64 | + original=None, | ||
| 65 | + replacement=None, | ||
| 66 | + description=message, | ||
| 67 | + ) | ||
| 68 | + | ||
| 69 | + | ||
| 70 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 71 | + path = Path(filename).absolute() | ||
| 72 | + try: | ||
| 73 | + pyproject = tomllib.loads(path.read_text(encoding="utf-8")) | ||
| 74 | + except (tomllib.TOMLDecodeError, OSError) as err: | ||
| 75 | + return [format_error_message(filename, err)] | ||
| 76 | + | ||
| 77 | + if not (isinstance(pyproject, dict) and isinstance(pyproject.get("project"), dict)): | ||
| 78 | + return [ | ||
| 79 | + format_error_message( | ||
| 80 | + filename, | ||
| 81 | + message=( | ||
| 82 | + "'project' section in pyproject.toml must present and be a table." | ||
| 83 | + ), | ||
| 84 | + ) | ||
| 85 | + ] | ||
| 86 | + | ||
| 87 | + project = pyproject["project"] | ||
| 88 | + requires_python = project.get("requires-python") | ||
| 89 | + if requires_python is not None: | ||
| 90 | + if not isinstance(requires_python, str): | ||
| 91 | + return [ | ||
| 92 | + format_error_message( | ||
| 93 | + filename, | ||
| 94 | + message="'project.requires-python' must be a string.", | ||
| 95 | + ) | ||
| 96 | + ] | ||
| 97 | + | ||
| 98 | + python_major = 3 | ||
| 99 | + specifier_set = SpecifierSet(requires_python) | ||
| 100 | + for specifier in specifier_set: | ||
| 101 | + if Version(specifier.version).major != python_major: | ||
| 102 | + return [ | ||
| 103 | + format_error_message( | ||
| 104 | + filename, | ||
| 105 | + message=( | ||
| 106 | + "'project.requires-python' must only specify " | ||
| 107 | + f"Python {python_major} versions, but found {specifier.version}." | ||
| 108 | + ), | ||
| 109 | + ) | ||
| 110 | + ] | ||
| 111 | + | ||
| 112 | + large_minor = 1000 | ||
| 113 | + supported_python_versions = list( | ||
| 114 | + specifier_set.filter( | ||
| 115 | + f"{python_major}.{minor}" for minor in range(large_minor + 1) | ||
| 116 | + ) | ||
| 117 | + ) | ||
| 118 | + if not supported_python_versions: | ||
| 119 | + return [ | ||
| 120 | + format_error_message( | ||
| 121 | + filename, | ||
| 122 | + message=( | ||
| 123 | + "'project.requires-python' must specify at least one " | ||
| 124 | + f"Python {python_major} version, but found {requires_python!r}." | ||
| 125 | + ), | ||
| 126 | + ) | ||
| 127 | + ] | ||
| 128 | + if f"{python_major}.0" in supported_python_versions: | ||
| 129 | + return [ | ||
| 130 | + format_error_message( | ||
| 131 | + filename, | ||
| 132 | + message=( | ||
| 133 | + "'project.requires-python' must specify a minimum version, " | ||
| 134 | + f"but found {requires_python!r}." | ||
| 135 | + ), | ||
| 136 | + ) | ||
| 137 | + ] | ||
| 138 | + # if f"{python_major}.{large_minor}" in supported_python_versions: | ||
| 139 | + # return [ | ||
| 140 | + # format_error_message( | ||
| 141 | + # filename, | ||
| 142 | + # message=( | ||
| 143 | + # "'project.requires-python' must specify a maximum version, " | ||
| 144 | + # f"but found {requires_python!r}." | ||
| 145 | + # ), | ||
| 146 | + # ) | ||
| 147 | + # ] | ||
| 148 | + | ||
| 149 | + classifiers = project.get("classifiers") | ||
| 150 | + if not ( | ||
| 151 | + isinstance(classifiers, list) | ||
| 152 | + and all(isinstance(c, str) for c in classifiers) | ||
| 153 | + ): | ||
| 154 | + return [ | ||
| 155 | + format_error_message( | ||
| 156 | + filename, | ||
| 157 | + message="'project.classifiers' must be an array of strings.", | ||
| 158 | + ) | ||
| 159 | + ] | ||
| 160 | + if len(set(classifiers)) != len(classifiers): | ||
| 161 | + return [ | ||
| 162 | + format_error_message( | ||
| 163 | + filename, | ||
| 164 | + message="'project.classifiers' must not contain duplicates.", | ||
| 165 | + ) | ||
| 166 | + ] | ||
| 167 | + | ||
| 168 | + # python_version_classifiers = [ | ||
| 169 | + # c | ||
| 170 | + # for c in classifiers | ||
| 171 | + # if ( | ||
| 172 | + # c.startswith("Programming Language :: Python :: ") | ||
| 173 | + # and not c.endswith((f":: {python_major}", f":: {python_major} :: Only")) | ||
| 174 | + # ) | ||
| 175 | + # ] | ||
| 176 | + # if python_version_classifiers: | ||
| 177 | + # python_version_classifier_set = set(python_version_classifiers) | ||
| 178 | + # supported_python_version_classifier_set = { | ||
| 179 | + # f"Programming Language :: Python :: {v}" | ||
| 180 | + # for v in supported_python_versions | ||
| 181 | + # } | ||
| 182 | + # if python_version_classifier_set != supported_python_version_classifier_set: | ||
| 183 | + # missing_classifiers = sorted( | ||
| 184 | + # supported_python_version_classifier_set | ||
| 185 | + # - python_version_classifier_set | ||
| 186 | + # ) | ||
| 187 | + # extra_classifiers = sorted( | ||
| 188 | + # python_version_classifier_set | ||
| 189 | + # - supported_python_version_classifier_set | ||
| 190 | + # ) | ||
| 191 | + # if missing_classifiers: | ||
| 192 | + # return [ | ||
| 193 | + # format_error_message( | ||
| 194 | + # filename, | ||
| 195 | + # message=( | ||
| 196 | + # "'project.classifiers' is missing the following classifier(s):\n" | ||
| 197 | + # + "\n".join(f" {c!r}" for c in missing_classifiers) | ||
| 198 | + # ), | ||
| 199 | + # ) | ||
| 200 | + # ] | ||
| 201 | + # if extra_classifiers: | ||
| 202 | + # return [ | ||
| 203 | + # format_error_message( | ||
| 204 | + # filename, | ||
| 205 | + # message=( | ||
| 206 | + # "'project.classifiers' contains extra classifier(s):\n" | ||
| 207 | + # + "\n".join(f" {c!r}" for c in extra_classifiers) | ||
| 208 | + # ), | ||
| 209 | + # ) | ||
| 210 | + # ] | ||
| 211 | + | ||
| 212 | + return [] | ||
| 213 | + | ||
| 214 | + | ||
| 215 | +def main() -> None: | ||
| 216 | + parser = argparse.ArgumentParser( | ||
| 217 | + description="Check consistency of pyproject.toml files.", | ||
| 218 | + fromfile_prefix_chars="@", | ||
| 219 | + ) | ||
| 220 | + parser.add_argument( | ||
| 221 | + "--verbose", | ||
| 222 | + action="store_true", | ||
| 223 | + help="verbose logging", | ||
| 224 | + ) | ||
| 225 | + parser.add_argument( | ||
| 226 | + "filenames", | ||
| 227 | + nargs="+", | ||
| 228 | + help="paths to lint", | ||
| 229 | + ) | ||
| 230 | + args = parser.parse_args() | ||
| 231 | + | ||
| 232 | + logging.basicConfig( | ||
| 233 | + format="<%(processName)s:%(levelname)s> %(message)s", | ||
| 234 | + level=logging.NOTSET | ||
| 235 | + if args.verbose | ||
| 236 | + else logging.DEBUG | ||
| 237 | + if len(args.filenames) < 1000 | ||
| 238 | + else logging.INFO, | ||
| 239 | + stream=sys.stderr, | ||
| 240 | + ) | ||
| 241 | + | ||
| 242 | + with concurrent.futures.ProcessPoolExecutor( | ||
| 243 | + max_workers=os.cpu_count(), | ||
| 244 | + ) as executor: | ||
| 245 | + futures = {executor.submit(check_file, x): x for x in args.filenames} | ||
| 246 | + for future in concurrent.futures.as_completed(futures): | ||
| 247 | + try: | ||
| 248 | + for lint_message in future.result(): | ||
| 249 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 250 | + except Exception: | ||
| 251 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 252 | + raise | ||
| 253 | + | ||
| 254 | + | ||
| 255 | +if __name__ == "__main__": | ||
| 256 | + main() | ||
| @@ -0,0 +1,299 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "numpy==1.26.4 ; python_version >= '3.10' and python_version <= '3.11'", | ||
| 5 | +# "numpy==2.1.0 ; python_version >= '3.12' and python_version <= '3.13'", | ||
| 6 | +# "numpy==2.3.4 ; python_version >= '3.14'", | ||
| 7 | +# "expecttest==0.3.0", | ||
| 8 | +# "pyrefly==0.52.0", | ||
| 9 | +# "sympy==1.13.3", | ||
| 10 | +# "types-requests==2.27.25", | ||
| 11 | +# "types-pyyaml==6.0.2", | ||
| 12 | +# "types-tabulate==0.8.8", | ||
| 13 | +# "types-protobuf==5.29.1.20250403", | ||
| 14 | +# "types-setuptools==79.0.0.20250422", | ||
| 15 | +# "types-jinja2==2.11.9", | ||
| 16 | +# "types-colorama==0.4.6", | ||
| 17 | +# "filelock==3.18.0", | ||
| 18 | +# "junitparser==2.1.1", | ||
| 19 | +# "rich==14.1.0", | ||
| 20 | +# "optree==0.17.0", | ||
| 21 | +# "types-openpyxl==3.1.5.20250919", | ||
| 22 | +# "types-python-dateutil==2.9.0.20251008", | ||
| 23 | +# "packaging", | ||
| 24 | +# "libcst", | ||
| 25 | +# "isort", | ||
| 26 | +# "usort", | ||
| 27 | +# ] | ||
| 28 | +# /// | ||
| 29 | +from __future__ import annotations | ||
| 30 | + | ||
| 31 | +import argparse | ||
| 32 | +import json | ||
| 33 | +import logging | ||
| 34 | +import os | ||
| 35 | +import re | ||
| 36 | +import subprocess | ||
| 37 | +import sys | ||
| 38 | +import time | ||
| 39 | +from enum import Enum | ||
| 40 | +from typing import NamedTuple | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +class LintSeverity(str, Enum): | ||
| 44 | + ERROR = "error" | ||
| 45 | + WARNING = "warning" | ||
| 46 | + ADVICE = "advice" | ||
| 47 | + DISABLED = "disabled" | ||
| 48 | + | ||
| 49 | + | ||
| 50 | +class LintMessage(NamedTuple): | ||
| 51 | + path: str | None | ||
| 52 | + line: int | None | ||
| 53 | + char: int | None | ||
| 54 | + code: str | ||
| 55 | + severity: LintSeverity | ||
| 56 | + name: str | ||
| 57 | + original: str | None | ||
| 58 | + replacement: str | None | ||
| 59 | + description: str | None | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +# Note: This regex pattern is kept for reference but not used for pyrefly JSON parsing | ||
| 63 | +RESULTS_RE: re.Pattern[str] = re.compile( | ||
| 64 | + r"""(?mx) | ||
| 65 | + ^ | ||
| 66 | + (?P<file>.*?): | ||
| 67 | + (?P<line>\d+): | ||
| 68 | + (?:(?P<column>-?\d+):)? | ||
| 69 | + \s(?P<severity>\S+?):? | ||
| 70 | + \s(?P<message>.*) | ||
| 71 | + \s(?P<code>\[.*\]) | ||
| 72 | + $ | ||
| 73 | + """ | ||
| 74 | +) | ||
| 75 | + | ||
| 76 | +# torch/_dynamo/variables/tensor.py:363: error: INTERNAL ERROR | ||
| 77 | +INTERNAL_ERROR_RE: re.Pattern[str] = re.compile( | ||
| 78 | + r"""(?mx) | ||
| 79 | + ^ | ||
| 80 | + (?P<file>.*?): | ||
| 81 | + (?P<line>\d+): | ||
| 82 | + \s(?P<severity>\S+?):? | ||
| 83 | + \s(?P<message>INTERNAL\sERROR.*) | ||
| 84 | + $ | ||
| 85 | + """ | ||
| 86 | +) | ||
| 87 | + | ||
| 88 | + | ||
| 89 | +def run_command( | ||
| 90 | + args: list[str], | ||
| 91 | + *, | ||
| 92 | + extra_env: dict[str, str] | None, | ||
| 93 | + retries: int, | ||
| 94 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 95 | + logging.debug("$ %s", " ".join(args)) | ||
| 96 | + start_time = time.monotonic() | ||
| 97 | + try: | ||
| 98 | + return subprocess.run( | ||
| 99 | + args, | ||
| 100 | + capture_output=True, | ||
| 101 | + ) | ||
| 102 | + finally: | ||
| 103 | + end_time = time.monotonic() | ||
| 104 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 105 | + | ||
| 106 | + | ||
| 107 | +# Severity mapping (currently only used for stderr internal errors) | ||
| 108 | +# Pyrefly JSON output doesn't include severity, so all errors default to ERROR | ||
| 109 | +severities = { | ||
| 110 | + "error": LintSeverity.ERROR, | ||
| 111 | + "note": LintSeverity.ADVICE, | ||
| 112 | +} | ||
| 113 | + | ||
| 114 | + | ||
| 115 | +def check_pyrefly_installed(code: str) -> list[LintMessage]: | ||
| 116 | + cmd = ["pyrefly", "--version"] | ||
| 117 | + try: | ||
| 118 | + subprocess.run(cmd, check=True, capture_output=True) | ||
| 119 | + return [] | ||
| 120 | + except subprocess.CalledProcessError as e: | ||
| 121 | + msg = e.stderr.decode(errors="replace") | ||
| 122 | + return [ | ||
| 123 | + LintMessage( | ||
| 124 | + path=None, | ||
| 125 | + line=None, | ||
| 126 | + char=None, | ||
| 127 | + code=code, | ||
| 128 | + severity=LintSeverity.ERROR, | ||
| 129 | + name="command-failed", | ||
| 130 | + original=None, | ||
| 131 | + replacement=None, | ||
| 132 | + description=f"Could not run '{' '.join(cmd)}': {msg}", | ||
| 133 | + ) | ||
| 134 | + ] | ||
| 135 | + | ||
| 136 | + | ||
| 137 | +def in_github_actions() -> bool: | ||
| 138 | + return bool(os.getenv("GITHUB_ACTIONS")) | ||
| 139 | + | ||
| 140 | + | ||
| 141 | +def check_files( | ||
| 142 | + code: str, config: str, remove_unused_ignores: bool, suppress: bool | ||
| 143 | +) -> list[LintMessage]: | ||
| 144 | + try: | ||
| 145 | + pyrefly_commands = [ | ||
| 146 | + "pyrefly", | ||
| 147 | + "check", | ||
| 148 | + "--config", | ||
| 149 | + config, | ||
| 150 | + "--output-format=json", | ||
| 151 | + ] | ||
| 152 | + if remove_unused_ignores: | ||
| 153 | + pyrefly_commands.append("--remove-unused-ignores") | ||
| 154 | + if suppress: | ||
| 155 | + pyrefly_commands.append("--suppress-errors") | ||
| 156 | + proc = run_command( | ||
| 157 | + [*pyrefly_commands], | ||
| 158 | + extra_env={}, | ||
| 159 | + retries=0, | ||
| 160 | + ) | ||
| 161 | + except OSError as err: | ||
| 162 | + return [ | ||
| 163 | + LintMessage( | ||
| 164 | + path=None, | ||
| 165 | + line=None, | ||
| 166 | + char=None, | ||
| 167 | + code=code, | ||
| 168 | + severity=LintSeverity.ERROR, | ||
| 169 | + name="command-failed", | ||
| 170 | + original=None, | ||
| 171 | + replacement=None, | ||
| 172 | + description=(f"Failed due to {err.__class__.__name__}:\n{err}"), | ||
| 173 | + ) | ||
| 174 | + ] | ||
| 175 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 176 | + stderr = str(proc.stderr, "utf-8").strip() | ||
| 177 | + if proc.returncode not in (0, 1): | ||
| 178 | + return [ | ||
| 179 | + LintMessage( | ||
| 180 | + path=None, | ||
| 181 | + line=None, | ||
| 182 | + char=None, | ||
| 183 | + code=code, | ||
| 184 | + severity=LintSeverity.ERROR, | ||
| 185 | + name="command-failed", | ||
| 186 | + original=None, | ||
| 187 | + replacement=None, | ||
| 188 | + description=stderr, | ||
| 189 | + ) | ||
| 190 | + ] | ||
| 191 | + | ||
| 192 | + # Parse JSON output from pyrefly. In GitHub Actions, pyrefly appends | ||
| 193 | + # ::error commands to stdout after the JSON, so use raw_decode to parse | ||
| 194 | + # only the first JSON object and ignore trailing output. | ||
| 195 | + try: | ||
| 196 | + if stdout: | ||
| 197 | + result, _ = json.JSONDecoder().raw_decode(stdout) | ||
| 198 | + errors = result.get("errors", []) | ||
| 199 | + else: | ||
| 200 | + errors = [] | ||
| 201 | + errors = [error for error in errors if error["name"] != "deprecated"] | ||
| 202 | + rc = [ | ||
| 203 | + LintMessage( | ||
| 204 | + path=error["path"], | ||
| 205 | + name=error["name"], | ||
| 206 | + description=error.get( | ||
| 207 | + "description", error.get("concise_description", "") | ||
| 208 | + ), | ||
| 209 | + line=error["line"], | ||
| 210 | + char=error["column"], | ||
| 211 | + code=code, | ||
| 212 | + severity=LintSeverity.ADVICE | ||
| 213 | + if error["name"] == "deprecated" | ||
| 214 | + else LintSeverity.ERROR, | ||
| 215 | + original=None, | ||
| 216 | + replacement=None, | ||
| 217 | + ) | ||
| 218 | + for error in errors | ||
| 219 | + ] | ||
| 220 | + except (json.JSONDecodeError, KeyError, TypeError) as e: | ||
| 221 | + return [ | ||
| 222 | + LintMessage( | ||
| 223 | + path=None, | ||
| 224 | + line=None, | ||
| 225 | + char=None, | ||
| 226 | + code=code, | ||
| 227 | + severity=LintSeverity.ERROR, | ||
| 228 | + name="json-parse-error", | ||
| 229 | + original=None, | ||
| 230 | + replacement=None, | ||
| 231 | + description=f"Failed to parse pyrefly JSON output: {e}", | ||
| 232 | + ) | ||
| 233 | + ] | ||
| 234 | + | ||
| 235 | + # Still check stderr for internal errors | ||
| 236 | + rc += [ | ||
| 237 | + LintMessage( | ||
| 238 | + path=match["file"], | ||
| 239 | + name="INTERNAL ERROR", | ||
| 240 | + description=match["message"], | ||
| 241 | + line=int(match["line"]), | ||
| 242 | + char=None, | ||
| 243 | + code=code, | ||
| 244 | + severity=severities.get(match["severity"], LintSeverity.ERROR), | ||
| 245 | + original=None, | ||
| 246 | + replacement=None, | ||
| 247 | + ) | ||
| 248 | + for match in INTERNAL_ERROR_RE.finditer(stderr) | ||
| 249 | + ] | ||
| 250 | + return rc | ||
| 251 | + | ||
| 252 | + | ||
| 253 | +def main() -> None: | ||
| 254 | + parser = argparse.ArgumentParser( | ||
| 255 | + description="pyrefly wrapper linter.", | ||
| 256 | + fromfile_prefix_chars="@", | ||
| 257 | + ) | ||
| 258 | + parser.add_argument( | ||
| 259 | + "--code", | ||
| 260 | + default="PYREFLY", | ||
| 261 | + help="the code this lint should report as", | ||
| 262 | + ) | ||
| 263 | + parser.add_argument( | ||
| 264 | + "--verbose", | ||
| 265 | + action="store_true", | ||
| 266 | + help="verbose logging", | ||
| 267 | + ) | ||
| 268 | + parser.add_argument( | ||
| 269 | + "--config", | ||
| 270 | + required=True, | ||
| 271 | + help="path to an mypy .ini config file", | ||
| 272 | + ) | ||
| 273 | + parser.add_argument( | ||
| 274 | + "--remove-unused-ignores", | ||
| 275 | + action="store_true", | ||
| 276 | + help="clean up unused ignores", | ||
| 277 | + ) | ||
| 278 | + parser.add_argument( | ||
| 279 | + "--suppress", | ||
| 280 | + action="store_true", | ||
| 281 | + help="add suppressions", | ||
| 282 | + ) | ||
| 283 | + args = parser.parse_args() | ||
| 284 | + | ||
| 285 | + logging.basicConfig( | ||
| 286 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 287 | + level=logging.INFO, | ||
| 288 | + stream=sys.stderr, | ||
| 289 | + ) | ||
| 290 | + | ||
| 291 | + lint_messages = check_pyrefly_installed(args.code) + check_files( | ||
| 292 | + args.code, args.config, args.remove_unused_ignores, args.suppress | ||
| 293 | + ) | ||
| 294 | + for lint_message in lint_messages: | ||
| 295 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 296 | + | ||
| 297 | + | ||
| 298 | +if __name__ == "__main__": | ||
| 299 | + main() | ||
| @@ -0,0 +1,469 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "ruff==0.14.4", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +"""Adapter for https://github.com/charliermarsh/ruff.""" | ||
| 8 | + | ||
| 9 | +from __future__ import annotations | ||
| 10 | + | ||
| 11 | +import argparse | ||
| 12 | +import concurrent.futures | ||
| 13 | +import dataclasses | ||
| 14 | +import enum | ||
| 15 | +import json | ||
| 16 | +import logging | ||
| 17 | +import os | ||
| 18 | +import subprocess | ||
| 19 | +import sys | ||
| 20 | +import time | ||
| 21 | +from typing import Any, BinaryIO | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +LINTER_CODE = "RUFF" | ||
| 25 | +SYNTAX_ERROR = "E999" | ||
| 26 | +IS_WINDOWS: bool = os.name == "nt" | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +class LintSeverity(str, enum.Enum): | ||
| 30 | + """Severity of a lint message.""" | ||
| 31 | + | ||
| 32 | + ERROR = "error" | ||
| 33 | + WARNING = "warning" | ||
| 34 | + ADVICE = "advice" | ||
| 35 | + DISABLED = "disabled" | ||
| 36 | + | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +class LintMessage: | ||
| 40 | + """A lint message defined by https://docs.rs/lintrunner/latest/lintrunner/lint_message/struct.LintMessage.html.""" | ||
| 41 | + | ||
| 42 | + path: str | None | ||
| 43 | + line: int | None | ||
| 44 | + char: int | None | ||
| 45 | + code: str | ||
| 46 | + severity: LintSeverity | ||
| 47 | + name: str | ||
| 48 | + original: str | None | ||
| 49 | + replacement: str | None | ||
| 50 | + description: str | None | ||
| 51 | + | ||
| 52 | + def asdict(self) -> dict[str, Any]: | ||
| 53 | + return dataclasses.asdict(self) | ||
| 54 | + | ||
| 55 | + def display(self) -> None: | ||
| 56 | + """Print to stdout for lintrunner to consume.""" | ||
| 57 | + print(json.dumps(self.asdict()), flush=True) | ||
| 58 | + | ||
| 59 | + | ||
| 60 | +def as_posix(name: str) -> str: | ||
| 61 | + return name.replace("\\", "/") if IS_WINDOWS else name | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def _run_command( | ||
| 65 | + args: list[str], | ||
| 66 | + *, | ||
| 67 | + timeout: int | None, | ||
| 68 | + stdin: BinaryIO | None, | ||
| 69 | + input: bytes | None, | ||
| 70 | + check: bool, | ||
| 71 | + cwd: os.PathLike[Any] | None, | ||
| 72 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 73 | + logging.debug("$ %s", " ".join(args)) | ||
| 74 | + start_time = time.monotonic() | ||
| 75 | + try: | ||
| 76 | + if input is not None: | ||
| 77 | + return subprocess.run( | ||
| 78 | + args, | ||
| 79 | + capture_output=True, | ||
| 80 | + shell=False, | ||
| 81 | + input=input, | ||
| 82 | + timeout=timeout, | ||
| 83 | + check=check, | ||
| 84 | + cwd=cwd, | ||
| 85 | + ) | ||
| 86 | + | ||
| 87 | + return subprocess.run( | ||
| 88 | + args, | ||
| 89 | + stdin=stdin, | ||
| 90 | + capture_output=True, | ||
| 91 | + shell=False, | ||
| 92 | + timeout=timeout, | ||
| 93 | + check=check, | ||
| 94 | + cwd=cwd, | ||
| 95 | + ) | ||
| 96 | + finally: | ||
| 97 | + end_time = time.monotonic() | ||
| 98 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 99 | + | ||
| 100 | + | ||
| 101 | +def run_command( | ||
| 102 | + args: list[str], | ||
| 103 | + *, | ||
| 104 | + retries: int = 0, | ||
| 105 | + timeout: int | None = None, | ||
| 106 | + stdin: BinaryIO | None = None, | ||
| 107 | + input: bytes | None = None, | ||
| 108 | + check: bool = False, | ||
| 109 | + cwd: os.PathLike[Any] | None = None, | ||
| 110 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 111 | + remaining_retries = retries | ||
| 112 | + while True: | ||
| 113 | + try: | ||
| 114 | + return _run_command( | ||
| 115 | + args, timeout=timeout, stdin=stdin, input=input, check=check, cwd=cwd | ||
| 116 | + ) | ||
| 117 | + except subprocess.TimeoutExpired as err: | ||
| 118 | + if remaining_retries == 0: | ||
| 119 | + raise err | ||
| 120 | + remaining_retries -= 1 | ||
| 121 | + logging.warning( # noqa: G200 | ||
| 122 | + "(%s/%s) Retrying because command failed with: %r", | ||
| 123 | + retries - remaining_retries, | ||
| 124 | + retries, | ||
| 125 | + err, | ||
| 126 | + ) | ||
| 127 | + time.sleep(1) | ||
| 128 | + | ||
| 129 | + | ||
| 130 | +def add_default_options(parser: argparse.ArgumentParser) -> None: | ||
| 131 | + """Add default options to a parser. | ||
| 132 | + | ||
| 133 | + This should be called the last in the chain of add_argument calls. | ||
| 134 | + """ | ||
| 135 | + parser.add_argument( | ||
| 136 | + "--retries", | ||
| 137 | + type=int, | ||
| 138 | + default=3, | ||
| 139 | + help="number of times to retry if the linter times out.", | ||
| 140 | + ) | ||
| 141 | + parser.add_argument( | ||
| 142 | + "--verbose", | ||
| 143 | + action="store_true", | ||
| 144 | + help="verbose logging", | ||
| 145 | + ) | ||
| 146 | + parser.add_argument( | ||
| 147 | + "filenames", | ||
| 148 | + nargs="+", | ||
| 149 | + help="paths to lint", | ||
| 150 | + ) | ||
| 151 | + | ||
| 152 | + | ||
| 153 | +def explain_rule(code: str) -> str: | ||
| 154 | + proc = run_command( | ||
| 155 | + ["ruff", "rule", "--output-format=json", code], | ||
| 156 | + check=True, | ||
| 157 | + ) | ||
| 158 | + rule = json.loads(str(proc.stdout, "utf-8").strip()) | ||
| 159 | + return f"\n{rule['linter']}: {rule['summary']}" | ||
| 160 | + | ||
| 161 | + | ||
| 162 | +def get_issue_severity(code: str) -> LintSeverity: | ||
| 163 | + # "B901": `return x` inside a generator | ||
| 164 | + # "B902": Invalid first argument to a method | ||
| 165 | + # "B903": __slots__ efficiency | ||
| 166 | + # "B950": Line too long | ||
| 167 | + # "C4": Flake8 Comprehensions | ||
| 168 | + # "C9": Cyclomatic complexity | ||
| 169 | + # "E2": PEP8 horizontal whitespace "errors" | ||
| 170 | + # "E3": PEP8 blank line "errors" | ||
| 171 | + # "E5": PEP8 line length "errors" | ||
| 172 | + # "T400": type checking Notes | ||
| 173 | + # "T49": internal type checker errors or unmatched messages | ||
| 174 | + if any( | ||
| 175 | + code.startswith(x) | ||
| 176 | + for x in ( | ||
| 177 | + "B9", | ||
| 178 | + "C4", | ||
| 179 | + "C9", | ||
| 180 | + "E2", | ||
| 181 | + "E3", | ||
| 182 | + "E5", | ||
| 183 | + "T400", | ||
| 184 | + "T49", | ||
| 185 | + "PLC", | ||
| 186 | + "PLR", | ||
| 187 | + ) | ||
| 188 | + ): | ||
| 189 | + return LintSeverity.ADVICE | ||
| 190 | + | ||
| 191 | + # "F821": Undefined name | ||
| 192 | + # "E999": syntax error | ||
| 193 | + if any(code.startswith(x) for x in ("F821", SYNTAX_ERROR, "PLE")): | ||
| 194 | + return LintSeverity.ERROR | ||
| 195 | + | ||
| 196 | + # "F": PyFlakes Error | ||
| 197 | + # "B": flake8-bugbear Error | ||
| 198 | + # "E": PEP8 "Error" | ||
| 199 | + # "W": PEP8 Warning | ||
| 200 | + # possibly other plugins... | ||
| 201 | + return LintSeverity.WARNING | ||
| 202 | + | ||
| 203 | + | ||
| 204 | +def format_lint_message( | ||
| 205 | + message: str, code: str, rules: dict[str, str], show_disable: bool | ||
| 206 | +) -> str: | ||
| 207 | + if rules: | ||
| 208 | + message += f".\n{rules.get(code) or ''}" | ||
| 209 | + message += ".\nSee https://beta.ruff.rs/docs/rules/" | ||
| 210 | + if show_disable: | ||
| 211 | + message += f".\n\nTo disable, use ` # noqa: {code}`" | ||
| 212 | + return message | ||
| 213 | + | ||
| 214 | + | ||
| 215 | +def check_files( | ||
| 216 | + filenames: list[str], | ||
| 217 | + severities: dict[str, LintSeverity], | ||
| 218 | + *, | ||
| 219 | + config: str | None, | ||
| 220 | + retries: int, | ||
| 221 | + timeout: int, | ||
| 222 | + explain: bool, | ||
| 223 | + show_disable: bool, | ||
| 224 | +) -> list[LintMessage]: | ||
| 225 | + try: | ||
| 226 | + proc = run_command( | ||
| 227 | + [ | ||
| 228 | + sys.executable, | ||
| 229 | + "-m", | ||
| 230 | + "ruff", | ||
| 231 | + "check", | ||
| 232 | + "--exit-zero", | ||
| 233 | + "--quiet", | ||
| 234 | + "--output-format=json", | ||
| 235 | + *([f"--config={config}"] if config else []), | ||
| 236 | + *filenames, | ||
| 237 | + ], | ||
| 238 | + retries=retries, | ||
| 239 | + timeout=timeout, | ||
| 240 | + check=True, | ||
| 241 | + ) | ||
| 242 | + except (OSError, subprocess.CalledProcessError) as err: | ||
| 243 | + return [ | ||
| 244 | + LintMessage( | ||
| 245 | + path=None, | ||
| 246 | + line=None, | ||
| 247 | + char=None, | ||
| 248 | + code=LINTER_CODE, | ||
| 249 | + severity=LintSeverity.ERROR, | ||
| 250 | + name="command-failed", | ||
| 251 | + original=None, | ||
| 252 | + replacement=None, | ||
| 253 | + description=( | ||
| 254 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 255 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 256 | + else ( | ||
| 257 | + f"COMMAND (exit code {err.returncode})\n" | ||
| 258 | + f"{' '.join(as_posix(x) for x in err.cmd)}\n\n" | ||
| 259 | + f"STDERR\n{err.stderr.decode('utf-8').strip() or '(empty)'}\n\n" | ||
| 260 | + f"STDOUT\n{err.stdout.decode('utf-8').strip() or '(empty)'}" | ||
| 261 | + ) | ||
| 262 | + ), | ||
| 263 | + ) | ||
| 264 | + ] | ||
| 265 | + | ||
| 266 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 267 | + vulnerabilities = json.loads(stdout) | ||
| 268 | + | ||
| 269 | + if explain: | ||
| 270 | + all_codes = {v["code"] for v in vulnerabilities} | ||
| 271 | + rules = {code: explain_rule(code) for code in all_codes} | ||
| 272 | + else: | ||
| 273 | + rules = {} | ||
| 274 | + | ||
| 275 | + def lint_message(vuln: dict[str, Any]) -> LintMessage: | ||
| 276 | + code = vuln["code"] or SYNTAX_ERROR | ||
| 277 | + return LintMessage( | ||
| 278 | + path=vuln["filename"], | ||
| 279 | + name=code, | ||
| 280 | + description=( | ||
| 281 | + format_lint_message( | ||
| 282 | + vuln["message"], | ||
| 283 | + code, | ||
| 284 | + rules, | ||
| 285 | + show_disable and bool(vuln["code"]), | ||
| 286 | + ) | ||
| 287 | + ), | ||
| 288 | + line=int(vuln["location"]["row"]), | ||
| 289 | + char=int(vuln["location"]["column"]), | ||
| 290 | + code=LINTER_CODE, | ||
| 291 | + severity=severities.get(code, get_issue_severity(code)), | ||
| 292 | + original=None, | ||
| 293 | + replacement=None, | ||
| 294 | + ) | ||
| 295 | + | ||
| 296 | + return [lint_message(v) for v in vulnerabilities] | ||
| 297 | + | ||
| 298 | + | ||
| 299 | +def check_file_for_fixes( | ||
| 300 | + filename: str, | ||
| 301 | + *, | ||
| 302 | + config: str | None, | ||
| 303 | + retries: int, | ||
| 304 | + timeout: int, | ||
| 305 | +) -> list[LintMessage]: | ||
| 306 | + try: | ||
| 307 | + with open(filename, "rb") as f: | ||
| 308 | + original = f.read() | ||
| 309 | + with open(filename, "rb") as f: | ||
| 310 | + proc_fix = run_command( | ||
| 311 | + [ | ||
| 312 | + sys.executable, | ||
| 313 | + "-m", | ||
| 314 | + "ruff", | ||
| 315 | + "check", | ||
| 316 | + "--fix-only", | ||
| 317 | + "--exit-zero", | ||
| 318 | + *([f"--config={config}"] if config else []), | ||
| 319 | + "--stdin-filename", | ||
| 320 | + filename, | ||
| 321 | + "-", | ||
| 322 | + ], | ||
| 323 | + stdin=f, | ||
| 324 | + retries=retries, | ||
| 325 | + timeout=timeout, | ||
| 326 | + check=True, | ||
| 327 | + ) | ||
| 328 | + except (OSError, subprocess.CalledProcessError) as err: | ||
| 329 | + return [ | ||
| 330 | + LintMessage( | ||
| 331 | + path=None, | ||
| 332 | + line=None, | ||
| 333 | + char=None, | ||
| 334 | + code=LINTER_CODE, | ||
| 335 | + severity=LintSeverity.ERROR, | ||
| 336 | + name="command-failed", | ||
| 337 | + original=None, | ||
| 338 | + replacement=None, | ||
| 339 | + description=( | ||
| 340 | + f"Failed due to {err.__class__.__name__}:\n{err}" | ||
| 341 | + if not isinstance(err, subprocess.CalledProcessError) | ||
| 342 | + else ( | ||
| 343 | + f"COMMAND (exit code {err.returncode})\n" | ||
| 344 | + f"{' '.join(as_posix(x) for x in err.cmd)}\n\n" | ||
| 345 | + f"STDERR\n{err.stderr.decode('utf-8').strip() or '(empty)'}\n\n" | ||
| 346 | + f"STDOUT\n{err.stdout.decode('utf-8').strip() or '(empty)'}" | ||
| 347 | + ) | ||
| 348 | + ), | ||
| 349 | + ) | ||
| 350 | + ] | ||
| 351 | + | ||
| 352 | + replacement = proc_fix.stdout | ||
| 353 | + if original == replacement: | ||
| 354 | + return [] | ||
| 355 | + | ||
| 356 | + return [ | ||
| 357 | + LintMessage( | ||
| 358 | + path=filename, | ||
| 359 | + name="format", | ||
| 360 | + description="Run `lintrunner -a` to apply this patch.", | ||
| 361 | + line=None, | ||
| 362 | + char=None, | ||
| 363 | + code=LINTER_CODE, | ||
| 364 | + severity=LintSeverity.WARNING, | ||
| 365 | + original=original.decode("utf-8"), | ||
| 366 | + replacement=replacement.decode("utf-8"), | ||
| 367 | + ) | ||
| 368 | + ] | ||
| 369 | + | ||
| 370 | + | ||
| 371 | +def main() -> None: | ||
| 372 | + parser = argparse.ArgumentParser( | ||
| 373 | + description=f"Ruff linter. Linter code: {LINTER_CODE}. Use with RUFF-FIX to auto-fix issues.", | ||
| 374 | + fromfile_prefix_chars="@", | ||
| 375 | + ) | ||
| 376 | + parser.add_argument( | ||
| 377 | + "--config", | ||
| 378 | + default=None, | ||
| 379 | + help="Path to the `pyproject.toml` or `ruff.toml` file to use for configuration", | ||
| 380 | + ) | ||
| 381 | + parser.add_argument( | ||
| 382 | + "--explain", | ||
| 383 | + action="store_true", | ||
| 384 | + help="Explain a rule", | ||
| 385 | + ) | ||
| 386 | + parser.add_argument( | ||
| 387 | + "--show-disable", | ||
| 388 | + action="store_true", | ||
| 389 | + help="Show how to disable a lint message", | ||
| 390 | + ) | ||
| 391 | + parser.add_argument( | ||
| 392 | + "--timeout", | ||
| 393 | + default=90, | ||
| 394 | + type=int, | ||
| 395 | + help="Seconds to wait for ruff", | ||
| 396 | + ) | ||
| 397 | + parser.add_argument( | ||
| 398 | + "--severity", | ||
| 399 | + action="append", | ||
| 400 | + help="map code to severity (e.g. `F401:advice`). This option can be used multiple times.", | ||
| 401 | + ) | ||
| 402 | + parser.add_argument( | ||
| 403 | + "--no-fix", | ||
| 404 | + action="store_true", | ||
| 405 | + help="Do not suggest fixes", | ||
| 406 | + ) | ||
| 407 | + add_default_options(parser) | ||
| 408 | + args = parser.parse_args() | ||
| 409 | + | ||
| 410 | + logging.basicConfig( | ||
| 411 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 412 | + level=logging.NOTSET | ||
| 413 | + if args.verbose | ||
| 414 | + else logging.DEBUG | ||
| 415 | + if len(args.filenames) < 1000 | ||
| 416 | + else logging.INFO, | ||
| 417 | + stream=sys.stderr, | ||
| 418 | + ) | ||
| 419 | + | ||
| 420 | + severities: dict[str, LintSeverity] = {} | ||
| 421 | + if args.severity: | ||
| 422 | + for severity in args.severity: | ||
| 423 | + parts = severity.split(":", 1) | ||
| 424 | + if len(parts) != 2: | ||
| 425 | + raise AssertionError(f"invalid severity `{severity}`") | ||
| 426 | + severities[parts[0]] = LintSeverity(parts[1]) | ||
| 427 | + | ||
| 428 | + lint_messages = check_files( | ||
| 429 | + args.filenames, | ||
| 430 | + severities=severities, | ||
| 431 | + config=args.config, | ||
| 432 | + retries=args.retries, | ||
| 433 | + timeout=args.timeout, | ||
| 434 | + explain=args.explain, | ||
| 435 | + show_disable=args.show_disable, | ||
| 436 | + ) | ||
| 437 | + for lint_message in lint_messages: | ||
| 438 | + lint_message.display() | ||
| 439 | + | ||
| 440 | + if args.no_fix or not lint_messages: | ||
| 441 | + # If we're not fixing, we can exit early | ||
| 442 | + return | ||
| 443 | + | ||
| 444 | + files_with_lints = {lint.path for lint in lint_messages if lint.path is not None} | ||
| 445 | + with concurrent.futures.ThreadPoolExecutor( | ||
| 446 | + max_workers=os.cpu_count(), | ||
| 447 | + thread_name_prefix="Thread", | ||
| 448 | + ) as executor: | ||
| 449 | + futures = { | ||
| 450 | + executor.submit( | ||
| 451 | + check_file_for_fixes, | ||
| 452 | + path, | ||
| 453 | + config=args.config, | ||
| 454 | + retries=args.retries, | ||
| 455 | + timeout=args.timeout, | ||
| 456 | + ): path | ||
| 457 | + for path in files_with_lints | ||
| 458 | + } | ||
| 459 | + for future in concurrent.futures.as_completed(futures): | ||
| 460 | + try: | ||
| 461 | + for lint_message in future.result(): | ||
| 462 | + lint_message.display() | ||
| 463 | + except Exception: # Catch all exceptions for lintrunner | ||
| 464 | + logging.critical('Failed at "%s".', futures[future]) | ||
| 465 | + raise | ||
| 466 | + | ||
| 467 | + | ||
| 468 | +if __name__ == "__main__": | ||
| 469 | + main() | ||
| @@ -0,0 +1,218 @@ | |||
| 1 | +import argparse | ||
| 2 | +import hashlib | ||
| 3 | +import json | ||
| 4 | +import logging | ||
| 5 | +import os | ||
| 6 | +import platform | ||
| 7 | +import stat | ||
| 8 | +import subprocess | ||
| 9 | +import sys | ||
| 10 | +import urllib.error | ||
| 11 | +import urllib.request | ||
| 12 | +from pathlib import Path | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +# String representing the host platform (e.g. Linux, Darwin). | ||
| 16 | +HOST_PLATFORM = platform.system() | ||
| 17 | +HOST_PLATFORM_ARCH = platform.system() + "-" + platform.machine() | ||
| 18 | + | ||
| 19 | +# PyTorch directory root | ||
| 20 | +try: | ||
| 21 | + result = subprocess.run( | ||
| 22 | + ["git", "rev-parse", "--show-toplevel"], | ||
| 23 | + stdout=subprocess.PIPE, | ||
| 24 | + check=True, | ||
| 25 | + ) | ||
| 26 | + PYTORCH_ROOT = result.stdout.decode("utf-8").strip() | ||
| 27 | +except subprocess.CalledProcessError: | ||
| 28 | + # If git is not installed, compute repo root as 3 folders up from this file | ||
| 29 | + PYTORCH_ROOT = str(Path(__file__).absolute().parents[3]) | ||
| 30 | + | ||
| 31 | +DRY_RUN = False | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +def compute_file_sha256(path: str) -> str: | ||
| 35 | + """Compute the SHA256 hash of a file and return it as a hex string.""" | ||
| 36 | + # If the file doesn't exist, return an empty string. | ||
| 37 | + if not os.path.exists(path): | ||
| 38 | + return "" | ||
| 39 | + | ||
| 40 | + hash = hashlib.sha256() | ||
| 41 | + | ||
| 42 | + # Open the file in binary mode and hash it. | ||
| 43 | + with open(path, "rb") as f: | ||
| 44 | + for b in f: | ||
| 45 | + hash.update(b) | ||
| 46 | + | ||
| 47 | + # Return the hash as a hexadecimal string. | ||
| 48 | + return hash.hexdigest() | ||
| 49 | + | ||
| 50 | + | ||
| 51 | +def report_download_progress( | ||
| 52 | + chunk_number: int, chunk_size: int, file_size: int | ||
| 53 | +) -> None: | ||
| 54 | + """ | ||
| 55 | + Pretty printer for file download progress. | ||
| 56 | + """ | ||
| 57 | + if file_size != -1: | ||
| 58 | + # pyrefly: ignore [no-matching-overload] | ||
| 59 | + percent = min(1, (chunk_number * chunk_size) / file_size) | ||
| 60 | + bar = "#" * int(64 * percent) | ||
| 61 | + sys.stdout.write(f"\r0% |{bar:<64}| {int(percent * 100)}%") | ||
| 62 | + | ||
| 63 | + | ||
| 64 | +def check(binary_path: Path, reference_hash: str) -> bool: | ||
| 65 | + """Check whether the binary exists and is the right one. | ||
| 66 | + | ||
| 67 | + If there is hash difference, delete the actual binary. | ||
| 68 | + """ | ||
| 69 | + if not binary_path.exists(): | ||
| 70 | + logging.info("%s does not exist.", binary_path) | ||
| 71 | + return False | ||
| 72 | + | ||
| 73 | + existing_binary_hash = compute_file_sha256(str(binary_path)) | ||
| 74 | + if existing_binary_hash == reference_hash: | ||
| 75 | + return True | ||
| 76 | + | ||
| 77 | + logging.warning( | ||
| 78 | + """\ | ||
| 79 | +Found binary hash does not match reference! | ||
| 80 | + | ||
| 81 | +Found hash: %s | ||
| 82 | +Reference hash: %s | ||
| 83 | + | ||
| 84 | +Deleting %s just to be safe. | ||
| 85 | +""", | ||
| 86 | + existing_binary_hash, | ||
| 87 | + reference_hash, | ||
| 88 | + binary_path, | ||
| 89 | + ) | ||
| 90 | + if DRY_RUN: | ||
| 91 | + logging.critical( | ||
| 92 | + "In dry run mode, so not actually deleting the binary. But consider deleting it ASAP!" | ||
| 93 | + ) | ||
| 94 | + return False | ||
| 95 | + | ||
| 96 | + try: | ||
| 97 | + binary_path.unlink() | ||
| 98 | + except OSError: | ||
| 99 | + logging.critical("Failed to delete binary", exc_info=True) | ||
| 100 | + logging.critical( | ||
| 101 | + "Delete this binary as soon as possible and do not execute it!" | ||
| 102 | + ) | ||
| 103 | + | ||
| 104 | + return False | ||
| 105 | + | ||
| 106 | + | ||
| 107 | +def download( | ||
| 108 | + name: str, | ||
| 109 | + output_dir: str, | ||
| 110 | + url: str, | ||
| 111 | + reference_bin_hash: str, | ||
| 112 | +) -> bool: | ||
| 113 | + """ | ||
| 114 | + Download a platform-appropriate binary if one doesn't already exist at the expected location and verifies | ||
| 115 | + that it is the right binary by checking its SHA256 hash against the expected hash. | ||
| 116 | + """ | ||
| 117 | + # First check if we need to do anything | ||
| 118 | + binary_path = Path(output_dir, name) | ||
| 119 | + if check(binary_path, reference_bin_hash): | ||
| 120 | + logging.info("Correct binary already exists at %s. Exiting.", binary_path) | ||
| 121 | + return True | ||
| 122 | + | ||
| 123 | + # Create the output folder | ||
| 124 | + binary_path.parent.mkdir(parents=True, exist_ok=True) | ||
| 125 | + | ||
| 126 | + # Download the binary | ||
| 127 | + logging.info("Downloading %s to %s", url, binary_path) | ||
| 128 | + | ||
| 129 | + if DRY_RUN: | ||
| 130 | + logging.info("Exiting as there is nothing left to do in dry run mode") | ||
| 131 | + return True | ||
| 132 | + | ||
| 133 | + urllib.request.urlretrieve( | ||
| 134 | + url, | ||
| 135 | + binary_path, | ||
| 136 | + reporthook=report_download_progress if sys.stdout.isatty() else None, | ||
| 137 | + ) | ||
| 138 | + | ||
| 139 | + logging.info("Downloaded %s successfully.", name) | ||
| 140 | + | ||
| 141 | + # Check the downloaded binary | ||
| 142 | + if not check(binary_path, reference_bin_hash): | ||
| 143 | + logging.critical("Downloaded binary %s failed its hash check", name) | ||
| 144 | + return False | ||
| 145 | + | ||
| 146 | + # Ensure that executable bits are set | ||
| 147 | + mode = os.stat(binary_path).st_mode | ||
| 148 | + mode |= stat.S_IXUSR | ||
| 149 | + os.chmod(binary_path, mode) | ||
| 150 | + | ||
| 151 | + logging.info("Using %s located at %s", name, binary_path) | ||
| 152 | + return True | ||
| 153 | + | ||
| 154 | + | ||
| 155 | +if __name__ == "__main__": | ||
| 156 | + parser = argparse.ArgumentParser( | ||
| 157 | + description="downloads and checks binaries from s3", | ||
| 158 | + ) | ||
| 159 | + parser.add_argument( | ||
| 160 | + "--config-json", | ||
| 161 | + required=True, | ||
| 162 | + help="Path to config json that describes where to find binaries and hashes", | ||
| 163 | + ) | ||
| 164 | + parser.add_argument( | ||
| 165 | + "--linter", | ||
| 166 | + required=True, | ||
| 167 | + help="Which linter to initialize from the config json", | ||
| 168 | + ) | ||
| 169 | + parser.add_argument( | ||
| 170 | + "--output-dir", | ||
| 171 | + required=True, | ||
| 172 | + help="place to put the binary", | ||
| 173 | + ) | ||
| 174 | + parser.add_argument( | ||
| 175 | + "--output-name", | ||
| 176 | + required=True, | ||
| 177 | + help="name of binary", | ||
| 178 | + ) | ||
| 179 | + parser.add_argument( | ||
| 180 | + "--dry-run", | ||
| 181 | + default=False, | ||
| 182 | + help="do not download, just print what would be done", | ||
| 183 | + ) | ||
| 184 | + | ||
| 185 | + args = parser.parse_args() | ||
| 186 | + if args.dry_run == "0": | ||
| 187 | + DRY_RUN = False | ||
| 188 | + else: | ||
| 189 | + DRY_RUN = True | ||
| 190 | + | ||
| 191 | + logging.basicConfig( | ||
| 192 | + format="[DRY_RUN] %(levelname)s: %(message)s" | ||
| 193 | + if DRY_RUN | ||
| 194 | + else "%(levelname)s: %(message)s", | ||
| 195 | + level=logging.INFO, | ||
| 196 | + stream=sys.stderr, | ||
| 197 | + ) | ||
| 198 | + with open(args.config_json) as f: | ||
| 199 | + config = json.load(f) | ||
| 200 | + config = config[args.linter] | ||
| 201 | + | ||
| 202 | + # Allow processor specific binaries for platform (e.g. Intel/M1 for macOS, x86_64/aarch64 for Linux) | ||
| 203 | + # Try arch-specific first, then fall back to generic platform | ||
| 204 | + host_platform = ( | ||
| 205 | + HOST_PLATFORM_ARCH if HOST_PLATFORM_ARCH in config else HOST_PLATFORM | ||
| 206 | + ) | ||
| 207 | + # If the host platform is not in platform_to_hash, it is unsupported. | ||
| 208 | + if host_platform not in config: | ||
| 209 | + logging.error("Unsupported platform: %s/%s", HOST_PLATFORM, HOST_PLATFORM_ARCH) | ||
| 210 | + sys.exit(1) | ||
| 211 | + | ||
| 212 | + url = config[host_platform]["download_url"] | ||
| 213 | + hash = config[host_platform]["hash"] | ||
| 214 | + | ||
| 215 | + ok = download(args.output_name, args.output_dir, url, hash) | ||
| 216 | + if not ok: | ||
| 217 | + logging.critical("Unable to initialize %s", args.linter) | ||
| 218 | + sys.exit(1) | ||
| @@ -0,0 +1,73 @@ | |||
| 1 | +{ | ||
| 2 | + "HOW TO UPDATE THE BINARIES": [ | ||
| 3 | + "Upload the new file to S3 under a new folder with the version number embedded in (see actionlint for an example).", | ||
| 4 | + "(Don't override the old files, otherwise you'll break `lintrunner install` for anyone using an older commit of pytorch.)", | ||
| 5 | + "'Hash' is the sha256 of the uploaded file.", | ||
| 6 | + "Validate the new download url and hash by running 'lintrunner init' to pull the new binaries and then run 'lintrunner' to try linting the files.", | ||
| 7 | + "Some binaries have custom builds; see https://github.com/pytorch/test-infra/blob/main/.github/workflows/clang-tidy-linux.yml and https://github.com/pytorch/test-infra/blob/main/.github/workflows/clang-tidy-macos.yml" | ||
| 8 | + ], | ||
| 9 | + "clang-format": { | ||
| 10 | + "Darwin-arm64": { | ||
| 11 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos-arm/19.1.4/clang-format", | ||
| 12 | + "hash": "f0da3ecf0ab1e9b50e8c27bd2d7ca0baa619e2f4b824b35d79d46356581fa552" | ||
| 13 | + }, | ||
| 14 | + "Darwin-x86_64": { | ||
| 15 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos-i386/19.1.4/clang-format", | ||
| 16 | + "hash": "f5eb5037b9aa9d1d2de650fb2e0fe1a2517768a462fae8e98791a67b698302f4" | ||
| 17 | + }, | ||
| 18 | + "Linux": { | ||
| 19 | + "download_url": "https://pytorch-package.obs.cn-north-4.myhuaweicloud.com/pta-codecheck/clang-format", | ||
| 20 | + "hash": "bfa9ef6eccb372f79ffcb6196af966fd84519ea9567f5ae7b6ad30208cd82109" | ||
| 21 | + }, | ||
| 22 | + "Linux-aarch64": { | ||
| 23 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux-aarch64/19.1.4/clang-format", | ||
| 24 | + "hash": "9cfa17f68100f4cbb3ba6180df9db6d0a32b3a8b45e122aa20e2dda1feaf9902" | ||
| 25 | + } | ||
| 26 | + }, | ||
| 27 | + "clang-tidy": { | ||
| 28 | + "Darwin-x86_64": { | ||
| 29 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos-i386/19.1.4/clang-tidy", | ||
| 30 | + "hash": "7b5da17d3f8b1c18c77d043999f05293f43402affb16de15dfcb276971984a3e" | ||
| 31 | + }, | ||
| 32 | + "Darwin-arm64": { | ||
| 33 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/macos-arm/19.1.4/clang-tidy", | ||
| 34 | + "hash": "04243f4044fe6d95f6d51d15be803331c3cbb61f2d8fcfeba5a5dec1e7ae6dfb" | ||
| 35 | + }, | ||
| 36 | + "Linux": { | ||
| 37 | + "download_url": "https://pytorch-package.obs.cn-north-4.myhuaweicloud.com/pta-codecheck/clang-tidy", | ||
| 38 | + "hash": "5637bd0fca665d2797926fedf53ca5ad4655bb9dbed1e1c8654c8e032ce1e7a8" | ||
| 39 | + }, | ||
| 40 | + "Linux-aarch64": { | ||
| 41 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/linux-aarch64/19.1.4/clang-tidy", | ||
| 42 | + "hash": "cd6708ca9731002abd8ecc1b616716d8a21cf682a3cb931dd3265c3f3f600d30" | ||
| 43 | + } | ||
| 44 | + }, | ||
| 45 | + "actionlint": { | ||
| 46 | + "Darwin-x86_64": { | ||
| 47 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/actionlint/1.7.7/Darwin_amd64/actionlint", | ||
| 48 | + "hash": "996affd492c57441c5ecfe00dedaef1fde056872d242c0cf7cc15de058d59d03" | ||
| 49 | + }, | ||
| 50 | + "Darwin-arm64": { | ||
| 51 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/actionlint/1.7.7/Darwin_arm64/actionlint", | ||
| 52 | + "hash": "00aba386d026da33be6e85dd5a46d7af4dd9e4d6cbdb02335f4b267162fd2d9e" | ||
| 53 | + }, | ||
| 54 | + "Linux": { | ||
| 55 | + "download_url": "https://pytorch-package.obs.cn-north-4.myhuaweicloud.com/pta-codecheck/actionlint", | ||
| 56 | + "hash": "9f7dedb4e23f89f2922073d1a6720405b7b520d4f5832ebb96f0d55a2958886c" | ||
| 57 | + }, | ||
| 58 | + "Linux-aarch64": { | ||
| 59 | + "download_url": "https://oss-clang-format.s3.us-east-2.amazonaws.com/actionlint/1.7.7/Linux_arm64/actionlint", | ||
| 60 | + "hash": "446687e63fac45472b0a66bae28975c28678af062670af119c11a7087baf35cc" | ||
| 61 | + } | ||
| 62 | + }, | ||
| 63 | + "bazel": { | ||
| 64 | + "Darwin": { | ||
| 65 | + "download_url": "https://raw.githubusercontent.com/bazelbuild/bazelisk/v1.16.0/bazelisk.py", | ||
| 66 | + "hash": "1f6d76d023ddd5f1625f34d934418e7334a267318d084f31be09df8a8835ed16" | ||
| 67 | + }, | ||
| 68 | + "Linux": { | ||
| 69 | + "download_url": "https://pytorch-package.obs.cn-north-4.myhuaweicloud.com/pta-codecheck/bazelisk.py", | ||
| 70 | + "hash": "1f6d76d023ddd5f1625f34d934418e7334a267318d084f31be09df8a8835ed16" | ||
| 71 | + } | ||
| 72 | + } | ||
| 73 | +} | ||
| @@ -0,0 +1,92 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import sys | ||
| 4 | +from pathlib import Path | ||
| 5 | +from typing import TYPE_CHECKING | ||
| 6 | + | ||
| 7 | + | ||
| 8 | +_PARENT = Path(__file__).parent.absolute() | ||
| 9 | +_PATH = [Path(p).absolute() for p in sys.path] | ||
| 10 | + | ||
| 11 | +if TYPE_CHECKING or _PARENT not in _PATH: | ||
| 12 | + from . import _linter | ||
| 13 | +else: | ||
| 14 | + import _linter | ||
| 15 | + | ||
| 16 | +if TYPE_CHECKING: | ||
| 17 | + from collections.abc import Iterator | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +ERROR = "Builtin `set` is deprecated" | ||
| 21 | +IMPORT_LINE = "from torch.utils._ordered_set import OrderedSet\n\n" | ||
| 22 | + | ||
| 23 | +DESCRIPTION = """`set_linter` is a lintrunner linter which finds usages of the | ||
| 24 | +Python built-in class `set` in Python code, and optionally replaces them with | ||
| 25 | +`OrderedSet`. | ||
| 26 | +""" | ||
| 27 | + | ||
| 28 | +EPILOG = """ | ||
| 29 | +`lintrunner` operates on whole commits. If you want to remove uses of `set` | ||
| 30 | +from existing files not part of a commit, call `set_linter` directly: | ||
| 31 | + | ||
| 32 | + python tools/linter/adapters/set_linter.py --fix [... python files ...] | ||
| 33 | + | ||
| 34 | +--- | ||
| 35 | + | ||
| 36 | +To omit a line of Python code from `set_linter` checking, append a comment: | ||
| 37 | + | ||
| 38 | + s = set() # noqa: set_linter | ||
| 39 | + t = { # noqa: set_linter | ||
| 40 | + "one", | ||
| 41 | + "two", | ||
| 42 | + } | ||
| 43 | + | ||
| 44 | +--- | ||
| 45 | + | ||
| 46 | +Running set_linter in fix mode (though either `lintrunner -a` or `--fix` | ||
| 47 | +should not significantly change the behavior of working code, but will still | ||
| 48 | +usually needs some manual intervention: | ||
| 49 | + | ||
| 50 | +1. Replacing `set` with `OrderedSet` will sometimes introduce new typechecking | ||
| 51 | +errors because `OrderedSet` is imperfectly generic. Find a common type for its | ||
| 52 | +elements (in the worst case, `typing.Any` always works), and use | ||
| 53 | +`OrderedSet[YourCommonTypeHere]`. | ||
| 54 | + | ||
| 55 | +2. The fix mode doesn't recognize generator expressions, so it replaces: | ||
| 56 | + | ||
| 57 | + s = {i for i in range(3)} | ||
| 58 | + | ||
| 59 | +with | ||
| 60 | + | ||
| 61 | + s = OrderedSet([i for i in range(3)]) | ||
| 62 | + | ||
| 63 | +You can and should delete the square brackets in every such case. | ||
| 64 | + | ||
| 65 | +3. There is a common pattern of set usage where a set is created and then only | ||
| 66 | +used for testing inclusion. For small collections, up to around 12 elements, a | ||
| 67 | +tuple is more time-efficient than an OrderedSet and also has less visual clutter | ||
| 68 | +(see https://github.com/rec/test/blob/master/python/time_access.py). | ||
| 69 | +""" | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +class SetLinter(_linter.FileLinter): | ||
| 73 | + linter_name = "set_linter" | ||
| 74 | + description = DESCRIPTION | ||
| 75 | + epilog = EPILOG | ||
| 76 | + report_column_numbers = True | ||
| 77 | + | ||
| 78 | + def _lint(self, pf: _linter.PythonFile) -> Iterator[_linter.LintResult]: | ||
| 79 | + if (pf.sets or pf.braced_sets) and (ins := pf.insert_import_line) is not None: | ||
| 80 | + yield _linter.LintResult( | ||
| 81 | + "Add import for OrderedSet", ins, 0, IMPORT_LINE, 0 | ||
| 82 | + ) | ||
| 83 | + for b in pf.braced_sets: | ||
| 84 | + yield _linter.LintResult(ERROR, *b[0].start, "OrderedSet([", 1) | ||
| 85 | + yield _linter.LintResult(ERROR, *b[-1].start, "])", 1) | ||
| 86 | + | ||
| 87 | + for s in pf.sets: | ||
| 88 | + yield _linter.LintResult(ERROR, *s.start, "OrderedSet", 3) | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +if __name__ == "__main__": | ||
| 92 | + SetLinter.run() | ||
| @@ -0,0 +1,161 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "shellcheck-py==0.7.2.1; platform_machine == 'x86_64'", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +from __future__ import annotations | ||
| 8 | + | ||
| 9 | +import argparse | ||
| 10 | +import json | ||
| 11 | +import logging | ||
| 12 | +import os | ||
| 13 | +import platform | ||
| 14 | +import subprocess | ||
| 15 | +import sys | ||
| 16 | +import time | ||
| 17 | +from enum import Enum | ||
| 18 | +from typing import NamedTuple | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +LINTER_CODE = "SHELLCHECK" | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +class LintSeverity(str, Enum): | ||
| 25 | + ERROR = "error" | ||
| 26 | + WARNING = "warning" | ||
| 27 | + ADVICE = "advice" | ||
| 28 | + DISABLED = "disabled" | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +class LintMessage(NamedTuple): | ||
| 32 | + path: str | None | ||
| 33 | + line: int | None | ||
| 34 | + char: int | None | ||
| 35 | + code: str | ||
| 36 | + severity: LintSeverity | ||
| 37 | + name: str | ||
| 38 | + original: str | None | ||
| 39 | + replacement: str | None | ||
| 40 | + description: str | None | ||
| 41 | + | ||
| 42 | + | ||
| 43 | +def run_command( | ||
| 44 | + args: list[str], | ||
| 45 | +) -> subprocess.CompletedProcess[bytes]: | ||
| 46 | + logging.debug("$ %s", " ".join(args)) | ||
| 47 | + start_time = time.monotonic() | ||
| 48 | + try: | ||
| 49 | + return subprocess.run( | ||
| 50 | + args, | ||
| 51 | + capture_output=True, | ||
| 52 | + ) | ||
| 53 | + finally: | ||
| 54 | + end_time = time.monotonic() | ||
| 55 | + logging.debug("took %dms", (end_time - start_time) * 1000) | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +def _is_x86_64() -> bool: | ||
| 59 | + return platform.machine() == "x86_64" | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def _shellcheck_candidates() -> list[str]: | ||
| 63 | + path_env = os.environ.get("PATH", "") | ||
| 64 | + candidates: list[str] = [] | ||
| 65 | + for directory in path_env.split(os.pathsep): | ||
| 66 | + if not directory: | ||
| 67 | + continue | ||
| 68 | + candidate = os.path.join(directory, "shellcheck") | ||
| 69 | + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): | ||
| 70 | + candidates.append(candidate) | ||
| 71 | + return candidates | ||
| 72 | + | ||
| 73 | + | ||
| 74 | +def check_files( | ||
| 75 | + files: list[str], | ||
| 76 | +) -> list[LintMessage]: | ||
| 77 | + args = ["--external-sources", "--format=json1"] + files | ||
| 78 | + | ||
| 79 | + proc: subprocess.CompletedProcess[bytes] | None = None | ||
| 80 | + last_error: OSError | None = None | ||
| 81 | + | ||
| 82 | + for shellcheck in _shellcheck_candidates(): | ||
| 83 | + try: | ||
| 84 | + proc = run_command([shellcheck] + args) | ||
| 85 | + break | ||
| 86 | + except OSError as err: | ||
| 87 | + last_error = err | ||
| 88 | + | ||
| 89 | + if proc is None: | ||
| 90 | + if last_error is not None and last_error.errno == 8: | ||
| 91 | + return [] | ||
| 92 | + if not _is_x86_64(): | ||
| 93 | + return [] | ||
| 94 | + return [ | ||
| 95 | + LintMessage( | ||
| 96 | + path=None, | ||
| 97 | + line=None, | ||
| 98 | + char=None, | ||
| 99 | + code=LINTER_CODE, | ||
| 100 | + severity=LintSeverity.ERROR, | ||
| 101 | + name="command-failed", | ||
| 102 | + original=None, | ||
| 103 | + replacement=None, | ||
| 104 | + description=( | ||
| 105 | + f"Failed to execute shellcheck.\n{last_error.__class__.__name__}: {last_error}" | ||
| 106 | + if last_error is not None | ||
| 107 | + else "Failed to find a usable shellcheck executable." | ||
| 108 | + ), | ||
| 109 | + ) | ||
| 110 | + ] | ||
| 111 | + stdout = str(proc.stdout, "utf-8").strip() | ||
| 112 | + results = json.loads(stdout)["comments"] | ||
| 113 | + return [ | ||
| 114 | + LintMessage( | ||
| 115 | + path=result["file"], | ||
| 116 | + name=f"SC{result['code']}", | ||
| 117 | + description=result["message"], | ||
| 118 | + line=result["line"], | ||
| 119 | + char=result["column"], | ||
| 120 | + code=LINTER_CODE, | ||
| 121 | + severity=LintSeverity.ERROR, | ||
| 122 | + original=None, | ||
| 123 | + replacement=None, | ||
| 124 | + ) | ||
| 125 | + for result in results | ||
| 126 | + ] | ||
| 127 | + | ||
| 128 | + | ||
| 129 | +if __name__ == "__main__": | ||
| 130 | + parser = argparse.ArgumentParser( | ||
| 131 | + description="shellcheck runner", | ||
| 132 | + fromfile_prefix_chars="@", | ||
| 133 | + ) | ||
| 134 | + parser.add_argument( | ||
| 135 | + "filenames", | ||
| 136 | + nargs="+", | ||
| 137 | + help="paths to lint", | ||
| 138 | + ) | ||
| 139 | + | ||
| 140 | + if not _shellcheck_candidates(): | ||
| 141 | + if not _is_x86_64(): | ||
| 142 | + sys.exit(0) | ||
| 143 | + err_msg = LintMessage( | ||
| 144 | + path="<none>", | ||
| 145 | + line=None, | ||
| 146 | + char=None, | ||
| 147 | + code=LINTER_CODE, | ||
| 148 | + severity=LintSeverity.ERROR, | ||
| 149 | + name="command-failed", | ||
| 150 | + original=None, | ||
| 151 | + replacement=None, | ||
| 152 | + description="shellcheck is not installed, did you forget to run `lintrunner init`?", | ||
| 153 | + ) | ||
| 154 | + print(json.dumps(err_msg._asdict()), flush=True) | ||
| 155 | + sys.exit(0) | ||
| 156 | + | ||
| 157 | + args = parser.parse_args() | ||
| 158 | + | ||
| 159 | + lint_messages = check_files(args.filenames) | ||
| 160 | + for lint_message in lint_messages: | ||
| 161 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,295 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +""" | ||
| 3 | +STABLE_SHIM_USAGE: Ensures that calls to versioned shim functions from | ||
| 4 | +torch/csrc/stable/c/shim.h in torch/csrc/stable are properly wrapped in | ||
| 5 | +TORCH_FEATURE_VERSION macros corresponding to the version when those | ||
| 6 | +functions were introduced. | ||
| 7 | +""" | ||
| 8 | + | ||
| 9 | +from __future__ import annotations | ||
| 10 | + | ||
| 11 | +import argparse | ||
| 12 | +import json | ||
| 13 | +import logging | ||
| 14 | +import re | ||
| 15 | +import sys | ||
| 16 | +from enum import Enum | ||
| 17 | +from pathlib import Path | ||
| 18 | +from typing import NamedTuple | ||
| 19 | + | ||
| 20 | + | ||
| 21 | +# Add repo root to sys.path so we can import from tools | ||
| 22 | +REPO_ROOT = Path(__file__).resolve().parents[3] | ||
| 23 | +sys.path.insert(0, str(REPO_ROOT)) | ||
| 24 | + | ||
| 25 | +from tools.linter.adapters.stable_shim_version_linter import PreprocessorTracker | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +LINTER_CODE = "STABLE_SHIM_USAGE" | ||
| 29 | + | ||
| 30 | + | ||
| 31 | +class LintSeverity(str, Enum): | ||
| 32 | + ERROR = "error" | ||
| 33 | + WARNING = "warning" | ||
| 34 | + ADVICE = "advice" | ||
| 35 | + DISABLED = "disabled" | ||
| 36 | + | ||
| 37 | + | ||
| 38 | +class LintMessage(NamedTuple): | ||
| 39 | + path: str | None | ||
| 40 | + line: int | None | ||
| 41 | + char: int | None | ||
| 42 | + code: str | ||
| 43 | + severity: LintSeverity | ||
| 44 | + name: str | ||
| 45 | + original: str | None | ||
| 46 | + replacement: str | None | ||
| 47 | + description: str | None | ||
| 48 | + | ||
| 49 | + | ||
| 50 | +def get_shim_functions( | ||
| 51 | + shim_files: list[Path | str] | None = None, | ||
| 52 | +) -> dict[str, tuple[int, int]]: | ||
| 53 | + """ | ||
| 54 | + Extract function names from shim header files and their required version. | ||
| 55 | + Returns a dict mapping function name to (major, minor) version tuple. | ||
| 56 | + | ||
| 57 | + Only functions defined inside TORCH_FEATURE_VERSION blocks are extracted. | ||
| 58 | + Functions without version guards are ignored. | ||
| 59 | + | ||
| 60 | + Args: | ||
| 61 | + shim_files: List of paths to shim header files. If None, will use the default | ||
| 62 | + paths to torch/csrc/stable/c/shim.h and | ||
| 63 | + torch/csrc/inductor/aoti_torch/c/shim.h based on the repository root. | ||
| 64 | + """ | ||
| 65 | + if shim_files is None: | ||
| 66 | + repo_root = Path(__file__).resolve().parents[3] | ||
| 67 | + shim_files_to_check = [ | ||
| 68 | + repo_root / "torch/csrc/stable/c/shim.h", | ||
| 69 | + repo_root / "torch/csrc/inductor/aoti_torch/c/shim.h", | ||
| 70 | + ] | ||
| 71 | + else: | ||
| 72 | + shim_files_to_check = [Path(f) for f in shim_files] | ||
| 73 | + | ||
| 74 | + # Assert that all shim files exist | ||
| 75 | + missing_files = [f for f in shim_files_to_check if not f.exists()] | ||
| 76 | + if missing_files: | ||
| 77 | + raise RuntimeError( | ||
| 78 | + f"The following shim files do not exist: {missing_files}. " | ||
| 79 | + "Ensure all shim header files exist in the repository." | ||
| 80 | + ) | ||
| 81 | + | ||
| 82 | + functions: dict[str, tuple[int, int]] = {} | ||
| 83 | + | ||
| 84 | + # Match function declarations like: AOTI_TORCH_EXPORT ... function_name( | ||
| 85 | + function_pattern = re.compile(r"AOTI_TORCH_EXPORT\s+\w+\s+(\w+)\s*\(") | ||
| 86 | + # Also match typedef function pointers | ||
| 87 | + typedef_pattern = re.compile(r"typedef\s+.*\(\*(\w+)\)") | ||
| 88 | + # Match using declarations like: using TypeName = ... | ||
| 89 | + using_pattern = re.compile(r"using\s+(\w+)\s*=") | ||
| 90 | + # Match struct/class declarations like: struct StructName or class ClassName | ||
| 91 | + struct_class_pattern = re.compile(r"(?:struct|class)\s+(\w+)") | ||
| 92 | + | ||
| 93 | + for shim_file in shim_files_to_check: | ||
| 94 | + with open(shim_file) as f: | ||
| 95 | + lines = f.readlines() | ||
| 96 | + | ||
| 97 | + tracker = PreprocessorTracker() | ||
| 98 | + | ||
| 99 | + for line in lines: | ||
| 100 | + is_directive_or_comment = tracker.process_line(line) | ||
| 101 | + | ||
| 102 | + # Only look for function declarations if not a comment/directive and inside a version block | ||
| 103 | + if not is_directive_or_comment: | ||
| 104 | + version_of_block = tracker.get_version_of_block() | ||
| 105 | + if version_of_block: | ||
| 106 | + stripped = line.strip() | ||
| 107 | + func_match = function_pattern.search(stripped) | ||
| 108 | + if func_match: | ||
| 109 | + func_name = func_match.group(1) | ||
| 110 | + functions[func_name] = version_of_block | ||
| 111 | + continue | ||
| 112 | + | ||
| 113 | + typedef_match = typedef_pattern.search(stripped) | ||
| 114 | + if typedef_match: | ||
| 115 | + func_name = typedef_match.group(1) | ||
| 116 | + functions[func_name] = version_of_block | ||
| 117 | + continue | ||
| 118 | + | ||
| 119 | + using_match = using_pattern.search(stripped) | ||
| 120 | + if using_match: | ||
| 121 | + type_name = using_match.group(1) | ||
| 122 | + functions[type_name] = version_of_block | ||
| 123 | + continue | ||
| 124 | + | ||
| 125 | + struct_class_match = struct_class_pattern.search(stripped) | ||
| 126 | + if struct_class_match: | ||
| 127 | + type_name = struct_class_match.group(1) | ||
| 128 | + functions[type_name] = version_of_block | ||
| 129 | + continue | ||
| 130 | + | ||
| 131 | + if not functions: | ||
| 132 | + raise RuntimeError( | ||
| 133 | + "Could not find any versioned shim functions. " | ||
| 134 | + "Ensure at least one of the shim files exists and contains versioned functions." | ||
| 135 | + ) | ||
| 136 | + | ||
| 137 | + return functions | ||
| 138 | + | ||
| 139 | + | ||
| 140 | +def write_shim_function_versions( | ||
| 141 | + functions: dict[str, tuple[int, int]], | ||
| 142 | + output_file: Path | str | None = None, | ||
| 143 | +) -> None: | ||
| 144 | + """ | ||
| 145 | + Write the shim function versions to a text file. | ||
| 146 | + | ||
| 147 | + Args: | ||
| 148 | + functions: Dictionary mapping function name to (major, minor) version tuple. | ||
| 149 | + output_file: Path to the output file. If None, will write to | ||
| 150 | + torch/csrc/stable/c/shim_function_versions.txt in the repository. | ||
| 151 | + """ | ||
| 152 | + if output_file is None: | ||
| 153 | + repo_root = Path(__file__).resolve().parents[3] | ||
| 154 | + output_file = repo_root / "torch/csrc/stable/c/shim_function_versions.txt" | ||
| 155 | + else: | ||
| 156 | + output_file = Path(output_file) | ||
| 157 | + | ||
| 158 | + # Sort functions by version, then by name for consistency | ||
| 159 | + sorted_functions = sorted(functions.items(), key=lambda x: (x[1], x[0])) | ||
| 160 | + | ||
| 161 | + with open(output_file, "w") as f: | ||
| 162 | + f.write( | ||
| 163 | + "# Auto-generated file listing shim functions and their minimum required versions\n" | ||
| 164 | + ) | ||
| 165 | + f.write("# Format: function_name: TORCH_VERSION_MAJOR_MINOR_PATCH\n") | ||
| 166 | + f.write("#\n") | ||
| 167 | + f.write( | ||
| 168 | + "# This file is automatically updated by the stable_shim_usage_linter.\n" | ||
| 169 | + ) | ||
| 170 | + f.write( | ||
| 171 | + "# If a function is not in this file, it was available before 2.10.0.\n" | ||
| 172 | + ) | ||
| 173 | + f.write("# DO NOT EDIT MANUALLY.\n\n") | ||
| 174 | + | ||
| 175 | + for func_name, (major, minor) in sorted_functions: | ||
| 176 | + f.write(f"{func_name}: TORCH_VERSION_{major}_{minor}_0\n") | ||
| 177 | + | ||
| 178 | + | ||
| 179 | +def check_file( | ||
| 180 | + filename: str, shim_functions: dict[str, tuple[int, int]] | ||
| 181 | +) -> list[LintMessage]: | ||
| 182 | + """ | ||
| 183 | + Check the input file for proper usage of versioned shim functions. | ||
| 184 | + | ||
| 185 | + Args: | ||
| 186 | + filename: File in torch/csrc/stable that calls functions from shim. | ||
| 187 | + shim_functions: Dictionary mapping function name to (major, minor) version tuple. | ||
| 188 | + """ | ||
| 189 | + lint_messages: list[LintMessage] = [] | ||
| 190 | + | ||
| 191 | + with open(filename) as f: | ||
| 192 | + lines = f.readlines() | ||
| 193 | + | ||
| 194 | + tracker = PreprocessorTracker() | ||
| 195 | + | ||
| 196 | + for line_num, line in enumerate(lines, 1): | ||
| 197 | + is_directive_or_comment = tracker.process_line(line) | ||
| 198 | + | ||
| 199 | + if is_directive_or_comment: | ||
| 200 | + continue | ||
| 201 | + | ||
| 202 | + version_of_block = tracker.get_version_of_block() | ||
| 203 | + | ||
| 204 | + for func_name, required_version in shim_functions.items(): | ||
| 205 | + # Look for: | ||
| 206 | + # 1. Function calls like: func_name( | ||
| 207 | + # 2. Type usage like: func_name variable_name | ||
| 208 | + # Use word boundaries to avoid matching partial names | ||
| 209 | + | ||
| 210 | + if re.search(rf"\b{re.escape(func_name)}\b", line): | ||
| 211 | + major, minor = required_version | ||
| 212 | + required_macro = f"TORCH_VERSION_{major}_{minor}_0" | ||
| 213 | + | ||
| 214 | + if version_of_block is None: | ||
| 215 | + # Not inside any version block | ||
| 216 | + lint_messages.append( | ||
| 217 | + LintMessage( | ||
| 218 | + path=filename, | ||
| 219 | + line=line_num, | ||
| 220 | + char=None, | ||
| 221 | + code=LINTER_CODE, | ||
| 222 | + severity=LintSeverity.ERROR, | ||
| 223 | + name="unversioned-shim-call", | ||
| 224 | + original=None, | ||
| 225 | + replacement=None, | ||
| 226 | + description=( | ||
| 227 | + f"Usage '{func_name}' from shim.h is not wrapped " | ||
| 228 | + f"in a TORCH_FEATURE_VERSION block. This function requires at least:\n" | ||
| 229 | + f"#if TORCH_FEATURE_VERSION >= {required_macro}\n" | ||
| 230 | + f" // ... your code calling {func_name} ...\n" | ||
| 231 | + f"#endif // TORCH_FEATURE_VERSION >= {required_macro}" | ||
| 232 | + ), | ||
| 233 | + ) | ||
| 234 | + ) | ||
| 235 | + elif version_of_block < required_version: | ||
| 236 | + # Inside a version block, but version is too old | ||
| 237 | + current_major, current_minor = version_of_block | ||
| 238 | + current_macro = f"TORCH_VERSION_{current_major}_{current_minor}_0" | ||
| 239 | + lint_messages.append( | ||
| 240 | + LintMessage( | ||
| 241 | + path=filename, | ||
| 242 | + line=line_num, | ||
| 243 | + char=None, | ||
| 244 | + code=LINTER_CODE, | ||
| 245 | + severity=LintSeverity.ERROR, | ||
| 246 | + name="insufficient-version-for-shim-call", | ||
| 247 | + original=None, | ||
| 248 | + replacement=None, | ||
| 249 | + description=( | ||
| 250 | + f"Use of '{func_name}' is wrapped in {current_macro}, " | ||
| 251 | + f"but this function requires at least {required_macro}. " | ||
| 252 | + f"The version guard must be at least the required version:\n" | ||
| 253 | + f"#if TORCH_FEATURE_VERSION >= {required_macro}\n" | ||
| 254 | + f" // ... your code calling {func_name} ...\n" | ||
| 255 | + f"#endif // TORCH_FEATURE_VERSION >= {required_macro}" | ||
| 256 | + ), | ||
| 257 | + ) | ||
| 258 | + ) | ||
| 259 | + | ||
| 260 | + return lint_messages | ||
| 261 | + | ||
| 262 | + | ||
| 263 | +if __name__ == "__main__": | ||
| 264 | + parser = argparse.ArgumentParser( | ||
| 265 | + description="stable shim usage linter", | ||
| 266 | + fromfile_prefix_chars="@", | ||
| 267 | + ) | ||
| 268 | + parser.add_argument( | ||
| 269 | + "--verbose", | ||
| 270 | + action="store_true", | ||
| 271 | + ) | ||
| 272 | + parser.add_argument( | ||
| 273 | + "filenames", | ||
| 274 | + nargs="+", | ||
| 275 | + help="paths to lint", | ||
| 276 | + ) | ||
| 277 | + | ||
| 278 | + args = parser.parse_args() | ||
| 279 | + | ||
| 280 | + logging.basicConfig( | ||
| 281 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 282 | + level=logging.NOTSET if args.verbose else logging.DEBUG, | ||
| 283 | + stream=sys.stderr, | ||
| 284 | + ) | ||
| 285 | + | ||
| 286 | + # Update the shim function versions file | ||
| 287 | + shim_functions = get_shim_functions() | ||
| 288 | + write_shim_function_versions(shim_functions) | ||
| 289 | + | ||
| 290 | + lint_messages = [] | ||
| 291 | + for filename in args.filenames: | ||
| 292 | + lint_messages.extend(check_file(filename, shim_functions)) | ||
| 293 | + | ||
| 294 | + for lint_message in lint_messages: | ||
| 295 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,469 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +""" | ||
| 3 | +STABLE_SHIM_VERSION: Ensures that function declarations in stable/c/shim.h | ||
| 4 | +are properly wrapped in TORCH_FEATURE_VERSION macros corresponding to the | ||
| 5 | +current TORCH_ABI_VERSION. | ||
| 6 | +""" | ||
| 7 | + | ||
| 8 | +from __future__ import annotations | ||
| 9 | + | ||
| 10 | +import argparse | ||
| 11 | +import json | ||
| 12 | +import logging | ||
| 13 | +import re | ||
| 14 | +import sys | ||
| 15 | +from enum import Enum | ||
| 16 | +from pathlib import Path | ||
| 17 | +from typing import NamedTuple | ||
| 18 | + | ||
| 19 | + | ||
| 20 | +# Add repo root to sys.path so we can import from tools.setup_helpers | ||
| 21 | +REPO_ROOT = Path(__file__).resolve().parents[3] | ||
| 22 | +sys.path.insert(0, str(REPO_ROOT)) | ||
| 23 | + | ||
| 24 | +from tools.setup_helpers.gen_version_header import parse_version | ||
| 25 | + | ||
| 26 | + | ||
| 27 | +LINTER_CODE = "STABLE_SHIM_VERSION" | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +class PreprocessorTracker: | ||
| 31 | + """ | ||
| 32 | + Helper class to track preprocessor directives and version blocks. | ||
| 33 | + | ||
| 34 | + This class maintains state as it processes C/C++ preprocessor directives | ||
| 35 | + (#if, #elif, #else, #endif) and tracks which code is inside version blocks. | ||
| 36 | + """ | ||
| 37 | + | ||
| 38 | + def __init__(self): | ||
| 39 | + """Initialize the preprocessor tracker.""" | ||
| 40 | + # Stack of (is_version_block, version_tuple) tuples | ||
| 41 | + # is_version_block: True if this is a TORCH_FEATURE_VERSION >= TORCH_VERSION_X_Y_0 block | ||
| 42 | + # version_tuple: (major, minor) if is_version_block is True, else None | ||
| 43 | + self.preprocessor_stack: list[tuple[bool, tuple[int, int] | None]] = [] | ||
| 44 | + | ||
| 45 | + # Current version requirement (if inside a version block) | ||
| 46 | + self.version_of_block: tuple[int, int] | None = None | ||
| 47 | + | ||
| 48 | + # Track if we're inside a block comment | ||
| 49 | + self.in_block_comment: bool = False | ||
| 50 | + | ||
| 51 | + # Regex to match version conditions in #if or #elif | ||
| 52 | + self.version_pattern = re.compile( | ||
| 53 | + r"#(?:if|elif)\s+TORCH_FEATURE_VERSION\s*>=\s*TORCH_VERSION_(\d+)_(\d+)_\d+" | ||
| 54 | + ) | ||
| 55 | + | ||
| 56 | + def process_line(self, line: str) -> bool: | ||
| 57 | + """ | ||
| 58 | + Process a line and update the preprocessor state. | ||
| 59 | + | ||
| 60 | + Args: | ||
| 61 | + line: The line to process | ||
| 62 | + | ||
| 63 | + Returns: | ||
| 64 | + True if the line was processed (is a preprocessor directive or comment), | ||
| 65 | + False if it's a regular code line that should be further analyzed. | ||
| 66 | + """ | ||
| 67 | + stripped = line.strip() | ||
| 68 | + | ||
| 69 | + # Handle block comments (/* ... */) | ||
| 70 | + # Check if we're entering a block comment | ||
| 71 | + if "/*" in line: | ||
| 72 | + self.in_block_comment = True | ||
| 73 | + | ||
| 74 | + # If we're in a block comment, check if we're exiting | ||
| 75 | + if self.in_block_comment: | ||
| 76 | + if "*/" in line: | ||
| 77 | + self.in_block_comment = False | ||
| 78 | + return True # Skip the line if we're in a block comment | ||
| 79 | + | ||
| 80 | + # Skip line comments - they're not active code | ||
| 81 | + if stripped.startswith("//"): | ||
| 82 | + return True | ||
| 83 | + | ||
| 84 | + # Track #if directives | ||
| 85 | + if stripped.startswith("#if"): | ||
| 86 | + version_match = self.version_pattern.match(stripped) | ||
| 87 | + if version_match: | ||
| 88 | + major = int(version_match.group(1)) | ||
| 89 | + minor = int(version_match.group(2)) | ||
| 90 | + version_tuple = (major, minor) | ||
| 91 | + self.preprocessor_stack.append((True, version_tuple)) | ||
| 92 | + self.version_of_block = version_tuple | ||
| 93 | + else: | ||
| 94 | + # Regular #if (not a version block) | ||
| 95 | + self.preprocessor_stack.append((False, None)) | ||
| 96 | + return True | ||
| 97 | + | ||
| 98 | + # Track #ifdef and #ifndef directives (not version blocks) | ||
| 99 | + if stripped.startswith(("#ifdef", "#ifndef")): | ||
| 100 | + self.preprocessor_stack.append((False, None)) | ||
| 101 | + return True | ||
| 102 | + | ||
| 103 | + # Track #endif directives | ||
| 104 | + if stripped.startswith("#endif"): | ||
| 105 | + if self.preprocessor_stack: | ||
| 106 | + is_version_block, _ = self.preprocessor_stack.pop() | ||
| 107 | + if is_version_block: | ||
| 108 | + # Restore previous version block if any | ||
| 109 | + self.version_of_block = None | ||
| 110 | + for i in range(len(self.preprocessor_stack) - 1, -1, -1): | ||
| 111 | + if self.preprocessor_stack[i][0]: | ||
| 112 | + self.version_of_block = self.preprocessor_stack[i][1] | ||
| 113 | + break | ||
| 114 | + return True | ||
| 115 | + | ||
| 116 | + # Track #else directives | ||
| 117 | + # #else replaces the previous #if or #elif, so we pop and push | ||
| 118 | + if stripped.startswith("#else"): | ||
| 119 | + if self.preprocessor_stack: | ||
| 120 | + self.preprocessor_stack.pop() | ||
| 121 | + # #else is never versioned, so push (False, None) | ||
| 122 | + self.preprocessor_stack.append((False, None)) | ||
| 123 | + self.version_of_block = None | ||
| 124 | + return True | ||
| 125 | + | ||
| 126 | + # Track #elif directives | ||
| 127 | + # #elif replaces the previous #if or #elif, so we pop and push | ||
| 128 | + if stripped.startswith("#elif"): | ||
| 129 | + if self.preprocessor_stack: | ||
| 130 | + self.preprocessor_stack.pop() | ||
| 131 | + | ||
| 132 | + self.version_of_block = None | ||
| 133 | + | ||
| 134 | + # Check if this #elif has a version condition | ||
| 135 | + version_match_elif = self.version_pattern.match(stripped) | ||
| 136 | + if version_match_elif: | ||
| 137 | + major = int(version_match_elif.group(1)) | ||
| 138 | + minor = int(version_match_elif.group(2)) | ||
| 139 | + version_tuple = (major, minor) | ||
| 140 | + self.preprocessor_stack.append((True, version_tuple)) | ||
| 141 | + self.version_of_block = version_tuple | ||
| 142 | + else: | ||
| 143 | + # Not a version elif, treat as regular conditional | ||
| 144 | + self.preprocessor_stack.append((False, None)) | ||
| 145 | + return True | ||
| 146 | + | ||
| 147 | + # Not a preprocessor directive or comment | ||
| 148 | + return False | ||
| 149 | + | ||
| 150 | + def is_in_version_block(self) -> bool: | ||
| 151 | + """Check if currently inside any version block.""" | ||
| 152 | + return self.version_of_block is not None | ||
| 153 | + | ||
| 154 | + def get_version_of_block(self) -> tuple[int, int] | None: | ||
| 155 | + """Get the current version requirement, or None if not in a version block.""" | ||
| 156 | + return self.version_of_block | ||
| 157 | + | ||
| 158 | + | ||
| 159 | +class LintSeverity(str, Enum): | ||
| 160 | + ERROR = "error" | ||
| 161 | + WARNING = "warning" | ||
| 162 | + ADVICE = "advice" | ||
| 163 | + DISABLED = "disabled" | ||
| 164 | + | ||
| 165 | + | ||
| 166 | +class LintMessage(NamedTuple): | ||
| 167 | + path: str | None | ||
| 168 | + line: int | None | ||
| 169 | + char: int | None | ||
| 170 | + code: str | ||
| 171 | + severity: LintSeverity | ||
| 172 | + name: str | ||
| 173 | + original: str | None | ||
| 174 | + replacement: str | None | ||
| 175 | + description: str | None | ||
| 176 | + | ||
| 177 | + | ||
| 178 | +def get_current_version() -> tuple[int, int]: | ||
| 179 | + """ | ||
| 180 | + Get the current PyTorch version from version.txt. | ||
| 181 | + This uses the same logic as tools/setup_helpers/gen_version_header.py | ||
| 182 | + which is used to generate torch/headeronly/version.h from version.h.in. | ||
| 183 | + | ||
| 184 | + Returns (major, minor) tuple or None if not found. | ||
| 185 | + """ | ||
| 186 | + repo_root = Path(__file__).resolve().parents[3] | ||
| 187 | + version_file = repo_root / "version.txt" | ||
| 188 | + | ||
| 189 | + if not version_file.exists(): | ||
| 190 | + raise RuntimeError( | ||
| 191 | + "Could not find version.txt. This linter requires version.txt to run" | ||
| 192 | + ) | ||
| 193 | + | ||
| 194 | + with open(version_file) as f: | ||
| 195 | + version = f.read().strip() | ||
| 196 | + major, minor, patch = parse_version(version) | ||
| 197 | + | ||
| 198 | + return (major, minor) | ||
| 199 | + | ||
| 200 | + | ||
| 201 | +def get_added_lines(filename: str) -> set[int]: | ||
| 202 | + """ | ||
| 203 | + Get the line numbers of added lines in: | ||
| 204 | + 1. Current uncommitted changes (git diff HEAD) | ||
| 205 | + 2. All commits in the current PR (git diff merge-base..HEAD) | ||
| 206 | + | ||
| 207 | + This ensures that in CI we catch version macro issues across all PR commits. | ||
| 208 | + | ||
| 209 | + Returns: | ||
| 210 | + Set of line numbers (1-indexed) that are new additions. | ||
| 211 | + """ | ||
| 212 | + import subprocess | ||
| 213 | + | ||
| 214 | + added_lines = set() | ||
| 215 | + | ||
| 216 | + def parse_diff(diff_output: str) -> set[int]: | ||
| 217 | + """Parse git diff output and return line numbers of added lines.""" | ||
| 218 | + lines = set() | ||
| 219 | + current_line = 0 | ||
| 220 | + for line in diff_output.split("\n"): | ||
| 221 | + # Unified diff format: @@ -old_start,old_count +new_start,new_count @@ | ||
| 222 | + if line.startswith("@@"): | ||
| 223 | + match = re.search(r"\+(\d+)", line) | ||
| 224 | + if match: | ||
| 225 | + current_line = int(match.group(1)) | ||
| 226 | + elif line.startswith("+") and not line.startswith("+++"): | ||
| 227 | + # This is an added line | ||
| 228 | + lines.add(current_line) | ||
| 229 | + current_line += 1 | ||
| 230 | + elif not line.startswith("-"): | ||
| 231 | + # Context line or unchanged line | ||
| 232 | + current_line += 1 | ||
| 233 | + return lines | ||
| 234 | + | ||
| 235 | + try: | ||
| 236 | + # Check uncommitted changes (working directory vs HEAD) | ||
| 237 | + result = subprocess.run( | ||
| 238 | + ["git", "diff", "HEAD", filename], | ||
| 239 | + capture_output=True, | ||
| 240 | + text=True, | ||
| 241 | + timeout=5, | ||
| 242 | + ) | ||
| 243 | + if result.returncode == 0: | ||
| 244 | + added_lines.update(parse_diff(result.stdout)) | ||
| 245 | + | ||
| 246 | + # Get merge-base with origin/main to check all PR commits | ||
| 247 | + result = subprocess.run( | ||
| 248 | + ["git", "fetch", "origin", "main"], | ||
| 249 | + capture_output=True, | ||
| 250 | + text=True, | ||
| 251 | + timeout=600, | ||
| 252 | + ) | ||
| 253 | + if result.returncode != 0: | ||
| 254 | + raise RuntimeError( | ||
| 255 | + f"Failed to fetch origin. Error: {result.stderr.strip()}" | ||
| 256 | + ) | ||
| 257 | + | ||
| 258 | + result = subprocess.run( | ||
| 259 | + ["git", "merge-base", "HEAD", "origin/main"], | ||
| 260 | + capture_output=True, | ||
| 261 | + text=True, | ||
| 262 | + timeout=5, | ||
| 263 | + ) | ||
| 264 | + if result.returncode != 0: | ||
| 265 | + raise RuntimeError( | ||
| 266 | + f"Failed to find merge-base with origin/main. " | ||
| 267 | + f"Make sure origin/main exists (run 'git fetch origin main'). " | ||
| 268 | + f"Error: {result.stderr.strip()}" | ||
| 269 | + ) | ||
| 270 | + | ||
| 271 | + merge_base = result.stdout.strip() | ||
| 272 | + result = subprocess.run( | ||
| 273 | + ["git", "diff", f"{merge_base}..HEAD", filename], | ||
| 274 | + capture_output=True, | ||
| 275 | + text=True, | ||
| 276 | + timeout=5, | ||
| 277 | + ) | ||
| 278 | + if result.returncode != 0: | ||
| 279 | + raise RuntimeError( | ||
| 280 | + f"Failed to get git diff information for {filename}. Error: {result.stderr}" | ||
| 281 | + ) | ||
| 282 | + added_lines.update(parse_diff(result.stdout)) | ||
| 283 | + | ||
| 284 | + except Exception as e: | ||
| 285 | + raise RuntimeError( | ||
| 286 | + f"Failed to get git diff information for {filename}. Error: {e}" | ||
| 287 | + ) from e | ||
| 288 | + | ||
| 289 | + return added_lines | ||
| 290 | + | ||
| 291 | + | ||
| 292 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 293 | + """ | ||
| 294 | + Parse the stable/c/shim.h file and check that: | ||
| 295 | + 1. All function declarations are within TORCH_FEATURE_VERSION blocks | ||
| 296 | + 2. New functions added in this commit use the current version macro | ||
| 297 | + | ||
| 298 | + For the AOTI shim (torch/csrc/inductor/aoti_torch/c/shim.h), we only | ||
| 299 | + enforce versioning on NEW function declarations, since existing functions | ||
| 300 | + are intentionally not version-guarded. | ||
| 301 | + """ | ||
| 302 | + lint_messages: list[LintMessage] = [] | ||
| 303 | + | ||
| 304 | + # Check if this is the AOTI shim - only enforce versioning on new lines | ||
| 305 | + is_aoti_shim = "torch/csrc/inductor/aoti_torch/c/shim.h" in filename | ||
| 306 | + | ||
| 307 | + # Get current version | ||
| 308 | + current_version = get_current_version() | ||
| 309 | + major, minor = current_version | ||
| 310 | + expected_version_macro = f"TORCH_VERSION_{major}_{minor}_0" | ||
| 311 | + expected_version_check = f"#if TORCH_FEATURE_VERSION >= {expected_version_macro}" | ||
| 312 | + | ||
| 313 | + # Get lines that are uncommitted or added in the most recent commit | ||
| 314 | + added_lines = get_added_lines(filename) | ||
| 315 | + | ||
| 316 | + with open(filename) as f: | ||
| 317 | + lines = f.readlines() | ||
| 318 | + | ||
| 319 | + # Use PreprocessorTracker to handle preprocessor directives | ||
| 320 | + tracker = PreprocessorTracker() | ||
| 321 | + | ||
| 322 | + # Track extern "C" blocks separately | ||
| 323 | + inside_extern_c = False | ||
| 324 | + | ||
| 325 | + # Patterns for extern "C" blocks | ||
| 326 | + extern_c_pattern = re.compile(r'extern\s+"C"\s*{') | ||
| 327 | + extern_c_end_pattern = re.compile(r'}\s*//\s*extern\s+"C"') | ||
| 328 | + | ||
| 329 | + # Function declaration patterns - looking for AOTI_TORCH_EXPORT or typedef | ||
| 330 | + function_decl_patterns = [ | ||
| 331 | + re.compile(r"^\s*AOTI_TORCH_EXPORT\s+\w+"), # AOTI_TORCH_EXPORT functions | ||
| 332 | + re.compile(r"^\s*typedef\s+.*\(\*\w+\)"), # typedef function pointers | ||
| 333 | + re.compile(r"^\s*using\s+\w+\s*="), # using declarations | ||
| 334 | + ] | ||
| 335 | + | ||
| 336 | + for line_num, line in enumerate(lines, 1): | ||
| 337 | + stripped = line.strip() | ||
| 338 | + | ||
| 339 | + # Skip empty lines | ||
| 340 | + if not stripped: | ||
| 341 | + continue | ||
| 342 | + | ||
| 343 | + # Let the tracker process preprocessor directives and comments | ||
| 344 | + is_directive_or_comment = tracker.process_line(line) | ||
| 345 | + | ||
| 346 | + if is_directive_or_comment: | ||
| 347 | + continue | ||
| 348 | + | ||
| 349 | + # Track extern "C" blocks | ||
| 350 | + if extern_c_pattern.search(stripped): | ||
| 351 | + inside_extern_c = True | ||
| 352 | + continue | ||
| 353 | + if extern_c_end_pattern.search(stripped): | ||
| 354 | + inside_extern_c = False | ||
| 355 | + continue | ||
| 356 | + | ||
| 357 | + # Check for function declarations | ||
| 358 | + if inside_extern_c: | ||
| 359 | + is_function_decl = any( | ||
| 360 | + pattern.match(stripped) for pattern in function_decl_patterns | ||
| 361 | + ) | ||
| 362 | + | ||
| 363 | + if is_function_decl: | ||
| 364 | + # Check if this is a newly added line | ||
| 365 | + is_new_line = line_num in added_lines | ||
| 366 | + | ||
| 367 | + # Get current version state from tracker | ||
| 368 | + inside_version_block = tracker.is_in_version_block() | ||
| 369 | + tracker_version = tracker.get_version_of_block() | ||
| 370 | + version_of_block_macro = ( | ||
| 371 | + f"TORCH_VERSION_{tracker_version[0]}_{tracker_version[1]}_0" | ||
| 372 | + if tracker_version | ||
| 373 | + else None | ||
| 374 | + ) | ||
| 375 | + | ||
| 376 | + if not inside_version_block: | ||
| 377 | + # Function declaration outside of version block | ||
| 378 | + if not is_new_line: | ||
| 379 | + # Existing function declaration outside of version block in aoti shim is ignored | ||
| 380 | + if is_aoti_shim: | ||
| 381 | + continue | ||
| 382 | + expected_version_macro_str = "TORCH_VERSION_X_Y_Z" | ||
| 383 | + expected_version_check_str = ( | ||
| 384 | + f"#if TORCH_FEATURE_VERSION >= {expected_version_macro}" | ||
| 385 | + ) | ||
| 386 | + additional_text = "\nX, Y, and Z correspond to the TORCH_ABI_VERSION when the function was added." | ||
| 387 | + else: | ||
| 388 | + expected_version_macro_str = expected_version_macro | ||
| 389 | + expected_version_check_str = expected_version_check | ||
| 390 | + additional_text = "" | ||
| 391 | + lint_messages.append( | ||
| 392 | + LintMessage( | ||
| 393 | + path=filename, | ||
| 394 | + line=line_num, | ||
| 395 | + char=None, | ||
| 396 | + code=LINTER_CODE, | ||
| 397 | + severity=LintSeverity.ERROR, | ||
| 398 | + name="unversioned-function-declaration", | ||
| 399 | + original=None, | ||
| 400 | + replacement=None, | ||
| 401 | + description=( | ||
| 402 | + f"Function declaration found outside of TORCH_FEATURE_VERSION block. " | ||
| 403 | + f"All function declarations must be wrapped in:\n" | ||
| 404 | + f"{expected_version_check_str}\n" | ||
| 405 | + f"// ... your declarations ...\n" | ||
| 406 | + f"#endif // TORCH_FEATURE_VERSION >= {expected_version_macro_str}" | ||
| 407 | + f"{additional_text}" | ||
| 408 | + ), | ||
| 409 | + ) | ||
| 410 | + ) | ||
| 411 | + elif is_new_line and version_of_block_macro != expected_version_macro: | ||
| 412 | + # New function declaration using wrong version macro | ||
| 413 | + lint_messages.append( | ||
| 414 | + LintMessage( | ||
| 415 | + path=filename, | ||
| 416 | + line=line_num, | ||
| 417 | + char=None, | ||
| 418 | + code=LINTER_CODE, | ||
| 419 | + severity=LintSeverity.ERROR, | ||
| 420 | + name="wrong-version-for-new-function", | ||
| 421 | + original=None, | ||
| 422 | + replacement=None, | ||
| 423 | + description=( | ||
| 424 | + f"New function declaration should use {expected_version_macro}, " | ||
| 425 | + f"but is wrapped in {version_of_block_macro}. " | ||
| 426 | + f"New additions in this commit must use the current version:\n" | ||
| 427 | + f"{expected_version_check}\n" | ||
| 428 | + f"// ... your declarations ...\n" | ||
| 429 | + f"#endif // TORCH_FEATURE_VERSION >= {expected_version_macro}" | ||
| 430 | + ), | ||
| 431 | + ) | ||
| 432 | + ) | ||
| 433 | + | ||
| 434 | + return lint_messages | ||
| 435 | + | ||
| 436 | + | ||
| 437 | +if __name__ == "__main__": | ||
| 438 | + parser = argparse.ArgumentParser( | ||
| 439 | + description="stable shim version linter", | ||
| 440 | + fromfile_prefix_chars="@", | ||
| 441 | + ) | ||
| 442 | + parser.add_argument( | ||
| 443 | + "--verbose", | ||
| 444 | + action="store_true", | ||
| 445 | + ) | ||
| 446 | + parser.add_argument( | ||
| 447 | + "filenames", | ||
| 448 | + nargs="+", | ||
| 449 | + help="paths to lint", | ||
| 450 | + ) | ||
| 451 | + | ||
| 452 | + args = parser.parse_args() | ||
| 453 | + | ||
| 454 | + logging.basicConfig( | ||
| 455 | + format="<%(threadName)s:%(levelname)s> %(message)s", | ||
| 456 | + level=logging.NOTSET | ||
| 457 | + if args.verbose | ||
| 458 | + else logging.DEBUG | ||
| 459 | + if len(args.filenames) < 1000 | ||
| 460 | + else logging.INFO, | ||
| 461 | + stream=sys.stderr, | ||
| 462 | + ) | ||
| 463 | + | ||
| 464 | + lint_messages = [] | ||
| 465 | + for filename in args.filenames: | ||
| 466 | + lint_messages.extend(check_file(filename)) | ||
| 467 | + | ||
| 468 | + for lint_message in lint_messages: | ||
| 469 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| @@ -0,0 +1,245 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +""" | ||
| 3 | +This lint verifies that every Python test file (file that matches test_*.py or | ||
| 4 | +*_test.py in the test folder) has a cuda hard code in `requires_gpu()` or | ||
| 5 | +`requires_triton()` decorated function or `if HAS_GPU:` guarded main section, | ||
| 6 | +to ensure that the test not fail on other GPU devices. | ||
| 7 | +""" | ||
| 8 | + | ||
| 9 | +from __future__ import annotations | ||
| 10 | + | ||
| 11 | +import argparse | ||
| 12 | +import ast | ||
| 13 | +import json | ||
| 14 | +import multiprocessing as mp | ||
| 15 | +from enum import Enum | ||
| 16 | +from typing import NamedTuple | ||
| 17 | + | ||
| 18 | + | ||
| 19 | +LINTER_CODE = "TEST_DEVICE_BIAS" | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +class LintSeverity(str, Enum): | ||
| 23 | + ERROR = "error" | ||
| 24 | + WARNING = "warning" | ||
| 25 | + ADVICE = "advice" | ||
| 26 | + DISABLED = "disabled" | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +class LintMessage(NamedTuple): | ||
| 30 | + path: str | None | ||
| 31 | + line: int | None | ||
| 32 | + char: int | None | ||
| 33 | + code: str | ||
| 34 | + severity: LintSeverity | ||
| 35 | + name: str | ||
| 36 | + original: str | None | ||
| 37 | + replacement: str | None | ||
| 38 | + description: str | None | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +DEVICE_BIAS = ["cuda", "xpu", "mps"] | ||
| 42 | +GPU_RELATED_DECORATORS = {"requires_gpu", "requires_triton"} | ||
| 43 | + | ||
| 44 | + | ||
| 45 | +def is_main_has_gpu(tree: ast.AST) -> bool: | ||
| 46 | + def _contains_has_gpu(node: ast.AST) -> bool: | ||
| 47 | + if isinstance(node, ast.Name) and node.id in ["HAS_GPU", "RUN_GPU"]: | ||
| 48 | + return True | ||
| 49 | + elif isinstance(node, ast.BoolOp): | ||
| 50 | + return any(_contains_has_gpu(value) for value in node.values) | ||
| 51 | + elif isinstance(node, ast.UnaryOp): | ||
| 52 | + return _contains_has_gpu(node.operand) | ||
| 53 | + elif isinstance(node, ast.Compare): | ||
| 54 | + return _contains_has_gpu(node.left) or any( | ||
| 55 | + _contains_has_gpu(comp) for comp in node.comparators | ||
| 56 | + ) | ||
| 57 | + elif isinstance(node, (ast.IfExp, ast.Call)): | ||
| 58 | + return False | ||
| 59 | + return False | ||
| 60 | + | ||
| 61 | + for node in ast.walk(tree): | ||
| 62 | + # Detect if __name__ == "__main__": | ||
| 63 | + if isinstance(node, ast.If): | ||
| 64 | + if ( | ||
| 65 | + isinstance(node.test, ast.Compare) | ||
| 66 | + and isinstance(node.test.left, ast.Name) | ||
| 67 | + and node.test.left.id == "__name__" | ||
| 68 | + ): | ||
| 69 | + if any( | ||
| 70 | + isinstance(comp, ast.Constant) and comp.value == "__main__" | ||
| 71 | + for comp in node.test.comparators | ||
| 72 | + ): | ||
| 73 | + for inner_node in node.body: | ||
| 74 | + if isinstance(inner_node, ast.If) and _contains_has_gpu( | ||
| 75 | + inner_node.test | ||
| 76 | + ): | ||
| 77 | + return True | ||
| 78 | + return False | ||
| 79 | + | ||
| 80 | + | ||
| 81 | +class DeviceBiasVisitor(ast.NodeVisitor): | ||
| 82 | + def __init__(self, filename: str, is_gpu_test_suite: bool) -> None: | ||
| 83 | + self.filename = filename | ||
| 84 | + self.lint_messages: list[LintMessage] = [] | ||
| 85 | + self.is_gpu_test_suite = is_gpu_test_suite | ||
| 86 | + | ||
| 87 | + def _has_proper_decorator(self, node: ast.FunctionDef) -> bool: | ||
| 88 | + for d in node.decorator_list: | ||
| 89 | + if isinstance(d, ast.Name) and d.id in GPU_RELATED_DECORATORS: | ||
| 90 | + return True | ||
| 91 | + if ( | ||
| 92 | + isinstance(d, ast.Call) | ||
| 93 | + and isinstance(d.func, ast.Name) | ||
| 94 | + and d.func.id in GPU_RELATED_DECORATORS | ||
| 95 | + ): | ||
| 96 | + return True | ||
| 97 | + return False | ||
| 98 | + | ||
| 99 | + # check device = "cuda" or torch.device("cuda") | ||
| 100 | + def _check_keyword_device(self, subnode: ast.keyword, msg_prefix: str) -> None: | ||
| 101 | + if subnode.arg != "device": | ||
| 102 | + return | ||
| 103 | + val = subnode.value | ||
| 104 | + if isinstance(val, ast.Constant) and any( | ||
| 105 | + # pyrefly: ignore [not-iterable, unsupported-operation] | ||
| 106 | + bias in val.value | ||
| 107 | + for bias in DEVICE_BIAS | ||
| 108 | + ): | ||
| 109 | + self.record( | ||
| 110 | + subnode, | ||
| 111 | + f"{msg_prefix} device='{val.value}', suggest to use device=GPU_TYPE", | ||
| 112 | + ) | ||
| 113 | + elif isinstance(val, ast.Call): | ||
| 114 | + if ( | ||
| 115 | + isinstance(val.func, ast.Attribute) | ||
| 116 | + and val.func.attr == "device" | ||
| 117 | + and len(val.args) > 0 | ||
| 118 | + and isinstance(val.args[0], ast.Constant) | ||
| 119 | + # pyrefly: ignore [not-iterable, unsupported-operation] | ||
| 120 | + and any(bias in val.args[0].value for bias in DEVICE_BIAS) | ||
| 121 | + ): | ||
| 122 | + self.record( | ||
| 123 | + val, | ||
| 124 | + f"{msg_prefix} torch.device('{val.args[0].value}'), suggest to use torch.device(GPU_TYPE)", | ||
| 125 | + ) | ||
| 126 | + | ||
| 127 | + # check .cuda() or .to("cuda") | ||
| 128 | + def _check_device_methods(self, subnode: ast.Call, msg_prefix: str) -> None: | ||
| 129 | + func = subnode.func | ||
| 130 | + if not isinstance(func, ast.Attribute): | ||
| 131 | + return | ||
| 132 | + method_name = func.attr | ||
| 133 | + if method_name in DEVICE_BIAS: | ||
| 134 | + self.record( | ||
| 135 | + subnode, | ||
| 136 | + f"{msg_prefix} .{method_name}(), suggest to use .to(GPU_TYPE)", | ||
| 137 | + ) | ||
| 138 | + elif method_name == "to" and subnode.args: | ||
| 139 | + arg = subnode.args[0] | ||
| 140 | + if isinstance(arg, ast.Constant) and any( | ||
| 141 | + # pyrefly: ignore [not-iterable, unsupported-operation] | ||
| 142 | + bias in arg.value | ||
| 143 | + for bias in DEVICE_BIAS | ||
| 144 | + ): | ||
| 145 | + self.record( | ||
| 146 | + subnode, | ||
| 147 | + f"{msg_prefix} .to('{arg.value}'), suggest to use .to(GPU_TYPE)", | ||
| 148 | + ) | ||
| 149 | + | ||
| 150 | + def _check_with_statement(self, node: ast.With, msg_prefix: str) -> None: | ||
| 151 | + for item in node.items: | ||
| 152 | + ctx_expr = item.context_expr | ||
| 153 | + if isinstance(ctx_expr, ast.Call): | ||
| 154 | + func = ctx_expr.func | ||
| 155 | + if ( | ||
| 156 | + isinstance(func, ast.Attribute) | ||
| 157 | + and func.attr == "device" | ||
| 158 | + and isinstance(func.value, ast.Name) | ||
| 159 | + and func.value.id == "torch" | ||
| 160 | + and ctx_expr.args | ||
| 161 | + and isinstance(ctx_expr.args[0], ast.Constant) | ||
| 162 | + # pyrefly: ignore [not-iterable, unsupported-operation] | ||
| 163 | + and any(bias in ctx_expr.args[0].value for bias in DEVICE_BIAS) | ||
| 164 | + ): | ||
| 165 | + self.record( | ||
| 166 | + ctx_expr, | ||
| 167 | + f"{msg_prefix} `with torch.device('{ctx_expr.args[0].value}')`, suggest to use torch.device(GPU_TYPE)", | ||
| 168 | + ) | ||
| 169 | + | ||
| 170 | + def _check_node(self, node: ast.AST, msg_prefix: str) -> None: | ||
| 171 | + for subnode in ast.walk(node): | ||
| 172 | + if isinstance(subnode, ast.keyword): | ||
| 173 | + self._check_keyword_device(subnode, msg_prefix) | ||
| 174 | + elif isinstance(subnode, ast.Call) and isinstance( | ||
| 175 | + subnode.func, ast.Attribute | ||
| 176 | + ): | ||
| 177 | + self._check_device_methods(subnode, msg_prefix) | ||
| 178 | + elif isinstance(subnode, ast.With): | ||
| 179 | + self._check_with_statement(subnode, msg_prefix) | ||
| 180 | + | ||
| 181 | + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: | ||
| 182 | + if self._has_proper_decorator(node): | ||
| 183 | + msg_prefix = ( | ||
| 184 | + "`@requires_gpu` or `@requires_triton` function should not hardcode" | ||
| 185 | + ) | ||
| 186 | + self._check_node(node, msg_prefix) | ||
| 187 | + elif self.is_gpu_test_suite: | ||
| 188 | + # If the function is guarded by HAS_GPU in main(), we still need to check for device bias | ||
| 189 | + msg_prefix = "The test suites is shared amount GPUS, should not hardcode" | ||
| 190 | + self._check_node(node, msg_prefix) | ||
| 191 | + self.generic_visit(node) | ||
| 192 | + | ||
| 193 | + def record(self, node: ast.AST, message: str) -> None: | ||
| 194 | + self.lint_messages.append( | ||
| 195 | + LintMessage( | ||
| 196 | + path=self.filename, | ||
| 197 | + line=getattr(node, "lineno", None), | ||
| 198 | + char=None, | ||
| 199 | + code=LINTER_CODE, | ||
| 200 | + severity=LintSeverity.ERROR, | ||
| 201 | + name="[device-bias]", | ||
| 202 | + original=None, | ||
| 203 | + replacement=None, | ||
| 204 | + description=message, | ||
| 205 | + ) | ||
| 206 | + ) | ||
| 207 | + | ||
| 208 | + | ||
| 209 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 210 | + with open(filename) as f: | ||
| 211 | + source = f.read() | ||
| 212 | + tree = ast.parse(source, filename=filename) | ||
| 213 | + is_gpu_test_suite = is_main_has_gpu(tree) | ||
| 214 | + checker = DeviceBiasVisitor(filename, is_gpu_test_suite) | ||
| 215 | + checker.visit(tree) | ||
| 216 | + return checker.lint_messages | ||
| 217 | + | ||
| 218 | + | ||
| 219 | +def main() -> None: | ||
| 220 | + parser = argparse.ArgumentParser( | ||
| 221 | + description="Detect Device bias in functions decorated with requires_gpu/requires_triton" | ||
| 222 | + " or guarded by HAS_GPU block in main() that may break other GPU devices.", | ||
| 223 | + fromfile_prefix_chars="@", | ||
| 224 | + ) | ||
| 225 | + parser.add_argument( | ||
| 226 | + "filenames", | ||
| 227 | + nargs="+", | ||
| 228 | + help="paths to lint", | ||
| 229 | + ) | ||
| 230 | + | ||
| 231 | + args = parser.parse_args() | ||
| 232 | + | ||
| 233 | + with mp.Pool(8) as pool: | ||
| 234 | + lint_messages = pool.map(check_file, args.filenames) | ||
| 235 | + | ||
| 236 | + flat_lint_messages = [] | ||
| 237 | + for sublist in lint_messages: | ||
| 238 | + flat_lint_messages.extend(sublist) | ||
| 239 | + | ||
| 240 | + for lint_message in flat_lint_messages: | ||
| 241 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 242 | + | ||
| 243 | + | ||
| 244 | +if __name__ == "__main__": | ||
| 245 | + main() | ||
| @@ -0,0 +1,144 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "libcst", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +""" | ||
| 8 | +This lint verifies that every Python test file (file that matches test_*.py or | ||
| 9 | +*_test.py in the test folder) has a main block which raises an exception or | ||
| 10 | +calls run_tests to ensure that the test will be run in OSS CI. | ||
| 11 | + | ||
| 12 | +Takes ~2 minuters to run without the multiprocessing, probably overkill. | ||
| 13 | +""" | ||
| 14 | + | ||
| 15 | +from __future__ import annotations | ||
| 16 | + | ||
| 17 | +import argparse | ||
| 18 | +import json | ||
| 19 | +import multiprocessing as mp | ||
| 20 | +from enum import Enum | ||
| 21 | +from typing import NamedTuple | ||
| 22 | + | ||
| 23 | +# pyrefly: ignore [import-error] | ||
| 24 | +import libcst as cst | ||
| 25 | + | ||
| 26 | +# pyrefly: ignore [import-error] | ||
| 27 | +import libcst.matchers as m | ||
| 28 | + | ||
| 29 | + | ||
| 30 | +LINTER_CODE = "TEST_HAS_MAIN" | ||
| 31 | + | ||
| 32 | + | ||
| 33 | +class HasMainVisiter(cst.CSTVisitor): | ||
| 34 | + def __init__(self) -> None: | ||
| 35 | + super().__init__() | ||
| 36 | + self.found = False | ||
| 37 | + | ||
| 38 | + def visit_Module(self, node: cst.Module) -> bool: | ||
| 39 | + name = m.Name("__name__") | ||
| 40 | + main = m.SimpleString('"__main__"') | m.SimpleString("'__main__'") | ||
| 41 | + run_test_call = m.Call( | ||
| 42 | + func=m.Name("run_tests") | m.Attribute(attr=m.Name("run_tests")) | ||
| 43 | + ) | ||
| 44 | + # Distributed tests (i.e. MultiProcContinuousTest) calls `run_rank` | ||
| 45 | + # instead of `run_tests` in main | ||
| 46 | + run_rank_call = m.Call( | ||
| 47 | + func=m.Name("run_rank") | m.Attribute(attr=m.Name("run_rank")) | ||
| 48 | + ) | ||
| 49 | + raise_block = m.Raise() | ||
| 50 | + | ||
| 51 | + # name == main or main == name | ||
| 52 | + if_main1 = m.Comparison( | ||
| 53 | + name, | ||
| 54 | + [m.ComparisonTarget(m.Equal(), main)], | ||
| 55 | + ) | ||
| 56 | + if_main2 = m.Comparison( | ||
| 57 | + main, | ||
| 58 | + [m.ComparisonTarget(m.Equal(), name)], | ||
| 59 | + ) | ||
| 60 | + for child in node.children: | ||
| 61 | + if m.matches(child, m.If(test=if_main1 | if_main2)): | ||
| 62 | + if m.findall(child, raise_block | run_test_call | run_rank_call): | ||
| 63 | + self.found = True | ||
| 64 | + break | ||
| 65 | + | ||
| 66 | + return False | ||
| 67 | + | ||
| 68 | + | ||
| 69 | +class LintSeverity(str, Enum): | ||
| 70 | + ERROR = "error" | ||
| 71 | + WARNING = "warning" | ||
| 72 | + ADVICE = "advice" | ||
| 73 | + DISABLED = "disabled" | ||
| 74 | + | ||
| 75 | + | ||
| 76 | +class LintMessage(NamedTuple): | ||
| 77 | + path: str | None | ||
| 78 | + line: int | None | ||
| 79 | + char: int | None | ||
| 80 | + code: str | ||
| 81 | + severity: LintSeverity | ||
| 82 | + name: str | ||
| 83 | + original: str | None | ||
| 84 | + replacement: str | None | ||
| 85 | + description: str | None | ||
| 86 | + | ||
| 87 | + | ||
| 88 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 89 | + lint_messages = [] | ||
| 90 | + | ||
| 91 | + with open(filename) as f: | ||
| 92 | + file = f.read() | ||
| 93 | + v = HasMainVisiter() | ||
| 94 | + cst.parse_module(file).visit(v) | ||
| 95 | + if not v.found: | ||
| 96 | + message = ( | ||
| 97 | + "Test files need to have a main block which either calls run_tests " | ||
| 98 | + + "(to ensure that the tests are run during OSS CI) or raises an exception " | ||
| 99 | + + "and added to the blocklist in test/run_test.py" | ||
| 100 | + ) | ||
| 101 | + lint_messages.append( | ||
| 102 | + LintMessage( | ||
| 103 | + path=filename, | ||
| 104 | + line=None, | ||
| 105 | + char=None, | ||
| 106 | + code=LINTER_CODE, | ||
| 107 | + severity=LintSeverity.ERROR, | ||
| 108 | + name="[no-main]", | ||
| 109 | + original=None, | ||
| 110 | + replacement=None, | ||
| 111 | + description=message, | ||
| 112 | + ) | ||
| 113 | + ) | ||
| 114 | + return lint_messages | ||
| 115 | + | ||
| 116 | + | ||
| 117 | +def main() -> None: | ||
| 118 | + parser = argparse.ArgumentParser( | ||
| 119 | + description="test files should have main block linter", | ||
| 120 | + fromfile_prefix_chars="@", | ||
| 121 | + ) | ||
| 122 | + parser.add_argument( | ||
| 123 | + "filenames", | ||
| 124 | + nargs="+", | ||
| 125 | + help="paths to lint", | ||
| 126 | + ) | ||
| 127 | + | ||
| 128 | + args = parser.parse_args() | ||
| 129 | + | ||
| 130 | + pool = mp.Pool(8) | ||
| 131 | + lint_messages = pool.map(check_file, args.filenames) | ||
| 132 | + pool.close() | ||
| 133 | + pool.join() | ||
| 134 | + | ||
| 135 | + flat_lint_messages = [] | ||
| 136 | + for sublist in lint_messages: | ||
| 137 | + flat_lint_messages.extend(sublist) | ||
| 138 | + | ||
| 139 | + for lint_message in flat_lint_messages: | ||
| 140 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 141 | + | ||
| 142 | + | ||
| 143 | +if __name__ == "__main__": | ||
| 144 | + main() | ||
| @@ -0,0 +1,167 @@ | |||
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +""" | ||
| 3 | +Test ownership was introduced in https://github.com/pytorch/pytorch/issues/66232. | ||
| 4 | + | ||
| 5 | +This lint verifies that every Python test file (file that matches test_*.py or *_test.py in the test folder) | ||
| 6 | +has valid ownership information in a comment header. Valid means: | ||
| 7 | + - The format of the header follows the pattern "# Owner(s): ["list", "of owner", "labels"] | ||
| 8 | + - Each owner label actually exists in PyTorch | ||
| 9 | + - Each owner label starts with "module: " or "oncall: " or is in ACCEPTABLE_OWNER_LABELS | ||
| 10 | +""" | ||
| 11 | + | ||
| 12 | +from __future__ import annotations | ||
| 13 | + | ||
| 14 | +import argparse | ||
| 15 | +import json | ||
| 16 | +import urllib.error | ||
| 17 | +from enum import Enum | ||
| 18 | +from typing import Any, NamedTuple | ||
| 19 | +from urllib.request import urlopen | ||
| 20 | + | ||
| 21 | + | ||
| 22 | +LINTER_CODE = "TESTOWNERS" | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +class LintSeverity(str, Enum): | ||
| 26 | + ERROR = "error" | ||
| 27 | + WARNING = "warning" | ||
| 28 | + ADVICE = "advice" | ||
| 29 | + DISABLED = "disabled" | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +class LintMessage(NamedTuple): | ||
| 33 | + path: str | None | ||
| 34 | + line: int | None | ||
| 35 | + char: int | None | ||
| 36 | + code: str | ||
| 37 | + severity: LintSeverity | ||
| 38 | + name: str | ||
| 39 | + original: str | None | ||
| 40 | + replacement: str | None | ||
| 41 | + description: str | None | ||
| 42 | + | ||
| 43 | + | ||
| 44 | +def get_pytorch_labels() -> Any: | ||
| 45 | + url = "https://ossci-metrics.s3.amazonaws.com/pytorch_labels.json" | ||
| 46 | + try: | ||
| 47 | + labels = urlopen(url).read().decode("utf-8") | ||
| 48 | + except urllib.error.URLError: | ||
| 49 | + # This is an FB-only hack, if the json isn't available we may | ||
| 50 | + # need to use a forwarding proxy to get out | ||
| 51 | + proxy_url = "http://fwdproxy:8080" | ||
| 52 | + proxy_handler = urllib.request.ProxyHandler( | ||
| 53 | + {"http": proxy_url, "https": proxy_url} | ||
| 54 | + ) | ||
| 55 | + context = urllib.request.build_opener(proxy_handler) | ||
| 56 | + labels = context.open(url).read().decode("utf-8") | ||
| 57 | + return json.loads(labels) | ||
| 58 | + | ||
| 59 | + | ||
| 60 | +PYTORCH_LABELS = get_pytorch_labels() | ||
| 61 | +# Team/owner labels usually start with "module: " or "oncall: ", but the following are acceptable exceptions | ||
| 62 | +ACCEPTABLE_OWNER_LABELS = ["NNC", "high priority"] | ||
| 63 | +OWNERS_PREFIX = "# Owner(s): " | ||
| 64 | +GLOB_EXCEPTIONS = ["**/test/run_test.py"] | ||
| 65 | + | ||
| 66 | + | ||
| 67 | +def check_labels( | ||
| 68 | + labels: list[str], filename: str, line_number: int | ||
| 69 | +) -> list[LintMessage]: | ||
| 70 | + lint_messages = [] | ||
| 71 | + for label in labels: | ||
| 72 | + if label not in PYTORCH_LABELS: | ||
| 73 | + lint_messages.append( | ||
| 74 | + LintMessage( | ||
| 75 | + path=filename, | ||
| 76 | + line=line_number, | ||
| 77 | + char=None, | ||
| 78 | + code=LINTER_CODE, | ||
| 79 | + severity=LintSeverity.ERROR, | ||
| 80 | + name="[invalid-label]", | ||
| 81 | + original=None, | ||
| 82 | + replacement=None, | ||
| 83 | + description=( | ||
| 84 | + f"{label} is not a PyTorch label " | ||
| 85 | + "(please choose from https://github.com/pytorch/pytorch/labels)" | ||
| 86 | + ), | ||
| 87 | + ) | ||
| 88 | + ) | ||
| 89 | + | ||
| 90 | + if label.startswith(("module:", "oncall:")) or label in ACCEPTABLE_OWNER_LABELS: | ||
| 91 | + continue | ||
| 92 | + | ||
| 93 | + lint_messages.append( | ||
| 94 | + LintMessage( | ||
| 95 | + path=filename, | ||
| 96 | + line=line_number, | ||
| 97 | + char=None, | ||
| 98 | + code=LINTER_CODE, | ||
| 99 | + severity=LintSeverity.ERROR, | ||
| 100 | + name="[invalid-owner]", | ||
| 101 | + original=None, | ||
| 102 | + replacement=None, | ||
| 103 | + description=( | ||
| 104 | + f"{label} is not an acceptable owner " | ||
| 105 | + "(please update to another label or edit ACCEPTABLE_OWNERS_LABELS " | ||
| 106 | + "in tools/linters/adapters/testowners_linter.py)" | ||
| 107 | + ), | ||
| 108 | + ) | ||
| 109 | + ) | ||
| 110 | + | ||
| 111 | + return lint_messages | ||
| 112 | + | ||
| 113 | + | ||
| 114 | +def check_file(filename: str) -> list[LintMessage]: | ||
| 115 | + lint_messages = [] | ||
| 116 | + has_ownership_info = False | ||
| 117 | + | ||
| 118 | + with open(filename) as f: | ||
| 119 | + for idx, line in enumerate(f): | ||
| 120 | + if not line.startswith(OWNERS_PREFIX): | ||
| 121 | + continue | ||
| 122 | + | ||
| 123 | + has_ownership_info = True | ||
| 124 | + labels = json.loads(line[len(OWNERS_PREFIX) :]) | ||
| 125 | + lint_messages.extend(check_labels(labels, filename, idx + 1)) | ||
| 126 | + | ||
| 127 | + if has_ownership_info is False: | ||
| 128 | + lint_messages.append( | ||
| 129 | + LintMessage( | ||
| 130 | + path=filename, | ||
| 131 | + line=None, | ||
| 132 | + char=None, | ||
| 133 | + code=LINTER_CODE, | ||
| 134 | + severity=LintSeverity.ERROR, | ||
| 135 | + name="[no-owner-info]", | ||
| 136 | + original=None, | ||
| 137 | + replacement=None, | ||
| 138 | + description="Missing a comment header with ownership information.", | ||
| 139 | + ) | ||
| 140 | + ) | ||
| 141 | + | ||
| 142 | + return lint_messages | ||
| 143 | + | ||
| 144 | + | ||
| 145 | +def main() -> None: | ||
| 146 | + parser = argparse.ArgumentParser( | ||
| 147 | + description="test ownership linter", | ||
| 148 | + fromfile_prefix_chars="@", | ||
| 149 | + ) | ||
| 150 | + parser.add_argument( | ||
| 151 | + "filenames", | ||
| 152 | + nargs="+", | ||
| 153 | + help="paths to lint", | ||
| 154 | + ) | ||
| 155 | + | ||
| 156 | + args = parser.parse_args() | ||
| 157 | + lint_messages = [] | ||
| 158 | + | ||
| 159 | + for filename in args.filenames: | ||
| 160 | + lint_messages.extend(check_file(filename)) | ||
| 161 | + | ||
| 162 | + for lint_message in lint_messages: | ||
| 163 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 164 | + | ||
| 165 | + | ||
| 166 | +if __name__ == "__main__": | ||
| 167 | + main() | ||
| @@ -0,0 +1,98 @@ | |||
| 1 | +"""Uploads a new binary to s3 and updates its hash in the config file. | ||
| 2 | + | ||
| 3 | +You'll need to have appropriate credentials on the PyTorch AWS buckets, see: | ||
| 4 | +https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration | ||
| 5 | +for how to configure them. | ||
| 6 | +""" | ||
| 7 | + | ||
| 8 | +import argparse | ||
| 9 | +import hashlib | ||
| 10 | +import json | ||
| 11 | +import logging | ||
| 12 | +import os | ||
| 13 | + | ||
| 14 | +import boto3 # type: ignore[import] | ||
| 15 | + | ||
| 16 | + | ||
| 17 | +def compute_file_sha256(path: str) -> str: | ||
| 18 | + """Compute the SHA256 hash of a file and return it as a hex string.""" | ||
| 19 | + # If the file doesn't exist, return an empty string. | ||
| 20 | + if not os.path.exists(path): | ||
| 21 | + return "" | ||
| 22 | + | ||
| 23 | + hash = hashlib.sha256() | ||
| 24 | + | ||
| 25 | + # Open the file in binary mode and hash it. | ||
| 26 | + with open(path, "rb") as f: | ||
| 27 | + for b in f: | ||
| 28 | + hash.update(b) | ||
| 29 | + | ||
| 30 | + # Return the hash as a hexadecimal string. | ||
| 31 | + return hash.hexdigest() | ||
| 32 | + | ||
| 33 | + | ||
| 34 | +def main() -> None: | ||
| 35 | + parser = argparse.ArgumentParser( | ||
| 36 | + description="s3 binary updater", | ||
| 37 | + fromfile_prefix_chars="@", | ||
| 38 | + ) | ||
| 39 | + parser.add_argument( | ||
| 40 | + "--config-json", | ||
| 41 | + required=True, | ||
| 42 | + help="path to config json that you are trying to update", | ||
| 43 | + ) | ||
| 44 | + parser.add_argument( | ||
| 45 | + "--linter", | ||
| 46 | + required=True, | ||
| 47 | + help="name of linter you're trying to update", | ||
| 48 | + ) | ||
| 49 | + parser.add_argument( | ||
| 50 | + "--platform", | ||
| 51 | + required=True, | ||
| 52 | + help="which platform you are uploading the binary for", | ||
| 53 | + ) | ||
| 54 | + parser.add_argument( | ||
| 55 | + "--file", | ||
| 56 | + required=True, | ||
| 57 | + help="file to upload", | ||
| 58 | + ) | ||
| 59 | + parser.add_argument( | ||
| 60 | + "--dry-run", | ||
| 61 | + action="store_true", | ||
| 62 | + help="if set, don't actually upload/write hash", | ||
| 63 | + ) | ||
| 64 | + args = parser.parse_args() | ||
| 65 | + logging.basicConfig(level=logging.INFO) | ||
| 66 | + with open(args.config_json) as f: | ||
| 67 | + config = json.load(f) | ||
| 68 | + linter_config = config[args.linter][args.platform] | ||
| 69 | + bucket = linter_config["s3_bucket"] | ||
| 70 | + object_name = linter_config["object_name"] | ||
| 71 | + | ||
| 72 | + # Upload the file | ||
| 73 | + logging.info( | ||
| 74 | + "Uploading file %s to s3 bucket: %s, object name: %s", | ||
| 75 | + args.file, | ||
| 76 | + bucket, | ||
| 77 | + object_name, | ||
| 78 | + ) | ||
| 79 | + if not args.dry_run: | ||
| 80 | + s3_client = boto3.client("s3") | ||
| 81 | + s3_client.upload_file(args.file, bucket, object_name) | ||
| 82 | + | ||
| 83 | + # Update hash in repo | ||
| 84 | + hash_of_new_binary = compute_file_sha256(args.file) | ||
| 85 | + logging.info("Computed new hash for binary %s", hash_of_new_binary) | ||
| 86 | + | ||
| 87 | + linter_config["hash"] = hash_of_new_binary | ||
| 88 | + config_dump = json.dumps(config, indent=4, sort_keys=True) | ||
| 89 | + | ||
| 90 | + logging.info("Writing out new config:") | ||
| 91 | + logging.info(config_dump) | ||
| 92 | + if not args.dry_run: | ||
| 93 | + with open(args.config_json, "w") as f: | ||
| 94 | + f.write(config_dump) | ||
| 95 | + | ||
| 96 | + | ||
| 97 | +if __name__ == "__main__": | ||
| 98 | + main() | ||
| @@ -0,0 +1,191 @@ | |||
| 1 | +# /// script | ||
| 2 | +# requires-python = ">=3.10" | ||
| 3 | +# dependencies = [ | ||
| 4 | +# "pyyaml==6.0.2", | ||
| 5 | +# ] | ||
| 6 | +# /// | ||
| 7 | +"""Checks for consistency of jobs between different GitHub workflows. | ||
| 8 | + | ||
| 9 | +Any job with a specific `sync-tag` must match all other jobs with the same `sync-tag`. | ||
| 10 | +""" | ||
| 11 | + | ||
| 12 | +from __future__ import annotations | ||
| 13 | + | ||
| 14 | +import argparse | ||
| 15 | +import itertools | ||
| 16 | +import json | ||
| 17 | +from collections import defaultdict | ||
| 18 | +from enum import Enum | ||
| 19 | +from pathlib import Path | ||
| 20 | +from typing import Any, NamedTuple, TYPE_CHECKING | ||
| 21 | + | ||
| 22 | +from yaml import dump, load | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +if TYPE_CHECKING: | ||
| 29 | + from collections.abc import Iterable | ||
| 30 | + | ||
| 31 | + | ||
| 32 | +# Safely load fast C Yaml loader/dumper if they are available | ||
| 33 | +try: | ||
| 34 | + from yaml import CSafeLoader as Loader | ||
| 35 | +except ImportError: | ||
| 36 | + from yaml import SafeLoader as Loader # type: ignore[assignment, misc] | ||
| 37 | + | ||
| 38 | + | ||
| 39 | +class LintSeverity(str, Enum): | ||
| 40 | + ERROR = "error" | ||
| 41 | + WARNING = "warning" | ||
| 42 | + ADVICE = "advice" | ||
| 43 | + DISABLED = "disabled" | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +class LintMessage(NamedTuple): | ||
| 47 | + path: str | None | ||
| 48 | + line: int | None | ||
| 49 | + char: int | None | ||
| 50 | + code: str | ||
| 51 | + severity: LintSeverity | ||
| 52 | + name: str | ||
| 53 | + original: str | None | ||
| 54 | + replacement: str | None | ||
| 55 | + description: str | None | ||
| 56 | + | ||
| 57 | + | ||
| 58 | +def glob_yamls(path: Path) -> Iterable[Path]: | ||
| 59 | + return itertools.chain(path.glob("**/*.yml"), path.glob("**/*.yaml")) | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def load_yaml(path: Path) -> Any: | ||
| 63 | + with open(path) as f: | ||
| 64 | + return load(f, Loader) | ||
| 65 | + | ||
| 66 | + | ||
| 67 | +def is_workflow(yaml: Any) -> bool: | ||
| 68 | + return yaml.get("jobs") is not None | ||
| 69 | + | ||
| 70 | + | ||
| 71 | +def print_lint_message( | ||
| 72 | + path: Path, | ||
| 73 | + job: dict[str, Any], | ||
| 74 | + sync_tag: str, | ||
| 75 | + baseline_path: Path, | ||
| 76 | + baseline_job_id: str, | ||
| 77 | +) -> None: | ||
| 78 | + job_id = next(iter(job.keys())) | ||
| 79 | + with open(path) as f: | ||
| 80 | + lines = f.readlines() | ||
| 81 | + for i, line in enumerate(lines): | ||
| 82 | + if f"{job_id}:" in line: | ||
| 83 | + line_number = i + 1 | ||
| 84 | + | ||
| 85 | + lint_message = LintMessage( | ||
| 86 | + path=str(path), | ||
| 87 | + # pyrefly: ignore [unbound-name] | ||
| 88 | + line=line_number, | ||
| 89 | + char=None, | ||
| 90 | + code="WORKFLOWSYNC", | ||
| 91 | + severity=LintSeverity.ERROR, | ||
| 92 | + name="workflow-inconsistency", | ||
| 93 | + original=None, | ||
| 94 | + replacement=None, | ||
| 95 | + description=f"Job doesn't match other job {baseline_job_id} in file {baseline_path} with sync-tag: '{sync_tag}'", | ||
| 96 | + ) | ||
| 97 | + print(json.dumps(lint_message._asdict()), flush=True) | ||
| 98 | + | ||
| 99 | + | ||
| 100 | +def get_jobs_with_sync_tag( | ||
| 101 | + job: dict[str, Any], | ||
| 102 | +) -> tuple[str, str, dict[str, Any]] | None: | ||
| 103 | + sync_tag = job.get("with", {}).get("sync-tag") | ||
| 104 | + if sync_tag is None: | ||
| 105 | + return None | ||
| 106 | + | ||
| 107 | + # remove the "if" field, which we allow to be different between jobs | ||
| 108 | + # (since you might have different triggering conditions on pull vs. | ||
| 109 | + # trunk, say.) | ||
| 110 | + if "if" in job: | ||
| 111 | + del job["if"] | ||
| 112 | + | ||
| 113 | + # same is true for ['with']['test-matrix'] | ||
| 114 | + if "test-matrix" in job.get("with", {}): | ||
| 115 | + del job["with"]["test-matrix"] | ||
| 116 | + # and ['with']['tests-to-include'], since dispatch filters differ | ||
| 117 | + if "tests-to-include" in job.get("with", {}): | ||
| 118 | + del job["with"]["tests-to-include"] | ||
| 119 | + # and ['with']['build-environment'], since GPU-specific suffixes differ for ROCm | ||
| 120 | + if ( | ||
| 121 | + "build-environment" in job.get("with", {}) | ||
| 122 | + and "rocm" in job["with"]["build-environment"] | ||
| 123 | + ): | ||
| 124 | + del job["with"]["build-environment"] | ||
| 125 | + # and ['name'], since ROCm jobs append a GPU-specific suffix to the job name | ||
| 126 | + if "name" in job and "rocm" in job.get("name", ""): | ||
| 127 | + del job["name"] | ||
| 128 | + | ||
| 129 | + # normalize needs: remove helper job-filter so comparisons ignore it | ||
| 130 | + needs = job.get("needs") | ||
| 131 | + if needs: | ||
| 132 | + needs_list = [needs] if isinstance(needs, str) else list(needs) | ||
| 133 | + needs_list = [n for n in needs_list if n != "job-filter"] | ||
| 134 | + if not needs_list: | ||
| 135 | + job.pop("needs", None) | ||
| 136 | + elif len(needs_list) == 1: | ||
| 137 | + job["needs"] = needs_list[0] | ||
| 138 | + else: | ||
| 139 | + job["needs"] = needs_list | ||
| 140 | + | ||
| 141 | + return (sync_tag, job_id, job) | ||
| 142 | + | ||
| 143 | + | ||
| 144 | +if __name__ == "__main__": | ||
| 145 | + parser = argparse.ArgumentParser( | ||
| 146 | + description="workflow consistency linter.", | ||
| 147 | + fromfile_prefix_chars="@", | ||
| 148 | + ) | ||
| 149 | + parser.add_argument( | ||
| 150 | + "filenames", | ||
| 151 | + nargs="+", | ||
| 152 | + help="paths to lint", | ||
| 153 | + ) | ||
| 154 | + args = parser.parse_args() | ||
| 155 | + | ||
| 156 | + # Go through all files, aggregating jobs with the same sync tag | ||
| 157 | + tag_to_jobs = defaultdict(list) | ||
| 158 | + for path in REPO_ROOT.glob(".github/workflows/*"): | ||
| 159 | + if not path.is_file() or path.suffix not in {".yml", ".yaml"}: | ||
| 160 | + continue | ||
| 161 | + workflow = load_yaml(path) | ||
| 162 | + if not is_workflow(workflow): | ||
| 163 | + continue | ||
| 164 | + clean_path = path.relative_to(REPO_ROOT) | ||
| 165 | + jobs = workflow.get("jobs", {}) | ||
| 166 | + for job_id, job in jobs.items(): | ||
| 167 | + res = get_jobs_with_sync_tag(job) | ||
| 168 | + if res is None: | ||
| 169 | + continue | ||
| 170 | + sync_tag, job_id, job_dict = res | ||
| 171 | + tag_to_jobs[sync_tag].append((clean_path, job_id, job_dict)) | ||
| 172 | + | ||
| 173 | + # Check the files passed as arguments | ||
| 174 | + for path in args.filenames: | ||
| 175 | + workflow = load_yaml(Path(path)) | ||
| 176 | + jobs = workflow["jobs"] | ||
| 177 | + for job_id, job in jobs.items(): | ||
| 178 | + res = get_jobs_with_sync_tag(job) | ||
| 179 | + if res is None: | ||
| 180 | + continue | ||
| 181 | + sync_tag, job_id, job_dict = res | ||
| 182 | + job_str = dump(job_dict) | ||
| 183 | + | ||
| 184 | + # For each sync tag, check that all the jobs have the same code. | ||
| 185 | + for baseline_path, baseline_job_id, baseline_dict in tag_to_jobs[sync_tag]: | ||
| 186 | + baseline_str = dump(baseline_dict) | ||
| 187 | + | ||
| 188 | + if job_id != baseline_job_id or job_str != baseline_str: | ||
| 189 | + print_lint_message( | ||
| 190 | + path, job_dict, sync_tag, baseline_path, baseline_job_id | ||
| 191 | + ) | ||
| @@ -0,0 +1,68 @@ | |||
| 1 | +aLoad | ||
| 2 | +aLoads | ||
| 3 | +ans | ||
| 4 | +aStore | ||
| 5 | +aStores | ||
| 6 | +belows | ||
| 7 | +bLoad | ||
| 8 | +bLoads | ||
| 9 | +bStore | ||
| 10 | +bStores | ||
| 11 | +BU | ||
| 12 | +contiguities | ||
| 13 | +contiguity | ||
| 14 | +coo | ||
| 15 | +DEPENDEES | ||
| 16 | +deser | ||
| 17 | +din | ||
| 18 | +dout | ||
| 19 | +ElementE | ||
| 20 | +followings | ||
| 21 | +fro | ||
| 22 | +froms | ||
| 23 | +Halfs | ||
| 24 | +hsa | ||
| 25 | +indexT | ||
| 26 | +inH | ||
| 27 | +inp | ||
| 28 | +inps | ||
| 29 | +inpt | ||
| 30 | +inpts | ||
| 31 | +mata | ||
| 32 | +matb | ||
| 33 | +matc | ||
| 34 | +nd | ||
| 35 | +nin | ||
| 36 | +NotIn | ||
| 37 | +nout | ||
| 38 | +NowNs | ||
| 39 | +numer | ||
| 40 | +OffsetT | ||
| 41 | +oH | ||
| 42 | +optins | ||
| 43 | +ot | ||
| 44 | +overrideable | ||
| 45 | +oW | ||
| 46 | +padD | ||
| 47 | +posIn | ||
| 48 | +ptd | ||
| 49 | +rebuild | ||
| 50 | +rebuilt | ||
| 51 | +reenable | ||
| 52 | +reenabled | ||
| 53 | +requestor | ||
| 54 | +ser | ||
| 55 | +serde | ||
| 56 | +serder | ||
| 57 | +serdes | ||
| 58 | +sme | ||
| 59 | +statics | ||
| 60 | +strat | ||
| 61 | +subtile | ||
| 62 | +subtiles | ||
| 63 | +supercede | ||
| 64 | +supercedes | ||
| 65 | +te | ||
| 66 | +THW | ||
| 67 | +tne | ||
| 68 | +WONT | ||
| @@ -0,0 +1,98 @@ | |||
| 1 | +from __future__ import annotations | ||
| 2 | + | ||
| 3 | +import os | ||
| 4 | +import platform | ||
| 5 | +import struct | ||
| 6 | +from itertools import chain | ||
| 7 | +from typing import cast, TYPE_CHECKING | ||
| 8 | + | ||
| 9 | + | ||
| 10 | +if TYPE_CHECKING: | ||
| 11 | + from collections.abc import Iterable | ||
| 12 | + | ||
| 13 | + | ||
| 14 | +CMAKE_MINIMUM_VERSION_STRING = "3.27" | ||
| 15 | + | ||
| 16 | +IS_WINDOWS = platform.system() == "Windows" | ||
| 17 | +IS_DARWIN = platform.system() == "Darwin" | ||
| 18 | +IS_LINUX = platform.system() == "Linux" | ||
| 19 | + | ||
| 20 | +IS_64BIT = struct.calcsize("P") == 8 | ||
| 21 | + | ||
| 22 | +BUILD_DIR = "build" | ||
| 23 | + | ||
| 24 | + | ||
| 25 | +def check_env_flag(name: str, default: str = "") -> bool: | ||
| 26 | + return os.getenv(name, default).upper() in ["ON", "1", "YES", "TRUE", "Y"] | ||
| 27 | + | ||
| 28 | + | ||
| 29 | +def check_negative_env_flag(name: str, default: str = "") -> bool: | ||
| 30 | + return os.getenv(name, default).upper() in ["OFF", "0", "NO", "FALSE", "N"] | ||
| 31 | + | ||
| 32 | + | ||
| 33 | +def gather_paths(env_vars: Iterable[str]) -> list[str]: | ||
| 34 | + return list(chain(*(os.getenv(v, "").split(os.pathsep) for v in env_vars))) | ||
| 35 | + | ||
| 36 | + | ||
| 37 | +def lib_paths_from_base(base_path: str) -> list[str]: | ||
| 38 | + return [os.path.join(base_path, s) for s in ["lib/x64", "lib", "lib64"]] | ||
| 39 | + | ||
| 40 | + | ||
| 41 | +# We promised that CXXFLAGS should also be affected by CFLAGS | ||
| 42 | +if "CFLAGS" in os.environ and "CXXFLAGS" not in os.environ: | ||
| 43 | + os.environ["CXXFLAGS"] = os.environ["CFLAGS"] | ||
| 44 | + | ||
| 45 | + | ||
| 46 | +class BuildType: | ||
| 47 | + """Checks build type. The build type will be given in :attr:`cmake_build_type_env`. If :attr:`cmake_build_type_env` | ||
| 48 | + is ``None``, then the build type will be inferred from ``CMakeCache.txt``. If ``CMakeCache.txt`` does not exist, | ||
| 49 | + os.environ['CMAKE_BUILD_TYPE'] will be used. | ||
| 50 | + | ||
| 51 | + Args: | ||
| 52 | + cmake_build_type_env (str): The value of os.environ['CMAKE_BUILD_TYPE']. If None, the actual build type will be | ||
| 53 | + inferred. | ||
| 54 | + | ||
| 55 | + """ | ||
| 56 | + | ||
| 57 | + def __init__(self, cmake_build_type_env: str | None = None) -> None: | ||
| 58 | + if cmake_build_type_env is not None: | ||
| 59 | + self.build_type_string = cmake_build_type_env | ||
| 60 | + return | ||
| 61 | + | ||
| 62 | + cmake_cache_txt = os.path.join(BUILD_DIR, "CMakeCache.txt") | ||
| 63 | + if os.path.isfile(cmake_cache_txt): | ||
| 64 | + # Found CMakeCache.txt. Use the build type specified in it. | ||
| 65 | + from .cmake_utils import get_cmake_cache_variables_from_file | ||
| 66 | + | ||
| 67 | + with open(cmake_cache_txt) as f: | ||
| 68 | + cmake_cache_vars = get_cmake_cache_variables_from_file(f) | ||
| 69 | + # Normally it is anti-pattern to determine build type from CMAKE_BUILD_TYPE because it is not used for | ||
| 70 | + # multi-configuration build tools, such as Visual Studio and XCode. But since we always communicate with | ||
| 71 | + # CMake using CMAKE_BUILD_TYPE from our Python scripts, this is OK here. | ||
| 72 | + self.build_type_string = cast(str, cmake_cache_vars["CMAKE_BUILD_TYPE"]) | ||
| 73 | + else: | ||
| 74 | + self.build_type_string = os.environ.get("CMAKE_BUILD_TYPE", "Release") | ||
| 75 | + | ||
| 76 | + def is_debug(self) -> bool: | ||
| 77 | + "Checks Debug build." | ||
| 78 | + return self.build_type_string == "Debug" | ||
| 79 | + | ||
| 80 | + def is_rel_with_deb_info(self) -> bool: | ||
| 81 | + "Checks RelWithDebInfo build." | ||
| 82 | + return self.build_type_string == "RelWithDebInfo" | ||
| 83 | + | ||
| 84 | + def is_release(self) -> bool: | ||
| 85 | + "Checks Release build." | ||
| 86 | + return self.build_type_string == "Release" | ||
| 87 | + | ||
| 88 | + | ||
| 89 | +# hotpatch environment variable 'CMAKE_BUILD_TYPE'. 'CMAKE_BUILD_TYPE' always prevails over DEBUG or REL_WITH_DEB_INFO. | ||
| 90 | +if "CMAKE_BUILD_TYPE" not in os.environ: | ||
| 91 | + if check_env_flag("DEBUG"): | ||
| 92 | + os.environ["CMAKE_BUILD_TYPE"] = "Debug" | ||
| 93 | + elif check_env_flag("REL_WITH_DEB_INFO"): | ||
| 94 | + os.environ["CMAKE_BUILD_TYPE"] = "RelWithDebInfo" | ||
| 95 | + else: | ||
| 96 | + os.environ["CMAKE_BUILD_TYPE"] = "Release" | ||
| 97 | + | ||
| 98 | +build_type = BuildType() | ||
| @@ -0,0 +1,92 @@ | |||
| 1 | +# Ideally, there would be a way in Bazel to parse version.txt | ||
| 2 | +# and use the version numbers from there as substitutions for | ||
| 3 | +# an expand_template action. Since there isn't, this silly script exists. | ||
| 4 | + | ||
| 5 | +from __future__ import annotations | ||
| 6 | + | ||
| 7 | +import argparse | ||
| 8 | +import os | ||
| 9 | +from typing import cast | ||
| 10 | + | ||
| 11 | + | ||
| 12 | +Version = tuple[int, int, int] | ||
| 13 | + | ||
| 14 | + | ||
| 15 | +def parse_version(version: str) -> Version: | ||
| 16 | + """ | ||
| 17 | + Parses a version string into (major, minor, patch) version numbers. | ||
| 18 | + | ||
| 19 | + Args: | ||
| 20 | + version: Full version number string, possibly including revision / commit hash. | ||
| 21 | + | ||
| 22 | + Returns: | ||
| 23 | + An int 3-tuple of (major, minor, patch) version numbers. | ||
| 24 | + """ | ||
| 25 | + # Extract version number part (i.e. toss any revision / hash parts). | ||
| 26 | + version_number_str = version | ||
| 27 | + for i in range(len(version)): | ||
| 28 | + c = version[i] | ||
| 29 | + if not (c.isdigit() or c == "."): | ||
| 30 | + version_number_str = version[:i] | ||
| 31 | + break | ||
| 32 | + | ||
| 33 | + return cast(Version, tuple([int(n) for n in version_number_str.split(".")])) | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def apply_replacements(replacements: dict[str, str], text: str) -> str: | ||
| 37 | + """ | ||
| 38 | + Applies the given replacements within the text. | ||
| 39 | + | ||
| 40 | + Args: | ||
| 41 | + replacements (dict): Mapping of str -> str replacements. | ||
| 42 | + text (str): Text in which to make replacements. | ||
| 43 | + | ||
| 44 | + Returns: | ||
| 45 | + Text with replacements applied, if any. | ||
| 46 | + """ | ||
| 47 | + for before, after in replacements.items(): | ||
| 48 | + text = text.replace(before, after) | ||
| 49 | + return text | ||
| 50 | + | ||
| 51 | + | ||
| 52 | +def main(args: argparse.Namespace) -> None: | ||
| 53 | + with open(args.version_path) as f: | ||
| 54 | + version = f.read().strip() | ||
| 55 | + (major, minor, patch) = parse_version(version) | ||
| 56 | + | ||
| 57 | + replacements = { | ||
| 58 | + "@TORCH_VERSION_MAJOR@": str(major), | ||
| 59 | + "@TORCH_VERSION_MINOR@": str(minor), | ||
| 60 | + "@TORCH_VERSION_PATCH@": str(patch), | ||
| 61 | + } | ||
| 62 | + | ||
| 63 | + # Create the output dir if it doesn't exist. | ||
| 64 | + os.makedirs(os.path.dirname(args.output_path), exist_ok=True) | ||
| 65 | + | ||
| 66 | + with open(args.template_path) as input: | ||
| 67 | + with open(args.output_path, "w") as output: | ||
| 68 | + for line in input: | ||
| 69 | + output.write(apply_replacements(replacements, line)) | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +if __name__ == "__main__": | ||
| 73 | + parser = argparse.ArgumentParser( | ||
| 74 | + description="Generate version.h from version.h.in template", | ||
| 75 | + ) | ||
| 76 | + parser.add_argument( | ||
| 77 | + "--template-path", | ||
| 78 | + required=True, | ||
| 79 | + help="Path to the template (i.e. version.h.in)", | ||
| 80 | + ) | ||
| 81 | + parser.add_argument( | ||
| 82 | + "--version-path", | ||
| 83 | + required=True, | ||
| 84 | + help="Path to the file specifying the version", | ||
| 85 | + ) | ||
| 86 | + parser.add_argument( | ||
| 87 | + "--output-path", | ||
| 88 | + required=True, | ||
| 89 | + help="Output path for expanded template (i.e. version.h)", | ||
| 90 | + ) | ||
| 91 | + args = parser.parse_args() | ||
| 92 | + main(args) | ||