From dd9f436a36486b4cb8a4c70a2321548b0be09b8f Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 27 Jun 2026 10:46:34 -0400
Subject: [PATCH] Normalize Unicode form when matching MANIFEST.in patterns
FileList matched MANIFEST.in patterns against on-disk names byte-for-byte
with no Unicode normalization. On macOS APFS/HFS+, a file stored NFD and a
pattern authored NFC denote the same file but differ byte-for-byte, so an
exclude/global-exclude/recursive-exclude/prune rule could silently fail to
drop a non-ASCII-named file, publishing it in the sdist despite the rule.
Normalize both the pattern and the candidate path to NFC before matching,
via a new unicode_utils.normalize() helper and a _NormalizedMatcher wrapper
around the compiled pattern in translate_pattern.
Fixes GHSA-h35f-9h28-mq5c.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
setuptools/command/egg_info.py | 28 +++++++++++-
setuptools/tests/test_manifest.py | 45 +++++++++++++++++++
setuptools/unicode_utils.py | 14 ++++++
3 files changed, 86 insertions(+), 1 deletion(-)
@@ -37,6 +37,27 @@
PY_MAJOR = f'{sys.version_info.major}.{sys.version_info.minor}'
+class _NormalizedMatcher:
+ """
+ Wrap a compiled pattern so that matching is insensitive to Unicode
+ normalization form.
+
+ File names walked from disk (NFD on macOS APFS/HFS+) and patterns from
+ ``MANIFEST.in`` (typically NFC) can denote the same file while differing
+ byte-for-byte. Normalizing both sides before matching keeps an exclusion
+ (or inclusion) from silently failing. See GHSA-h35f-9h28-mq5c.
+ """
+
+ def __init__(self, pattern: re.Pattern) -> None:
+ self._pattern = pattern
+
+ def match(self, path):
+ return self._pattern.match(unicode_utils.normalize(path))
+
+ def search(self, path):
+ return self._pattern.search(unicode_utils.normalize(path))
+
+
def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
"""
Translate a file path glob like '*.txt' in to a regular expression.
@@ -46,6 +67,11 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
"""
pat = ''
+ # Normalize the pattern so it matches paths regardless of the Unicode
+ # normalization form used on disk (GHSA-h35f-9h28-mq5c). Candidate paths
+ # are normalized to the same form by ``_NormalizedMatcher``.
+ glob = unicode_utils.normalize(glob)
+
# This will split on '/' within [character classes]. This is deliberate.
chunks = glob.split(os.path.sep)
@@ -117,7 +143,7 @@ def translate_pattern(glob): # noqa: C901 # is too complex (14) # FIXME
pat += sep
pat += r'\Z'
- return re.compile(pat, flags=re.MULTILINE | re.DOTALL)
+ return _NormalizedMatcher(re.compile(pat, flags=re.MULTILINE | re.DOTALL))
class InfoCommon:
@@ -10,6 +10,7 @@
import shutil
import sys
import tempfile
+import unicodedata
import pytest
@@ -157,6 +158,21 @@ def test_translated_pattern_mismatch(pattern_mismatch):
assert not translate_pattern(pattern).match(target)
+def test_translate_pattern_unicode_normalization():
+ """
+ Matching is insensitive to Unicode normalization form: a pattern authored
+ in one form matches a path stored on disk in another (and vice versa), so
+ that an exclusion cannot be bypassed by an NFC/NFD mismatch.
+
+ Regression test for GHSA-h35f-9h28-mq5c.
+ """
+ nfc = unicodedata.normalize('NFC', 'café.txt') # 'café.txt' composed
+ nfd = unicodedata.normalize('NFD', 'café.txt') # 'café.txt' decomposed
+ assert nfc != nfd # the two byte forms genuinely differ
+ assert translate_pattern(nfc).match(nfd)
+ assert translate_pattern(nfd).match(nfc)
+
+
class TempDirTestCase:
def setup_method(self, method):
self.temp_dir = tempfile.mkdtemp()
@@ -331,6 +347,35 @@ def test_graft_prune(self):
files = default_files | set([ml('app/a.txt'), ml('app/b.txt'), ml('app/c.rst')])
assert files == self.get_files()
+ def test_global_exclude_unicode_normalization(self):
+ """
+ A ``global-exclude`` authored NFC must drop a file whose on-disk name
+ is NFD: on macOS APFS/HFS+ the two are the same file, and even on
+ case/normalization-exact filesystems the decomposed name can be
+ committed and reach the build. Otherwise the file is published in the
+ sdist despite the exclusion.
+
+ Regression test for GHSA-h35f-9h28-mq5c.
+ """
+ nfc_name = unicodedata.normalize('NFC', 'café.txt')
+ nfd_name = unicodedata.normalize('NFD', 'café.txt')
+ assert nfc_name != nfd_name
+ # write the file under its decomposed (NFD) name ...
+ touch(os.path.join(self.temp_dir, 'app', nfd_name))
+ # ... and exclude it with the composed (NFC) form.
+ self.make_manifest(
+ f"""
+ global-include *.txt
+ global-exclude {nfc_name}
+ """
+ )
+ leaked = {
+ f
+ for f in self.get_files()
+ if unicodedata.normalize('NFC', os.path.basename(f)) == nfc_name
+ }
+ assert not leaked, f"excluded file leaked into manifest: {leaked}"
+
class TestFileListTest(TempDirTestCase):
"""
@@ -19,6 +19,20 @@ def decompose(path):
return path
+def normalize(text):
+ """
+ Return *text* in a canonical Unicode form (NFC) so that names which are
+ visually identical but encoded differently compare equal.
+
+ macOS APFS/HFS+ store file names in decomposed form (NFD), while patterns
+ in ``MANIFEST.in`` are typically authored composed (NFC). The two denote
+ the same file but differ byte-for-byte, so matching them directly lets an
+ exclusion silently fail. Normalizing both the walked path and the pattern
+ to a single form before matching avoids that (GHSA-h35f-9h28-mq5c).
+ """
+ return unicodedata.normalize('NFC', text) if isinstance(text, str) else text
+
+
def filesys_decode(path):
"""
Ensure that the given path is decoded,