check-peps.py

#!/usr/bin/env python3

# This file is placed in the public domain or under the
# CC0-1.0-Universal license, whichever is more permissive.

"""check-peps: Check PEPs for common mistakes.

Usage: check-peps [-d | --detailed] <PEP files...>

Only the PEPs specified are checked.
If none are specified, all PEPs are checked.

Use "--detailed" to show the contents of lines where errors were found.
"""

from __future__ importannotations

importdatetimeasdt
importre
importsys
frompathlibimportPath

TYPE_CHECKING=False
ifTYPE_CHECKING:
fromcollections.abcimportIterable, Iterator, KeysView, Sequence
fromtypingimportTypeAlias

# (line number, warning message)
Message: TypeAlias=tuple[int, str]
MessageIterator: TypeAlias=Iterator[Message]


# get the directory with the PEP sources
ROOT_DIR=Path(__file__).resolve().parent
PEP_ROOT=ROOT_DIR/"peps"

# See PEP 12 for the order
# Note we retain "BDFL-Delegate"
ALL_HEADERS= (
"PEP",
"Title",
"Author",
"Sponsor",
"BDFL-Delegate", "PEP-Delegate",
"Discussions-To",
"Status",
"Type",
"Topic",
"Requires",
"Created",
"Python-Version",
"Post-History",
"Replaces",
"Superseded-By",
"Resolution",
)
REQUIRED_HEADERS=frozenset({"PEP", "Title", "Author", "Status", "Type", "Created"})

# See PEP 1 for the full list
ALL_STATUSES=frozenset({
"Accepted",
"Active",
"April Fool!",
"Deferred",
"Draft",
"Final",
"Provisional",
"Rejected",
"Superseded",
"Withdrawn",
})

# PEPs that are allowed to link directly to PEPs
SKIP_DIRECT_PEP_LINK_CHECK=frozenset({"0009", "0287", "0676", "0684", "8001"})

DEFAULT_FLAGS=re.ASCII|re.IGNORECASE# Insensitive latin

# any sequence of letters or '-', followed by a single ':' and a space or end of line
HEADER_PATTERN=re.compile(r"^([a-z\-]+):(?: |$)", DEFAULT_FLAGS)
# any sequence of unicode letters or legal special characters
NAME_PATTERN=re.compile(r"(?:[^\W\d_]|[ ',\-.])+(?: |$)")
# any sequence of ASCII letters, digits, or legal special characters
EMAIL_LOCAL_PART_PATTERN=re.compile(r"[\w!#$%&'*+\-/=?^{|}~.]+", DEFAULT_FLAGS)

DISCOURSE_THREAD_PATTERN=re.compile(r"([\w\-]+/)?\d+", DEFAULT_FLAGS)
DISCOURSE_POST_PATTERN=re.compile(r"([\w\-]+/)?\d+(/\d+)?", DEFAULT_FLAGS)

MAILMAN_2_PATTERN=re.compile(r"[\w\-]+/\d{4}-[a-z]+/\d+\.html", DEFAULT_FLAGS)
MAILMAN_3_THREAD_PATTERN=re.compile(r"[\w\-]+@python\.org/thread/[a-z0-9]+/?", DEFAULT_FLAGS)
MAILMAN_3_MESSAGE_PATTERN=re.compile(r"[\w\-]+@python\.org/message/[a-z0-9]+/?(#[a-z0-9]+)?", DEFAULT_FLAGS)

# Controlled by the "--detailed" flag
DETAILED_ERRORS=False


defcheck(filenames: Sequence[str] = (), /) ->int:
"""The main entry-point."""
iffilenames:
filenames=map(Path, filenames)
else:
filenames=PEP_ROOT.glob("pep-????.rst")
if (count:=sum(map(check_file, filenames))) >0:
s="s"* (count!=1)
print(f"check-peps failed: {count} error{s}", file=sys.stderr)
return1
return0


defcheck_file(filename: Path, /) ->int:
filename=filename.resolve()
try:
content=filename.read_text(encoding="utf-8")
exceptFileNotFoundError:
return_output_error(filename, [""], [(0, "Could not read PEP!")])
else:
lines=content.splitlines()
return_output_error(filename, lines, check_peps(filename, lines))


defcheck_peps(filename: Path, lines: Sequence[str], /) ->MessageIterator:
yieldfromcheck_headers(lines)
forline_num, lineinenumerate(lines, start=1):
iffilename.stem.removeprefix("pep-") inSKIP_DIRECT_PEP_LINK_CHECK:
continue
yieldfromcheck_direct_links(line_num, line.lstrip())


defcheck_headers(lines: Sequence[str], /) ->MessageIterator:
yieldfrom_validate_pep_number(next(iter(lines), ""))

found_headers= {}
found_header_lines: list[tuple[str, int]] = []
line_num=0
forline_num, lineinenumerate(lines, start=1):
ifline.strip() =="":
headers_end_line_num=line_num
break
ifmatch:=HEADER_PATTERN.match(line):
header=match[1]
found_header_lines.append((header, line_num))
ifheaderinALL_HEADERS:
ifheadernotinfound_headers:
found_headers[header] =None
else:
yieldline_num, f"Must not have duplicate header: {header} "
else:
yieldline_num, f"Must not have invalid header: {header}"
else:
headers_end_line_num=line_num

yieldfrom_validate_required_headers(found_headers.keys())

shifted_line_nums= [linefor_, lineinfound_header_lines[1:]]
fori, (header, line_num) inenumerate(found_header_lines):
start=line_num-1
end=headers_end_line_num-1
ifi<len(found_header_lines) -1:
end=shifted_line_nums[i] -1
remainder="\n".join(lines[start:end]).removeprefix(f"{header}:")
ifremainder!="":
ifremainder[0] notin {" ", "\n"}:
yieldline_num, f"Headers must have a space after the colon: {header}"
remainder=remainder.lstrip()
yieldfrom_validate_header(header, line_num, remainder)


def_validate_header(header: str, line_num: int, content: str) ->MessageIterator:
ifheader=="Title":
yieldfrom_validate_title(line_num, content)
elifheader=="Author":
yieldfrom_validate_author(line_num, content)
elifheader=="Sponsor":
yieldfrom_validate_sponsor(line_num, content)
elifheaderin {"BDFL-Delegate", "PEP-Delegate"}:
yieldfrom_validate_delegate(line_num, content)
elifheader=="Discussions-To":
yieldfrom_validate_discussions_to(line_num, content)
elifheader=="Status":
yieldfrom_validate_status(line_num, content)
elifheader=="Type":
yieldfrom_validate_type(line_num, content)
elifheader=="Topic":
yieldfrom_validate_topic(line_num, content)
elifheaderin {"Requires", "Replaces", "Superseded-By"}:
yieldfrom_validate_pep_references(line_num, content)
elifheader=="Created":
yieldfrom_validate_created(line_num, content)
elifheader=="Python-Version":
yieldfrom_validate_python_version(line_num, content)
elifheader=="Post-History":
yieldfrom_validate_post_history(line_num, content)
elifheader=="Resolution":
yieldfrom_validate_resolution(line_num, content)


defcheck_direct_links(line_num: int, line: str) ->MessageIterator:
"""Check that PEPs and RFCs aren't linked directly"""

line=line.lower()
if"dev/peps/pep-"inlineor"peps.python.org/pep-"inline:
yieldline_num, "Use the :pep:`NNN` role to refer to PEPs"
if"rfc-editor.org/rfc/"inlineor"ietf.org/doc/html/rfc"inline:
yieldline_num, "Use the :rfc:`NNN` role to refer to RFCs"


def_output_error(filename: Path, lines: Sequence[str], errors: Iterable[Message]) ->int:
relative_filename=filename.relative_to(ROOT_DIR)
err_count=0
forline_num, msginerrors:
err_count+=1

print(f"{relative_filename}:{line_num}: {msg}")
ifnotDETAILED_ERRORS:
continue

line=lines[line_num-1]
print(" |")
print(f"{line_num: >4} | '{line}'")
print(" |")

returnerr_count


###########################
# PEP Header Validators #
###########################


def_validate_required_headers(found_headers: KeysView[str]) ->MessageIterator:
"""PEPs must have all required headers, in the PEP 12 order"""

ifmissing:=REQUIRED_HEADERS.difference(found_headers):
formissing_headerinsorted(missing, key=ALL_HEADERS.index):
yield1, f"Must have required header: {missing_header}"

ordered_headers=sorted(found_headers, key=ALL_HEADERS.index)
iflist(found_headers) !=ordered_headers:
order_str=", ".join(ordered_headers)
yield1, "Headers must be in PEP 12 order. Correct order: "+order_str


def_validate_pep_number(line: str) ->MessageIterator:
"""'PEP' header must be a number 1-9999"""

ifnotline.startswith("PEP: "):
yield1, "PEP must begin with the 'PEP:' header"
return

pep_number=line.removeprefix("PEP: ").lstrip()
yieldfrom_pep_num(1, pep_number, "'PEP:' header")


def_validate_title(line_num: int, line: str) ->MessageIterator:
"""'Title' must be 1-79 characters"""

iflen(line) ==0:
yieldline_num, "PEP must have a title"
eliflen(line) >79:
yieldline_num, "PEP title must be less than 80 characters"


def_validate_author(line_num: int, body: str) ->MessageIterator:
"""'Author' must be list of 'Name <email@example.com>, …'"""

lines=body.split("\n")
foroffset, lineinenumerate(lines):
ifoffset>=1andline[:9].isspace():
# Checks for:
# Author: Alice
# Bob
# ^^^^
# Note that len("Author: ") == 8
yieldline_num+offset, "Author line must not be over-indented"
ifoffset<len(lines) -1:
ifnotline.endswith(","):
yieldline_num+offset, "Author continuation lines must end with a comma"
forpartinline.removesuffix(",").split(", "):
yieldfrom_email(line_num+offset, part, "Author")


def_validate_sponsor(line_num: int, line: str) ->MessageIterator:
"""'Sponsor' must have format 'Name <email@example.com>'"""

yieldfrom_email(line_num, line, "Sponsor")


def_validate_delegate(line_num: int, line: str) ->MessageIterator:
"""'Delegate' must have format 'Name <email@example.com>'"""

ifline=="":
return

# PEP 451
if", "inline:
forpartinline.removesuffix(",").split(", "):
yieldfrom_email(line_num, part, "Delegate")
return

yieldfrom_email(line_num, line, "Delegate")


def_validate_discussions_to(line_num: int, line: str) ->MessageIterator:
"""'Discussions-To' must be a thread URL"""

yieldfrom_thread(line_num, line, "Discussions-To", discussions_to=True)
ifline=="Pending":
return
ifline.startswith("https://"):
return
forsuffixin"@python.org", "@googlegroups.com":
ifline.endswith(suffix):
remainder=line.removesuffix(suffix)
ifre.fullmatch(r"[\w\-]+", remainder) isNone:
yieldline_num, "Discussions-To must be a valid mailing list"
return
yieldline_num, "Discussions-To must be a valid thread URL, mailing list, or 'Pending'"


def_validate_status(line_num: int, line: str) ->MessageIterator:
"""'Status' must be a valid PEP status"""

iflinenotinALL_STATUSES:
yieldline_num, "Status must be a valid PEP status"


def_validate_type(line_num: int, line: str) ->MessageIterator:
"""'Type' must be a valid PEP type"""

iflinenotin {"Standards Track", "Informational", "Process"}:
yieldline_num, "Type must be a valid PEP type"


def_validate_topic(line_num: int, line: str) ->MessageIterator:
"""'Topic' must be for a valid sub-index"""

topics=line.split(", ")
unique_topics=set(topics)
iflen(topics) >len(unique_topics):
yieldline_num, "Topic must not contain duplicates"

ifunique_topics- {"Governance", "Packaging", "Typing", "Release"}:
ifnotall(map(str.istitle, unique_topics)):
yieldline_num, "Topic must be properly capitalised (Title Case)"
ifunique_topics- {"governance", "packaging", "typing", "release"}:
yieldline_num, "Topic must be for a valid sub-index"
ifsorted(topics) !=topics:
yieldline_num, "Topic must be sorted lexicographically"


def_validate_pep_references(line_num: int, line: str) ->MessageIterator:
"""`Requires`/`Replaces`/`Superseded-By` must be 'NNN' PEP IDs"""

line=line.removesuffix(",").rstrip()
ifline.count(", ") !=line.count(","):
yieldline_num, "PEP references must be separated by comma-spaces (', ')"
return

references=line.split(", ")
forreferenceinreferences:
yieldfrom_pep_num(line_num, reference, "PEP reference")


def_validate_created(line_num: int, line: str) ->MessageIterator:
"""'Created' must be a 'DD-mmm-YYYY' date"""

yieldfrom_date(line_num, line, "Created")


def_validate_python_version(line_num: int, line: str) ->MessageIterator:
"""'Python-Version' must be an ``X.Y[.Z]`` version"""

versions=line.split(", ")
forversioninversions:
ifversion.count(".") notin {1, 2}:
yieldline_num, f"Python-Version must have two or three segments: {version}"
continue

try:
major, minor, micro=version.split(".", 2)
exceptValueError:
major, minor=version.split(".", 1)
micro=""

ifmajornotin"123":
yieldline_num, f"Python-Version major part must be 1, 2, or 3: {version}"
ifnot_is_digits(minor) andminor!="x":
yieldline_num, f"Python-Version minor part must be numeric: {version}"
elifminor!="0"andminor[0] =="0":
yieldline_num, f"Python-Version minor part must not have leading zeros: {version}"

ifmicro=="":
return
ifminor=="x":
yieldline_num, f"Python-Version micro part must be empty if minor part is 'x': {version}"
elifmicro[0] =="0":
yieldline_num, f"Python-Version micro part must not have leading zeros: {version}"
elifnot_is_digits(micro):
yieldline_num, f"Python-Version micro part must be numeric: {version}"


def_validate_post_history(line_num: int, body: str) ->MessageIterator:
"""'Post-History' must be '`DD-mmm-YYYY <Thread URL>`__, …' or `DD-mmm-YYYY`"""

ifbody=="":
return

foroffset, lineinenumerate(body.removesuffix(",").split("\n"), start=line_num):
forpostinline.removesuffix(",").strip().split(", "):
prefix, postfix= (post.startswith("`"), post.endswith(">`__"))
ifnotprefixandnotpostfix:
yieldfrom_date(offset, post, "Post-History")
elifprefixandpostfix:
post_date, post_url=post[1:-4].split(" <")
yieldfrom_date(offset, post_date, "Post-History")
yieldfrom_thread(offset, post_url, "Post-History")
else:
yieldoffset, "post line must be a date or both start with “`” and end with “>`__”"


def_validate_resolution(line_num: int, line: str) ->MessageIterator:
"""'Resolution' must be a direct thread/message URL or a link with a date."""

prefix, postfix= (line.startswith("`"), line.endswith(">`__"))
ifnotprefixandnotpostfix:
yieldfrom_thread(line_num, line, "Resolution", allow_message=True)
elifprefixandpostfix:
post_date, post_url=line[1:-4].split(" <")
yieldfrom_date(line_num, post_date, "Resolution")
yieldfrom_thread(line_num, post_url, "Resolution", allow_message=True)
else:
yieldline_num, "Resolution line must be a link or both start with “`” and end with “>`__”"


########################
# Validation Helpers #
########################


def_pep_num(line_num: int, pep_number: str, prefix: str) ->MessageIterator:
ifpep_number=="":
yieldline_num, f"{prefix} must not be blank: {pep_number!r}"
return
ifpep_number.startswith("0") andpep_number!="0":
yieldline_num, f"{prefix} must not contain leading zeros: {pep_number!r}"
ifnot_is_digits(pep_number):
yieldline_num, f"{prefix} must be numeric: {pep_number!r}"
elifnot0<=int(pep_number) <=9999:
yieldline_num, f"{prefix} must be between 0 and 9999: {pep_number!r}"


def_is_digits(string: str) ->bool:
"""Match a string of ASCII digits ([0-9]+)."""
returnstring.isascii() andstring.isdigit()


def_email(line_num: int, author_email: str, prefix: str) ->MessageIterator:
author_email=author_email.strip()

ifauthor_email.count("<") >1:
msg=f"{prefix} entries must not contain multiple '<': {author_email!r}"
yieldline_num, msg
ifauthor_email.count(">") >1:
msg=f"{prefix} entries must not contain multiple '>': {author_email!r}"
yieldline_num, msg
ifauthor_email.count("@") >1:
msg=f"{prefix} entries must not contain multiple '@': {author_email!r}"
yieldline_num, msg

author=author_email.split("<", 1)[0].rstrip()
ifNAME_PATTERN.fullmatch(author) isNone:
msg=f"{prefix} entries must begin with a valid 'Name': {author_email!r}"
yieldline_num, msg
return

email_text=author_email.removeprefix(author)
ifnotemail_text:
# Does not have the optional email part
return

ifnotemail_text.startswith(" <") ornotemail_text.endswith(">"):
msg=f"{prefix} entries must be formatted as 'Name <email@example.com>': {author_email!r}"
yieldline_num, msg
email_text=email_text.removeprefix(" <").removesuffix(">")

if"@"inemail_text:
local, domain=email_text.rsplit("@", 1)
elif" at "inemail_text:
local, domain=email_text.rsplit(" at ", 1)
else:
yieldline_num, f"{prefix} entries must contain a valid email address: {author_email!r}"
return
ifEMAIL_LOCAL_PART_PATTERN.fullmatch(local) isNoneor_invalid_domain(domain):
yieldline_num, f"{prefix} entries must contain a valid email address: {author_email!r}"


def_invalid_domain(domain_part: str) ->bool:
*labels, root=domain_part.split(".")
forlabelinlabels:
ifnotlabel.replace("-", "").isalnum():
returnTrue
returnnotroot.isalnum() ornotroot.isascii()


def_thread(line_num: int, url: str, prefix: str, *, allow_message: bool=False, discussions_to: bool=False) ->MessageIterator:
ifallow_messageanddiscussions_to:
msg="allow_message and discussions_to cannot both be True"
raiseValueError(msg)

msg=f"{prefix} must be a valid thread URL"

ifnoturl.startswith("https://"):
ifnotdiscussions_to:
yieldline_num, msg
return

ifurl.startswith("https://discuss.python.org/t/"):
remainder=url.removeprefix("https://discuss.python.org/t/").removesuffix("/")

# Discussions-To links must be the thread itself, not a post
ifdiscussions_to:
# The equivalent pattern is similar to '([\w\-]+/)?\d+',
# but the topic name must contain a non-numeric character

# We use ``str.rpartition`` as the topic name is optional
topic_name, _, topic_id=remainder.rpartition("/")
iftopic_name==''and_is_digits(topic_id):
return
topic_name=topic_name.replace("-", "0").replace("_", "0")
# the topic name must not be entirely numeric
valid_topic_name=not_is_digits(topic_name) andtopic_name.isalnum()
ifvalid_topic_nameand_is_digits(topic_id):
return
else:
# The equivalent pattern is similar to '([\w\-]+/)?\d+(/\d+)?',
# but the topic name must contain a non-numeric character
ifremainder.count("/") ==2:
# When there are three parts, the URL must be "topic-name/topic-id/post-id".
topic_name, topic_id, post_id=remainder.rsplit("/", 2)
topic_name=topic_name.replace("-", "0").replace("_", "0")
valid_topic_name=not_is_digits(topic_name) andtopic_name.isalnum()
ifvalid_topic_nameand_is_digits(topic_id) and_is_digits(post_id):
# the topic name must not be entirely numeric
return
elifremainder.count("/") ==1:
# When there are only two parts, there's an ambiguity between
# "topic-name/topic-id" and "topic-id/post-id".
# We disambiguate by checking if the LHS is a valid name and
# the RHS is a valid topic ID (for the former),
# and then if both the LHS and RHS are valid IDs (for the latter).
left, right=remainder.rsplit("/")
left=left.replace("-", "0").replace("_", "0")
# the topic name must not be entirely numeric
left_is_name=not_is_digits(left) andleft.isalnum()
ifleft_is_nameand_is_digits(right):
return
elif_is_digits(left) and_is_digits(right):
return
else:
# When there's only one part, it must be a valid topic ID.
if_is_digits(remainder):
return

ifurl.startswith("https://mail.python.org/pipermail/"):
remainder=url.removeprefix("https://mail.python.org/pipermail/")
ifMAILMAN_2_PATTERN.fullmatch(remainder) isnotNone:
return

ifurl.startswith("https://mail.python.org/archives/list/"):
remainder=url.removeprefix("https://mail.python.org/archives/list/")
ifallow_messageandMAILMAN_3_MESSAGE_PATTERN.fullmatch(remainder) isnotNone:
return
ifMAILMAN_3_THREAD_PATTERN.fullmatch(remainder) isnotNone:
return

yieldline_num, msg


def_date(line_num: int, date_str: str, prefix: str) ->MessageIterator:
try:
parsed_date=dt.datetime.strptime(date_str, "%d-%b-%Y")
exceptValueError:
yieldline_num, f"{prefix} must be a 'DD-mmm-YYYY' date: {date_str!r}"
return
else:
ifdate_str[1] =="-": # Date must be zero-padded
yieldline_num, f"{prefix} must be a 'DD-mmm-YYYY' date: {date_str!r}"
return

ifparsed_date.year<1990:
yieldline_num, f"{prefix} must not be before Python was invented: {date_str!r}"
ifparsed_date> (dt.datetime.now() +dt.timedelta(days=14)):
yieldline_num, f"{prefix} must not be in the future: {date_str!r}"


if__name__=="__main__":
if {"-h", "--help", "-?"}.intersection(sys.argv[1:]):
print(__doc__, file=sys.stderr)
raiseSystemExit(0)

files= {}
forarginsys.argv[1:]:
ifnotarg.startswith("-"):
files[arg] =None
elifargin {"-d", "--detailed"}:
DETAILED_ERRORS=True
else:
print(f"Unknown option: {arg!r}", file=sys.stderr)
raiseSystemExit(1)
raiseSystemExit(check(files))