common.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

# This file is adpated from PyTorch Core
# https://github.com/pytorch/pytorch/blob/master/scripts/release_notes/common.py

importjson
importlocale
importos
importre
importsubprocess
fromcollectionsimportnamedtuple

importrequests


topics= [
"bc_breaking",
"deprecations",
"new_features",
"improvements",
"bug_fixes",
"performance",
"docs",
"devs",
"Untopiced",
]


Features=namedtuple(
"Features",
 [
"title",
"body",
"pr_number",
"files_changed",
"labels",
 ],
)


defdict_to_features(dct):
returnFeatures(
title=dct["title"],
body=dct["body"],
pr_number=dct["pr_number"],
files_changed=dct["files_changed"],
labels=dct["labels"],
 )


deffeatures_to_dict(features):
returndict(features._asdict())


defrun(command):
"""Returns (return-code, stdout, stderr)"""
p=subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, err=p.communicate()
rc=p.returncode
enc=locale.getpreferredencoding()
output=output.decode(enc)
err=err.decode(enc)
returnrc, output.strip(), err.strip()


defcommit_body(commit_hash):
cmd=f"git log -n 1 --pretty=format:%b {commit_hash}"
ret, out, err=run(cmd)
returnoutifret==0elseNone


defcommit_title(commit_hash):
cmd=f"git log -n 1 --pretty=format:%s {commit_hash}"
ret, out, err=run(cmd)
returnoutifret==0elseNone


defcommit_files_changed(commit_hash):
cmd=f"git diff-tree --no-commit-id --name-only -r {commit_hash}"
ret, out, err=run(cmd)
returnout.split("\n") ifret==0elseNone


defparse_pr_number(body, commit_hash, title):
regex=r"Pull Request resolved: https://github.com/pytorch/data/pull/([0-9]+)"
matches=re.findall(regex, body)
iflen(matches) ==0:
if"revert"notintitle.lower() and"updating submodules"notintitle.lower():
print(f"[{commit_hash}: {title}] Could not parse PR number, ignoring PR")
returnNone
iflen(matches) >1:
print(f"[{commit_hash}: {title}] Got two PR numbers, using the first one")
returnmatches[0]
returnmatches[0]


defget_ghstack_token():
pattern="github_oauth = (.*)"
withopen(os.path.expanduser("~/.ghstackrc"), "r+") asf:
config=f.read()
matches=re.findall(pattern, config)
iflen(matches) ==0:
raiseRuntimeError("Can't find a github oauth token")
returnmatches[0]


token=get_ghstack_token()
headers= {"Authorization": f"token {token}"}


defrun_query(query):
request=requests.post("https://api.github.com/graphql", json={"query": query}, headers=headers)
ifrequest.status_code==200:
returnrequest.json()
else:
raiseException(f"Query failed to run by returning code of {request.status_code}. {query}")


defgh_labels(pr_number):
query=f"""
 {{
 repository(owner: "pytorch", name: "data") {{
 pullRequest(number: {pr_number}) {{
 labels(first: 10) {{
 edges {{
 node {{
 name
 }}
 }}
 }}
 }}
 }}
 }}
 """
query=run_query(query)
edges=query["data"]["repository"]["pullRequest"]["labels"]["edges"]
return [edge["node"]["name"] foredgeinedges]


defget_features(commit_hash, return_dict=False):
title, body, files_changed= (
commit_title(commit_hash),
commit_body(commit_hash),
commit_files_changed(commit_hash),
 )
pr_number=parse_pr_number(body, commit_hash, title)
labels= []
ifpr_numberisnotNone:
labels=gh_labels(pr_number)
result=Features(title, body, pr_number, files_changed, labels)
ifreturn_dict:
returnfeatures_to_dict(result)
returnresult


classCommitDataCache:
def__init__(self, path="results/data.json"):
self.path=path
self.data= {}
ifos.path.exists(path):
self.data=self.read_from_disk()

defget(self, commit):
ifcommitnotinself.data.keys():
# Fetch and cache the data
self.data[commit] =get_features(commit)
self.write_to_disk()
returnself.data[commit]

defread_from_disk(self):
withopen(self.path) asf:
data=json.load(f)
data= {commit: dict_to_features(dct) forcommit, dctindata.items()}
returndata

defwrite_to_disk(self):
data= {commit: features._asdict() forcommit, featuresinself.data.items()}
withopen(self.path, "w") asf:
json.dump(data, f)