check_paper_formats.py

"""Checks that the paper format in the DB matches what is in the FS.

This script checks the arXiv_metadata.source_format plausibly matches the file extension of the source files."""
fromflaskimportBlueprint
fromsqlalchemyimportor_
importjson

importclick

fromarxiv.identifierimportIdentifier
fromarxiv.dbimportSession
fromarxiv.db.modelsimportMetadata
frombrowse.services.disseminationimportget_article_store, ArticleStore
fromarxiv.formatsimportformats_from_source_flag
fromarxiv.filesimportFileObj
bp=Blueprint("check", __name__)

@bp.cli.command("paper_formats")
@click.argument("yymm")
defcheck_paper_formats(yymm: str) ->None:
"""Checks formats for yymm."""
query= (Session.query(Metadata.paper_id, Metadata.version,
Metadata.source_format,
Metadata.source_flags, Metadata.source_size)
 .filter(or_(Metadata.paper_id.like(f"%/{yymm}%"),
Metadata.paper_id.like(f"{yymm}.%"))))

rows=query.all()
ifnotrows:
print(f"No rows found.")
a_store: ArticleStore=get_article_store()
results= {}
forrowinrows:
paper_id, version, src_fmt, source_flags, source_size=row[0], row[1], row[2], row[3], row[4]
print(f"Doing {paper_id}v{version}")
result= {"paper_id": paper_id, "version":version, "db_source_format":src_fmt, "db_source_flags":source_flags, "db_source_size":source_size}
results[f"{paper_id}v{version}"] =result
arxiv_id=Identifier(f"{paper_id}v{version}")
src=a_store.get_source(arxiv_id)
ifisinstance(src, str):
result["src_file_problem"] =src
continue
elifisinstance(src, tuple):
result["src_file_problem"] =""
result["source_flag_only_formats"] =formats_from_source_flag(source_flags)

fileobj, docmeta, version=src
ifisinstance(fileobj, FileObj):
result["sizes_match"] =source_size==fileobj.size
result["fs_size"] =fileobj.size
result["fs_name"] =fileobj.name
result["dissemination_formats"] =a_store.get_dissemination_formats(docmeta, fileobj)
result["formats_match"] =result["dissemination_formats"] ==result["source_flag_only_formats"]
else:
result["src_file_problem"] ="MULTIPLE FILES"
else:
result["src_file_problem"] ="UNKNOWN SOURCE FORMAT TYPE {type(src)}"
continue

withopen("paper_formats.json", "w") asfh:
json.dump(results, fh, indent=4, default=str)