pmh_record.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

importdatetime
importhtml
importre

fromsqlalchemyimportnullslast, or_, orm, text
fromsqlalchemy.dialects.postgresqlimportJSONB

importpage
fromappimportdb
fromappimportlogger
fromrecordthresher.record_makerimportPmhRecordMaker
fromutilimportNoDoiException
fromutilimportclean_doi
fromutilimportis_doi_url
fromutilimportnormalize_title

DEBUG_BASE=False

_too_common_normalized_titles=None


deftoo_common_normalized_titles():
global_too_common_normalized_titles
if_too_common_normalized_titlesisNone:
_too_common_normalized_titles=set([
titlefor (title, ) in
db.engine.execute(text('select normalized_title from common_normalized_titles'))
 ])
return_too_common_normalized_titles


deftitle_is_too_short(normalized_title):
ifnotnormalized_title:
returnTrue
returnlen(normalized_title) <=21


deftitle_is_too_common(normalized_title):
returnnormalized_titleintoo_common_normalized_titles()


defis_known_mismatch(doi, pmh_record):
# screen figshare supplemental items, e.g.
# https://doi.org/10.1021/ac035352d vs
# https://api.figshare.com/v2/oai?verb=GetRecord&metadataPrefix=oai_dc&identifier=oai:figshare.com:article/3339307

ifpmh_record.bare_pmh_idandpmh_record.bare_pmh_id.startswith('oai:figshare.com'):
ifpmh_record.doiandpmh_record.doi.startswith('{}.s'.format(doi)):
returnTrue

mismatches= {
'10.1063/1.4818552': [
'hdl:10068/886851'# thesis with same title
 ],
'10.1016/j.conbuildmat.2019.116939': [
'oai:repositorio.cuc.edu.co:11323/5292'# abstract
 ],
'10.1111/j.1439-0396.2010.01055.x': [
'oai:dspace.uevora.pt:10174/3888'# abstract
 ],
'10.3233/ves-200717': [
'oai:pure.atira.dk:publications/0eb5fb9c-4e41-4879-970a-78b53b7b078e'# poster with same title
 ],
'10.1139/p79-106': [
'oai:tsukuba.repo.nii.ac.jp:00011538'# thesis with same title
 ],
'10.1057/s41296-019-00346-8': [
'oai:eprints.ucl.ac.uk.OAI2:10087912'# wrong pdf on landing page
 ],
'10.1201/9781315151823': [
'oai:openresearch.lsbu.ac.uk:86zv0'# doi belongs to book, pmh id belongs to chapter
 ],
'10.1007/s00221-007-1163-1': [
'oai:wrap.warwick.ac.uk:54523'# thesis with same title
 ],
'10.1080/08989621.2020.1860764': [
'oai:generic.eprints.org:86138'# pdf links don't work, http://irep.iium.edu.my/86138/
 ],
'10.15866/irease.v11i4.14675': [
'oai:generic.eprints.org:67164'# screenshot of scopus
 ],
'10.1016/b978-0-12-805393-5.00012-9': [
'oai:CiteSeerX.psu:10.1.1.885.6937'# title parsed incorrectly
 ],
'10.1007/s10494-020-00126-0': [
'oai:HAL:hal-02195059v1'# conference paper with same title
 ],
'10.1007/978-1-4614-7163-9_110149-1': [
'oai:HAL:hal-01343052v1'# conference paper with same title
 ],
'10.1093/epolic/eiaa015': [
# all different papers with same title
'oai::82981',
'oai::92620',
'oai:RePEc:pra:mprapa:82981',
 ],
'10.1063/1.5114468': [
'oai:www.ucm.es:27329'# conference paper and article with same title
 ],
'10.1007/978-981-32-9620-6_9': [
'oai:CiteSeerX.psu:10.1.1.434.2367'# citeseerx misfire
 ],
'10.1093/clinchem/hvaa290': [
'oai:research-information.bris.ac.uk:publications/a61e230c-1d1d-45f6-bdfa-0ed944b93804'
 ],
 }
returnpmh_record.bare_pmh_idinmismatches.get(doi, [])


defoai_tag_match(tagname, record, return_list=False):
ifnottagnameinrecord.metadata:
return [] ifreturn_listelseNone
matches=record.metadata[tagname]
ifreturn_list:
returnmatches# will be empty list if we found naught
else:
try:
returnmatches[0]
exceptIndexError: # no matches.
returnNone


deftitle_match_limit_exceptions():
return {
'abinitiomoleculardynamicscdsequantumdotdopedglasses',
'speedingupdiscoveryauxeticzeoliteframeworksmachinelearning',
'epigeneticregulationkappaopioidreceptorgeneinsertiondeletionpromoterregion',
 }


classPmhRecord(db.Model):
id=db.Column(db.Text, primary_key=True)
repo_id=db.Column(db.Text) # delete once endpoint_ids are all populated
endpoint_id=db.Column(db.Text)
doi=db.Column(db.Text)
record_timestamp=db.Column(db.DateTime)
api_raw=db.Column(JSONB)
title=db.Column(db.Text)
license=db.Column(db.Text)
oa=db.Column(db.Text)
urls=db.Column(JSONB)
authors=db.Column(JSONB)
relations=db.Column(JSONB)
sources=db.Column(JSONB)
updated=db.Column(db.DateTime)
rand=db.Column(db.Numeric)
pmh_id=db.Column(db.Text)

pages=db.relationship(
# 'Page',
'PageNew',
lazy='select', # lazy load
cascade="all, delete-orphan",
# don't want a backref because don't want page to link to this
foreign_keys="PageNew.pmh_id"
 )

@property
defbare_pmh_id(self):
returnself.pmh_idorself.id

def__init__(self, **kwargs):
self.updated=datetime.datetime.utcnow().isoformat()
super(self.__class__, self).__init__(**kwargs)

defpopulate(self, endpoint_id, pmh_input_record, metadata_prefix='oai_dc'):
self.updated=datetime.datetime.utcnow().isoformat()
self.id='{}:{}'.format(endpoint_id, pmh_input_record.header.identifier)
self.endpoint_id=endpoint_id
self.pmh_id=pmh_input_record.header.identifier
self.api_raw=pmh_input_record.raw
self.record_timestamp=pmh_input_record.header.datestamp
self.title=oai_tag_match("title", pmh_input_record)
self.authors=oai_tag_match("creator", pmh_input_record, return_list=True)
self.relations=oai_tag_match("relation", pmh_input_record, return_list=True)
self.oa=oai_tag_match("oa", pmh_input_record)

ifmetadata_prefix=='qdc':
self.license=oai_tag_match("rights.license", pmh_input_record)
else:
self.license=oai_tag_match("rights", pmh_input_record)

self.sources=oai_tag_match("collname", pmh_input_record, return_list=True)

identifier_matches=oai_tag_match("identifier", pmh_input_record, return_list=True)
ifself.pmh_idandself.pmh_id.startswith('oai:authors.library.caltech.edu'):
identifier_matches= []

ifself.pmh_idandself.pmh_id.startswith('oai:deepblue.lib.umich.edu'):
# lots of identifiers and this item's is first
identifier_matches.reverse()

ifself.pmh_idandself.pmh_id.startswith('oai:alma.44SUR_INST') orself.pmh_id.startswith('oai:alma.44SUR_INST'):
identifier_matches=oai_tag_match("file.download.url", pmh_input_record, return_list=True)
self.authors=oai_tag_match("creatorname", pmh_input_record, return_list=True)

ifself.pmh_idandself.pmh_id.startswith('oai:japanlinkcenter.org'):
identifier_matches=oai_tag_match("url", pmh_input_record, return_list=True)
self.relations=oai_tag_match("related_content", pmh_input_record, return_list=True)
self.authors= []

identifier_doi_matches=oai_tag_match("identifier.doi", pmh_input_record, return_list=True)
identifier_uri_matches=oai_tag_match("identifier.uri", pmh_input_record, return_list=True)
self.urls=self.get_good_urls(identifier_matches+identifier_uri_matches)

ifnotself.urls:
self.urls=self.get_good_urls(self.relations)

ifnotself.urlsandself.pmh_idandself.pmh_id.startswith('oai:repozytorium.biblos.pk.edu.pl:'):
rpk_id=self.pmh_id.split(':')[-1]
self.urls= [f'https://repozytorium.biblos.pk.edu.pl/resources/{rpk_id}']

possible_dois= []

ignore_relation_prefixes= [
'oai:pantheon.ufrj.br:',
'oai:repository.ucatolica.edu.co:',
'oai:repository.kulib.kyoto-u.ac.jp:',
 ]

ifself.relationsandnotany(self.bare_pmh_id.startswith(p) forpinignore_relation_prefixes):
possible_dois+= [sforsinself.relationsifsand'/*ref*/'notinsandnots.startswith('reference')]

ifself.pmh_idandself.pmh_id.startswith('oai:pure.mpg.de:'):
possible_dois.reverse()

ifself.bare_pmh_idandself.bare_pmh_id.startswith('oai:openarchive.ki.se:'):
# ticket 22247, relation DOIs are only for this article with this prefix
possible_dois= [sforsinpossible_doisifs.startswith('info:eu-repo/semantics/altIdentifier/doi/')]

ifidentifier_matches:
possible_dois+= [sforsinidentifier_matchesifs]
ifidentifier_doi_matches:
possible_dois+= [sforsinidentifier_doi_matchesifs]

ifself.pmh_idandself.pmh_id.startswith('oai:escholarship.mcgill.ca:'):
possible_dois+=oai_tag_match('source', pmh_input_record, return_list=True)

self.set_doi(possible_dois)

self.doi=self._doi_override_by_id().get(self.bare_pmh_id, self.doi)
self.title=self._title_override_by_id().get(self.bare_pmh_id, self.title)

defset_doi(self, possible_dois):
ifpossible_dois:
forpossible_doiinpossible_dois:
if (
is_doi_url(possible_doi)
orpossible_doi.startswith("doi:")
orre.findall(r"10\.\d", possible_doi)
 ):
try:
doi_candidate=clean_doi(possible_doi)

ifnotdoi_candidate:
continue

skip_these_doi_snippets= [
'10.17605/osf.io',
'10.14279/depositonce',
'/(issn)',
'10.17169/refubium',
'10.3929/ethz-',
 ]
skip_these_dois= [
'10.1002/9781118786352', # journal
 ]
fordoi_snippetinskip_these_doi_snippets:
ifdoi_snippet.lower() indoi_candidate.lower():
doi_candidate=None
break

forskip_doiinskip_these_dois:
ifskip_doianddoi_candidateandskip_doi.lower() ==doi_candidate.lower():
doi_candidate=None
break

ifdoi_candidate:
self.doi=doi_candidate
exceptNoDoiException:
pass

@staticmethod
def_title_override_by_id():
return {
# wrong title
'oai:RePEc:feb:natura:00655': 'Do Workers Value Flexible Jobs? A Field Experiment On Compensating Differentials',
# reviews of books with same title
'oai:ir.uiowa.edu:annals-of-iowa-11115': '(Book Notice) The Bull Moose Years: Theodore Roosevelt and the Progressive Party',
'oai:ir.uiowa.edu:annals-of-iowa-9228': '(Book Review) Land, Piety, Peoplehood: The Establishment of Mennonite Communities in America, 1683-1790',

# published title changed slightly
'oai:figshare.com:article/10272041': 'Ab initio molecular dynamics of CdSe Quantum Dot-Doped Glasses',

# ticket 6010. record links to PDF for different article.
'oai:eprints.uwe.ac.uk:33511': 'The Bristol-Bath Urban freight Consolidation Centre from the perspective of its users',

'oai:www.duo.uio.no:10852/77974': 'Chronic pain among the hospitalized patients after the 22nd july-2011 terror attacks in Oslo and at Utøya Island.',

'oai:lirias2repo.kuleuven.be:123456789/647375': '',

'oai:share.osf.io:460F8-921-B0C': 'How optimal is word recognition under multimodal uncertainty?',

'oai:zenodo.org:5772413': 'Compensation Strategies for Gait Impairments in Parkinson Disease',
 }

@staticmethod
def_doi_override_by_id():
return {
# wrong DOI in identifier url
'oai:dspace.flinders.edu.au:2328/36108': '10.1002/eat.22455',

# picked up wrong DOI in relation
'oai:oai.kemsu.elpub.ru:article/2590': '10.21603/2078-8975-2018-4-223-231',

# junk in identifier
'oai:scholarspace.manoa.hawaii.edu:10125/42031': '10.18357/ijih122201717783',

# wrong DOI in relation
'oai:oai.perinatology.elpub.ru:article/560': '10.21508/1027-4065-2017-62-5-111-118',

'oai:HAL:hal-00927061v2': '10.1090/memo/1247',

'oai:revistas.ucm.es:article/62495': '10.5209/clac.62495',

'oai:oro.open.ac.uk:57403': '10.1090/hmath/011',

'oai:eprints.soas.ac.uk:22576': '10.4324/9781315762210-8',

'oai:oai.mir.elpub.ru:article/838': '10.18184/2079-4665.2018.9.3.338-350',

'oai:arXiv.org:1605.06120': None,

'oai:research-repository.griffith.edu.au:10072/80920': None,

'oai:HAL:cea-01550620v1': '10.1103/physrevb.93.214414',

'oai:ora.ox.ac.uk:uuid:f5740dd3-0b45-4e7b-8f2e-d4872a6c326c': '10.1016/j.jclinepi.2017.12.022',

'oai:ora.ox.ac.uk:uuid:a78ee943-6cfe-4fb9-859e-d7ec82ebec85': '10.1016/j.jclinepi.2019.05.033',

'oai:archive.ugent.be:3125191': None,

'oai:scholar.sun.ac.za:10019.1/95408': '10.4102/sajpsychiatry.v19i3.951',

'oai:rcin.org.pl:60213': None,

'oai:rcin.org.pl:48382': None,

'oai:philarchive.org/rec/LOGSTC': '10.1093/analys/anw051',

'oai:philarchive.org/rec/LOGMBK': '10.1111/1746-8361.12258',

'oai:CiteSeerX.psu:10.1.1.392.2251': None,

'oai:serval.unil.ch:BIB_289289AA7E27': None, # oai:serval.unil.ch:duplicate of BIB_98991EE549F6

'oai:deepblue.lib.umich.edu:2027.42/141967': '10.1111/asap.12132',

'oai:eprints.lancs.ac.uk:80508': None, # says 10.1057/978-1-137-58629-2, but that's the book holding this chapter

'oai:zenodo.org:3994623': '10.1007/978-3-319-29791-0',

'oai:elib.dlr.de:136158': None, # chapter of 10.1007/978-3-030-48340-1

'oai:wrap.warwick.ac.uk:147355': '10.1177/0022242921992052',

'oai:www.zora.uzh.ch:133251': None,

'oai:ray.yorksj.ac.uk:2511': None, # record is chapter, DOI is book

'oai:intellectum.unisabana.edu.co:10818/20216': None, # all DOIs are citations

'oai:cris.maastrichtuniversity.nl:publications/494aa88b-4a2b-4b81-b926-8005af3f85d5': None, # record is chapter, DOI is book

'oai:repository.ucatolica.edu.co:10983/22919': '10.14718/revarq.2018.20.2.1562',

'oai:HAL:halshs-03107637v1': None, # record is chapter, DOI 10.1515/9781614514909 is book

'oai:eprints.bbk.ac.uk.oai2:28966': '10.1007/978-3-030-29736-7_11', # book chapter

'oai:dspace.library.uvic.ca:1828/11889': None,

'oai:real.mtak.hu:122999': None,

'oai:real.mtak.hu:121388': None,

'oai:arXiv.org:1306.1461': None,

'oai:lirias2repo.kuleuven.be:123456789/647375': None,

'oai:strathprints.strath.ac.uk:65258': None, # record is chapter, DOI is book

'oai:research-repository.griffith.edu.au:10072/389013': None, # only a summary

'oai:e-space.mmu.ac.uk:618539': None, # bad DOI fom relation

'oai:alma61RMIT.INST:11247217330001341': None, # book/chapter

'oai:repository.unab.edu.co:20.500.12749/2107': None,

'oai:HAL:hal-03157274v1': '10.1080/00220388.2020.1715942',

'oai:zenodo.org:2414272': None,

# book chapters
'oai:trepo.tuni.fi:10024/118421': None,
'oai:www.duo.uio.no:10852/76264': None,
'oai:www.duo.uio.no:10852/76567': None,
'oai:research-repository.griffith.edu.au:10072/380027': None,
'oai:eprints.leedsbeckett.ac.uk:5899': None,
'oai:openresearch.lsbu.ac.uk:8yyz8': None,

'oai:openaccess.city.ac.uk:15995': '10.1016/j.insmatheco.2012.12.009',

# book chapter
'oai:air.unimi.it:2434/707996': None,
'oai:iris.unive.it:10278/3734529': '10.1007/978-3-319-20791-9_314-1',
'oai:repozytorium.amu.edu.pl:10593/6693': None,
'b2002691-9888-4c9e-900a-91bdad900d8b/oai:repozytorium.amu.edu.pl:10593/6693': None,
'oai:boa.unimib.it:10281/3095': None,
'oai:eprints.nottingham.ac.uk:42938': None,
'oai:nottingham-repository.worktribe.com:744398': None,
'oai:centaur.reading.ac.uk:37024': None,
'oai:sure.sunderland.ac.uk:9399': None,
'oai:sure.sunderland.ac.uk:9330': None,
'oai:sure.sunderland.ac.uk:10188': None,
'oai:sure.sunderland.ac.uk:10190': None,
'oai:sure.sunderland.ac.uk:9176': None,
'oai:sure.sunderland.ac.uk:10630': None,
'oai:sure.sunderland.ac.uk:9304': None,
'oai:sure.sunderland.ac.uk:9175': None,
'oai:sure.sunderland.ac.uk:9873': None,
'oai:sure.sunderland.ac.uk:9930': None,
'oai:sure.sunderland.ac.uk:9755': None,

'oai:arXiv.org:hep-ph/0404005': None,
'oai:arXiv.org:1212.3347': '10.4169/math.mag.86.2.143',
'oai:westminsterresearch.westminster.ac.uk:929yw' : None,
'oai:research-information.bris.ac.uk:publications/a61e230c-1d1d-45f6-bdfa-0ed944b93804': None,
'oai:tara.tcd.ie:2262/91798': '10.1017/ipm.2017.80',
'oai:gupea.ub.gu.se:2077/66303': None,
'oai:repository.ucatolica.edu.co:10983/26325': None,
 }

defget_good_urls(self, candidate_urls):
valid_urls= []
# pmc can only add pmc urls. otherwise has junk about dois that aren't actually open.
ifcandidate_urls:
if"oai:pubmedcentral.nih.gov"inself.id:
forurlincandidate_urls:
if"/pmc/"inurlandurl!="http://www.ncbi.nlm.nih.gov/pmc/articles/PMC":
pmcid_matches=re.findall(".*(PMC\d+).*", url)
ifpmcid_matches:
pmcid=pmcid_matches[0]
url="https://www.ncbi.nlm.nih.gov/pmc/articles/{}".format(pmcid)
valid_urls.append(url)
else:
ifself.endpoint_id=='ycf3gzxeiyuw3jqwjmx3': # https://lirias.kuleuven.be
candidate_urls= [re.sub(r'^\d+;http', 'http', url) forurlincandidate_urlsifurl]

valid_urls+= [urlforurlincandidate_urlsifurlandurl.startswith("http")]

ifself.pmh_idandself.pmh_id.startswith('oai:RePEc:'):
repec_handle=re.sub(r'^oai:', '', self.pmh_id)
return [f'https://econpapers.repec.org/{repec_handle}']

# filter out doi urls unless they are the only url
# might be a figshare url etc, but otherwise is usually to a publisher page which
# may or may not be open, and we are handling through hybrid path

use_doi_url_id_prefixes= [
'cdr.lib.unc.edu:',
 ]

use_doi_url=False
foruse_doi_url_id_prefixinuse_doi_url_id_prefixes:
ifself.bare_pmh_idandself.bare_pmh_id.startswith(use_doi_url_id_prefix):
use_doi_url=True
break

ifnotuse_doi_urlandlen(valid_urls) >1:
valid_urls= [urlforurlinvalid_urlsif"doi.org/"notinurl]

valid_urls= [urlforurlinvalid_urlsif"doi.org/10.1111/"notinurl]

ifself.bare_pmh_idandself.bare_pmh_id.startswith('oai:alma.61RMIT_INST:'):
valid_urls= [urlforurlinvalid_urlsif'rmit.edu.au'inurl]

ifself.bare_pmh_idandself.bare_pmh_id.startswith('oai:pure.rug.nl:'):
valid_urls= [urlforurlinvalid_urlsif'rug.nl'inurl]

# filter out some urls that we know are closed or otherwise not useful
blacklist_url_snippets= [
"/10.1093/analys/",
"academic.oup.com/analysis",
"analysis.oxfordjournals.org/",
"ncbi.nlm.nih.gov/pubmed/",
"gateway.webofknowledge.com/",
"orcid.org/",
"researchgate.net/",
"academia.edu/",
"europepmc.org/abstract/",
"ftp://",
"api.crossref",
"api.elsevier",
"api.osf",
"eprints.soton.ac.uk/413275",
"eprints.qut.edu.au/91459/3/91460.pdf",
"hdl.handle.net/2117/168732",
"hdl.handle.net/10044/1/81238", # wrong article
"journals.elsevier.com",
"https://hdl.handle.net/10037/19572", # copyright violation. ticket 22259
"http://irep.iium.edu.my/58547/9/Antibiotic%20dosing%20during%20extracorporeal%20membrane%20oxygenation.pdf",
"oceanrep.geomar.de/52096/7/s41586-021-03496-1.pdf",
"eprints.lmu.edu.ng/3516/",
"springerlink.com/content/",
 ]

backlist_url_patterns=list(map(re.escape, blacklist_url_snippets)) + [
r'springer.com/.*/journal/\d+$',
r'springer.com/journal/\d+$',
r'supinfo\.pdf$',
r'\dfigures\.pdf$',
r'\dsupplemental\.pdf$',
r'Appendix[^/]*\.pdf$',
r'^https?://www\.icgip\.org/?$',
r'^https?://(www\.)?agu.org/journals/',
r'issue/current$',
r'/809AB601-EF05-4DD1-9741-E33D7847F8E5\.pdf$',
r'onlinelibrary\.wiley\.com/doi/',
r'https?://doi\.org/10\.1002/', # wiley
r'https?://doi\.org/10\.1111/', # wiley
r'https?://doi\.org/10\.1007/', # springer
r'authors\.library\.caltech\.edu/93971/\d+/41562_2019_595_MOESM',
r'aeaweb\.org/.*\.ds$',
r'aeaweb\.org/.*\.data$',
r'aeaweb\.org/.*\.appx$',
r'https?://dspace\.stir\.ac\.uk/.*\.jpg$',
r'https?://dspace\.stir\.ac\.uk/.*\.tif$',
r'/table_final\.pdf$',
r'/supplemental_final\.pdf$',
r'psasir\.upm\.edu\.my/id/eprint/36880/1/Conceptualizing%20and%20measuring%20youth\.pdf',
r'psasir\.upm\.edu\.my/id/eprint/53326/1/Conceptualizing%20and%20measuring%20youth\.pdf',
r'^https?://(www\.)?tandfonline\.com/toc/',
r'\dSuppl\.pdf$',
r'^https://lirias\.kuleuven\.be/handle/\d+/\d+$',
r'^https?://eu\.wiley\.com/',
r'^https?://www\.wiley\.com/',
r'hull-repository\.worktribe\.com/(\w+/)?437540(/|$)',
r'researchonline\.jcu\.edu\.au/.*_cover.pdf',
 ]

forurl_snippetinbacklist_url_patterns:
valid_urls= [urlforurlinvalid_urlsifnotre.search(url_snippet, url)]

supplemental_url_patterns= [
r'Figures.pdf$',
 ]

iflen(valid_urls) >1:
forurl_patterninsupplemental_url_patterns:
valid_urls= [urlforurlinvalid_urlsifnotre.search(url_pattern, url)]

# and then html unescape them, because some are html escaped
valid_urls= [html.unescape(url) forurlinvalid_urls]

# make sure they are actually urls
valid_urls= [urlforurlinvalid_urlsifurl.startswith("http")]

ifself.bare_pmh_id.startswith('oai:ora.ox.ac.uk:uuid:') andnotvalid_urls:
# https://ora.ox.ac.uk
# pmh records don't have page urls but we can guess them
# remove 'oai:ora.ox.ac.uk:' prefix and append to base URL
valid_urls.append('https://ora.ox.ac.uk/objects/{}'.format(self.bare_pmh_id[len('oai:ora.ox.ac.uk:'):]))

valid_urls=list(set(valid_urls))
returnvalid_urls

defmint_repo_page_for_url(self, url):
my_repo_page=self.mint_page_for_url(page.RepoPage, url)

# get the most recent scrape data
most_recent_old_page=page.PageNew.query.filter(
page.PageNew.endpoint_id==self.endpoint_id,
page.PageNew.url==url
 ).order_by(
nullslast(page.PageNew.scrape_updated.desc())
 ).options(orm.noload('*')).first()

ifmost_recent_old_page:
my_repo_page.scrape_updated=most_recent_old_page.scrape_updated
my_repo_page.scrape_metadata_url=most_recent_old_page.scrape_metadata_url
my_repo_page.scrape_pdf_url=most_recent_old_page.scrape_pdf_url
my_repo_page.scrape_license=most_recent_old_page.scrape_license
my_repo_page.scrape_version=most_recent_old_page.scrape_version

returnmy_repo_page

defmint_page_for_url(self, page_class, url):
frompageimportPageNew
# this is slow, but no slower than looking for titles before adding pages
existing_page=PageNew.query.filter(PageNew.normalized_title==self.calc_normalized_title(),
PageNew.match_type==page_class.__mapper_args__["polymorphic_identity"],
PageNew.url==url,
PageNew.endpoint_id==self.endpoint_id
 ).options(orm.noload('*')).first()
ifexisting_page:
my_page=existing_page
else:
my_page=page_class()
my_page.url=url
my_page.normalized_title=self.calc_normalized_title()
my_page.endpoint_id=self.endpoint_id

my_page.doi=self.doi
my_page.title=self.title
my_page.authors=self.authors
my_page.record_timestamp=self.record_timestamp
my_page.pmh_id=self.id
my_page.repo_id=self.repo_id# delete once endpoint_ids are all populated
my_page.pmh_record=self
returnmy_page

defcalc_normalized_title(self):
ifnotself.title:
returnNone

ifself.endpoint_id=='63d70f0f03831f36129':
# figshare. the record is for a figure but the title is from its parent article.
ifnotself.doiandre.match(r'10\.36227/techrxiv\.\d+(?:\.v\n+)?', self.doi):
returnNone

working_title=self.title

# repo specific rules
# AMNH adds biblio to the end of titles, which ruins match. remove this.
# example http://digitallibrary.amnh.org/handle/2246/6816 oai:digitallibrary.amnh.org:2246/6816
if"amnh.org"inself.id:
# cut off the last part, after an openning paren
working_title=re.sub("(Bulletin of.+no.+\d+)", "", working_title, re.IGNORECASE|re.MULTILINE)
working_title=re.sub("(American Museum nov.+no.+\d+)", "", working_title, re.IGNORECASE|re.MULTILINE)

# for endpoint 0dde28a908329849966, adds this to end of all titles, so remove (eg http://hdl.handle.net/11858/00-203Z-0000-002E-72BD-3)
working_title=re.sub("vollständige digitalisierte Ausgabe", "", working_title, re.IGNORECASE|re.MULTILINE)
returnnormalize_title(working_title)

defdelete_old_record(self):
# old records used the bare record_id as pmh_record.id
# delete the old record before merging, instead of conditionally updating or creating the new record
db.session.query(PmhRecord).filter(
PmhRecord.id==self.bare_pmh_id, PmhRecord.endpoint_id==self.endpoint_id
 ).delete()

defmint_pages(self, reset_scrape_date=False):
ifself.endpoint_id=='ac9de7698155b820de7':
# NIH PMC. Don't mint pages because we use a CSV dump to make OA locations. See Pub.ask_pmc
return []

ifself.bare_pmh_idandself.bare_pmh_id.startswith('oai:openarchive.ki.se:'):
# ticket 22247, only type=art can match DOIs
if'<dc:type>art</dc:type>'notinself.api_raw:
return []

self.pages= []

# this should have already been done when setting .urls, but do it again in case there were improvements
# case in point: new url patterns added to the blacklist
good_urls=self.get_good_urls(self.urls)

ifre.compile(r'<dc:rights>Limited Access</dc:rights>', re.MULTILINE).findall(self.api_raw):
logger.info('found limited access label, not minting pages')
else:
forurlingood_urls:
my_repo_page=self.mint_repo_page_for_url(url)

ifself.doi:
my_repo_page.match_doi=True

normalized_title=self.calc_normalized_title()
ifnormalized_title:
num_pages_with_this_normalized_title=db.session.query(page.RepoPage.id).filter(
page.RepoPage.match_title==True,
page.RepoPage.normalized_title==normalized_title
 ).count()

if (
num_pages_with_this_normalized_title>=20
andnormalized_titlenotintitle_match_limit_exceptions()
and"oai:HAL:"notinself.bare_pmh_id
 ):
logger.info(
"not allowing title matches because too many with this title: {}".format(
normalized_title
 )
 )
elifself.bare_pmh_idandself.bare_pmh_id.startswith("oai:mdpi.com:"):
# publisher site, don't match to other DOIs by title
pass
else:
my_repo_page.match_title=True

self.pages.append(my_repo_page)

# logger.info(u"minted pages: {}".format(self.pages))

# delete pages with this pmh_id that aren't being updated
db.session.query(page.PageNew).filter(
page.PageNew.endpoint_id==self.endpoint_id,
or_(page.PageNew.pmh_id==self.id, page.PageNew.pmh_id==self.pmh_id),
page.PageNew.id.notin_([p.idforpinself.pages])
 ).delete(synchronize_session=False)

ifreset_scrape_dateandself.pages:
# move already queued-pages at the front of the queue
# if the record was updated the oa status might have changed
query_text='''
 update page_green_scrape_queue
 set finished = null
 where id = any(:ids) and started is null
 '''

reset_query=text(query_text).bindparams(ids=[p.idforpinself.pages])

db.session.execute(reset_query)

returnself.pages

defenqueue_representative_page(self):
# if recordthresher is going to try to make a record based on a page, enqueue it now
ifrp:=PmhRecordMaker.representative_page(self):
db.session.merge(page.PageGreenScrapeQueue(id=rp.id, endpoint_id=rp.endpoint_id))

def__repr__(self):
return"<PmhRecord ({}) doi:{} '{}...'>".format(self.id, self.doi, self.title[0:20])

defto_dict(self):
response= {
"oaipmh_id": self.bare_pmh_id,
"oaipmh_record_timestamp": self.record_timestampandself.record_timestamp.isoformat(),
"urls": self.urls,
"title": self.title
 }
returnresponse