open_location.py

# -*- coding: utf8 -*-
#

importre
fromenumimportEnum
fromurllib.parseimportunquote

importshortuuid

importoa_evidence
fromappimportdb
fromappimportlogger
frompdf_urlimportPdfUrl
fromreported_noncompliant_copiesimportis_reported_noncompliant_url
fromutilimportis_doi_url
fromutilimportnormalize_doi


defurl_sort_score(url):
url_lower=url.lower()

score=0

# pmc results are better than IR results, if we've got them
if"europepmc.org"inurl_lower:
score+=-50

if"/pmc/"inurl_lower:
score+=-45

# arxiv results are better than IR results, if we've got them
if"arxiv"inurl_lower:
score+=-40

if".edu"inurl_lower:
score+=-30

if"citeseerx"inurl_lower:
score+=+10

# ftp is really bad
if"ftp"inurl_lower:
score+=+60

# break ties
if"pdf"inurl_lower:
score+=-1

if"biodiversitylibrary.org/part/281352"inurl_lower:
# just enough to beat a PDF
score+=-101

# otherwise whatever we've got
returnscore


defvalidate_pdf_urls(open_locations):
unvalidated= [xforxinopen_locationsifx.pdf_url_validisNone]

ifunvalidated:
bad_pdf_urls= {
x.urlforxin
PdfUrl.query.filter(
PdfUrl.url.in_([x.pdf_urlforxinunvalidated]),
PdfUrl.is_pdf.is_(False)
 ).all()
 }

forlocationinunvalidated:
location.pdf_url_valid= (
location.pdf_urlnotinbad_pdf_urls
orlocation.oa_status==OAStatus.gold
orlocation.oa_status==OAStatus.hybrid
or'journal.csj.jp/doi/pdf'inlocation.pdf_url
 )

ifnotlocation.pdf_url_valid:
logger.info('excluding location with bad pdf url: {}'.format(location))


classOAStatus(Enum):
closed='closed'
green='green'
bronze='bronze'
hybrid='hybrid'
gold='gold'


defoa_status_sort_key(location):
keys= {
OAStatus.closed: 0,
OAStatus.green: 1,
OAStatus.bronze: 2,
OAStatus.hybrid: 3,
OAStatus.gold: 4,
 }

returnkeys.get(location.oa_status, 0)


classOpenLocation(db.Model):
id=db.Column(db.Text, primary_key=True)
pub_id=db.Column(db.Text, db.ForeignKey('pub.id'))
doi=db.Column(db.Text) # denormalized from Publication for ease of interpreting

pdf_url=db.Column(db.Text)
metadata_url=db.Column(db.Text)
license=db.Column(db.Text)
evidence=db.Column(db.Text)
updated=db.Column(db.DateTime)
error=db.Column(db.Text)

def__init__(self, **kwargs):
self.id=shortuuid.uuid()[0:20]
self.doi=""
self.match= {}
self.pmh_id=None
self.endpoint_id=None
self.base_doc=None
self.version=None
self.error=""
self.pdf_url_valid=None
self.institution=None
self.oa_date=None
self.publisher_specific_license=None
super(OpenLocation, self).__init__(**kwargs)

@property
defhas_open_license(self):
ifnotself.license:
returnFalse
ifself.licensein (
"unknown",
"elsevier-specific: oa user license",
 ) orself.publisher_specific_licensein (
"http://onlinelibrary.wiley.com/termsAndConditions#am",
"http://www.apa.org/pubs/journals/resources/open-access.aspx",
 ) or (hasattr(self, 'not_ol_exception_func') andself.not_ol_exception_func()):
returnFalse
returnTrue

@property
defbest_url(self):
returnself.pdf_urlorself.metadata_url

@property
defbest_url_is_pdf(self):
ifnotself.best_url:
returnNone
ifself.pdf_url:
returnTrue
returnFalse

@property
defis_reported_noncompliant(self):
ifis_reported_noncompliant_url(self.doi, self.pdf_url) oris_reported_noncompliant_url(self.doi, self.metadata_url):
returnTrue
returnFalse

@property
defis_gold(self):
returnself.best_urlandself.display_evidence.startswith(oa_evidence.oa_journal_prefix)

@property
defis_green(self):
returnself.best_urlandself.display_evidence.startswith('oa repository')

@property
defis_hybrid(self):
returnself.best_urlandnot (self.is_goldorself.is_green) andself.has_open_license

@property
defis_bronze(self):
ifself.best_urlandnot (self.is_goldorself.is_green) andnotself.has_open_license:
returnTrue

ifis_doi_url(self.best_url):
url_doi=normalize_doi(self.best_url, return_none_if_error=True)
unquoted_doi=normalize_doi(unquote(self.best_url), return_none_if_error=True)

return (
self.doiin (url_doi, unquoted_doi)
andnot (self.is_goldorself.is_hybridorself.is_green)
 )

returnFalse

@property
defdisplay_evidence(self):
returnself.evidenceor''

@property
defhost_type_calculated(self):
ifself.is_goldorself.is_hybridorself.is_bronze:
return"publisher"
return"repository"

@property
defhost_type(self):
ifhasattr(self, "host_type_set"):
returnself.host_type_set
else:
returnself.host_type_calculated

@property
defis_doaj_journal(self):
return"doaj"inself.display_evidence

@property
defdisplay_updated(self):
ifself.updated:
try:
returnself.updated.isoformat()
exceptAttributeError:
returnself.updated
returnNone

@property
defoa_status(self):
ifhasattr(self, "oa_status_set"):
returnOAStatus(self.oa_status_set)
ifself.is_gold:
returnOAStatus.gold
ifself.is_hybrid:
returnOAStatus.hybrid
ifself.is_bronze:
returnOAStatus.bronze
ifself.is_green:
returnOAStatus.green
ifnotself.best_url:
returnOAStatus.closed
ifnotself.display_evidence:
logger.info("should have evidence for {} but none".format(self.id))

returnOAStatus.green

@property
defis_pmc(self):
ifself.best_urlandre.findall("ncbi.nlm.nih.gov/pmc", self.best_url):
returnTrue
returnFalse

@property
defsort_score(self):

score=0

ifself.host_type=="publisher":
score+=-1000
ifself.display_evidencein [oa_evidence.oa_journal_manual, oa_evidence.oa_journal_observed]:
score+=100
ifself.has_open_license:
# give gold/hybrid locations a boost so they aren't removed by deduplication
score+=-50
else:
# doi.org urls for preprints are called repositories. prefer them over other repos.
ifself.doiandself.metadata_urlandself.doiinself.metadata_url:
score+=-50

ifself.version=="publishedVersion":
score+=-600
ifself.metadata_url=="https://doi.org/{}".format(self.doi):
score+=-200
elifself.version=="acceptedVersion":
score+=-400
elifself.version=="submittedVersion":
score+=-200
# otherwise maybe version is null. sort that to the bottom

# this is very important
ifself.pdf_url:
score+=-100

# if had a doi match, give it a little boost because more likely a perfect match (negative is good)
if"doi"inself.display_evidence:
score+=-10

# penalize versioned preprint dois like 10.26434/chemrxiv.12073869.v1 to let 10.26434/chemrxiv.12073869 win
ifself.host_type=='repository'andself.metadata_urlandre.findall(r'\.v\d+$', self.metadata_url):
score+=5

# let the repos sort themselves out
score+=url_sort_score(self.best_url)

returnscore

def__repr__(self):
return"<OpenLocation ({}) {} {} {} {}>".format(self.id, self.doi, self.display_evidence, self.pdf_url, self.metadata_url)

defto_dict(self):
response= {
"pdf_url": self.pdf_url,
"metadata_url": self.metadata_url,
"license": self.license,
"evidence": self.display_evidence,
"pmh_id": self.pmh_id,
"endpoint_id": self.endpoint_id,
"oa_color": self.oa_status.value,
"version": self.version
 }

ifself.is_reported_noncompliant:
response["reported_noncompliant"] =True

returnresponse

defto_dict_v2(self):
ifhasattr(self, "is_best"):
is_best=self.is_best
else:
is_best=False

response= {
"updated": self.display_updated,
"url": self.best_url,
"url_for_pdf": self.pdf_url,
"url_for_landing_page": self.metadata_url,
"evidence": self.display_evidence,
"license": self.license,
"version": self.version,
"host_type": self.host_type,
"is_best": is_best,
"pmh_id": self.pmh_id,
"endpoint_id": self.endpoint_id,
"repository_institution": self.institution,
"oa_date": self.oa_dateandself.oa_date.isoformat(),
 }

ifself.is_reported_noncompliant:
response["reported_noncompliant"] =True

returnresponse