page.py

importdatetime
importgzip
importrandom
importre

importboto3
importdateutil.parser
importshortuuid
fromsqlalchemyimportsql, or_
fromsqlalchemy.dialects.postgresqlimportJSONB
fromsqlalchemy.ext.declarativeimportdeclared_attr

importoa_page
fromappimportdb
fromappimportlogger
fromhttp_cacheimporthttp_get
fromoa_localimportfind_normalized_license
fromoa_pmcimportquery_pmc
frompdf_to_textimportconvert_pdf_to_txt_pages
frompdf_utilimportPDFVersion, save_pdf, enqueue_pdf_parsing, save_pdf_new
fromutilimportclean_url, is_pmc, save_landing_page_new
fromwebpageimportPmhRepoWebpage, PublisherWebpage

DEBUG_BASE=False


classPmhVersionFirstAvailable(db.Model):
endpoint_id=db.Column(db.Text, primary_key=True)
pmh_id=db.Column(db.Text, primary_key=True)
scrape_version=db.Column(db.Text, primary_key=True)
first_available=db.Column(db.DateTime)
url=db.Column(db.Text)


classPageBase(db.Model):
__abstract__=True

id=db.Column(db.Text, primary_key=True)
url=db.Column(db.Text)

@declared_attr
defpmh_id(self):
returndb.Column(db.Text, db.ForeignKey("pmh_record.id"))

@declared_attr
defendpoint_id(self):
returndb.Column(db.Text, db.ForeignKey("endpoint.id"))

title=db.Column(db.Text)

@declared_attr
defnormalized_title(self):
returndb.Column(db.Text, db.ForeignKey("pub.normalized_title"))

scrape_updated=db.Column(db.DateTime)
scrape_pdf_url=db.Column(db.Text)
scrape_metadata_url=db.Column(db.Text)
scrape_version=db.Column(db.Text)
scrape_license=db.Column(db.Text)

record_timestamp=db.Column(db.DateTime)

error=db.Column(db.Text)
updated=db.Column(db.DateTime)

rand=db.Column(db.Numeric)

@declared_attr
defendpoint(self):
returndb.relationship(
'Endpoint',
lazy='subquery',
uselist=None,
cascade="",
viewonly=True
 )

@declared_attr
defpmh_record(self):
returndb.relationship(
'PmhRecord',
lazy='subquery',
uselist=None,
cascade="",
viewonly=True
 )

@property
deffirst_available(self):
ifself.scrape_versionandself.pmh_record:
lookup=PmhVersionFirstAvailable.query.filter(
PmhVersionFirstAvailable.pmh_id==self.pmh_record.bare_pmh_id,
PmhVersionFirstAvailable.endpoint_id==self.pmh_record.endpoint_id,
PmhVersionFirstAvailable.scrape_version==self.scrape_version
 ).first()

iflookup:
returnlookup.first_available.date()

returnself.save_first_version_availability()

@property
defis_open(self):
returnself.scrape_metadata_urlorself.scrape_pdf_url

@property
defis_pmc(self):
returnself.urlandis_pmc(self.url)

@property
defpmcid(self):
ifnotself.is_pmc:
returnNone
matches=re.findall("(pmc\d+)", self.url, re.IGNORECASE)
ifnotmatches:
returnNone
returnmatches[0].lower()

defstore_fulltext(self, fulltext_bytes, fulltext_type):
pass

defstore_landing_page(self, landing_page_markup):
pass

defget_pmh_record_url(self):
returnself.endpointandself.pmh_recordand"{}?verb=GetRecord&metadataPrefix={}&identifier={}".format(
self.endpoint.pmh_url, self.endpoint.metadata_prefix,
self.pmh_record.bare_pmh_id
 )

@property
defrepository_display_name(self):
ifself.endpointandself.endpoint.repo:
returnself.endpoint.repo.display_name()
else:
returnNone

@property
defbare_pmh_id(self):
returnself.pmh_recordandself.pmh_record.bare_pmh_id

@property
defhas_no_error(self):
returnself.errorisNoneorself.error==""

@property
defscrape_updated_datetime(self):
ifisinstance(self.scrape_updated, str):
returndatetime.datetime.strptime(self.scrape_updated,
"%Y-%m-%dT%H:%M:%S.%f")
elifisinstance(self.scrape_updated, datetime.datetime):
returnself.scrape_updated
else:
returnNone

defnot_scraped_in(self, interval):
return (
notself.scrape_updated_datetime
orself.scrape_updated_datetime< (
datetime.datetime.now() -interval)
 )

defscrape_eligible(self):
return (
 (self.has_no_errororself.not_scraped_in(
datetime.timedelta(weeks=1))) and
 (
self.pmh_idand"oai:open-archive.highwire.org"notinself.pmh_id) and
not (
self.urland'//hdl.handle.net/10454/'inself.url) and# https://support.unpaywall.org/a/tickets/22695
not (self.urlandself.url.startswith(
'https://biblio.vub.ac.be/vubir/') andself.url.endswith(
'.html'))
 )

defset_info_for_pmc_page(self):
ifnotself.pmcid:
return

result_list=query_pmc(self.pmcid)
ifnotresult_list:
return
result=result_list[0]
has_pdf=result.get("hasPDF", None)
is_author_manuscript=result.get("authMan", None)
is_open_access=result.get("isOpenAccess", None)
raw_license=result.get("license", None)

self.scrape_metadata_url="http://europepmc.org/articles/{}".format(
self.pmcid)
ifhas_pdf=="Y":
self.scrape_pdf_url="http://europepmc.org/articles/{}?pdf=render".format(
self.pmcid)
ifself.pmcid=='pmc2126438':
self.scrape_pdf_url+='#page=8'
ifis_author_manuscript=="Y":
self.scrape_version="acceptedVersion"
else:
self.scrape_version="publishedVersion"
ifraw_license:
self.scrape_license=find_normalized_license(raw_license)
elifis_open_access=="Y":
self.scrape_license="unspecified-oa"

defscrape(self):
ifnotself.scrape_eligible():
logger.info('refusing to scrape this page')
return

self.updated=datetime.datetime.utcnow().isoformat()
self.scrape_updated=datetime.datetime.utcnow().isoformat()
self.scrape_pdf_url=None
self.scrape_metadata_url=None
self.scrape_license=None
self.scrape_version=None
self.error=""

ifself.pmh_id!=oa_page.publisher_equivalent_pmh_id:
self.scrape_green()
else:
self.scrape_publisher_equivalent()

defscrape_publisher_equivalent(self):
withPublisherWebpage(url=self.url) aspublisher_page:
publisher_page.scrape_for_fulltext_link()

ifpublisher_page.is_open:
self.scrape_version=publisher_page.open_version_source_string
self.scrape_pdf_url=publisher_page.scraped_pdf_url
self.scrape_metadata_url=publisher_page.scraped_open_metadata_url
self.scrape_license=publisher_page.scraped_license
ifpublisher_page.is_openandnotpublisher_page.scraped_pdf_url:
self.scrape_metadata_url=self.url

defscrape_green(self):
# handle these special cases, where we compute the pdf rather than looking for it
if"oai:arXiv.org"inself.pmh_id:
self.scrape_metadata_url=self.url
self.scrape_pdf_url=self.url.replace("abs", "pdf")

ifself.is_pmc:
self.set_info_for_pmc_page()
return

# https://ink.library.smu.edu.sg/do/oai/
ifself.endpointandself.endpoint.id=='ys9xnlw27yogrfsecedx'and'ink.library.smu.edu.sg'inself.url:
if'viewcontent.cgi?'inself.url:
return
ifself.pmh_recordandfind_normalized_license(
self.pmh_record.license):
self.scrape_metadata_url=self.url
self.set_version_and_license()
return

pdf_r=None
ifnotself.scrape_pdf_urlornotself.scrape_version:
withPmhRepoWebpage(url=self.url,
scraped_pdf_url=self.scrape_pdf_url) asmy_webpage:
ifnotself.scrape_pdf_url:
my_webpage.scrape_for_fulltext_link()
self.error+=my_webpage.error
ifmy_webpage.is_open:
logger.info("** found an open copy! {}".format(
my_webpage.fulltext_url))
pdf_r=my_webpage.r
self.scrape_updated=datetime.datetime.utcnow().isoformat()
self.scrape_metadata_url=self.url
ifmy_webpage.scraped_pdf_url:
self.scrape_pdf_url=my_webpage.scraped_pdf_url
ifmy_webpage.scraped_open_metadata_url:
self.scrape_metadata_url=my_webpage.scraped_open_metadata_url
ifmy_webpage.scraped_license:
self.scrape_license=my_webpage.scraped_license
ifmy_webpage.scraped_version:
self.scrape_version=my_webpage.scraped_version

ifself.scrape_pdf_urlandnotself.scrape_version:
self.set_version_and_license(r=my_webpage.r)
elifself.is_openandnotself.scrape_version:
self.update_with_local_info()

self.store_fulltext(my_webpage.fulltext_bytes,
my_webpage.fulltext_type)
version=PDFVersion.from_version_str(my_webpage.scraped_version)
save_pdf_new(my_webpage.fulltext_bytes, self.pmh_id.split(':', 1)[-1].lower(), 'pmh',
version.valueifversionelsePDFVersion.SUBMITTED, resolved_url=my_webpage.scraped_pdf_url)
self.store_landing_page(my_webpage.page_text)
save_landing_page_new(my_webpage.page_text, self.pmh_id.split(':', 1)[-1].lower(), 'pmh', my_webpage.url, my_webpage.resolved_url)

ifself.scrape_pdf_urlandnotself.scrape_version:
withPmhRepoWebpage(url=self.url,
scraped_pdf_url=self.scrape_pdf_url,
repo_id=self.repo_id) asmy_webpage:
my_webpage.set_r_for_pdf()
pdf_r=my_webpage.r
self.set_version_and_license(r=my_webpage.r)

ifself.is_openandnotself.scrape_version:
self.scrape_version=self.default_version()

# associate certain landing page URLs with PDFs
# https://repository.uantwerpen.be
ifself.endpointandself.endpoint.id=='mmv3envg3kaaztya9tmo':
ifself.scrape_pdf_urlandself.scrape_pdf_url==self.scrape_metadata_urlandself.pmh_record:
logger.info('looking for landing page for {}'.format(
self.scrape_pdf_url))
landing_urls= [uforuinself.pmh_record.urlsif
'hdl.handle.net'inu]
iflen(landing_urls) ==1:
logger.info(
'trying landing page {}'.format(landing_urls[0]))

try:
ifhttp_get(landing_urls[0]).status_code==200:
self.scrape_metadata_url=landing_urls[0]
except:
pass

ifself.scrape_metadata_url:
logger.info('set landing page {}'.format(
self.scrape_metadata_url))

# https://lirias.kuleuven.be
if (self.endpoint
andself.endpoint.id=='ycf3gzxeiyuw3jqwjmx3'
andself.scrape_pdf_url==self.scrape_metadata_url
andself.scrape_pdf_urland'lirias.kuleuven.be'inself.scrape_pdf_url
 ):
ifself.pmh_recordandself.pmh_record.bare_pmh_idand'oai:lirias2repo.kuleuven.be:'inself.pmh_record.bare_pmh_id:
self.scrape_metadata_url='https://lirias.kuleuven.be/handle/{}'.format(
self.pmh_record.bare_pmh_id.replace(
'oai:lirias2repo.kuleuven.be:', '')
 )
ifself.scrape_metadata_url:
self.scrape_metadata_url=clean_url(self.scrape_metadata_url)

ifself.scrape_pdf_url:
self.scrape_pdf_url=clean_url(self.scrape_pdf_url)

ifisinstance(self, PageNew) andself.scrape_version:
if (arxiv_dois:=re.findall(r'arxiv.org/abs/(.*?)$', self.url)):
# Save Arxiv PDF with Datacite issued DOI (preprint, self.doi is null)
arxiv_doi=f'10.48550/arxiv.{arxiv_dois[0]}'
logger.info(
f'Saving {self.scrape_version} PDF of DOI - {arxiv_doi}')
self.save_pdf(PDFVersion.SUBMITTED, doi=arxiv_doi)
ifself.doi:
logger.info(
f'Saving {self.scrape_version} PDF of DOI - {self.doi}')
# Save all other PDFs and Arxiv DOIs that have been published (self.doi is not null)
self.save_pdf(PDFVersion.from_version_str(self.scrape_version),
pdf_r=pdf_r)

defpmc_first_available_date(self):
ifself.pmcid:
pmc_result_list=query_pmc(self.pmcid)
ifpmc_result_list:
pmc_result=pmc_result_list[0]
received_date=pmc_result.get("fullTextReceivedDate", None)
ifreceived_date:
try:
returndatetime.datetime.strptime(received_date,
'%Y-%m-%d').date()
exceptException:
returnNone

returnNone

defsave_first_version_availability(self):
ifnotself.scrape_version:
returnNone

first_available=None

ifisinstance(self.record_timestamp, str):
first_available=dateutil.parser.parse(self.record_timestamp)
elifisinstance(self.record_timestamp, datetime.datetime):
first_available=self.record_timestamp.date()
elifisinstance(self.record_timestamp, datetime.date):
first_available=self.record_timestamp

ifself.pmcid:
first_available=self.pmc_first_available_date()

if (self.endpointandself.endpoint.idand
self.pmh_recordandself.pmh_record.bare_pmh_idand
self.urland
first_available):
stmt=sql.text('''
 insert into pmh_version_first_available
 (endpoint_id, pmh_id, url, scrape_version, first_available) values
 (:endpoint_id, :pmh_id, :url, :scrape_version, :first_available)
 on conflict do nothing
 ''').bindparams(
endpoint_id=self.endpoint.id,
pmh_id=self.pmh_record.bare_pmh_id,
url=self.url,
scrape_version=self.scrape_version,
first_available=first_available
 )
db.session.execute(stmt)

returnfirst_available

defdefault_version(self):
ifself.endpointandself.endpoint.policy_promises_no_submitted:
return"acceptedVersion"
elifself.bare_pmh_idand'oai:library.wur.nl:wurpubs'inself.bare_pmh_id:
return"acceptedVersion"
else:
return"submittedVersion"

defupdate_with_local_info(self):
scrape_version_old=self.scrape_version
scrape_license_old=self.scrape_license

# if this repo has told us they will never have submitted, set default to be accepted
ifself.endpointandself.endpoint.policy_promises_no_submittedandself.scrape_version!="publishedVersion":
self.scrape_version="acceptedVersion"

# now look at the pmh record
ifself.pmh_record:
# trust accepted in a variety of formats
accepted_patterns= [
re.compile(r"accepted.?version",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"version.?accepted",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"accepted.?manuscript",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"<dc:type>peer.?reviewed</dc:type>",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r"<dc:description>Refereed/Peer-reviewed</dc:description>",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
 ]
forpatterninaccepted_patterns:
ifpattern.findall(self.pmh_record.api_raw):
self.scrape_version="acceptedVersion"
# print u"version for is {}".format(self.scrape_version)

# trust a strict version of published version
published_patterns= [
re.compile(r"<dc:type>.*publishedVersion</dc:type>",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r"<dc:type\.version>.*publishedVersion</dc:type\.version>",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"<free_to_read>.*published.*</free_to_read>",
re.IGNORECASE|re.MULTILINE|re.DOTALL)
 ]
forpublished_patterninpublished_patterns:
ifpublished_pattern.findall(self.pmh_record.api_raw):
self.scrape_version="publishedVersion"

# get license if it is in pmh record
rights_pattern=re.compile(r"<dc:rights>(.*)</dc:rights>",
re.IGNORECASE|re.MULTILINE|re.DOTALL)
rights_matches=rights_pattern.findall(self.pmh_record.api_raw)
rights_license_pattern=re.compile(
r"<dc:rights\.license>(.*)</dc:rights\.license>",
re.IGNORECASE|re.MULTILINE|re.DOTALL)
rights_matches.extend(
rights_license_pattern.findall(self.pmh_record.api_raw))

forrights_textinrights_matches:
open_license=find_normalized_license(rights_text)
# only overwrite it if there is one, so doesn't overwrite anything scraped
ifopen_license:
self.scrape_license=open_license

self.scrape_license=_scrape_license_override().get(
self.pmh_record.bare_pmh_id, self.scrape_license)

ifself.scrape_pdf_urlandre.search(r'^https?://rke\.abertay\.ac\.uk',
self.scrape_pdf_url):
ifre.search(r'Publishe[dr]_?\d\d\d\d\.pdf$', self.scrape_pdf_url):
self.scrape_version="publishedVersion"
ifre.search(r'\d\d\d\d_?Publishe[dr].pdf$', self.scrape_pdf_url):
self.scrape_version="publishedVersion"

ifself.pmh_record:
self.scrape_version=_scrape_version_override().get(
self.pmh_record.bare_pmh_id, self.scrape_version)

ifscrape_version_old!=self.scrape_versionorscrape_license_old!=self.scrape_license:
self.updated=datetime.datetime.utcnow().isoformat()
print("based on OAI-PMH metadata, updated {} {} for {} {}".format(
self.scrape_version, self.scrape_license, self.url, self.id))
returnTrue

# print u"based on metadata, assuming {} {} for {} {}".format(self.scrape_version, self.scrape_license, self.url, self.id)

returnFalse

# use standards from https://wiki.surfnet.nl/display/DRIVERguidelines/Version+vocabulary
# submittedVersion, acceptedVersion, publishedVersion
defset_version_and_license(self, r=None):
self.updated=datetime.datetime.utcnow().isoformat()

ifself.is_pmc:
self.set_info_for_pmc_page()
return

# set as default
self.scrape_version=self.default_version()

self.update_with_local_info()

# now try to see what we can get out of the pdf itself
version_is_from_strict_metadata=self.pmh_recordandself.pmh_record.api_rawandre.compile(
r"<dc:type>{}</dc:type>".format(self.scrape_version),
re.IGNORECASE|re.MULTILINE|re.DOTALL
 ).findall(self.pmh_record.api_raw)

ifversion_is_from_strict_metadataornotr:
logger.info(
"before scrape returning {} with scrape_version: {}, license {}".format(
self.url, self.scrape_version, self.scrape_license))
return

try:
# http://crossmark.dyndns.org/dialog/?doi=10.1016/j.jml.2012 at http://dspace.mit.edu/bitstream/1721.1/102417/1/Gibson_The%20syntactic.pdf
ifre.findall("crossmark\.[^/]*\.org/", r.text_big(),
re.IGNORECASE):
self.scrape_version="publishedVersion"

pages_text=convert_pdf_to_txt_pages(r, max_pages=25)
first_page_text=pages_textandpages_text[0]
text='\n'.join(pages_text)

iftextandself.scrape_version!="publishedVersion"andnotversion_is_from_strict_metadata:
patterns= [
re.compile(r"©.?\d{4}", re.UNICODE),
re.compile(r"© The Author\(s\),? \d{4}", re.UNICODE),
re.compile(r"\(C\).?\d{4}", re.IGNORECASE),
re.compile(r"copyright.{0,6}\d{4}", re.IGNORECASE),
re.compile(
r"received.{0,100}revised.{0,100}accepted.{0,100}publication",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"all rights reserved", re.IGNORECASE),
re.compile(
r"This article is distributed under the terms of the Creative Commons",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r"This article is licensed under a Creative Commons",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r"this is an open access article",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r"This article is brought to you for free and open access by Works.",
re.IGNORECASE|re.MULTILINE|re.DOTALL),
 ]

forpatterninpatterns:
ifpattern.findall(text):
logger.info(
'found {}, decided PDF is published version'.format(
pattern.pattern))
self.scrape_version="publishedVersion"

iftextandself.scrape_version!='acceptedVersion':
patterns= [
re.compile(
r'This is a post-peer-review, pre-copyedit version',
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r'This is the peer reviewed version of the following article',
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r'The present manuscript as of \d\d \w+ \d\d\d\d has been accepted',
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(
r'Post-peer-review, pre-copyedit version of accepted manuscript',
re.IGNORECASE|re.MULTILINE|re.DOTALL),
re.compile(r'This is a "Post-Print" accepted manuscript',
re.IGNORECASE|re.MULTILINE|re.DOTALL),
 ]

forpatterninpatterns:
ifpattern.findall(text):
logger.info(
'found {}, decided PDF is accepted version'.format(
pattern.pattern))
self.scrape_version="acceptedVersion"

ifrandr.urland'61RMIT_INST'inr.url:
if'Version: Accepted'intext:
logger.info(
'found Version: Accepted, decided PDF is accepted version')
self.scrape_version="acceptedVersion"

iffirst_page_text:
if'Version: Accepted'infirst_page_text:
logger.info(
'found Version: Accepted, decided PDF is accepted version')
self.scrape_version="acceptedVersion"
ifre.compile(
r"Document Version\s+Final published version.",
re.IGNORECASE|re.MULTILINE|re.DOTALL).findall(
first_page_text):
logger.info(
'found Document Version - Final published version, decided PDF is published')
self.scrape_version="publishedVersion"
ifre.compile(r"Document Version:\s*Peer reviewed version",
re.IGNORECASE|re.MULTILINE|re.DOTALL).findall(
first_page_text):
logger.info(
'found Document Version: Peer reviewed version, decided PDF is accepted')
self.scrape_version="acceptedVersion"

heading_text=text[0:50].lower()
accepted_headings= [
"final accepted version",
"accepted manuscript",
 ]

forheadinginaccepted_headings:
ifheadinginheading_text:
logger.info(
'found {} in heading, decided PDF is accepted version'.format(
heading))
self.scrape_version="acceptedVersion"
break

ifnotself.scrape_license:
open_license=find_normalized_license(text)
ifopen_license:
logger.info('found license in PDF: {}'.format(open_license))
self.scrape_license=open_license

exceptExceptionase:
logger.exception(
"exception in convert_pdf_to_txt for {}".format(self.url))
self.error+="Exception doing convert_pdf_to_txt!"
logger.info(self.error)

ifself.pmh_record:
self.scrape_version=_scrape_version_override().get(
self.pmh_record.bare_pmh_id, self.scrape_version)

logger.info(
"scrape returning {} with scrape_version: {}, license {}".format(
self.url, self.scrape_version, self.scrape_license))

def__repr__(self):
return"<PageBase ( {} ) {}>".format(self.pmh_id, self.url)


classLandingPageArchiveKeyLookup(db.Model):
__tablename__='repo_page_landing_page_key_lookup'

id=db.Column(db.Text, db.ForeignKey('page_new.id'), primary_key=True)
key=db.Column(db.Text)


classFulltextArchiveKeyLookup(db.Model):
__tablename__='repo_page_fulltext_key_lookup'

id=db.Column(db.Text, db.ForeignKey('page_new.id'), primary_key=True)
key=db.Column(db.Text)


LANDING_PAGE_ARCHIVE_BUCKET='unpaywall-worksdb-repo-landing-page'
FULLTEXT_PDF_ARCHIVE_BUCKET='unpaywall-tier-2-fulltext'


classPageNew(PageBase):
repo_id=db.Column(db.Text) # delete once endpoint_id is populated
doi=db.Column(db.Text, db.ForeignKey("pub.id"))
authors=db.Column(JSONB)

num_pub_matches=db.Column(db.Numeric)
match_type=db.Column(db.Text)

landing_page_archive_key=db.relationship(
'LandingPageArchiveKeyLookup',
lazy='subquery',
uselist=False,
foreign_keys='LandingPageArchiveKeyLookup.id'
 )

fulltext_pdf_archive_key=db.relationship(
'FulltextArchiveKeyLookup',
lazy='subquery',
uselist=False,
foreign_keys='FulltextArchiveKeyLookup.id'
 )

__mapper_args__= {
"polymorphic_on": match_type,
"polymorphic_identity": "page_new"
 }

def__init__(self, **kwargs):
self.id=shortuuid.uuid()[0:20]
self.error=""
self.rand=random.random()
self.updated=datetime.datetime.utcnow().isoformat()
super(PageNew, self).__init__(**kwargs)

defstore_fulltext(self, fulltext_bytes, fulltext_type):
iffulltext_bytesand (
self.num_pub_matchesisNoneorself.num_pub_matches<1):
try:
ifnotself.fulltext_pdf_archive_key:
self.fulltext_pdf_archive_key=FulltextArchiveKeyLookup(
id=self.id, key=self.id)

logger.info(
f'saving {len(fulltext_bytes)}{fulltext_type} bytes to {self.fulltext_pdf_archive_url()}')
client=boto3.client('s3', verify=False)
client.put_object(
Body=gzip.compress(fulltext_bytes),
Bucket=FULLTEXT_PDF_ARCHIVE_BUCKET,
Key=self.fulltext_pdf_archive_key.key
 )
exceptExceptionase:
logger.error(f'failed to save fulltext bytes: {e}')

defstore_landing_page(self, landing_page_markup):
iflanding_page_markup:
try:
ifnotself.landing_page_archive_key:
self.landing_page_archive_key=LandingPageArchiveKeyLookup(
id=self.id, key=f'{self.id}.gz')

logger.info(
f'saving {len(landing_page_markup)} characters to {self.landing_page_archive_url()}')
client=boto3.client('s3', verify=False)
client.put_object(
Body=gzip.compress(landing_page_markup.encode('utf-8')),
Bucket=LANDING_PAGE_ARCHIVE_BUCKET,
Key=self.landing_page_archive_key.key
 )
exceptExceptionase:
logger.error(f'failed to save landing page text: {e}')

deflanding_page_archive_url(self):
ifnotself.landing_page_archive_key:
returnNone
else:
returnf's3://{LANDING_PAGE_ARCHIVE_BUCKET}/{self.landing_page_archive_key.key}'

deffulltext_pdf_archive_url(self):
ifnotself.fulltext_pdf_archive_key:
returnNone
else:
returnf's3://{FULLTEXT_PDF_ARCHIVE_BUCKET}/{self.fulltext_pdf_archive_key.key}'

defscrape_if_matches_pub(self):
self.num_pub_matches=self.query_for_num_pub_matches()

ifself.num_pub_matches>0andself.scrape_eligible():
returnself.scrape()

defenqueue_scrape_if_matches_pub(self):
self.num_pub_matches=self.query_for_num_pub_matches()

ifself.num_pub_matches>0andself.scrape_eligible():
stmt=sql.text(
'insert into page_green_scrape_queue (id, finished, endpoint_id) values (:id, :finished, :endpoint_id) on conflict do nothing'
 ).bindparams(id=self.id, finished=self.scrape_updated,
endpoint_id=self.endpoint_id)
db.session.execute(stmt)

def__repr__(self):
return"<PageNew ( {} ) {}>".format(self.pmh_id, self.url)

defto_dict(self, include_id=True):
response= {
"oaipmh_id": self.pmh_recordandself.pmh_record.bare_pmh_id,
"oaipmh_record_timestamp": self.record_timestamp.isoformat(),
"pdf_url": self.scrape_pdf_url,
"title": self.title,
"version": self.scrape_version,
"license": self.scrape_license,
"oaipmh_api_url": self.get_pmh_record_url()
 }
ifinclude_id:
response["id"] =self.id
returnresponse

defsave_pdf(self, version: PDFVersion, pdf_r=None, doi=None):
ifnotpdf_r:
pdf_r=http_get(self.scrape_pdf_url, ask_slowly=True, stream=False)
pdf_content=None
ifhasattr(pdf_r, 'content_big'):
pdf_content=pdf_r.content_big()
elifhasattr(pdf_r, 'content'):
pdf_content=pdf_r.content
ifpdf_content:
save_pdf_new(pdf_content, self.pmh_id.split(':', 1)[-1], 'pmh', version, url=pdf_r.url)
save_pdf(doiorself.doi, pdf_content, version)
enqueue_pdf_parsing(doiorself.doi, version)


classPage(db.Model):
url=db.Column(db.Text, primary_key=True)
id=db.Column(db.Text, db.ForeignKey("pmh_record.id"))
source=db.Column(db.Text)
doi=db.Column(db.Text, db.ForeignKey("pub.id"))
title=db.Column(db.Text)
normalized_title=db.Column(db.Text, db.ForeignKey("pub.normalized_title"))
authors=db.Column(JSONB)

scrape_updated=db.Column(db.DateTime)
scrape_evidence=db.Column(db.Text)
scrape_pdf_url=db.Column(db.Text)
scrape_metadata_url=db.Column(db.Text)
scrape_version=db.Column(db.Text)
scrape_license=db.Column(db.Text)

error=db.Column(db.Text)
updated=db.Column(db.DateTime)

started=db.Column(db.DateTime)
finished=db.Column(db.DateTime)
rand=db.Column(db.Numeric)

def__init__(self, **kwargs):
self.error=""
self.updated=datetime.datetime.utcnow().isoformat()
super(self.__class__, self).__init__(**kwargs)

@property
deffirst_available(self):
returnNone

@property
defpmh_id(self):
returnself.id

@property
defbare_pmh_id(self):
returnself.id

@property
defis_open(self):
returnself.scrape_metadata_urlorself.scrape_pdf_url

@property
defis_pmc(self):
ifnotself.url:
returnFalse
if"ncbi.nlm.nih.gov/pmc"inself.url:
returnTrue
if"europepmc.org/articles/"inself.url:
returnTrue
returnFalse

@property
defrepo_id(self):
ifnotself.pmh_idornot":"inself.pmh_id:
returnNone
returnself.pmh_id.split(":")[1]

@property
defendpoint_id(self):
ifnotself.pmh_idornot":"inself.pmh_id:
returnNone
returnself.pmh_id.split(":")[1]

@property
defpmcid(self):
ifnotself.is_pmc:
returnNone
returnself.url.rsplit("/", 1)[1].lower()

@property
defis_preprint_repo(self):
preprint_url_fragments= [
"precedings.nature.com",
"10.15200/winn.",
"/peerj.preprints",
".figshare.",
"10.1101/", # biorxiv
"10.15363/"# thinklab
 ]
forurl_fragmentinpreprint_url_fragments:
ifself.urlandurl_fragmentinself.url.lower():
returnTrue
returnFalse

@property
defrepository_display_name(self):
returnself.repo_id

defupdate_with_local_info(self):
pass

# examples
# https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3039489&resulttype=core&format=json&tool=oadoi
# https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=PMC3606428&resulttype=core&format=json&tool=oadoi
defset_info_for_pmc_page(self):
ifnotself.pmcid:
return

url_template="https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={}&resulttype=core&format=json&tool=oadoi"
url=url_template.format(self.pmcid)

# try:
r=http_get(url)
data=r.json()
result_list=data["resultList"]["result"]
ifnotresult_list:
return
result=result_list[0]
has_pdf=result.get("hasPDF", None)
is_author_manuscript=result.get("authMan", None)
is_open_access=result.get("isOpenAccess", None)
raw_license=result.get("license", None)

self.scrape_metadata_url="http://europepmc.org/articles/{}".format(
self.pmcid)
ifhas_pdf=="Y":
self.scrape_pdf_url="http://europepmc.org/articles/{}?pdf=render".format(
self.pmcid)
ifis_author_manuscript=="Y":
self.scrape_version="acceptedVersion"
else:
self.scrape_version="publishedVersion"
ifraw_license:
self.scrape_license=find_normalized_license(raw_license)
elifis_open_access=="Y":
self.scrape_license="unspecified-oa"

# except Exception as e:
# self.error += u"Exception in set_info_for_pmc_page"
# logger.info(u"Exception in set_info_for_pmc_page")

def__repr__(self):
return"<Page ( {} ) {} doi:{} '{}...'>".format(self.pmh_id, self.url,
self.doi,
self.title[0:20])


def_scrape_version_override():
return {
'oai:dspace.cvut.cz:10467/86163': 'submittedVersion',
'oai:repository.arizona.edu:10150/633848': 'acceptedVersion',
'oai:archive.ugent.be:6914822': 'acceptedVersion',
'oai:serval.unil.ch:BIB_E033703283B2': 'acceptedVersion',
'oai:serval.unil.ch:BIB_3108959306C9': 'acceptedVersion',
'oai:serval.unil.ch:BIB_08C9BAB31C2E': 'acceptedVersion',
'oai:serval.unil.ch:BIB_E8CC2511C152': 'acceptedVersion',
'oai:HAL:hal-01924005v1': 'acceptedVersion',
'oai:serval.unil.ch:BIB_FC320764865F': 'publishedVersion',
'oai:serval.unil.ch:BIB_12B5A0826BD9': 'acceptedVersion',
'oai:upcommons.upc.edu:2117/115471': 'acceptedVersion',
'oai:zenodo.org:4267175': 'acceptedVersion',
'oai:europepmc.org:7435056': 'acceptedVersion',
 }


def_scrape_license_override():
return {
'oai:academiccommons.columbia.edu:10.7916/D8D80PCQ': None,
 }


classRepoPage(PageNew):
__tablename__=None
__mapper_args__= {"polymorphic_identity": "any"}

match_title=db.Column(db.Boolean)
match_doi=db.Column(db.Boolean)

def__init__(self, **kwargs):
self.match_title=False
self.match_doi=False
super(RepoPage, self).__init__(**kwargs)

def__repr__(self):
return"<RepoPage ( {} ) {} match_title: {}, match_doi: {}>".format(
self.pmh_id, self.url, self.match_title, self.match_doi
 )

defto_dict(self, include_id=True):
response= {
"oaipmh_id": self.pmh_recordandself.pmh_record.bare_pmh_id,
"oaipmh_record_timestamp": self.record_timestamp.isoformat(),
"pdf_url": self.scrape_pdf_url,
"title": self.title,
"version": self.scrape_version,
"license": self.scrape_license,
"oaipmh_api_url": self.get_pmh_record_url(),
"match_title": self.match_title,
"match_doi": self.match_doi
 }
ifinclude_id:
response["id"] =self.id
returnresponse

defquery_for_num_pub_matches(self):
frompmh_recordimporttitle_is_too_common
frompmh_recordimporttitle_is_too_short
frompubimportPub

ifself.match_titleandnot (