pub.py

importdatetime
importgzip
importjson
importrandom
importre
importurllib.parse
fromcollectionsimportCounter
fromcollectionsimportOrderedDict
fromcollectionsimportdefaultdict
fromenumimportEnum
fromfunctoolsimportcached_property
fromthreadingimportThread

importboto3
importdateutil.parser
importrequests
fromdateutil.relativedeltaimportrelativedelta
fromlxmlimportetree
frompsycopg2.errorsimportUniqueViolation
fromsqlalchemyimportorm, sql, text, func
fromsqlalchemy.dialects.postgresqlimportJSONB
fromsqlalchemy.excimportIntegrityError
fromsqlalchemy.orm.attributesimportflag_modified

importoa_evidence
importoa_local
importoa_manual
importoa_page
importpage
fromappimportdb, oa_db_engine
fromappimportlogger
fromconstimportLANDING_PAGE_ARCHIVE_BUCKET, PDF_ARCHIVE_BUCKET, \
LANDING_PAGE_ARCHIVE_BUCKET_NEW
fromconvert_http_to_httpsimportfix_url_scheme
fromhttp_cacheimportget_session_id
fromurllib.parseimportquote
fromjournalimportJournal
fromoa_manualimportOAManual
fromopen_locationimportOpenLocation, validate_pdf_urls, OAStatus, \
oa_status_sort_key
frompdf_urlimportPdfUrl
frompdf_utilimportsave_pdf, enqueue_pdf_parsing, PDFVersion, save_pdf_new
frompmh_recordimportis_known_mismatch
frompmh_recordimporttitle_is_too_common
frompmh_recordimporttitle_is_too_short
fromrecordthresher.recordimportRecordthresherParentRecord
fromrecordthresher.record_makerimportCrossrefRecordMaker
fromrecordthresher.record_maker.parseland_record_makerimport \
ParselandRecordMaker
fromrecordthresher.record_makerimportPmhRecordMaker
fromrecordthresher.record_maker.pdf_record_makerimportPDFRecordMaker
fromreported_noncompliant_copiesimportreported_noncompliant_url_fragments
froms3_utilimportharvest_html_table
fromutilimportNoDoiException, enqueue_unpaywall_refresh, \
save_landing_page_new
fromutilimportis_pmc, clamp, clean_doi, normalize_doi
fromconvert_http_to_httpsimportfix_url_scheme
fromutilimportnormalize
fromutilimportnormalize_title
fromutilimportsafe_commit
fromwebpageimportPublisherWebpage

s2_endpoint_id='trmgzrn8eq4yx7ddvmzs'

s3=boto3.client('s3', verify=False)


defbuild_new_pub(doi, crossref_api):
my_pub=Pub(id=doi, crossref_api_raw_new=crossref_api)
my_pub.title=my_pub.crossref_title
my_pub.normalized_title=normalize_title(my_pub.title)
returnmy_pub


defadd_new_pubs(pubs_to_commit):
ifnotpubs_to_commit:
return []

pubs_indexed_by_id=dict((my_pub.id, my_pub) formy_pubinpubs_to_commit)
ids_already_in_db= [
id_tuple[0] forid_tupleindb.session.query(Pub.id).filter(
Pub.id.in_(list(pubs_indexed_by_id.keys()))).all()
 ]
pubs_to_add_to_db= []

for (pub_id, my_pub) inpubs_indexed_by_id.items():
ifpub_idinids_already_in_db:
# merge if we need to
pass
else:
pubs_to_add_to_db.append(my_pub)
# logger.info(u"adding new pub {}".format(my_pub.id))

ifpubs_to_add_to_db:
logger.info("adding {} pubs".format(len(pubs_to_add_to_db)))
db.session.add_all(pubs_to_add_to_db)
safe_commit(db)
db.session.execute(
text(
'''
 insert into recordthresher.doi_record_queue (doi, updated) (
 select id, (crossref_api_raw_new->'indexed'->>'date-time')::timestamp without time zone from pub
 where id = any (:dois)
 ) on conflict do nothing
 '''
 ).bindparams(dois=[p.idforpinpubs_to_add_to_db])
 )
safe_commit(db)
returnpubs_to_add_to_db


defcall_targets_in_parallel(targets):
ifnottargets:
return

# logger.info(u"calling", targets)
threads= []
fortargetintargets:
process=Thread(target=target, args=[])
process.start()
threads.append(process)
forprocessinthreads:
try:
process.join(timeout=60*10)
except (KeyboardInterrupt, SystemExit):
pass
exceptExceptionase:
logger.exception(
"thread Exception {} in call_targets_in_parallel. continuing.".format(
e))
# logger.info(u"finished the calls to {}".format(targets))


defcall_args_in_parallel(target, args_list):
# logger.info(u"calling", targets)
threads= []
forargsinargs_list:
process=Thread(target=target, args=args)
process.start()
threads.append(process)
forprocessinthreads:
try:
process.join(timeout=60*10)
except (KeyboardInterrupt, SystemExit):
pass
exceptExceptionase:
logger.exception(
"thread Exception {} in call_args_in_parallel. continuing.".format(
e))
# logger.info(u"finished the calls to {}".format(targets))


deflookup_product_by_doi(doi):
biblio= {"doi": doi}
returnlookup_product(**biblio)


deflookup_product(**biblio):
my_pub=None
if"doi"inbiblioandbiblio["doi"]:
doi=normalize_doi(biblio["doi"])

# map unregistered JSTOR DOIs to real articles
# for example https://www.jstor.org/stable/2244328?seq=1 says 10.2307/2244328 on the page
# but https://doi.org/10.2307/2244328 goes nowhere and the article is at https://doi.org/10.1214/aop/1176990626

jstor_overrides= {
'10.2307/2244328': '10.1214/aop/1176990626',
# https://www.jstor.org/stable/2244328
'10.2307/25151720': '10.1287/moor.1060.0190',
# https://www.jstor.org/stable/25151720
'10.2307/2237638': '10.1214/aoms/1177704711',
# https://www.jstor.org/stable/2237638
 }

other_overrides= {
# these seem to be the same thing but the first one doesn't work
# https://api.crossref.org/v1/works/http://dx.doi.org/10.3402/qhw.v1i3.4932
# https://api.crossref.org/v1/works/http://dx.doi.org/10.1080/17482620600881144
'10.3402/qhw.v1i3.4932': '10.1080/17482620600881144',
 }

doi=jstor_overrides.get(
doi,
other_overrides.get(doi, doi)
 )

my_pub=Pub.query.get(doi)

ifnotmy_pub:
# try cleaning DOI further
doi=clean_doi(doi)
my_pub=Pub.query.get(doi)
ifnotmy_pub:
raiseNoDoiException

my_pub.reset_vars()
returnmy_pub


defrefresh_pub(my_pub, do_commit=False):
my_pub.run_with_hybrid()
db.session.merge(my_pub)
ifdo_commit:
safe_commit(db)
returnmy_pub


defthread_result_wrapper(func, args, res):
res.append(func(*args))


# get rid of this when we get rid of POST endpoint
# for now, simplify it so it just calls the single endpoint
defget_pubs_from_biblio(biblios, run_with_hybrid=False):
returned_pubs= []
forbiblioinbiblios:
returned_pubs.append(
get_pub_from_biblio(biblio, run_with_hybrid=run_with_hybrid))
returnreturned_pubs


defget_pub_from_biblio(biblio, run_with_hybrid=False, skip_all_hybrid=False,
recalculate=True):
my_pub=lookup_product(**biblio)
ifrun_with_hybrid:
my_pub.run_with_hybrid()
safe_commit(db)
elifrecalculate:
my_pub.recalculate()
returnmy_pub


defmax_pages_from_one_repo(endpoint_ids):
endpoint_id_counter=Counter(endpoint_ids)
most_common=endpoint_id_counter.most_common(1)
ifmost_common:
returnmost_common[0][1]
return0


defget_citeproc_date(year=0, month=1, day=1):
try:
returndatetime.date(year, month, day)
exceptValueError:
returnNone


defcsv_dict_from_response_dict(data):
ifnotdata:
returnNone

response=defaultdict(str)
response["doi"] =data.get("doi", None)
response["doi_url"] =data.get("doi_url", None)
response["is_oa"] =data.get("is_oa", None)
response["oa_status"] =data.get("oa_status", None)
response["genre"] =data.get("genre", None)
response["is_paratext"] =data.get("is_paratext", None)
response["journal_name"] =data.get("journal_name", None)
response["journal_issns"] =data.get("journal_issns", None)
response["journal_issn_l"] =data.get("journal_issn_l", None)
response["journal_is_oa"] =data.get("journal_is_oa", None)
response["journal_is_in_doaj"] =data.get("journal_is_in_doaj", None)
response["publisher"] =data.get("publisher", None)
response["published_date"] =data.get("published_date", None)
response["data_standard"] =data.get("data_standard", None)

best_location_data=data.get("best_oa_location", None)
ifnotbest_location_data:
best_location_data=defaultdict(str)
response["best_oa_url"] =best_location_data.get("url", "")
response["best_oa_url_is_pdf"] =best_location_data.get("url_for_pdf",
"") !=""
response["best_oa_evidence"] =best_location_data.get("evidence", None)
response["best_oa_host"] =best_location_data.get("host_type", None)
response["best_oa_version"] =best_location_data.get("version", None)
response["best_oa_license"] =best_location_data.get("license", None)

returnresponse


defbuild_crossref_record(data):
ifnotdata:
returnNone

record= {}

simple_fields= [
"publisher",
"subject",
"link",
"license",
"funder",
"type",
"update-to",
"clinical-trial-number",
"ISSN", # needs to be uppercase
"ISBN", # needs to be uppercase
"alternative-id"
 ]

forfieldinsimple_fields:
iffieldindata:
record[field.lower()] =data[field]

if"title"indata:
ifisinstance(data["title"], str):
record["title"] =data["title"]
else:
ifdata["title"]:
record["title"] =data["title"][0] # first one
if"title"inrecordandrecord["title"]:
record["title"] =re.sub("\s+", " ", record["title"])

if"container-title"indata:
record["all_journals"] =data["container-title"]
ifisinstance(data["container-title"], str):
record["journal"] =data["container-title"]
else:
ifdata["container-title"]:
record["journal"] =data["container-title"][-1] # last one
# get rid of leading and trailing newlines
ifrecord.get("journal", None):
record["journal"] =record["journal"].strip()

if"author"indata:
# record["authors_json"] = json.dumps(data["author"])
record["all_authors"] =data["author"]
ifdata["author"]:
first_author=data["author"][0]
iffirst_authorand"family"infirst_author:
record["first_author_lastname"] =first_author["family"]
forauthorinrecord["all_authors"]:
ifauthorand"affiliation"inauthorandnotauthor.get(
"affiliation", None):
delauthor["affiliation"]

if"issued"indata:
# record["issued_raw"] = data["issued"]
try:
if"raw"indata["issued"]:
record["year"] =int(data["issued"]["raw"])
elif"date-parts"indata["issued"]:
record["year"] =int(data["issued"]["date-parts"][0][0])
date_parts=data["issued"]["date-parts"][0]
pubdate=get_citeproc_date(*date_parts)
ifpubdate:
record["pubdate"] =pubdate.isoformat()
except (IndexError, TypeError):
pass

if"deposited"indata:
try:
record["deposited"] =data["deposited"]["date-time"]
except (IndexError, TypeError):
pass

record["added_timestamp"] =datetime.datetime.utcnow().isoformat()
returnrecord


classPmcidPublishedVersionLookup(db.Model):
pmcid=db.Column(db.Text, db.ForeignKey('pmcid_lookup.pmcid'),
primary_key=True)


classPmcidLookup(db.Model):
doi=db.Column(db.Text, db.ForeignKey('pub.id'), primary_key=True)
pmcid=db.Column(db.Text)
release_date=db.Column(db.Text)

pmcid_pubished_version_link=db.relationship(
'PmcidPublishedVersionLookup',
lazy='subquery',
viewonly=True,
backref=db.backref("pmcid_lookup", lazy="subquery"),
foreign_keys="PmcidPublishedVersionLookup.pmcid"
 )

@property
defversion(self):
ifself.pmcid_pubished_version_link:
return"publishedVersion"
return"acceptedVersion"


classIssnlLookup(db.Model):
__tablename__='openalex_issn_to_issnl'

issn=db.Column(db.Text, primary_key=True)
issn_l=db.Column(db.Text)
journal_id=db.Column(db.Text)


classJournalOaStartYear(db.Model):
__tablename__='journal_oa_start_year_patched'

issn_l=db.Column(db.Text, primary_key=True)
title=db.Column(db.Text)
oa_year=db.Column(db.Integer)


classS2Lookup(db.Model):
__tablename__='semantic_scholar'

doi=db.Column(db.Text, primary_key=True)
s2_url=db.Column(db.Text)
s2_pdf_url=db.Column(db.Text)


classGreenScrapeAction(Enum):
scrape_now=1
queue=2
none=3


classPreprint(db.Model):
preprint_id=db.Column(db.Text, primary_key=True)
postprint_id=db.Column(db.Text, primary_key=True)

def__repr__(self):
return'<Preprint {}, {}>'.format(self.preprint_id, self.postprint_id)


classRetraction(db.Model):
retraction_doi=db.Column(db.Text, primary_key=True)
retracted_doi=db.Column(db.Text, primary_key=True)

def__repr__(self):
return'<Retraction {}, {}>'.format(self.retraction_doi,
self.retracted_doi)


classFilteredPreprint(db.Model):
preprint_id=db.Column(db.Text, primary_key=True)
postprint_id=db.Column(db.Text, primary_key=True)

def__repr__(self):
return'<FilteredPreprint {}, {}>'.format(self.preprint_id,
self.postprint_id)


classPubRefreshResult(db.Model):
id=db.Column(db.Text, primary_key=True)
refresh_time=db.Column(db.DateTime, primary_key=True)
oa_status_before=db.Column(db.Text)
oa_status_after=db.Column(db.Text)

def__repr__(self):
returnf'<PubRefreshResult({self.id}, {self.refresh_time}, {self.oa_status_before}, {self.oa_status_after})>'


classPub(db.Model):
id=db.Column(db.Text, primary_key=True)
updated=db.Column(db.DateTime)
crossref_api_raw_new=db.Column(JSONB)
published_date=db.Column(db.DateTime)
title=db.Column(db.Text)
normalized_title=db.Column(db.Text)
issns_jsonb=db.Column(JSONB)

last_changed_date=db.Column(db.DateTime)
response_jsonb=db.Column(JSONB)
response_is_oa=db.Column(db.Boolean)
response_best_evidence=db.Column(db.Text)
response_best_url=db.Column(db.Text)
response_best_host=db.Column(db.Text)
response_best_repo_id=db.Column(db.Text)
response_best_version=db.Column(db.Text)

scrape_updated=db.Column(db.DateTime)
scrape_evidence=db.Column(db.Text)
scrape_pdf_url=db.Column(db.Text)
scrape_metadata_url=db.Column(db.Text)
scrape_license=db.Column(db.Text)

resolved_doi_url=db.Column(db.Text)
resolved_doi_http_status=db.Column(db.SmallInteger)
doi_landing_page_is_archived=db.Column(db.Boolean)
recordthresher_id=db.Column(db.Text)

error=db.Column(db.Text)

rand=db.Column(db.Numeric)

pmcid_links=db.relationship(
'PmcidLookup',
lazy='subquery',
viewonly=True,
backref=db.backref("pub", lazy="subquery"),
foreign_keys="PmcidLookup.doi"
 )

page_matches_by_doi=db.relationship(
'Page',
lazy='subquery',
viewonly=True,
backref=db.backref("pub_by_doi", lazy="subquery"),
foreign_keys="Page.doi"
 )

repo_page_matches_by_doi=db.relationship(
'RepoPage',
lazy='subquery',
viewonly=True,
primaryjoin="and_(RepoPage.match_doi == True, RepoPage.doi == Pub.id)"
 )

repo_page_matches_by_title=db.relationship(
'RepoPage',
lazy='subquery',
viewonly=True,
primaryjoin="and_(RepoPage.match_title == True, RepoPage.normalized_title == Pub.normalized_title)"
 )

def__init__(self, **biblio):
self.reset_vars()
self.rand=random.random()
self.license=None
self.free_metadata_url=None
self.free_pdf_url=None
self.oa_status=None
self.evidence=None
self.open_locations= []
self.embargoed_locations= []
self.closed_urls= []
self.session_id=None
self.version=None
self.issn_l=None
self.openalex_journal_id=None
# self.updated = datetime.datetime.utcnow()
for (k, v) inbiblio.items():
self.__setattr__(k, v)

@orm.reconstructor
definit_on_load(self):
self.reset_vars()

defreset_vars(self):
ifself.idandself.id.startswith("10."):
self.id=normalize_doi(self.id)

self.license=None
self.free_metadata_url=None
self.free_pdf_url=None
self.oa_status=None
self.evidence=None
self.open_locations= []
self.embargoed_locations= []
self.closed_urls= []
self.session_id=None
self.version=None

issn_l_lookup=self.lookup_issn_l()
self.issn_l=issn_l_lookup.issn_lifissn_l_lookupelseNone
self.openalex_journal_id=issn_l_lookup.journal_idifissn_l_lookupelseNone

@property
defdoi(self):
returnself.id

@property
defunpaywall_api_url(self):
return"https://api.unpaywall.org/v2/{}?email=internal@impactstory.org".format(
self.id)

@property
deftdm_api(self):
returnNone

@property
defcrossref_api_raw(self):
record=None
try:
ifself.crossref_api_raw_new:
returnself.crossref_api_raw_new
exceptIndexError:
pass

returnrecord

@property
defcrossref_api_modified(self):
record=None
ifself.crossref_api_raw_new:
try:
returnbuild_crossref_record(self.crossref_api_raw_new)
exceptIndexError:
pass

ifself.crossref_api_raw:
try:
record=build_crossref_record(self.crossref_api_raw)
print("got record")
returnrecord
exceptIndexError:
pass

returnrecord

@property
defopen_urls(self):
# return sorted urls, without dups
urls= []
forlocationinself.sorted_locations:
iflocation.best_urlnotinurls:
urls.append(location.best_url)
returnurls

@property
defurl(self):
ifself.doiandself.doi.startswith('10.2218/forum.'):
article_id=self.doi.split('.')[-1]
returnf'http://journals.ed.ac.uk/forum/article/view/{article_id}'

return"https://doi.org/{}".format(self.id)

@property
defis_oa(self):
returnbool(self.fulltext_url)

@property
defis_paratext(self):
paratext_exprs= [
r'^Author Guidelines$',
r'^Author Index$'
r'^Back Cover',
r'^Back Matter',
r'^Contents$',
r'^Contents:',
r'^Cover Image',
r'^Cover Picture',
r'^Editorial Board',
r'Editor Report$',
r'^Front Cover',
r'^Frontispiece',
r'^Graphical Contents List$',
r'^Index$',
r'^Inside Back Cover',
r'^Inside Cover',
r'^Inside Front Cover',
r'^Issue Information',
r'^List of contents',
r'^List of Tables$',
r'^List of Figures$',
r'^List of Plates$',
r'^Masthead',
r'^Pages de début$',
r'^Title page',
r"^Editor's Preface",
 ]

forexprinparatext_exprs:
ifself.titleandre.search(expr, self.title, re.IGNORECASE):
returnTrue

returnFalse

@property
defis_retracted(self):
returnbool(
Retraction.query.filter(Retraction.retracted_doi==self.doi).all()
 )

defrecalculate(self, quiet=False, ask_preprint=True):
self.clear_locations()

ifself.publisher=="CrossRef Test Account":
self.error+="CrossRef Test Account"
raiseNoDoiException

ifself.journal=="CrossRef Listing of Deleted DOIs":
self.error+="CrossRef Deleted DOI"
raiseNoDoiException

self.find_open_locations(ask_preprint)
self.decide_if_open()
self.set_license_hacks()

ifself.is_oaandnotquiet:
logger.info(
"**REFRESH found a fulltext_url for {}! {}: {} **".format(
self.id, self.oa_status.value, self.fulltext_url))

defrefresh_crossref(self):
fromput_crossref_in_dbimportget_api_for_one_doi
self.crossref_api_raw_new=get_api_for_one_doi(self.doi)

defrefresh_including_crossref(self):
self.refresh_crossref()
returnself.refresh()

defdo_not_refresh(self):
current_oa_status=self.response_jsonbandself.response_jsonb.get(
'oa_status', None)
ifcurrent_oa_statusandcurrent_oa_status=="gold"orcurrent_oa_status=="hybrid":
issns_to_refresh= [
"2152-7180",
"1687-8507",
"1131-5598",
"2083-2931",
"2008-322X",
"2152-7180, 2152-7199, 0037-8046, 1545-6846, 0024-3949, 1613-396X, 1741-2862, 0047-1178",
"2598-0025"
 ]
r=requests.get(
f"https://parseland.herokuapp.com/parse-publisher?doi={self.id}")
ifr.status_code!=200:
logger.info(
f"need to refresh gold or hybrid because parseland is bad response {self.id}")
returnFalse
elifself.issn_linissns_to_refresh:
logger.info(
f"need to refresh gold or hybrid because of the journal {self.id}")
returnFalse
elifself.scrape_licenseandself.scrape_license=='mit':
returnFalse
returnTrue

defproblematic_scrape(self):
# query dynamodb to see if this has been scraped several times before
html_response=harvest_html_table.query(
IndexName='by_normalized_doi',
KeyConditionExpression='normalized_doi = :doi',
ExpressionAttributeValues={':doi': self.id}
 )
if'Count'inhtml_responseandhtml_response['Count'] >2:
logger.info(f"problematic scrape {self.id}, count in dynamodb is {html_response['Count']}")
returnTrue

defrefresh(self, session_id=None, force=False):
if (self.resolved_doi_http_statusisnotNoneorself.problematic_scrape()) andnotforce:
logger.info(
f"not refreshing {self.id} because it has been scraped once already")
self.store_or_remove_pdf_urls_for_validation()
self.store_refresh_priority()
self.create_or_update_recordthresher_record()
db.session.merge(self)
return

self.session_id=session_idorget_session_id()
refresh_result=PubRefreshResult(
id=self.id,
refresh_time=datetime.datetime.utcnow(),
oa_status_before=self.response_jsonbandself.response_jsonb.get(
'oa_status', None)
 )

ifself.is_closed_exception:
logger.info(f'{self.doi} is closed exception. Setting to closed and skipping hybrid refresh')
self.open_locations= []
else:
self.refresh_hybrid_scrape()

# and then recalculate everything, so can do to_dict() after this and it all works
self.update()

refresh_result.oa_status_after=self.response_jsonbandself.response_jsonb.get(
'oa_status', None)
db.session.merge(refresh_result)

# then do this so the recalculated stuff saves
# it's ok if this takes a long time... is a short time compared to refresh_hybrid_scrape
self.create_or_update_recordthresher_record()
db.session.merge(self)

defcreate_or_update_parseland_record(self):
ifpl_record:=ParselandRecordMaker.make_record(self):
db.session.merge(pl_record)

defcreate_or_update_recordthresher_record(self, all_records=True):
ifself.errorand"crossref deleted doi"inself.error.lower():
returnFalse
ifrt_record:=CrossrefRecordMaker.make_record(self):
db.session.merge(rt_record)
ifnotall_records:
returnTrue
self.recordthresher_id=rt_record.id
secondary_records=PmhRecordMaker.make_secondary_repository_responses(
rt_record)
forsecondary_recordinsecondary_records:
db.session.merge(secondary_record)
db.session.merge(
RecordthresherParentRecord(record_id=secondary_record.id,
parent_record_id=rt_record.id))
self.create_or_update_parseland_record()
ifself.is_oa:
enqueue_pdf_parsing(self.id, PDFVersion.from_version_str(self.response_best_version))
returnTrue
returnFalse

defset_results(self):
self.issns_jsonb=self.issns
self.response_jsonb=self.to_dict_v2()
self.response_is_oa=self.is_oa
self.response_best_url=self.best_url
self.response_best_evidence=self.best_evidence
self.response_best_version=self.best_version
self.response_best_host=self.best_host
self.response_best_repo_id=self.best_repo_id

defclear_results(self):
self.response_jsonb=None
self.response_is_oa=None
self.response_best_url=None
self.response_best_evidence=None
self.response_best_version=None
self.response_best_host=None
self.response_best_repo_id=None
self.error=""
self.issns_jsonb=None

@staticmethod
defignored_keys_for_internal_diff():
# remove these keys from comparison because their contents are volatile or we don't care about them
return ["updated", "last_changed_date",
"x_reported_noncompliant_copies", "x_error", "data_standard"]

@staticmethod
defignored_keys_for_external_diff():
# remove these keys because they have been added to the api response but we don't want to trigger a diff
returnPub.ignored_keys_for_internal_diff()

@staticmethod
defignored_top_level_keys_for_external_diff():
# existing ignored key regex method doesn't work for multiline keys
# but don't want to replace it yet because it works on nested rows
return ["z_authors", "oa_locations_embargoed"]

@staticmethod
defremove_response_keys(jsonb_response, keys):
response_copy=json.loads(json.dumps(jsonb_response))

forkeyinkeys:
try:
delresponse_copy[key]
exceptKeyError:
pass

returnresponse_copy

defhas_changed(self, old_response_jsonb, ignored_keys,
ignored_top_level_keys):
ifnotold_response_jsonb:
logger.info(
"response for {} has changed: no old response".format(self.id))
returnTrue

copy_of_new_response=Pub.remove_response_keys(self.response_jsonb,
ignored_top_level_keys)
copy_of_old_response=Pub.remove_response_keys(old_response_jsonb,
ignored_top_level_keys)

# have to sort to compare
copy_of_new_response_in_json=json.dumps(copy_of_new_response,
sort_keys=True, indent=2)
# have to sort to compare
copy_of_old_response_in_json=json.dumps(copy_of_old_response,
sort_keys=True, indent=2)

forkeyinignored_keys:
# remove it
copy_of_new_response_in_json=re.sub(
r'"{}":\s*".+?",?\s*'.format(key), '',
copy_of_new_response_in_json)
copy_of_old_response_in_json=re.sub(
r'"{}":\s*".+?",?\s*'.format(key), '',
copy_of_old_response_in_json)

# also remove it if it is an empty list
copy_of_new_response_in_json=re.sub(
r'"{}":\s*\[\],?\s*'.format(key), '',
copy_of_new_response_in_json)
copy_of_old_response_in_json=re.sub(
r'"{}":\s*\[\],?\s*'.format(key), '',
copy_of_old_response_in_json)

# also anything till a comma (gets data_standard)
copy_of_new_response_in_json=re.sub(
r'"{}":\s*.+?,\s*'.format(key), '',
copy_of_new_response_in_json)
copy_of_old_response_in_json=re.sub(
r'"{}":\s*.+?,\s*'.format(key), '',
copy_of_old_response_in_json)

returncopy_of_new_response_in_json!=copy_of_old_response_in_json

defupdate(self):
returnself.recalculate_and_store()

defrecalculate_and_store(self):
ifnotself.crossref_api_raw_new:
self.crossref_api_raw_new=self.crossref_api_raw

self.title=self.crossref_title
self.normalized_title=normalize_title(self.title)
ifnotself.published_date:
self.published_date=self.issued
ifnotself.rand:
self.rand=random.random()

old_response_jsonb=self.response_jsonb

self.clear_results()
try:
self.recalculate()
exceptNoDoiException:
logger.info("invalid doi {}".format(self))
self.error+="Invalid DOI"
pass

self.set_results()
self.mint_pages()
self.scrape_green_locations(GreenScrapeAction.queue)
self.store_or_remove_pdf_urls_for_validation()
self.store_refresh_priority()
self.store_preprint_relationships()
self.store_retractions()
response_changed=self.decide_if_response_changed(old_response_jsonb)

defdecide_if_response_changed(self, old_response_jsonb):
response_changed=False

ifself.has_changed(old_response_jsonb,
Pub.ignored_keys_for_external_diff(),
Pub.ignored_top_level_keys_for_external_diff()):
logger.info(
"changed! updating last_changed_date for this record! {}".format(
self.id))
self.last_changed_date=datetime.datetime.utcnow().isoformat()
response_changed=True

ifself.has_changed(old_response_jsonb,
Pub.ignored_keys_for_internal_diff(), []):
logger.info(
"changed! updating updated timestamp for this record! {}".format(
self.id))
self.updated=datetime.datetime.utcnow()
self.response_jsonb[
'updated'] =datetime.datetime.utcnow().isoformat()
response_changed=True

ifresponse_changed:
flag_modified(self, "response_jsonb") # force it to be saved
else:
self.response_jsonb=old_response_jsonb# don't save if only ignored fields changed

returnresponse_changed

defrun(self):
try:
self.recalculate_and_store()
exceptNoDoiException:
logger.info("invalid doi {}".format(self))
self.error+="Invalid DOI"
pass
# logger.info(json.dumps(self.response_jsonb, indent=4))

defrun_with_hybrid(self, quiet=False, shortcut_data=None):
logger.info("in run_with_hybrid")
self.clear_results()
try:
self.refresh()
exceptNoDoiException:
logger.info("invalid doi {}".format(self))
self.error+="Invalid DOI"
pass

# set whether changed or not
self.set_results()

@property
defhas_been_run(self):
ifself.evidence:
returnTrue
returnFalse

@property
defbest_redirect_url(self):
returnself.fulltext_urlorself.url

@property
defhas_fulltext_url(self):
returnself.fulltext_urlisnotNone

@property
defhas_license(self):
ifnotself.license:
returnFalse
ifself.license=="unknown":
returnFalse
returnTrue

@property
defclean_doi(self):
ifnotself.id:
returnNone
returnnormalize_doi(self.id)