parseobject.cpp

/* ============================================================
* QuiteRSS is a open-source cross-platform RSS/Atom news feeds reader
* Copyright (C) 2011-2020 QuiteRSS Team <quiterssteam@gmail.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
* ============================================================ */
#include"parseobject.h"

#include"mainapplication.h"
#include"database.h"
#include"VersionNo.h"
#include"common.h"

#include<QDebug>
#include<QDesktopServices>
#include<QTextDocumentFragment>
#if defined(Q_OS_WIN)
#include<windows.h>
#endif
#include<qzregexp.h>

ParseObject::ParseObject(QObject *parent)
 : QObject(parent)
{
setObjectName("parseObject_");

 db_ = Database::connection("secondConnection");

 parseTimer_ = newQTimer(this);
 parseTimer_->setSingleShot(true);
 parseTimer_->setInterval(10);
connect(parseTimer_, SIGNAL(timeout()), this, SLOT(getQueuedXml()),
 Qt::QueuedConnection);

connect(this, SIGNAL(signalReadyParse(QByteArray,int,QDateTime,QString)),
SLOT(slotParse(QByteArray,int,QDateTime,QString)));
}

ParseObject::~ParseObject()
{

}

voidParseObject::disconnectObjects()
{
disconnect(this);
}

/** @brief Queueing xml-data
 *----------------------------------------------------------------------------*/
voidParseObject::parseXml(QByteArray data, int feedId,
 QDateTime dtReply, QString codecName)
{
 idsQueue_.enqueue(feedId);
 xmlsQueue_.enqueue(data);
 dtReadyQueue_.enqueue(dtReply);
 codecNameQueue_.enqueue(codecName);
qDebug() << "xmlsQueue_ <<" << feedId << "count=" << xmlsQueue_.count();

if (!parseTimer_->isActive())
 parseTimer_->start();
}

/** @brief Process xml-data queue
 *----------------------------------------------------------------------------*/
voidParseObject::getQueuedXml()
{
if (!mutex_.tryLock())
return;

if (idsQueue_.count()) {
int feedId = idsQueue_.dequeue();
 QByteArray currentXml_ = xmlsQueue_.dequeue();
 QDateTime currentDtReady_ = dtReadyQueue_.dequeue();
 QString currentCodecName_ = codecNameQueue_.dequeue();
qDebug() << "xmlsQueue_ >>" << feedId << "count=" << xmlsQueue_.count();

 emit signalReadyParse(currentXml_, feedId, currentDtReady_, currentCodecName_);
 }

 mutex_.unlock();

if (idsQueue_.count())
 parseTimer_->start();
}

/** @brief Parse xml-data
 *----------------------------------------------------------------------------*/
voidParseObject::slotParse(const QByteArray &xmlData, constint &feedId,
const QDateTime &dtReply, const QString &codecName)
{
if (mainApp->isSaveDataLastFeed()) {
 QFile file(mainApp->dataDir() + "/lastfeed.dat");
 file.open(QIODevice::WriteOnly);
 file.write(xmlData);
 file.close();
 }

qDebug() << "=================== parseXml:start ============================";

 db_.transaction();

// extract feed id, duplicate news mode and date to avoid from feed table
 parseFeedId_ = feedId;
 QString feedUrl;
 duplicateNewsMode_ = false;
 addSingleNewsAnyDate_ = false;
 avoidedOldSingleNews_ = false;
 avoidedOldSingleNewsDate_ = QDate::currentDate();
 QSqlQuery q(db_);
 q.setForwardOnly(true);
 q.exec(QString("SELECT duplicateNewsMode, xmlUrl, addSingleNewsAnyDateOn, avoidedOldSingleNewsDateOn, avoidedOldSingleNewsDate"
" FROM feeds WHERE id=='%1'").arg(parseFeedId_));
if (q.first()) {
 duplicateNewsMode_ = q.value(0).toBool();
 feedUrl = q.value(1).toString();
 addSingleNewsAnyDate_ = q.value(2).toBool();
 avoidedOldSingleNews_ = q.value(3).toBool();
 avoidedOldSingleNewsDate_ = q.value(4).toDate();
 }

// id not found (ex. feed deleted while updating)
if (feedUrl.isEmpty()) {
qWarning() << QString("Feed with id = '%1' not found").arg(parseFeedId_);
 emit signalFinishUpdate(parseFeedId_, false, 0, "0");
 db_.commit();
return;
 }

qDebug() << QString("Feed '%1' found with id = %2").arg(feedUrl).arg(parseFeedId_);

// actually parsing
 feedChanged_ = false;
 lastBuildDate_ = dtReply;

bool codecOk = false;
 QString convertData(xmlData);
 QString feedType;
 QDomDocument doc;
 QString errorStr;
int errorLine;
int errorColumn;

 QzRegExp rx("encoding=\"([^\"]+)", Qt::CaseInsensitive);
int pos = rx.indexIn(xmlData);
if (pos == -1) {
 rx.setPattern("encoding='([^']+)");
 pos = rx.indexIn(xmlData);
 }
if (pos > -1) {
 QString codecNameT = rx.cap(1);
qDebug() << "Codec name (1):" << codecNameT;
 QTextCodec *codec = QTextCodec::codecForName(codecNameT.toUtf8());
if (codec) {
 convertData = codec->toUnicode(xmlData);
 } else {
qWarning() << "Codec not found (1): " << codecNameT << feedUrl;
if (codecNameT.contains("us-ascii", Qt::CaseInsensitive)) {
 QString str(xmlData);
 convertData = str.remove(rx.cap(0)+"\"");
 }
 }
 } else {
if (!codecName.isEmpty()) {
qDebug() << "Codec name (2):" << codecName;
 QTextCodec *codec = QTextCodec::codecForName(codecName.toUtf8());
if (codec) {
 convertData = codec->toUnicode(xmlData);
 codecOk = true;
 } else {
qWarning() << "Codec not found (2): " << codecName << feedUrl;
 }
 }
if (!codecOk) {
 codecOk = false;
 QStringList codecNameList;
 codecNameList << "UTF-8" << "Windows-1251" << "KOI8-R" << "KOI8-U"
 << "ISO 8859-5" << "IBM 866";
foreach (QString codecNameT, codecNameList) {
 QTextCodec *codec = QTextCodec::codecForName(codecNameT.toUtf8());
if (codec && codec->canEncode(xmlData)) {
qDebug() << "Codec name (3):" << codecNameT;
 convertData = codec->toUnicode(xmlData);
 codecOk = true;
break;
 }
 }
if (!codecOk) {
 convertData = QString::fromLocal8Bit(xmlData);
 }
 }
 }

if (!doc.setContent(convertData, false, &errorStr, &errorLine, &errorColumn)) {
qWarning() << QString("Parse data error (2): url %1, id %2, line %3, column %4: %5").
arg(feedUrl).arg(parseFeedId_).
arg(errorLine).arg(errorColumn).arg(errorStr);
 } else {
 QDomElement rootElem = doc.documentElement();
 feedType = rootElem.tagName();
qDebug() << "Feed type: " << feedType;

 q.exec(QString("SELECT id, guid, title, published, link_href FROM news WHERE feedId='%1'").
arg(parseFeedId_));
if (q.lastError().isValid()) {
qWarning() << __PRETTY_FUNCTION__ << __LINE__
 << "q.lastError(): " << q.lastError().text();
 }
else {
while (q.next()) {
 QString str = q.value(2).toString();
 titleList_.append(str);

 str = q.value(1).toString();
 guidList_.append(str);

 str = q.value(3).toString();
 publishedList_.append(str);
 str = q.value(4).toString();
 linkList_.append(str);
 }
 }
 q.finish();

if (feedType == "feed") {
parseAtom(feedUrl, doc);
 } elseif ((feedType == "rss") || (feedType == "rdf:RDF")) {
parseRss(feedUrl, doc);
 }

 guidList_.clear();
 titleList_.clear();
 publishedList_.clear();
 linkList_.clear();
 }

// Set feed update time and receive data from server time
 QString updated = QLocale::c().toString(QDateTime::currentDateTimeUtc(),
"yyyy-MM-ddTHH:mm:ss");
 QString lastBuildDate = lastBuildDate_.toString(Qt::ISODate);
 q.prepare("UPDATE feeds SET updated=?, lastBuildDate=?, status=0 WHERE id=?");
 q.addBindValue(updated);
 q.addBindValue(lastBuildDate);
 q.addBindValue(parseFeedId_);
 q.exec();

int newCount = 0;
if (feedChanged_) {
runUserFilter(parseFeedId_);
 newCount = recountFeedCounts(parseFeedId_, feedUrl, updated, lastBuildDate);
 }

 q.finish();
 db_.commit();

 emit signalFinishUpdate(parseFeedId_, feedChanged_, newCount, "0");
qDebug() << "=================== parseXml:finish ===========================";
}

voidParseObject::parseAtom(const QString &feedUrl, const QDomDocument &doc)
{
 QDomElement rootElem = doc.documentElement();
 FeedItemStruct feedItem;

 feedItem.linkBase = rootElem.attribute("xml:base");
 feedItem.title = toPlainText(rootElem.namedItem("title").toElement().text());
 feedItem.description = rootElem.namedItem("subtitle").toElement().text();
 feedItem.updated = rootElem.namedItem("updated").toElement().text();
 feedItem.updated = parseDate(feedItem.updated, feedUrl);
 QDomElement authorElem = rootElem.namedItem("author").toElement();
if (!authorElem.isNull()) {
 feedItem.author = toPlainText(authorElem.namedItem("name").toElement().text());
if (feedItem.author.isEmpty()) feedItem.author = toPlainText(authorElem.text());
 feedItem.authorUri = authorElem.namedItem("uri").toElement().text();
 feedItem.authorEmail = authorElem.namedItem("email").toElement().text();
 }
 feedItem.language = rootElem.namedItem("language").toElement().text();
 QDomNodeList linksList = rootElem.elementsByTagName("link");
for (int j = 0; j < linksList.size(); j++) {
if (linksList.at(j).toElement().attribute("rel") == "alternate") {
 feedItem.link = linksList.at(j).toElement().attribute("href");
break;
 }
 }
if (feedItem.link.isEmpty()) {
for (int j = 0; j < linksList.size(); j++) {
if (!(linksList.at(j).toElement().attribute("rel") == "self")) {
 feedItem.link = linksList.at(j).toElement().attribute("href");
break;
 }
 }
 }

if (QUrl(feedItem.link).host().isEmpty() || (QUrl(feedItem.link).host().indexOf('.')) == -1) {
if (!feedItem.linkBase.isEmpty() && !QUrl(feedItem.linkBase).host().isEmpty())
 feedItem.link = QUrl(feedItem.linkBase).scheme() % "://" % QUrl(feedItem.linkBase).host();
else
 feedItem.link = QUrl(feedUrl).scheme() % "://" % QUrl(feedUrl).host();
 }
if (feedItem.linkBase.isEmpty() && !QUrl(feedItem.link).host().isEmpty())
 feedItem.linkBase = QUrl(feedItem.link).scheme() % "://" % QUrl(feedItem.link).host();
if (QUrl(feedItem.link).host().isEmpty())
 feedItem.link = feedItem.linkBase + feedItem.link;
 feedItem.link = toPlainText(feedItem.link);
 QUrl url = QUrl(feedItem.link);
if (url.scheme().isEmpty())
 url.setScheme(QUrl(feedUrl).scheme());
 feedItem.link = url.toString();

 QSqlQuery q(db_);
 q.setForwardOnly(true);
 QString qStr ("UPDATE feeds "
"SET title=?, description=?, htmlUrl=?, "
"author_name=?, author_email=?, "
"author_uri=?, pubdate=?, language=? "
"WHERE id==?");
 q.prepare(qStr);
 q.addBindValue(feedItem.title);
 q.addBindValue(feedItem.description);
 q.addBindValue(feedItem.link);
 q.addBindValue(feedItem.author);
 q.addBindValue(feedItem.authorEmail);
 q.addBindValue(feedItem.authorUri);
 q.addBindValue(feedItem.updated);
 q.addBindValue(feedItem.language);
 q.addBindValue(parseFeedId_);
 q.exec();

 QDomNodeList newsList = doc.elementsByTagName("entry");
for (int i = 0; i < newsList.size(); i++) {
 NewsItemStruct newsItem;
 newsItem.id = newsList.item(i).namedItem("id").toElement().text();
 newsItem.title = toPlainText(newsList.item(i).namedItem("title").toElement().text());
 newsItem.updated = newsList.item(i).namedItem("published").toElement().text();
if (newsItem.updated.isEmpty())
 newsItem.updated = newsList.item(i).namedItem("updated").toElement().text();
 newsItem.updated = parseDate(newsItem.updated, feedUrl);
 QDomElement authorElem = newsList.item(i).namedItem("author").toElement();
if (!authorElem.isNull()) {
 newsItem.author = toPlainText(authorElem.namedItem("name").toElement().text());
if (newsItem.author.isEmpty()) newsItem.author = toPlainText(authorElem.text());
 newsItem.authorUri = authorElem.namedItem("uri").toElement().text();
 newsItem.authorEmail = authorElem.namedItem("email").toElement().text();
 }

 newsItem.description = newsList.item(i).namedItem("summary").toElement().text();
 QDomNode nodeSummary = newsList.item(i).namedItem("summary");
if (!nodeSummary.isNull() && newsItem.description.isEmpty()) {
 QTextStream in(&newsItem.description);
 nodeSummary.save(in, 0);
 }
 QDomNode nodeContent = newsList.item(i).namedItem("content");
if (nodeContent.toElement().attribute("type") == "xhtml") {
 QTextStream in(&newsItem.content);
 nodeContent.save(in, 0);
 } else {
 newsItem.content = nodeContent.toElement().text();
 }
 QString imgUrl = newsList.item(i).namedItem("media:thumbnail").toElement().attribute("url");
 QString community = getCommunity(newsList.item(i).namedItem("media:community"));
 nodeContent = newsList.item(i).namedItem("media:group");
if (!nodeContent.isNull()) {
 QString description = nodeContent.namedItem("media:description").toElement().text();
if (description.length() > newsItem.content.length())
 newsItem.content = description;
 newsItem.content = fromPlainText(newsItem.content);
if (imgUrl.isEmpty())
 imgUrl = nodeContent.namedItem("media:thumbnail").toElement().attribute("url");
if (community.isEmpty())
 community = getCommunity(nodeContent.namedItem("media:community"));
 }
if (!(newsItem.content.isEmpty() ||
 (newsItem.description.length() > newsItem.content.length()))) {
 newsItem.description = newsItem.content;
 }
 newsItem.content.clear();
if (!imgUrl.isEmpty()) {
 newsItem.description = "<p class=\"description\">" + newsItem.description + "</p>";
 newsItem.description += "<img src=\"" + imgUrl + "\" alt=\"image\"/>";
 }
if (!community.isEmpty())
 newsItem.description += community;

 QDomNodeList categoryElem = newsList.item(i).toElement().elementsByTagName("category");
for (int j = 0; j < categoryElem.size(); j++) {
if (!newsItem.category.isEmpty()) newsItem.category.append(", ");
 QString category = categoryElem.at(j).toElement().attribute("label");
if (category.isEmpty())
 category = categoryElem.at(j).toElement().attribute("term");
 newsItem.category.append(toPlainText(category));
 }
 QDomElement enclosureElem = newsList.item(i).namedItem("enclosure").toElement();
 newsItem.eUrl = enclosureElem.attribute("url");
 newsItem.eType = enclosureElem.attribute("type");
 newsItem.eLength = enclosureElem.attribute("length");
 QDomNodeList linksList = newsList.item(i).toElement().elementsByTagName("link");
for (int j = 0; j < linksList.size(); j++) {
if (linksList.at(j).toElement().attribute("type") == "text/html") {
if (linksList.at(j).toElement().attribute("rel") == "self")
 newsItem.link = linksList.at(j).toElement().attribute("href");
if (linksList.at(j).toElement().attribute("rel") == "alternate")
 newsItem.linkAlternate = linksList.at(j).toElement().attribute("href");
if (linksList.at(j).toElement().attribute("rel") == "replies")
 newsItem.comments = linksList.at(j).toElement().attribute("href");
 } elseif (newsItem.linkAlternate.isEmpty()) {
if (linksList.at(j).toElement().attribute("rel") == "alternate")
 newsItem.linkAlternate = linksList.at(j).toElement().attribute("href");
 }
 }
for (int j = 0; j < linksList.size(); j++) {
if (newsItem.linkAlternate.isEmpty()) {
if (!(linksList.at(j).toElement().attribute("rel") == "self")) {
 newsItem.linkAlternate = linksList.at(j).toElement().attribute("href");
break;
 }
 }
 }

if (!newsItem.link.isEmpty() && QUrl(newsItem.link).host().isEmpty())
 newsItem.link = feedItem.linkBase + newsItem.link;
 newsItem.link = toPlainText(newsItem.link);
if (!newsItem.linkAlternate.isEmpty() && QUrl(newsItem.linkAlternate).host().isEmpty())
 newsItem.linkAlternate = feedItem.linkBase + newsItem.linkAlternate;
 newsItem.linkAlternate = toPlainText(newsItem.linkAlternate);
if (newsItem.link.isEmpty()) {
 newsItem.link = newsItem.linkAlternate;
 newsItem.linkAlternate.clear();
 }
 url = QUrl(newsItem.link);
if (url.scheme().isEmpty())
 url.setScheme(QUrl(feedUrl).scheme());
 newsItem.link = url.toString();

addAtomNewsIntoBase(&newsItem);
 }
}

voidParseObject::addAtomNewsIntoBase(NewsItemStruct *newsItem)
{
Common::sleep(5);

// search news duplicates in base
 QSqlQuery q(db_);
 q.setForwardOnly(true);
 QString qStr;
qDebug() << "atomId:" << newsItem->id;
qDebug() << "title:" << newsItem->title;
qDebug() << "published:" << newsItem->updated;

bool isDuplicate = false;
for (int i = 0; i < guidList_.count(); ++i) {
if (!newsItem->id.isEmpty()) { // search by guid if present
if (guidList_.at(i) == newsItem->id) {
if (duplicateNewsMode_) { // autodelete duplicate news enabled
 isDuplicate = true;
 } else { // autodelete dupl. news disabled
if (!newsItem->updated.isEmpty()) { // search by pubDate if present
if (publishedList_.at(i) == newsItem->updated)
 isDuplicate = true;
 } else { // ... or by title
if (!newsItem->title.isEmpty() && (titleList_.at(i) == newsItem->title))
 isDuplicate = true;
 }
 }
 }
 } else { // guid is absent
if (!newsItem->updated.isEmpty()) { // search by pubDate if present
if (publishedList_.at(i) == newsItem->updated)
 isDuplicate = true;
 } else { // ... or by title
if (!newsItem->title.isEmpty() && (titleList_.at(i) == newsItem->title))
 isDuplicate = true;
 }
 }
if (isDuplicate) break;
 }

// Verify old news before a date to avoid adding them to base
bool isOld = false;
 QDateTime pubDate_ = QDateTime::fromString(newsItem->updated, "yyyy-MM-ddTHH:mm:ss");
 QDateTime avoidedDate_ = QDateTime(mainApp->mainWindow()->avoidedOldNewsDate_);
if (!addSingleNewsAnyDate_) { //
if (avoidedOldSingleNews_ ) { // avoid adding old single news
if (QDateTime(avoidedOldSingleNewsDate_) > pubDate_)
 isOld = true;
 } elseif (mainApp->mainWindow()->avoidOldNews_ && avoidedDate_ > pubDate_) { // avoid adding old news
 isOld = true;
 }
 }

// if duplicates not found and is old news, add them into base
if (!isDuplicate && !isOld) {
boolread = false;
if (mainApp->mainWindow()->markIdenticalNewsRead_) {
 q.prepare("SELECT id FROM news WHERE title LIKE :title AND feedId!=:id");
 q.bindValue(":id", parseFeedId_);
 q.bindValue(":title", newsItem->title);
 q.exec();
if (q.first()) read = true;
 }

 qStr = QString("INSERT INTO news("
"feedId, description, content, guid, title, author_name, "
"author_uri, author_email, published, received, "
"link_href, link_alternate, category, comments, "
"enclosure_url, enclosure_type, enclosure_length, new, read) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
 q.prepare(qStr);
 q.addBindValue(parseFeedId_);
 q.addBindValue(newsItem->description);
 q.addBindValue(newsItem->content);
 q.addBindValue(newsItem->id);
 q.addBindValue(newsItem->title);
 q.addBindValue(newsItem->author);
 q.addBindValue(newsItem->authorUri);
 q.addBindValue(newsItem->authorEmail);
 QString updated = newsItem->updated;
if (updated.isEmpty())
 updated = QDateTime::currentDateTimeUtc().toString(Qt::ISODate);
 q.addBindValue(updated);
 q.addBindValue(QDateTime::currentDateTime().toString(Qt::ISODate));
 q.addBindValue(newsItem->link);
 q.addBindValue(newsItem->linkAlternate);
 q.addBindValue(newsItem->category);
 q.addBindValue(newsItem->comments);
 q.addBindValue(newsItem->eUrl);
 q.addBindValue(newsItem->eType);
 q.addBindValue(newsItem->eLength);
 q.addBindValue(read ? 0 : 1);
 q.addBindValue(read ? 2 : 0);
if (!q.exec()) {
qWarning() << __PRETTY_FUNCTION__ << __LINE__
 << "q.lastError(): " << q.lastError().text();
 }
 q.finish();
qDebug() << "q.exec(" << q.lastQuery() << ")";
qDebug() << "" << parseFeedId_;
qDebug() << "" << newsItem->description;
qDebug() << "" << newsItem->content;
qDebug() << "" << newsItem->id;
qDebug() << "" << newsItem->title;
qDebug() << "" << newsItem->author;
qDebug() << "" << newsItem->authorUri;
qDebug() << "" << newsItem->authorEmail;
qDebug() << "" << newsItem->updated;
qDebug() << "" << QDateTime::currentDateTime().toString();
qDebug() << "" << newsItem->link;
qDebug() << "" << newsItem->linkAlternate;
qDebug() << "" << newsItem->category;
qDebug() << "" << newsItem->comments;
qDebug() << "" << newsItem->eUrl;
qDebug() << "" << newsItem->eType;
qDebug() << "" << newsItem->eLength;

if (lastBuildDate_ < QDateTime::fromString(newsItem->updated, Qt::ISODate))
 lastBuildDate_ = QDateTime::fromString(newsItem->updated, Qt::ISODate);
 feedChanged_ = true;
 }
}

voidParseObject::parseRss(const QString &feedUrl, const QDomDocument &doc)
{
 QDomNode channel = doc.documentElement().namedItem("channel");
if (channel.isNull())
 channel = doc.documentElement().namedItem("rss:channel");
 FeedItemStruct feedItem;

 feedItem.title = toPlainText(channel.namedItem("title").toElement().text());
if (feedItem.title.isEmpty())
 feedItem.title = toPlainText(channel.namedItem("rss:title").toElement().text());
 feedItem.description = channel.namedItem("description").toElement().text();
if (feedItem.description.isEmpty())
 feedItem.description = toPlainText(channel.namedItem("rss:description").toElement().text());
 feedItem.link = toPlainText(channel.namedItem("link").toElement().text());
if (feedItem.link.isEmpty())
 feedItem.link = toPlainText(channel.namedItem("rss:link").toElement().text());
 QUrl url = QUrl(feedItem.link);
if (url.host().isEmpty())
 url.setHost(QUrl(feedUrl).host());
if (url.scheme().isEmpty())
 url.setScheme(QUrl(feedUrl).scheme());
 feedItem.link = url.toString();

 feedItem.updated = channel.namedItem("pubDate").toElement().text();
if (feedItem.updated.isEmpty())
 feedItem.updated = channel.namedItem("pubdate").toElement().text();
 feedItem.updated = parseDate(feedItem.updated, feedUrl);
 feedItem.author = toPlainText(channel.namedItem("author").toElement().text());
 feedItem.language = channel.namedItem("language").toElement().text();
if (feedItem.language.isEmpty())
 feedItem.language = channel.namedItem("dc:language").toElement().text();

 QSqlQuery q(db_);
 q.setForwardOnly(true);
 QString qStr("UPDATE feeds "
"SET title=?, description=?, htmlUrl=?, "
"author_name=?, pubdate=?, language=? "
"WHERE id==?");
 q.prepare(qStr);
 q.addBindValue(feedItem.title);
 q.addBindValue(feedItem.description);
 q.addBindValue(feedItem.link);
 q.addBindValue(feedItem.author);
 q.addBindValue(feedItem.updated);
 q.addBindValue(feedItem.language);
 q.addBindValue(parseFeedId_);
 q.exec();

 QDomNodeList newsList = doc.elementsByTagName("item");
if (newsList.isEmpty())
 newsList = doc.elementsByTagName("rss:item");
for (int i = 0; i < newsList.size(); i++) {
 NewsItemStruct newsItem;
 newsItem.id = newsList.item(i).namedItem("guid").toElement().text();
 newsItem.title = toPlainText(newsList.item(i).namedItem("title").toElement().text());
if (newsItem.title.isEmpty())
 newsItem.title = toPlainText(newsList.item(i).namedItem("rss:title").toElement().text());
 newsItem.updated = newsList.item(i).namedItem("pubDate").toElement().text();
if (newsItem.updated.isEmpty())
 newsItem.updated = newsList.item(i).namedItem("pubdate").toElement().text();
if (newsItem.updated.isEmpty())
 newsItem.updated = newsList.item(i).namedItem("dc:date").toElement().text();
 newsItem.updated = parseDate(newsItem.updated, feedUrl);
 newsItem.author = toPlainText(newsList.item(i).namedItem("author").toElement().text());
if (newsItem.author.isEmpty())
 newsItem.author = toPlainText(newsList.item(i).namedItem("dc:creator").toElement().text());
 newsItem.link = toPlainText(newsList.item(i).namedItem("link").toElement().text());
if (newsItem.link.isEmpty()) {
 newsItem.link = toPlainText(newsList.item(i).namedItem("rss:link").toElement().text());
if (newsItem.link.isEmpty()) {
if (newsList.item(i).namedItem("guid").toElement().attribute("isPermaLink") == "true")
 newsItem.link = newsItem.id;
 }
 }
 url = QUrl(newsItem.link);
if (url.host().isEmpty())
 url.setHost(QUrl(feedUrl).host());
if (url.scheme().isEmpty())
 url.setScheme(QUrl(feedUrl).scheme());
 newsItem.link = url.toString();

 newsItem.description = newsList.item(i).namedItem("description").toElement().text();
 QDomNode nodeSummary = newsList.item(i).namedItem("description");
if (!nodeSummary.isNull() && newsItem.description.isEmpty()) {
 QTextStream in(&newsItem.description);
 nodeSummary.save(in, 0);
 }
 newsItem.content = newsList.item(i).namedItem("content:encoded").toElement().text();
 QDomNode nodeContent = newsList.item(i).namedItem("content:encoded");
if (!nodeContent.isNull() && newsItem.content.isEmpty()) {
 QTextStream in(&newsItem.content);
 nodeContent.save(in, 0);
 }
 QString imgUrl = newsList.item(i).namedItem("media:thumbnail").toElement().attribute("url");
 QString community = getCommunity(newsList.item(i).namedItem("media:community"));
 nodeContent = newsList.item(i).namedItem("media:group");
if (!nodeContent.isNull()) {
 QString description = nodeContent.namedItem("media:description").toElement().text();
if (description.length() > newsItem.content.length())
 newsItem.content = description;
 newsItem.content = fromPlainText(newsItem.content);
if (imgUrl.isEmpty())
 imgUrl = nodeContent.namedItem("media:thumbnail").toElement().attribute("url");
if (community.isEmpty())
 community = getCommunity(nodeContent.namedItem("media:community"));
 }
if (!(newsItem.content.isEmpty() ||
 (newsItem.description.length() > newsItem.content.length()))) {
 newsItem.description = newsItem.content;
 }
 newsItem.content.clear();
if (!imgUrl.isEmpty()) {
 newsItem.description = "<p class=\"description\">" + newsItem.description + "</p>";
 newsItem.description += "<img src=\"" + imgUrl + "\" alt=\"image\"/>";
 }
if (!community.isEmpty())
 newsItem.description += community;

 QDomNodeList categoryElem = newsList.item(i).toElement().elementsByTagName("category");
for (int j = 0; j < categoryElem.size(); j++) {
if (!newsItem.category.isEmpty()) newsItem.category.append(", ");
 newsItem.category.append(toPlainText(categoryElem.at(j).toElement().text()));
 }
 newsItem.comments = newsList.item(i).namedItem("comments").toElement().text();
 QDomElement enclosureElem = newsList.item(i).namedItem("enclosure").toElement();
 newsItem.eUrl = enclosureElem.attribute("url");
 newsItem.eType = enclosureElem.attribute("type");
 newsItem.eLength = enclosureElem.attribute("length");

if (newsItem.title.isEmpty()) {
 newsItem.title = toPlainText(newsItem.description);
if (newsItem.title.size() > 50) {
 newsItem.title.resize(50);
 newsItem.title = newsItem.title % "...";
 }
 }

addRssNewsIntoBase(&newsItem);
 }
}

voidParseObject::addRssNewsIntoBase(NewsItemStruct *newsItem)
{
Common::sleep(5);

// search news duplicates in base
 QSqlQuery q(db_);
 q.setForwardOnly(true);
 QString qStr;
qDebug() << "guid: " << newsItem->id;
qDebug() << "link_href:" << newsItem->link;
qDebug() << "title:" << newsItem->title;
qDebug() << "published:" << newsItem->updated;

bool isDuplicate = false;
for (int i = 0; i < guidList_.count(); ++i) {
if (!newsItem->id.isEmpty()) { // search by guid if present
if (guidList_.at(i) == newsItem->id) {
if (!newsItem->updated.isEmpty()) { // search by pubDate if present
if (!duplicateNewsMode_) {
if (publishedList_.at(i) == newsItem->updated)
 isDuplicate = true;
 }
else {
 isDuplicate = true;
 }
 } else { // ... or by title
if (!newsItem->title.isEmpty() && (titleList_.at(i) == newsItem->title))
 isDuplicate = true;
 }
 }
if (!isDuplicate) {
if (!newsItem->updated.isEmpty()) {
if ((publishedList_.at(i) == newsItem->updated) &&
 (titleList_.at(i) == newsItem->title)) {
 isDuplicate = true;
 }
 }
 }
 }
elseif (!newsItem->link.isEmpty()) { // search by link_href
if (linkList_.at(i) == newsItem->link) {
if (!newsItem->updated.isEmpty()) { // search by pubDate if present
if (!duplicateNewsMode_) {
if (publishedList_.at(i) == newsItem->updated)
 isDuplicate = true;
 }
else {
 isDuplicate = true;
 }
 } else { // ... or by title
if (!newsItem->title.isEmpty() && (titleList_.at(i) == newsItem->title))
 isDuplicate = true;
 }
 }
if (!isDuplicate) {
if (!newsItem->updated.isEmpty()) {
if ((publishedList_.at(i) == newsItem->updated) &&
 (titleList_.at(i) == newsItem->title)) {
 isDuplicate = true;
 }
 }
 }
 }
else { // guid is absent
if (!newsItem->updated.isEmpty()) { // search by pubDate if present
if (!duplicateNewsMode_) {
if (publishedList_.at(i) == newsItem->updated)
 isDuplicate = true;
 }
else {
 isDuplicate = true;
 }
 } else { // ... or by title
if (!newsItem->title.isEmpty() && (titleList_.at(i) == newsItem->title))
 isDuplicate = true;
 }
if (!isDuplicate) {
if (!newsItem->updated.isEmpty()) {
if ((publishedList_.at(i) == newsItem->updated) &&
 (titleList_.at(i) == newsItem->title)) {
 isDuplicate = true;
 }
 }
 }
 }
if (isDuplicate) break;
 }

// Verify old news before a date to avoid adding them to base
bool isOld = false;
 QDateTime pubDate_ = QDateTime::fromString(newsItem->updated, "yyyy-MM-ddTHH:mm:ss");
 QDateTime avoidedDate_ = QDateTime(mainApp->mainWindow()->avoidedOldNewsDate_);
if (!addSingleNewsAnyDate_) { //
if (avoidedOldSingleNews_ ) { // avoid adding old single news
if (QDateTime(avoidedOldSingleNewsDate_) > pubDate_)
 isOld = true;
 } elseif (mainApp->mainWindow()->avoidOldNews_ && avoidedDate_ > pubDate_) { // avoid adding old news
 isOld = true;
 }
 }

// if duplicates not found And old news, add them into base
if (!isDuplicate && !isOld) {
boolread = false;
if (mainApp->mainWindow()->markIdenticalNewsRead_) {
 q.prepare("SELECT id FROM news WHERE title LIKE :title AND feedId!=:id");
 q.bindValue(":id", parseFeedId_);
 q.bindValue(":title", newsItem->title);
 q.exec();
if (q.first()) read = true;
 }

 qStr = QString("INSERT INTO news("
"feedId, description, content, guid, title, author_name, "
"published, received, link_href, category, comments, "
"enclosure_url, enclosure_type, enclosure_length, new, read) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
 q.prepare(qStr);
 q.addBindValue(parseFeedId_);
 q.addBindValue(newsItem->description);
 q.addBindValue(newsItem->content);
 q.addBindValue(newsItem->id);
 q.addBindValue(newsItem->title);
 q.addBindValue(newsItem->author);
 QString updated = newsItem->updated;
if (updated.isEmpty())
 updated = QDateTime::currentDateTimeUtc().toString(Qt::ISODate);
 q.addBindValue(updated);
 q.addBindValue(QDateTime::currentDateTime().toString(Qt::ISODate));
 q.addBindValue(newsItem->link);
 q.addBindValue(newsItem->category);
 q.addBindValue(newsItem->comments);
 q.addBindValue(newsItem->eUrl);
 q.addBindValue(newsItem->eType);
 q.addBindValue(newsItem->eLength);
 q.addBindValue(read ? 0 : 1);
 q.addBindValue(read ? 2 : 0);
if (!q.exec()) {
qWarning() << __PRETTY_FUNCTION__ << __LINE__
 << "q.lastError(): " << q.lastError().text();
 }
 q.finish();
qDebug() << "q.exec(" << q.lastQuery() << ")";
qDebug() << "" << parseFeedId_;
qDebug() << "" << newsItem->description;
qDebug() << "" << newsItem->content;
qDebug() << "" << newsItem->id;
qDebug() << "" << newsItem->title;
qDebug() << "" << newsItem->author;
qDebug() << "" << newsItem->updated;
qDebug() << "" << QDateTime::currentDateTime().toString();
qDebug() << "" << newsItem->link;
qDebug() << "" << newsItem->category;
qDebug() << "" << newsItem->comments;
qDebug() << "" << newsItem->eUrl;
qDebug() << "" << newsItem->eType;
qDebug() << "" << newsItem->eLength;

if (lastBuildDate_ < QDateTime::fromString(newsItem->updated, Qt::ISODate))
 lastBuildDate_ = QDateTime::fromString(newsItem->updated, Qt::ISODate);
 feedChanged_ = true;
 }
}

QString ParseObject::toPlainText(const QString &text)
{
returnQTextDocumentFragment::fromHtml(text).toPlainText().simplified();
}

QString ParseObject::fromPlainText(QString text)
{
 text = text.replace("\r\n", "<br>");
 text = text.replace("\n", "<br>");
return text;
}

QString ParseObject::getCommunity(const QDomNode &nodeContent)
{
 QString community;
if (!nodeContent.isNull()) {
 QString count = nodeContent.namedItem("media:starRating").toElement().attribute("count");
 QString average = nodeContent.namedItem("media:starRating").toElement().attribute("average");
 QString min = nodeContent.namedItem("media:starRating").toElement().attribute("min");
 QString max = nodeContent.namedItem("media:starRating").toElement().attribute("max");
 QString views = nodeContent.namedItem("media:statistics").toElement().attribute("views");
if (!count.isEmpty())
 community = QString("Count: %1, average: %2, min: %3, max: %4<br>").
arg(count).arg(average).arg(min).arg(max);
if (!views.isEmpty())
 community += QString("Views: %1").arg(views);
if (!community.isEmpty())
 community = "<p><i>" + community + "</i></p>";
 }
return community;
}

/** @brief Date/time string parsing
 *----------------------------------------------------------------------------*/
QString ParseObject::parseDate(const QString &dateString, const QString &urlString)
{
 QDateTime dt;
 QString temp;
 QString timeZone;

if (dateString.isEmpty()) returnQString();

 QDateTime dtLocalTime = QDateTime::currentDateTime();
 QDateTime dtUTC = QDateTime(dtLocalTime.date(), dtLocalTime.time(), Qt::UTC);
int nTimeShift = dtLocalTime.secsTo(dtUTC)/3600;

 QString ds = dateString.simplified();
 QLocale locale(QLocale::C);

if (ds.indexOf(',') != -1) {
 ds = ds.remove(0, ds.indexOf(',')+1).simplified();
 }

for (int i = 0; i < 2; i++, locale = QLocale::system()) {
 temp = ds.left(23);
 timeZone = ds.mid(temp.length(), 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "yyyy-MM-ddTHH:mm:ss.z");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(19);
 timeZone = ds.mid(temp.length(), 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "yyyy-MM-ddTHH:mm:ss");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(23);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "yyyy-MM-dd HH:mm:ss.z");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(19);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "yyyy-MM-dd HH:mm:ss");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(20);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.contains("EDT"))
 timeZone="-4";
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "dd MMM yyyy HH:mm:ss");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(19);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "d MMM yyyy HH:mm:ss");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(11);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "dd MMM yyyy");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(10);
 timeZone = ds.mid(temp.length()+1, 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "d MMM yyyy");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

 temp = ds.left(10);
 timeZone = ds.mid(temp.length(), 3);
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "yyyy-MM-dd");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");

// @HACK(arhohryakov:2012.01.01):
// "dd MMM yy HH:mm:ss" format doesn/t parse automatically
// Reformat it to "dd MMM yyyy HH:mm:ss"
 QString temp2;
 temp2 = ds; // save ds for output in case of error
if (70 < ds.mid(7, 2).toInt()) temp2.insert(7, "19");
else temp2.insert(7, "20");
 temp = temp2.left(20);
 timeZone = ds.mid(temp.length()+1-2, 3); // "-2", cause 2 symbols inserted
if (timeZone.isEmpty()) timeZone = QString::number(nTimeShift);
 dt = locale.toDateTime(temp, "dd MMM yyyy HH:mm:ss");
if (dt.isValid()) return locale.toString(dt.addSecs(timeZone.toInt() * -3600), "yyyy-MM-ddTHH:mm:ss");
 }