基于Qt开发的爬虫工具.zip

#include "widget.h" #include "ui_widget.h" #include "fervor/fvupdater.h" #if (QT_VERSION >= QT_VERSION_CHECK(5, 0, 0)) #else #define QStringLiteral(str) QString::fromUtf8(str) #endif Widget::Widget(QWidget *parent) : QWidget(parent), ui(new Ui::Widget),m_View(NULL), isFirstBe(false), m_ARK(NULL), m_BEH(NULL), m_COMB(NULL), m_OAK(NULL) { ui->setupUi(this); #if defined(Q_OS_WIN) setWindowFlags(Qt::FramelessWindowHint); setAttribute(Qt::WA_TranslucentBackground); #endif #if defined(Q_OS_OSX) ui->closeBtn->setVisible(false); #endif //初始化为未按下鼠标左键 mouse_press = false; arkIsOK = false; behIsOk = false; comBlOk = false; oakOK = false; aokOK = false; connect(ui->updateButton, SIGNAL(clicked()), FvUpdater::sharedUpdater(), SLOT(CheckForUpdatesNotSilent())); m_View =new QWebView(); QStringList lst; initThreeThreads(); QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE"); db.setHostName("acidalia"); db.setDatabaseName("bigarkdb"); db.setUserName("sarah"); db.setPassword("123456tb"); bool ok = db.open(); qDebug()<<" opened? "<<ok; connect(ui->comboBox,SIGNAL(currentIndexChanged(QString)),this,SLOT(setCurrentCas(QString))); QSqlQuery query("SELECT cas FROM arkdata order by id limit 200"); while (query.next()) { QString country = query.value(0).toString(); lst.append(country); } ui->comboBox->addItems(lst); } Widget::~Widget() { if(m_View){ m_View->deleteLater(); m_View = NULL; } delete ui; } void Widget::initThreeThreads() { m_ARK = new BiJiaARKThread(0); connect(m_ARK,SIGNAL(signalSendFinalData(QByteArray)),this,SLOT(parseArk(QByteArray))); m_BEH = new BiJiaBehThread(0); connect(m_BEH,SIGNAL(signalSendFinalData(QByteArray,bool)),this,SLOT(parseBeh(QByteArray,bool))); connect(m_BEH,SIGNAL(signalMessageShow(QString)),this,SLOT(appendStatus(QString))); m_COMB = new BiJiaCOMBThread(0); connect(m_COMB,SIGNAL(signalSendFinalData(QByteArray,bool)),this,SLOT(parseComBlock(QByteArray,bool))); m_OAK = new BiJiaOAKThread(0); connect(m_OAK,SIGNAL(signalSendFinalData(QByteArray,bool)),this,SLOT(parseOAkChemical(QByteArray,bool))); connect(m_OAK,SIGNAL(signalMessageShow(QString)),this,SLOT(appendStatus(QString))); m_AOK = new BiJiaAOKThread(0); connect(m_AOK,SIGNAL(signalSendFinalData(QByteArray)),this,SLOT(parseAoK(QByteArray))); m_Astatech = new BiJiaAstatechThread(0); connect(m_Astatech,SIGNAL(signalSendFinalData(QByteArray,bool)),this,SLOT(parseAstatech(QByteArray,bool))); connect(m_Astatech,SIGNAL(signalMessageShow(QString)),this,SLOT(appendStatus(QString))); } void Widget::setUrl(const QUrl &url) { m_View->setUrl(url); } void Widget::setWebSiteType(Widget::WEBSITE_TYPE type) { iType = type; } void Widget::startLoad(const QUrl &url) { qDebug()<<" startLoad "<<url; } void Widget::startPost(const QUrl &url) { QByteArray content(ui->lineEdit->text().toUtf8()); int contentLength = content.length(); QNetworkRequest req; req.setUrl(QUrl(url)); req.setHeader(QNetworkRequest::ContentTypeHeader,"application/x-www-form-urlencoded"); req.setHeader(QNetworkRequest::ContentLengthHeader,contentLength); // manager->post(req,content); } void Widget::parseArk(const QByteArray &arr) { QWebFrame *frame =m_View->page()->mainFrame(); frame->setHtml((arr)); QWebElement document = frame->documentElement(); QWebElementCollection introSpans = document.findAll("input[type=hidden]"); QList<QWebElement> elst = introSpans.toList(); ui->textEdit->clear(); for(int i = 0;i<elst.count();i++){ QString value = elst.at(i).attribute("value"); QString id = elst.at(i).attribute("id"); //qDebug()<<"-------------ARKPHARM data--------------- "<<value<< " \n"<<id; if(id.contains("Hidden") || id.contains("WeightValue") || value.isEmpty()){ continue; } ui->textEdit->append(value); } if(elst.isEmpty()){ ui->textEdit->append(QStringLiteral("未找到相关产品")); } arkIsOK = true; qDebug()<<QStringLiteral("ARK价格获取完毕--------------------------------"); if(arkIsOK && behIsOk && comBlOk && oakOK && aokOK && astateChOK){ ui->pushButton->setEnabled(true); ui->label_5->setText(QStringLiteral("ARK最后搜索完成,请仔细查看搜索结果")); } } void Widget::parseBeh(const QByteArray &arr, bool isFirst) { QWebFrame *frame =m_View->page()->mainFrame(); frame->setHtml(arr); QWebElement document = frame->documentElement(); if(isFirst){ ui->textEdit_2->clear(); examineChildElements(document,true); }else{ QWebElementCollection introSpans = document.findAll("input[type=hidden]"); QList<QWebElement> elst = introSpans.toList(); QStringList secondResult; secondResult.clear(); for(int i = 0;i<elst.count();i++){ QString value = elst.at(i).attribute("value"); QString id = elst.at(i).attribute("id"); //qDebug()<<"-------------BEPHARM data--------------- "<<value<< " \n"<<id; if(id.isEmpty() || value.isEmpty() || id.contains("WeightValue")){ continue; }else{ // ui->textEdit_2->append(tr("未找到该产品价格")); // ui->label_5->setText(tr("未找到该产品价格")); ui->textEdit_2->append(value); } } behIsOk = true; qDebug()<<QStringLiteral("BEHM价格获取完毕--------------------------------"); if(arkIsOK && behIsOk && comBlOk && oakOK && aokOK && astateChOK){ ui->pushButton->setEnabled(true); ui->label_5->setText(QStringLiteral("BEHM最后搜索完成,请仔细查看搜索结果")); } } } void Widget::parseComBlock(const QByteArray &arr, bool isFirst) { // qDebug()<<"parseComBlock "<<arr.size(); // ui->textEdit_3->setText(QStringLiteral(arr)); QWebFrame *frame =m_View->page()->mainFrame(); frame->setHtml((arr)); QWebElement document = frame->documentElement(); QWebElementCollection introSpans = document.findAll("TD[ALIGN=right]"); QList<QWebElement> elst = introSpans.toList(); ui->textEdit_3->clear(); for(int i = 0;i<elst.count();i++){ QString value = elst.at(i).toInnerXml(); if(value.contains("<input") || value.isEmpty() || value.contains("Others")){ continue; } ui->textEdit_3->append(elst.at(i).toInnerXml()); } if(elst.isEmpty()){ ui->textEdit_3->append(QStringLiteral("未找到相关产品")); } qDebug()<<QStringLiteral("COMBLOCKS价格获取完毕--------------------------------"); comBlOk = true; if(arkIsOK && behIsOk && comBlOk && oakOK && aokOK && astateChOK){ ui->pushButton->setEnabled(true); ui->label_5->setText(QStringLiteral("COM最后搜索完成,请仔细查看搜索结果")); } } void Widget::parseOAkChemical(const QByteArray &arr, bool isFirst) { QWebFrame *frame =m_View->page()->mainFrame(); frame->setHtml((arr)); QWebElement document = frame->documentElement(); if(isFirst){ ui->textEdit_4->clear(); examineChildElements(document,false); }else{ QWebElementCollection introSpans = document.findAll("input[type=hidden]"); QList<QWebElement> elst = introSpans.toList(); QStringList secondResult; secondResult.clear(); for(int i = 0;i<elst.count();i++){ QString value = elst.at(i).attr