CFDA

cfda数据抓取

1.网站数据是加密的,需要浏览器进行数据解析

 2.网址url有js加密

3.PhantomJS无法解析数据, chrome无法获取数据,所有最终选择用Firefox浏览器

import pymysql
import time
import uuid
from lxml import etree
import logging
from selenium import webdriver
import threading
import queue
import re

logging.basicConfigfilename='shengchan.log', filemode="w", level=logging.INFO)


class App1Spiderobject):
    def __init__self):
        self.db = pymysql.connecthost='', port=, database='', user='',
                                  password='', charset='utf8')
        self.cursor = self.db.cursor)
        self.options = webdriver.FirefoxOptions)
        self.options.add_argument'--headless')
        # 谷歌文档提到需要加上这个属性来规避bug
        self.options.add_argument'--disable-gpu')
        # 设置默认编码为utf-8
        self.options.add_argument'lang=zh_CN.UTF-8')
        # 隐藏滚动条, 应对一些特殊页面
        self.options.add_argument'--hide-scrollbars')
        # 禁止加载图片
        self.options.add_argument'blink-settings=imagesEnabled=false')
        # 指定浏览器分辨率
        self.options.add_argument'window-size=1440x900')
        self.browser = webdriver.Firefoxfirefox_options=self.options)

    def mainself):
        """
        入口函数
        :param response:
        :return:
        """
        start = 1
        while True:
            browser = self.go_index)
            if browser:
                for i in rangestart, 520):
                    browser = self.go_pagebrowser, i)
                    if browser:
                        for j in range15):
                            if i > 511:
                                detail_html = self.go_detailbrowser, j)
                                if detail_html:
                                    id = i - 1) * 15 + j + 1
                                    self.parse_detaildetail_html, id)
                                else:
                                    break
                    else:
                        start = i - 1
                        break
            else:
                continue

    def go_indexself):
        """
        访问主页
        :return: 浏览器对象
        """
        # print"!-- start index --!")
        index_url = "http://app1.sfda.gov.cn/datasearch/face3/base.jsp?tableId=34&tableName=TABLE34&title=%D2%A9%C6%B7%C9%FA%B2%FA%C6%F3%D2%B5&bcId=118103348874362715907884020353"
        try:
            self.browser.getindex_url)
            time.sleep3)
        except:
            # print"!-- error to get index page --!")
            # print"网速不太好，休息1分钟")
            time.sleep30)
            return None
        else:
            html = self.browser.page_source
            condition = re.searchr"管理局--数据查询", html)
            if condition:
                # print"!-- success to get index page --!")
                return self.browser
            else:
                # print"!-- error to get index page --!----")
                # print"网速不太好，休息1分钟------")
                time.sleep30)
                return None

    def go_pageself, browser, page):
        """
        跳转到指定页面
        :param browser: 浏览器对象
        :param page: 要跳转的页码
        :return: 跳转后的浏览器对象
        """
        # logging.info"!-- start page %s --!" % page)
        print"!-- start page %s --!" % page)
        go_page_js = 'location.href="javascript:devPage%s)";' % page
        try:
            browser.execute_scriptgo_page_js)
            # 需要等待firefox页面加载完成
            time.sleep2)
        except Exception as e:
            print"!-- error to go page %s --!" % page)
            # logging.info"!-- error to go page %s --!" % page)
            return None
        else:
            html = browser.page_source
            condition = re.searchr"第 %s 页" % page, html)
            if condition:
                logging.info"!-- success to go page %s --!" % page)
                return browser
            else:
                logging.info"!-- error to go page %s --!" % page)
                return None

    def go_detailself, browser, number):
        """
        包含了提取详情页面数据信息，保存数据信息。
        :param browser: 浏览器对象
        :return: 详细数据生成器
        """
        # logging.info"!-- go detail %s --!" % number)
        print"!-- go detail %s --!" % number)
        go_detail_js = "var div=document.getElementById'content');" 
                       "var c=div.getElementsByTagName'a')[{detail_num}].click);"
        return_list_js = 'location.href = "javascript:viewList);"'
        _go_detail_js = go_detail_js.formatdetail_num=number)
        browser.execute_script_go_detail_js)
        time.sleep2)
        detail_html = browser.page_source
        condition = re.searchr"javascript:viewList", detail_html)
        if condition:
            browser.execute_scriptreturn_list_js)
            time.sleep2)
            return detail_html
        else:
            # logging.info"!-- error to get detail --! %s" % number)
            print"!-- error to get detail --! %s" % number)
            return None

    def parse_detailself, detail_html, id):
        # printid)
        """
        详情页面提取规则
        :param html: 被提取页面的html
        :return: data
        """
        response = etree.HTMLdetail_html)

        try:
            # 厂家编号
            number = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[2]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            number = '00000000'

        try:
            # 生产地址
            manufactureAddress = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[11]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            manufactureAddress = ''

        try:
            # 生产范围
            manufactureRange = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[12]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            manufactureRange = ''

        try:
            # 发证日期
            certificateDate = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[13]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            certificateDate = '2018-01-01'

        try:
            # 有效期
            validityDate = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[14]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            validityDate = '2018-01-01'

        try:
            # 发证机关
            certificateOrgan = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[15]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            certificateOrgan = ''

        try:
            # 签发人
            Signer = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[16]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            Signer = ''

        try:
            # 日常监管机构
            superviseAgency = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[17]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            superviseAgency = ''

        try:
            # 日常监管人员
            superviser = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[18]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            superviser = ''

        try:
            # 社会信用代码/组织机构代码
            socialCreditCode = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[3]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            socialCreditCode = ''

        try:
            # 监督举报电话
            reportTel = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[19]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            reportTel = ''

        try:
            # 备注
            comment = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[20]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            comment = ''

        try:
            # 分类码
            classificationCode = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[4]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            classificationCode = ''

        try:
            # 省份
            province = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[5]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            province = ''

        try:
            # 企业名称
            companyName = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[6]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            companyName = ''

        try:
            # 法定代表人
            legalPeople = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[7]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            legalPeople = ''

        try:
            # 企业负责人
            companyResponsioner = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[8]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            companyResponsioner = ''

        try:
            # 质量负责人
            qualityResponsioner = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[9]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            qualityResponsioner = ''

        try:
            # 注册地址
            registerAddress = response.xpath'//*[@id="content"]/div/div/table[1]/tbody/tr[10]/td[2]/text)')[0].strip).replace"'", "‘")
        except:
            registerAddress = ''

        cjrepetition = self.cursor.execute"select id from cfda_drug_company20181205 where numbers = %s" % id)
        if not cjrepetition:
            cjsql = "insert into cfda_drug_company20181205number, manufactureAddress,  manufactureRange, certificateDate, validityDate, certificateOrgan, Signer, superviseAgency, superviser, socialCreditCode, reportTel, comment, classificationCode, province, companyName, legalPeople, companyResponsioner, qualityResponsioner, registerAddress, numbers) values'{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', '{}', {})"
            cjsql_data = cjsql.formatnumber, manufactureAddress, manufactureRange,
                                      certificateDate, validityDate, certificateOrgan,
                                      Signer, superviseAgency, superviser,
                                      socialCreditCode, reportTel, comment,
                                      classificationCode, province, companyName,
                                      legalPeople, companyResponsioner, qualityResponsioner,
                                      registerAddress, intid))
            try:
                self.cursor.executecjsql_data)
                self.db.commit)
            except Exception as e:
                print'id:%s   e:%s' % id, e))


if __name__ == '__main__':
    sheng = App1Spider)
    sheng.main)