Crawling.py

import requests
from pathlib import Path
import ast

from selenium import webdriver
from bs4 import BeautifulSoup as bs
from selenium.webdriver import ActionChains
from multiprocessing import Process
import multiprocessing
import json
import pprint
from urllib import parse

def find_index(data, target):
    res = []
    lis = data
    while True:
        try:
            res.append(lis.index(target) + (res[-1] + 1 if len(res) != 0 else 0))  # +1의 이유 : 0부터 시작이니까
            lis = data[res[-1] + 1:]
        except:
            break
    return res

def get_date(target_url): # 필요
    """
    # 게시물의 posting 날짜를 가져오는 함수
    예시
    # https://blog.naver.com/ggtourkorea/222205886152
    # ttps://blog.naver.com/matsu2/222154600781
    """
    idst = find_index(target_url, '/')[2] + 1
    idend = find_index(target_url, '/')[3]  # logNo start
    blog_id = target_url[idst:idend]
    log_no = target_url[idend + 1:]
    response = requests.get(
        'https://blog.naver.com/PostView.nhn?blogId={}&logNo={}&redirect=Dlog&widgetTypeCall=true&directAccess=false'.format(
            blog_id, log_no))
    soup  = bs(str(response.text), 'html.parser')
    try:
        date = soup.find_all(attrs={'class': 'date fil5 pcol2 _postAddDate'})[0].get_text()
    except:
        try:
            date = soup.find_all(attrs={'class': 'se_publishDate pcol2'})[0].get_text()
        except:
            date = '오류발생, 해당 질의조건과 url을 알려주십시오'
    return date

def get_title(target_url):
    """
    # 해당 URL에 해당하는 게시물의 title과, 작성자 id를 리턴.
    """
    idst = find_index(target_url, '/')[2] + 1
    idend = find_index(target_url, '/')[3]  # logNo start
    response = requests.get(
        'https://blog.naver.com/PostView.nhn?blogId={}&logNo={}&redirect=Dlog&widgetTypeCall=true&directAccess=false'.format(
            target_url[idst:idend], target_url[idend + 1:]))
    soup = bs(str(response.text), 'html.parser')
    title = soup.find_all(attrs={'property': 'og:title'})[0]['content']
    return title, target_url[idst:idend]



def get_result(keywords, urlS, target, driver_path, processN, show_chrome):
    def view_back(target_url, keyword, driver, rank=False):
        """
        설명 : [파라미터 중 keyword]를 필수포함 검색어(검색연산자 "")로 네이버 검색시
                view 탭에서 [파라미터 중 target_url]을 URL로 가지는 포스팅이 있는지 확인함.
                추가적으로 그 포스팅이 검색결과에서 몇 번째 순위에 위치하는지도 추출할 수 있음.
        사용법 : view(대상 url, 필수포함 검색어, 크롬드라이버(chromium)위치, 순위 추출여부(기본 : Falsel))
        return 값
            경우
            1. rank=False(순위추출 x)의 경우, '노출' 혹은 '미노출'
            2. rank=True(순위추출 o)의 경우, 순위(숫자) 혹은 '미노출'
        """
        try:
            keyword = '{}'.format(keyword)
            driver.get("https://search.naver.com/search.naver?where=view&sm=tab_jum&query={}&qvt=0".format(keyword))  # 키워드 검색
            driver.implicitly_wait(time_to_wait=0.3)

            before_len_of_url = None
            n_for_break = 0
            while True:
                li = driver.find_element_by_xpath('//li[@class="bx _svp_item"]')
                li = driver.execute_script("return arguments[0].parentNode;", li)
                # print('//*[@id="main_pack"]/section[{}]/div/div[2]/panel-list/div/more-contents/div/ul'.format(1 if rank == True else 2))
                html = li.get_attribute('innerHTML')
                soup = bs(html, 'html.parser')

                urlS = [k.attrs['href'] for k in soup.find_all(attrs={'class': 'api_txt_lines total_tit'})]  # JUST 노출 확인
                if rank == False and target_url in urlS:
                    returns = '노출'
                    break
                elif rank == True and target_url in urlS:
                    child = driver.find_element_by_xpath('//a[@href="' + target_url + '"]')
                    parent = driver.execute_script("return arguments[0].parentNode;", child)
                    parent = driver.execute_script("return arguments[0].parentNode;", parent)
                    html = parent.get_attribute('outerHTML')
                    soup = bs(html, 'html.parser')
                    rank_n = soup.find('li').attrs['data-cr-rank']
                    returns = rank_n
                    break

                if before_len_of_url != None and before_len_of_url == len(urlS):
                    n_for_break += 1
                else:
                    n_for_break = 0
                # print(n_for_break)
                if n_for_break >= 10:
                    returns = '미노출'
                    # print(len(urlS))
                    break
                A = driver.find_elements_by_xpath('//li[@class="bx _svp_item"]')
                html = A[-1].get_attribute('outerHTML')
                soup = bs(html, 'html.parser')
                rank_n = soup.find('li').attrs['data-cr-rank']
                # print(rank_n)
                if int(rank_n) > 40:
                    returns = '미노출'
                    break

                before_len_of_url = len(urlS)

                last = driver.find_element_by_xpath('//*[@id="footer"]')
                action = ActionChains(driver)
                action.move_to_element(last).perform()
                driver.implicitly_wait(time_to_wait=0.3)
        # print(time.time()-st_t)
        except Exception as e:
            # print(e)
            returns = '미노출'

        return returns   #   #

    def view(keyword, driver):
        pass
    def find_index(data, target):
        res = []
        lis = data
        while True:
            try:
                res.append(lis.index(target) + (res[-1] + 1 if len(res) != 0 else 0)) #+1의 이유 : 0부터 시작이니까
                lis = data[res[-1]+1:]
            except:
                break
        return res

    def exposure(urlS, target_url):
        FlagExposure = False
        for i in urlS:
            if target_url[target_url.index('//')+2:] in i:
                FlagExposure = True
                break
        return FlagExposure

    def get_title(target_url):
        idst = find_index(target_url, '/')[2] + 1
        idend = find_index(target_url, '/')[3] # logNo start
        response = requests.get('https://blog.naver.com/PostView.nhn?blogId={}&logNo={}&redirect=Dlog&widgetTypeCall=true&directAccess=false'.format(target_url[idst:idend], target_url[idend+1:]))
        soup = bs(str(response.text), 'html.parser')
        title = soup.find_all(attrs={'property': 'og:title'})[0]['content']
        return title, target_url[idst:idend]

    def get_date(target_url):
        # https://blog.naver.com/ggtourkorea/222205886152
        # ttps://blog.naver.com/matsu2/222154600781
        idst = find_index(target_url, '/')[2] + 1
        idend = find_index(target_url, '/')[3] # logNo start
        blog_id = target_url[idst:idend]
        log_no = target_url[idend + 1:]
        response = requests.get('https://blog.naver.com/PostView.nhn?blogId={}&logNo={}&redirect=Dlog&widgetTypeCall=true&directAccess=false'.format(blog_id, log_no))
        soup = bs(str(response.text),'html.parser')
        try:
            date = soup.find_all(attrs={'class': 'date fil5 pcol2 _postAddDate'})[0].get_text()
        except:
            try:
                date = soup.find_all(attrs={'class': 'se_publishDate pcol2'})[0].get_text()
            except:
                date = '오류발생, 해당 질의조건과 url을 알려주십시오'
        return date

    def replaceURLCODE(keyword):
        keyword = keyword.replace('&', r'%26')
        keyword = keyword.replace('/', r'%2F')
        keyword = keyword.replace(':', r'%3A')
        keyword = keyword.replace('?', r'%3F')
        keyword = keyword.replace('=', r'%3D')
        return keyword
    if show_chrome == False:
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('headless')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('lang=ko_KR')
        driver = webdriver.Chrome(str(driver_path), chrome_options=chrome_options)  # 드라이버 설정
    else:
        driver = webdriver.Chrome(driver_path) # 드라이버 설정
    if target != 'viewlist':
        result = {'blog': [],
            'date': [],
            'img': [],
            'site': [],
            'url': urlS,
            'video': []
        }
    else:
        result = {'date': [],
                'keyword': keywords,
                'rank': [],
                'title': [],
                'url': urlS
        }
    for keyword, url in zip(keywords, urlS):
        if target != 'viewlist':
            result['blog'].append(view(url, replaceURLCODE(keyword), driver))
            result['date'].append(get_date(url))
            result['img'].append(image(url, replaceURLCODE(keyword), driver))
            result['site'].append(site(replaceURLCODE(keyword), url))
            result['video'].append(video(url, replaceURLCODE(keyword), driver))
        else:
            result['date'].append(get_date(url))
            result['rank'].append(view(url, replaceURLCODE(keyword), driver, rank=True))
            result['title'].append(get_title(url))
    driver.quit()
    print(multiprocessing.current_process().name, "'s chromium driver finished! :", processN)

    f = open('tmp/Process-{}.txt'.format(processN), 'w', encoding='UTF8')
    print(result, file=f)
    f.close()

    print('******************', multiprocessing.current_process().name, 'FINISHED******************')


def main(keywords, urlS, target, driver_path):
    """
    사용법
    main(keywords, urlS, target, driver_path)
    keywords : 키워드
    urlS : URLS
    target : 'urllist', 'newlist', 'viewlist' 중 하나
    driver_path : 드라이버 경로
    """
    # print(__name__)
    Path('tmp').mkdir(parents=True, exist_ok=True)
    def get_maxProcessNAndshowChrome():
        f = open('NumberOfProcess.txt', 'r',encoding='UTF-8')
        max_processN = f.readlines()[-2]
        show_chrome = f.readlines()[-1]
        f.close()
        try:
            max_processN = int(max_processN[max_processN.index('Number of process :') + len('Number of process :'):])
            show_chrome = show_chrome[show_chrome.index('Show Chrome Tab :') + len('Show Chrome Tab :'):]
            show_chrome = True if 'True' in show_chrome or 'true' in show_chrome else False
            return max_processN, show_chrome
        except:
            print(max_processN[max_processN.index('Number of process :') + len('Number of process :'):])
            return 'NumberOfProcess.txt에서의 잘못된 입력, 에러발생'

    def reading(i):
        with open('tmp/Process-{}.txt'.format(i), 'r', encoding='UTF8') as f:
            s = f.read()
            whip = ast.literal_eval(s)
        return whip

    def get_NPer_process(p, n):
        tmp_ = (n // p + 1) if n % p != 0 else n // p 
        res = [tmp_ for i in range(n // tmp_)] + ([n % tmp_] if n % tmp_ != 0 else [])
        return res, len(res)

    if __name__ == 'crawlingFunction':
        # multiprocessing.freeze_support()
        """
        Structure of both keywords and urlS
        [[],[]] -> newlist -> 한 원소당 하나의 질의
        [[]] -> urlist -> 한 원소에 모든 질의가 담김
        target : 'urllist' or 'newlist' or 'viewlist'
        """
        if target not in ['urllist', 'newlist', 'viewlist']: return 'Exception : Not valid target'
        if target != 'viewlist':
            result = {'blog': [],
                    'date': [],
                    'img': [],
                    'site': [],
                    'url': [],
                    'video': []
            }
        else:
            result = {'date': [],
                    'keyword': [],
                    'rank': [],
                    'title': [],
                    'url': []
            }
        # print(keywords, urlS)
        for keywords_, urlS_ in zip(keywords, urlS): # urllist : for문 1회 반복, newlist : for문 원소 개수만큼 반복
            procs = []
            len_of_data = len(keywords_) if len(keywords_) == len(urlS_) else 'Diff, Error'
            print("len_of_data :",len_of_data)

            if len_of_data == 'Diff, Error': return len_of_data

            max_processN, show_chrome = get_maxProcessNAndshowChrome()
            if type(max_processN) == str:
                return max_processN

            if len_of_data < 20: max_processN = 1
            from_, to_ = 0, 0
            intervalList, loop = get_NPer_process(max_processN, len_of_data)
            for interval, i in zip(intervalList, range(loop)):
                from_ = to_         # 0, 3, 6
                to_ = from_ + interval  # 3, 6, 9
                print(from_, to_, i+1)
                proc = Process(target=get_result, args=(keywords_[from_:to_], urlS_[from_:to_], target, driver_path, i + 1, show_chrome))6
                proc.start()
                procs.append(proc)

            print("Processes Started")

            for proc in procs:
                proc.join()
            print("Processes Joined")
            # print(loop)
            if target != 'viewlist':
                result['blog'].append([])
                result['date'].append([])
                result['img'].append([])
                result['site'].append([])
                result['url'].append([])
                result['video'].append([])
            else:
                result['date'].append([])
                result['keyword'].append([])
                result['rank'].append([])
                result['title'].append([])
                result['url'].append([])
            for i in range(loop):
                # print('in loop', i)
                dic = reading(i + 1)

                if target != 'viewlist':
                    result['blog'][-1] += (dic['blog'])
                    result['date'][-1] += (dic['date'])
                    result['img'][-1] += (dic['img'])
                    result['site'][-1] += (dic['site'])
                    result['url'][-1] += (dic['url'])
                    result['video'][-1] += (dic['video'])
                else:
                    result['date'][-1] += dic['date']
                    result['keyword'][-1] += dic['keyword']
                    result['rank'][-1] += dic['rank']
                    result['title'][-1] += dic['title']
                    result['url'][-1] += dic['url']
        # f = open('test1.txt','w')
        # print(result,file = f)
        # f.close()
        return result


# driver_dir = "..\\ROOT\\Programming\\AssociatedFiles\\chromedriver.exe"
# print(main(keywords, urlS, 'viewlist', driver_dir), file=f)


# cd "OneDrive - 인천광역시교육청\바탕 화면\외주 _ 블로그 크롤링"