# Twitter Search Ensemble

# load modules
from selenium import webdriver
import time,random,sys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
#
from bs4 import BeautifulSoup
from random import randint
from time import sleep
from anonBrowse import *


def TwitterSearch(seedUrl,client,browser,autoscroll,outfilename):
    if client == "web":
        if browser == "Firefox":
            # disable image and flash for Firefox
            firefox_profile = webdriver.FirefoxProfile()
            firefox_profile.set_preference('permissions.default.stylesheet', 2)
            firefox_profile.set_preference('permissions.default.image', 2)
            firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
            mydriver = webdriver.Firefox(firefox_profile=firefox_profile)
        else:
            # disable image and flash for Chrome
            from selenium.webdriver.chrome.options import Options
            chrome_options = Options()
            chrome_options.add_experimental_option("prefs", {'profile.default_content_settings.images': 2,'plugins.plugins_disabled': ['Adobe Flash Player']})
            chrome_options.add_argument("--disable-extensions")
            mydriver = webdriver.Chrome(chrome_options=chrome_options)
        # get URL
        mydriver.get(seedUrl)

        # scroll down the page iteratively with a delay [18+ per scroll, and 20 tweets per scroll]
        H=10000000000000
        idc=0
        rt=0
        while rt<3:
            try:
                if autoscroll:
                    mydriver.execute_script("window.scrollTo(0, Math.max(document.body.scrollHeight,document.documentElement.scrollHeight,document.documentElement.clientHeight));")
                else:
                    mydriver.execute_script("window.scrollTo(0, %s);" %H)
                r=mydriver.find_elements_by_xpath("//*[@id='stream-items-id']/li")
                tweetid=[x.get_attribute("data-item-id") for x in r]

                with open(outfilename, "w") as infile:
                    tweetid = [i for i in tweetid if i is not None]
                    infile.write("\n".join(tweetid))
                print (len(tweetid))

                if idc==len(tweetid):
                    time.sleep(30+random.random())
                    try:
                        mydriver.find_element_by_class_name('try-again-after-whale').click()
                    except:
                        if autoscroll:
                            mydriver.execute_script("window.scrollTo(0, Math.max(document.body.scrollHeight,document.documentElement.scrollHeight,document.documentElement.clientHeight));")
                        else:
                            mydriver.execute_script("window.scrollTo(0, %s);" %H)
                    rt=rt+1
                else:
                    time.sleep(1+random.random())
                    rt=0
            except:
                print "Unexpected error:", sys.exc_info()[0]
                time.sleep(60*15+random.random())
    else:
        url = 'https://mobile.twitter.com/'+seedUrl.split('/')[-1]
        br = anonBrowser()
        html = br.open(url).read()

        soup = BeautifulSoup(html)
        target = soup.find_all('div', {'class', 'tweet-text'})
        tweetids = [i['data-id'] for i in target]

        with open(outfilename, "a") as infile:
            tweetids = [i for i in tweetids if i is not None]
            infile.write("\n".join(tweetids)+"\n")

        n=len(tweetids)
        while (soup.find('div',{'class','w-button-more'}) is not None):
            nextp = 'https://mobile.twitter.com'+soup.find('div',{'class','w-button-more'}).a['href']
            try:
                html = br.open(nextp,timeout=30.0).read()
                time.sleep(1+random.random())
            except:
                sleep(15*60+1)
                html = br.open(nextp,timeout=30.0).read()

            soup = BeautifulSoup(html)
            target = soup.find_all('div', {'class', 'tweet-text'})
            tweetids = [i['data-id'] for i in target]

            with open(outfilename, "a") as infile:
                tweetids = [i for i in tweetids if i is not None]
                infile.write("\n".join(tweetids)+"\n")

            n=n+len(tweetids)
            print (str(n))
        print (soup.find('div',{'class','w-button-more'}))

# twitter urls
url="https://twitter.com/search?f=realtime&q=%22AIDS%22%20lang%3Aen%20since%3A2014-11-30%20until%3A2014-12-01&src=typd" # AIDS on 11-30, language=en,excl retweets
url="https://twitter.com/search?f=realtime&q=%22AIDS%22%20lang%3Aen%20since%3A2014-12-01%20until%3A2014-12-02&src=typd" # AIDS on 12-01, language=en,excl retweets
url="https://twitter.com/search?f=realtime&q=%22AIDS%22%20lang%3Aen%20since%3A2014-12-02%20until%3A2014-12-03&src=typd" # AIDS on 12-02, language=en,excl retweets
url="https://twitter.com/search?f=realtime&q=%22HIV%22%20lang%3Aen%20since%3A2014-11-30%20until%3A2014-12-01&src=typd" # HIV on 11-30, language=en,excl retweets
url="https://twitter.com/search?f=realtime&q=%22HIV%22%20lang%3Aen%20since%3A2014-12-01%20until%3A2014-12-02&src=typd" # HIV on 12-01, language=en,excl retweets
url="https://twitter.com/search?f=realtime&q=%22HIV%22%20lang%3Aen%20since%3A2014-12-02%20until%3A2014-12-03&src=typd" # HIV on 12-02, language=en,excl retweets

TwitterSearch(seedUrl=url,client="web",browser="Firefox",autoscroll=True,outfilename="AIDS_1201.txt")