source: trollforge/jylam/dlftroll/http.py @ 213 Tweet

import os
import sys
import urllib
import urllib2
import cookielib
import random



# Create an http object. First getPageWithReferer() call will get PHPSESSID (and others) cookie
# After that, each getPageWithReferer() call will send previously gotten cookie to server

class HTTP(object):
    sProxy = "void"
    sInited = 0
    def __init__(self, proxyfile):
        print "-> Getting random proxy from",proxyfile,"..."

        if not( os.path.isfile(proxyfile)):
            print "--> Error, '",  proxyfile, ",' is not a file"
            print ""
            sys.exit(2)

        
        proxyFile = file(proxyfile, "r")
        proxyList = proxyFile.readlines()

        num = random.randint(0, len(proxyList))
        self.sProxy = "http://"+proxyList[random.randint(0, len(proxyList))]
        print "--> Using proxy",  self.sProxy


    def getPageWithReferer(self, APage, AReferer, arguments):


        params        = urllib.urlencode(arguments)
        
        request      = urllib2.Request(APage, params)
        request.add_header("referer", AReferer)
        request.add_header("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.0; fr; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4")
        
#        Mozilla/5.0 (Firefox 3.1.1.7) Lunix Jewbuntu Diaper Drake
        proxy_support = urllib2.ProxyHandler({'http' : self.sProxy})
        opener        = urllib2.build_opener(proxy_support)
        urllib2.install_opener(opener)

        cj = cookielib.LWPCookieJar()
        if self.sInited == 1:
            cj.load("linuxfr.cookies");
        opener2 = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) 
        urllib2.install_opener(opener2)


        try: page =  urllib2.urlopen(request).read()
        except urllib2.HTTPError, e:
                print e


        if self.sInited == 0:
            cj.save("linuxfr.cookies");
            self.sInited = 1

        opener.close();
        return page