#! /usr/bin/env python

import sys
import pycurl
import re
import getopt

# Performs a number of web calling thingies
# uses pyCurl to perform network calls, and spoofs UA as ie
class curlAgent:
	def __init__(self):
		self.contents = ''

	def body_callback(self, buf):
		self.contents = self.contents + buf
	
	def retrieve_url(self, url):
		c = pycurl.Curl()
		c.setopt(c.URL, url)
		c.setopt(c.WRITEFUNCTION, self.body_callback)
		#c.setopt(c.VERBOSE,1)
		c.setopt(c.HTTPHEADER, ["User-Agent: Mozilla/5.001 (windows; U; NT4.0; en-us) Gecko/25250101", "Agent: "]) 
		#c.setopt(c.PROXY, "wwwcache.lancs.ac.uk") 
		#c.setopt(c.PROXYPORT,8080) 
		
		#c.setopt(PROXYUSERPWD,"dummyuser:dummypasswd") 
		#c.setopt(HTTPAUTH,8) 
		#NTLM c.setopt(WRITEFUNCTION, b.write) 
		c.setopt(c.FOLLOWLOCATION, 1) 
		c.setopt(c.MAXREDIRS, 5)
		#c.setopt(c.INTERFACE, "ath0")
		c.perform()
		c.close()

# Global counters for various things
global pagecount, hrefcount
pagecount = 0
hrefcount = 0

# Uses the curlAgent to retrieve many URLs and runs a given object with them
# basically t'is a link reteiever that wanders many a site
class URLRetriever:
	def __init__(self, curlAgent):
		self.curlAgent = curlAgent

	def get_href_tags(self, text):
		return re.findall("(?:[hH][rR][eE][fF]\\s*=)(?:[\\s\"\"']*)(?!#|[Mm]ailto|[lL]ocation.|[jJ]avascript|.*css|.*this\\.)(.*?)(?:[\\s>\"\"'])", text)

	def get_src_tags(self, text):
		return re.findall("(?:[sS][rR][cC]\\s*=)(?:[\\s\"\"']*)(.*?)(?:[\\s>\"\"'])", text)


	def build_array(self, url, recursionlevel, handler):
		global pagecount, hrefcount
		#print "----?" + url
		#sys.stdout.write("\rlevel: " + str(recursionlevel) + "\t page count: " + str(pagecount) + "\t href count: " + str(hrefcount))
		#sys.stdout.flush();
		pagecount += 1 
		try:
		    currentplace = url.rsplit("/", 1)[0]

		    self.curlAgent.retrieve_url(url)
		    href = self.get_href_tags(self.curlAgent.contents)
		    src = self.get_src_tags(self.curlAgent.contents)
		    
		    for image in src:
			    if(not image.startswith("http://")):
				    image = currentplace + "" + image
			    handler.acceptSRC(image)

		    recursionlevel = recursionlevel - 1
		    
		    for value in href:
			    if(not value.startswith("http://")):
				    value = currentplace + "" + value
			    

			    #if(value != None):
				    #if(len(value) > 4):
			    handler.acceptURL(value, self.curlAgent.contents)
			    hrefcount += 1
			    if(recursionlevel > 0):
				    try:
					    r = URLRetriever(curlAgent())
					    subtext = r.build_array(value, recursionlevel, handler)
				    except:
					    pass
				    #print value
		except x:
		    pass


# May be used to print links out to the terminal.
# Ought to be passed to the URL retiever in order to use
class Printer:
    def __init__(self):
	self.urlnoRepeats = []
	self.srcnoRepeats = []

    def acceptURL(self, url, text):
	pass
	#is this fails then the url is unique!
	#try:
	    #self.urlnoRepeats.index(url)
	#except ValueError:
	    ##print "[ " + str(pagecount) + " : " + str(hrefcount) + " ]" + url	
	    ##print "" + url 
	    #self.urlnoRepeats.append(url)
	    #pass

    def acceptSRC(self, url):
	#if this fails then the url is unique!
	try:
	    self.srcnoRepeats.index(url)
	except ValueError:
	    print "" + url
	    self.srcnoRepeats.append(url)




# Simply builds a big list of stuff
# Each time something calls acceptURL an item is added onto the end
class ArrayBuilder:
	def __init__(self):
		self.contents = []
	def acceptURL(self, url, text):
		self.contents.append(url)

#print >>sys.stderr, 'Testing', pycurl.version



# Create a new URLRetriever with the curlAgent in order to strip websites of their links
r = URLRetriever(curlAgent())

# create a new list of links 3 pages deep, and pass them into the printer object, which will output them to stdout
r.build_array("http://www.fark.com", 3, Printer());

#print r.text;
		
		
		


