import urllib
import csv
from bs4 import BeautifulSoup
def getPageText(url):
html = urllib.urlopen(url).read()
soup = BeautifulSoup(html, "html.parser")
# kill all script and style elements
for script in soup(["script", "style"]):
script.extract() # rip it out
for footer in soup(["footer"]):
footer.extract()
for header in soup(["header"]):
header.extract()
for title in soup(["title"]):
title.extract()
for h1 in soup(["h1"]):
h1.extract()
for div in soup('div', id='background'):
div.extract()
for span in soup('span', 'stuff'):
span.extract()
# get text
text = soup.get_text()
# break into lines and remove leading and trailing space on each
lines = (line.strip() for line in text.splitlines())
# break multi-headlines into a line each
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
# drop blank lines
text = '\n'.join(chunk for chunk in chunks if chunk)
tests = text.encode('utf-8')
f = csv.writer(open("output.csv", "a"))
print(text.encode('utf-8'))
f.writerow([url, tests])
def main():
urls = tuple(open("source.txt","rb").read().replace("\r\n","\n").split("\n"))
text = [getPageText(url) for url in urls]
if __name__=="__main__":
main()