Download as PDF, PPTX
![# parse resulting JSON and insert into a mongoDB collection!
for content in api.json()['response']['docs']:!
if not collection.find_one(content):!
collection.insert(content)!
!
!
# only returns 10 per page!
"There are only %i docuemtns returned 0_o" % !
! len(api.json()[‘response']['docs'])!
Questions? tweet @zipfianacademy #pydata
Acquire](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-44-2048.jpg)
![# there are many more than 10 articles however!
total_art = articles_left = api.json()['response']['meta']['hits']!
!
!
print "There are currently %s articles in the NYT archive" % total_art!
!
!
#=> There are currently 15277775 articles in the NYT archive
Questions? tweet @zipfianacademy #pydata
Acquire](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-45-2048.jpg)
![# let us loop (and hopefully not hit our rate limit)!
while articles_left > 0 and page_count < max_pages:!
more_articles = requests.get(url + "&page=" + str(page) + "&end_date=" + str(last_date))!
print "Inserting page " + str(page)!
# make sure it was successful!
if more_articles.status_code == 200:!
for content in more_articles.json()['response']['docs']:!
latest_article = parser.parse(content['pub_date']).strftime("%Y%m%d")!
if not collection.find_one(content) and content['document_type'] == 'article':!
print "No dups"!
try:!
print "Inserting article " + str(content['headline'])!
collection.insert(content)!
except errors.DuplicateKeyError:!
print "Duplicates"!
continue!
else:!
print "In collection already”!
! ! …
Iteration 0.5
Questions? tweet @zipfianacademy #pydata
Acquire](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-49-2048.jpg)
![Acquire
# now we can get some content!!
#limit = 100!
limit = 10000!
!
for article in collection.find({'html' : {'$exists' : False} }):!
if limit and limit > 0:!
if not article.has_key('html') and article['document_type'] == 'article':!
limit -= 1!
print article['web_url']!
html = requests.get(article['web_url'] + "?smid=tw-nytimes")!
!
if html.status_code == 200:!
soup = BeautifulSoup(html.text)!
!
# serialize html!
collection.update({ '_id' : article['_id'] }, { '$set' : !
! ! ! ! ! ! ! ! ! ! ! ! ! { 'html' : unicode(soup), 'content' : [] } !
! ! ! ! ! ! ! ! ! ! ! ! } )!
!
for p in soup.find_all('div', class_='articleBody'):!
collection.update({ '_id' : article['_id'] }, { '$push' : !
! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() !
! ! ! ! ! ! ! ! ! ! ! ! ! } })!
Questions? tweet @zipfianacademy #pydata](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-52-2048.jpg)
![# parse HTML content of articles!
for article in collection.find({'html' : {'$exists' : True} }):!
print article['web_url']!
soup = BeautifulSoup(article['html'], 'html.parser')!
arts = soup.find_all('div', class_='articleBody')!
!
if len(arts) == 0:!
arts = soup.find_all('p', class_=‘story-body-text')!
!
! ! …
Questions? tweet @zipfianacademy #pydata
Parse](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-55-2048.jpg)
![for p in arts:!
collection.update({ '_id' : article['_id'] }, { '$push' : !
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() } !
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! })!
Questions? tweet @zipfianacademy #pydata
Store](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-57-2048.jpg)
![articles.describe()!
# ! ! text section!
# count 1405 1405!
# unique 1397 10!
!
fig = plt.figure()!
# histogram of section counts!
articles['section'].value_counts().plot(kind='bar')
Questions? tweet @zipfianacademy #pydata
Explore](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-60-2048.jpg)
![Vectorize
wnl = nltk.WordNetLemmatizer()!
!
def tokenize_and_normalize(chunks):!
words = [ tokenize.word_tokenize(sent) for sent in
tokenize.sent_tokenize("".join(chunks)) ]!
flatten = [ inner for sublist in words for inner in sublist ]!
stripped = [] !
!
for word in flatten: !
if word not in stopwords.words('english'):!
try:!
stripped.append(word.encode('latin-1').decode('utf8').lower())!
except:!
print "Cannot encode: " + word!
!
no_punks = [ word for word in stripped if len(word) > 1 ] !
return [wnl.lemmatize(t) for t in no_punks]!
Questions? tweet @zipfianacademy #pydata](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-66-2048.jpg)
![class DocumentClassifier(YhatModel):!
@preprocess(in_type=dict, out_type=dict)!
def execute(self, data):!
featureBody = vectorizer.transform([data['content']])!
result = multi_bayes.predict(featureBody)!
list_res = result.tolist()!
return {"section_name": list_res}!
!
clf = DocumentClassifier()!
yh = Yhat("jonathan@zipfianacademy.com", “xxxxxx",!
! ! ! ! ! ! ! ! ! ! ! ! ! "http://cloud.yhathq.com/")!
yh.deploy("documentClassifier", DocumentClassifier, globals())
Questions? tweet @zipfianacademy #pydata
Exposé](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-83-2048.jpg)
![yh = Yhat("<USERNAME>", "<API KEY>", "http://cloud.yhathq.com/")
!
@app.route('/')
def index():
return app.send_static_file('index.html')
!
@app.route('/predict', methods=['POST'])
def predict():
article = request.form['article']
results = yh.predict("documentClf", { 'content': article })
return jsonify({"results": results})
Questions? tweet @zipfianacademy #pydata
Present](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-86-2048.jpg)
![• JSON is made up of hash tables and arrays
• Hash tables: { “foo” : 1, “bar” : 2, baz : “3” }
• Arrays: [1, 2, 3]
• Arrays of arrays: [[1, 2, 3], [‘foo’, ‘bar’, ‘baz’]]
• Array of hashes: [{‘foo’:1, ‘bar’:2}, {‘baz’:3}]
• Hashes of hashes: {‘foo’: {‘bar’: 2, ‘baz’: 3}}
Questions? tweet @zipfianacademy #pydata
Data Formats](https://image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-113-2048.jpg)
![# parse resulting JSON and insert into a mongoDB collection!
for content in api.json()['response']['docs']:!
if not collection.find_one(content):!
collection.insert(content)!
!
!
# only returns 10 per page!
"There are only %i docuemtns returned 0_o" % !
! len(api.json()[‘response']['docs'])!
Questions? tweet @zipfianacademy #pydata
Acquire](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-44-2048.jpg)
![# there are many more than 10 articles however!
total_art = articles_left = api.json()['response']['meta']['hits']!
!
!
print "There are currently %s articles in the NYT archive" % total_art!
!
!
#=> There are currently 15277775 articles in the NYT archive
Questions? tweet @zipfianacademy #pydata
Acquire](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-45-2048.jpg)
![# let us loop (and hopefully not hit our rate limit)!
while articles_left > 0 and page_count < max_pages:!
more_articles = requests.get(url + "&page=" + str(page) + "&end_date=" + str(last_date))!
print "Inserting page " + str(page)!
# make sure it was successful!
if more_articles.status_code == 200:!
for content in more_articles.json()['response']['docs']:!
latest_article = parser.parse(content['pub_date']).strftime("%Y%m%d")!
if not collection.find_one(content) and content['document_type'] == 'article':!
print "No dups"!
try:!
print "Inserting article " + str(content['headline'])!
collection.insert(content)!
except errors.DuplicateKeyError:!
print "Duplicates"!
continue!
else:!
print "In collection already”!
! ! …
Iteration 0.5
Questions? tweet @zipfianacademy #pydata
Acquire](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-49-2048.jpg)
![Acquire
# now we can get some content!!
#limit = 100!
limit = 10000!
!
for article in collection.find({'html' : {'$exists' : False} }):!
if limit and limit > 0:!
if not article.has_key('html') and article['document_type'] == 'article':!
limit -= 1!
print article['web_url']!
html = requests.get(article['web_url'] + "?smid=tw-nytimes")!
!
if html.status_code == 200:!
soup = BeautifulSoup(html.text)!
!
# serialize html!
collection.update({ '_id' : article['_id'] }, { '$set' : !
! ! ! ! ! ! ! ! ! ! ! ! ! { 'html' : unicode(soup), 'content' : [] } !
! ! ! ! ! ! ! ! ! ! ! ! } )!
!
for p in soup.find_all('div', class_='articleBody'):!
collection.update({ '_id' : article['_id'] }, { '$push' : !
! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() !
! ! ! ! ! ! ! ! ! ! ! ! ! } })!
Questions? tweet @zipfianacademy #pydata](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-52-2048.jpg)
![# parse HTML content of articles!
for article in collection.find({'html' : {'$exists' : True} }):!
print article['web_url']!
soup = BeautifulSoup(article['html'], 'html.parser')!
arts = soup.find_all('div', class_='articleBody')!
!
if len(arts) == 0:!
arts = soup.find_all('p', class_=‘story-body-text')!
!
! ! …
Questions? tweet @zipfianacademy #pydata
Parse](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-55-2048.jpg)
![for p in arts:!
collection.update({ '_id' : article['_id'] }, { '$push' : !
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() } !
! ! ! ! ! ! ! ! ! ! ! ! ! ! ! })!
Questions? tweet @zipfianacademy #pydata
Store](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-57-2048.jpg)
![articles.describe()!
# ! ! text section!
# count 1405 1405!
# unique 1397 10!
!
fig = plt.figure()!
# histogram of section counts!
articles['section'].value_counts().plot(kind='bar')
Questions? tweet @zipfianacademy #pydata
Explore](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-60-2048.jpg)
![Vectorize
wnl = nltk.WordNetLemmatizer()!
!
def tokenize_and_normalize(chunks):!
words = [ tokenize.word_tokenize(sent) for sent in
tokenize.sent_tokenize("".join(chunks)) ]!
flatten = [ inner for sublist in words for inner in sublist ]!
stripped = [] !
!
for word in flatten: !
if word not in stopwords.words('english'):!
try:!
stripped.append(word.encode('latin-1').decode('utf8').lower())!
except:!
print "Cannot encode: " + word!
!
no_punks = [ word for word in stripped if len(word) > 1 ] !
return [wnl.lemmatize(t) for t in no_punks]!
Questions? tweet @zipfianacademy #pydata](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-66-2048.jpg)
![class DocumentClassifier(YhatModel):!
@preprocess(in_type=dict, out_type=dict)!
def execute(self, data):!
featureBody = vectorizer.transform([data['content']])!
result = multi_bayes.predict(featureBody)!
list_res = result.tolist()!
return {"section_name": list_res}!
!
clf = DocumentClassifier()!
yh = Yhat("jonathan@zipfianacademy.com", “xxxxxx",!
! ! ! ! ! ! ! ! ! ! ! ! ! "http://cloud.yhathq.com/")!
yh.deploy("documentClassifier", DocumentClassifier, globals())
Questions? tweet @zipfianacademy #pydata
Exposé](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-83-2048.jpg)
![yh = Yhat("<USERNAME>", "<API KEY>", "http://cloud.yhathq.com/")
!
@app.route('/')
def index():
return app.send_static_file('index.html')
!
@app.route('/predict', methods=['POST'])
def predict():
article = request.form['article']
results = yh.predict("documentClf", { 'content': article })
return jsonify({"results": results})
Questions? tweet @zipfianacademy #pydata
Present](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-86-2048.jpg)
![• JSON is made up of hash tables and arrays
• Hash tables: { “foo” : 1, “bar” : 2, baz : “3” }
• Arrays: [1, 2, 3]
• Arrays of arrays: [[1, 2, 3], [‘foo’, ‘bar’, ‘baz’]]
• Array of hashes: [{‘foo’:1, ‘bar’:2}, {‘baz’:3}]
• Hashes of hashes: {‘foo’: {‘bar’: 2, ‘baz’: 3}}
Questions? tweet @zipfianacademy #pydata
Data Formats](https://crownmelresort.com/image.slidesharecdn.com/dataengineering101buildingyourfirstdataproductbyjonathandinu-140613094459-phpapp01/75/Data-Engineering-101-Building-your-first-data-product-by-Jonathan-Dinu-PyData-SV-2014-113-2048.jpg)
Uploaded byPyData
Data Engineering 101: Building your first data product by Jonathan Dinu PyData SV 2014
The document presents a seminar led by Jonathan Dinu on the fundamentals of data engineering and building a data product, focusing on the development of a news reader application called 'nws rdr.' It details the processes of data acquisition, pipeline construction, and the implementation of machine learning techniques such as Naive Bayes and collaborative filtering. The seminar also includes a Q&A section and emphasizes the value of data products in enhancing user experiences.
More Related Content
Similar to Data Engineering 101: Building your first data product by Jonathan Dinu PyData SV 2014
Data Engineering 101: Building your first data product by Jonathan Dinu PyData SV 2014
- 1. Jonathan Dinu Co-Founder, ZipfianAcademy jonathan@zipfianacademy.com @clearspandex @ZipfianAcademy Data Engineering 101: Building your first data product May 4th, 2014
- 2. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 3.
- 4.
- 5.
- 6. Today Disclaimer: All charactersappearing in this presentation are fictitious. Any resemblance to real persons, living or dead, is purely coincidental. Questions? tweet @zipfianacademy #pydata
- 7. Today Disclaimer: This presentationcontains strong opinions that you may or may not agree with. All thoughts are my own. Jonathan Dinu Co-Founder, Zipfian Academy jonathan@zipfianacademy.com @clearspandex Questions? tweet @zipfianacademy #pydata
- 8. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • CreatingValue for Users • Q&A Questions? tweet @zipfianacademy #pydata
- 9. nwsrdr (News Reader) Source:http://www.groovypost.com/wp-content/uploads/2013/05/Bookmark- Button.png OR nwsrdr + nwrsrdr + nwrsrdr + nwrsrdr nwsrdr getnews.com/bookmarklet When browsing the web simply click the +nwsrdr to save any page to nwsrdr Get nwsrdr on your desktop Questions? tweet @zipfianacademy #pydata
- 10. nwsrdr • Auto-categorize Articles •Find Similar Articles • Recommend articles • Suggest Feeds to Follow • No Ads! It’s like Prismatic + Pocket + Google Reader (RIP) + Delicious! Questions? tweet @zipfianacademy #pydata
- 11. nwsrdr It’s like Prismatic+ Pocket + Google Reader (RIP) + Delicious! • Naive Bayes (classification) • Clustering (unsupervised learning) • Collaborative Filtering • Triangle Closing • Real Business Model! Questions? tweet @zipfianacademy #pydata
- 12. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 13. OR Data Products Product Builton Data (that you sell) Questions? tweet @zipfianacademy #pydata
- 14. OR Data Products Product thatGenerates Data Questions? tweet @zipfianacademy #pydata
- 15. OR Data Products Product thatGenerates Data (that you sell) Questions? tweet @zipfianacademy #pydata
- 16. OR Data Products Product thatGenerates Data (that you sell) i.e. Facebook Questions? tweet @zipfianacademy #pydata
- 17. OR Data Products Questions? tweet@zipfianacademy #pydata Source: http://gifgif.media.mit.edu/
- 18. OR Data Products Source: http://www.adamlaiacano.com/post/57703317453/data-generating-productsQuestions?tweet @zipfianacademy #pydata
- 19.
- 20. Products that enhancea users’ experience the more “data” a user provides Data Generating Products Ex: Recommender Systems Questions? tweet @zipfianacademy #pydata
- 21. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 22. OR Data Science Questions? tweet@zipfianacademy #pydata
- 23. i.e. solve moreproblems than you create Data Science Questions? tweet @zipfianacademy #pydata
- 24. Source: http://estoyentretenido.com/wp-content/uploads/2012/11/Jackie-Chan- Meme.jpg But.... How?!?!?!!? DataScience Questions? tweet @zipfianacademy #pydata
- 25.
- 26.
- 27. OR Data Engineering Questions? tweet@zipfianacademy #pydata
- 28. Prepared Data Test Set Training Set Train Model Sampling Evaluate Cross Validation Data Science Questions? tweet @zipfianacademy #pydata
- 29. Raw Data Cleaned Data Scrubbing Prepared DataVectorization New Data Test Set Training Set Train Model Sampling Evaluate Cross Validation Cleaned Data Prepared DataVectorizationScrubbing Predict Labels/ Classes Data Engineering Questions? tweet @zipfianacademy #pydata
- 30. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 31. What • Naive Bayes(classification) • Clustering (unsupervised learning) • Collaborative Filtering • Triangle Closing • Real Business Model Questions? tweet @zipfianacademy #pydata
- 32. nwsrdr • Auto-categorize Articles •Find Similar Articles • Recommend articles • No Ads! It’s like Prismatic + Pocket + Google Reader (RIP) + Delicious! Questions? tweet @zipfianacademy #pydata
- 33. nwsrdr It’s like Prismatic+ Pocket + Google Reader (RIP) + Delicious! • Naive Bayes (classification) • Clustering (unsupervised learning) • Collaborative Filtering • Triangle Closing • Real Business Model! Questions? tweet @zipfianacademy #pydata
- 34. Source: http://media.tumblr.com/tumblr_lakcynCyG31qbzcoy.jpg Abstraction (Cake) How (ABK) Questions?tweet @zipfianacademy #pydata
- 35.
- 36.
- 37.
- 38. Obligatory Name Drop Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation requests BeautifulSoup4 pandas pymongo Flask yHat At ScaleLocally scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) Flask yHat scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata
- 39. Obligatory Name Drop Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy HadoopStreaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata
- 40. Pipeline Iteration 0: • Findout how much data • Run locally • Experiment Questions? tweet @zipfianacademy #pydata
- 41. Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata Acquire
- 42. Retrieve Meta-data forALL NYT articles Questions? tweet @zipfianacademy #pydata Acquire
- 43. api_key='xxxxxxxxxxxxx'! ! ! ! url = 'http://api.nytimes.com/svc/search/v2/ articlesearch.json?fq=section_name.contains:("Arts" "BusinessDay" "Opinion" "Sports" "U.S." "World")&sort=newest&api-key=' + api_key! ! ! ! # make an API request! api = requests.get(url)! Questions? tweet @zipfianacademy #pydata Acquire
- 44. # parse resultingJSON and insert into a mongoDB collection! for content in api.json()['response']['docs']:! if not collection.find_one(content):! collection.insert(content)! ! ! # only returns 10 per page! "There are only %i docuemtns returned 0_o" % ! ! len(api.json()[‘response']['docs'])! Questions? tweet @zipfianacademy #pydata Acquire
- 45. # there aremany more than 10 articles however! total_art = articles_left = api.json()['response']['meta']['hits']! ! ! print "There are currently %s articles in the NYT archive" % total_art! ! ! #=> There are currently 15277775 articles in the NYT archive Questions? tweet @zipfianacademy #pydata Acquire
- 46. Gotchas! • Rate Limiting •Page Limiting Questions? tweet @zipfianacademy #pydata Acquire
- 47. Iterate Iteration 1: • (Meaningful)Sample of Data • Prototype — “Close the Loop” Questions? tweet @zipfianacademy #pydata
- 48. Retrieve Meta-data forALL NYT articles Questions? tweet @zipfianacademy #pydata Acquire (take 2)
- 49. # let usloop (and hopefully not hit our rate limit)! while articles_left > 0 and page_count < max_pages:! more_articles = requests.get(url + "&page=" + str(page) + "&end_date=" + str(last_date))! print "Inserting page " + str(page)! # make sure it was successful! if more_articles.status_code == 200:! for content in more_articles.json()['response']['docs']:! latest_article = parser.parse(content['pub_date']).strftime("%Y%m%d")! if not collection.find_one(content) and content['document_type'] == 'article':! print "No dups"! try:! print "Inserting article " + str(content['headline'])! collection.insert(content)! except errors.DuplicateKeyError:! print "Duplicates"! continue! else:! print "In collection already”! ! ! … Iteration 0.5 Questions? tweet @zipfianacademy #pydata Acquire
- 50. articles_left -= 10! page+= 1! page_count += 1! cursor_count += 1! final_page = max(final_page, page)! else:! if more_articles.status_code == 403:! print "Sleepy..."! # account for rate limiting! time.sleep(2)! elif cursor_count > 100:! print "Adjusting date”! ! ! ! ! # account for page limiting! cursor_count = 0! page = 0! last_date = latest_article! else:! print "ERRORS: " + str(more_articles.status_code)! cursor_count = 0! page = 0! last_date = latest_article! Questions? tweet @zipfianacademy #pydata Acquire
- 51. Download HTML contentof articles from NYT.com Questions? tweet @zipfianacademy #pydata Acquire (and store in MongoDB™)
- 52. Acquire # now wecan get some content!! #limit = 100! limit = 10000! ! for article in collection.find({'html' : {'$exists' : False} }):! if limit and limit > 0:! if not article.has_key('html') and article['document_type'] == 'article':! limit -= 1! print article['web_url']! html = requests.get(article['web_url'] + "?smid=tw-nytimes")! ! if html.status_code == 200:! soup = BeautifulSoup(html.text)! ! # serialize html! collection.update({ '_id' : article['_id'] }, { '$set' : ! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'html' : unicode(soup), 'content' : [] } ! ! ! ! ! ! ! ! ! ! ! ! ! } )! ! for p in soup.find_all('div', class_='articleBody'):! collection.update({ '_id' : article['_id'] }, { '$push' : ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() ! ! ! ! ! ! ! ! ! ! ! ! ! ! } })! Questions? tweet @zipfianacademy #pydata
- 53. Parse Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo scikit-learn/NLTK Flask yHat Locally Questions? tweet @zipfianacademy #pydata
- 54. Parse HTML withBeautifulSoup and Extract the article Body Questions? tweet @zipfianacademy #pydata (and store in MongoDB™) Parse
- 55. # parse HTMLcontent of articles! for article in collection.find({'html' : {'$exists' : True} }):! print article['web_url']! soup = BeautifulSoup(article['html'], 'html.parser')! arts = soup.find_all('div', class_='articleBody')! ! if len(arts) == 0:! arts = soup.find_all('p', class_=‘story-body-text')! ! ! ! … Questions? tweet @zipfianacademy #pydata Parse
- 56. Store Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata
- 57. for p inarts:! collection.update({ '_id' : article['_id'] }, { '$push' : ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! { 'content' : p.get_text() } ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! })! Questions? tweet @zipfianacademy #pydata Store
- 58. Explore Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata
- 59. Exploratory Data Analysiswith pandas Questions? tweet @zipfianacademy #pydata Explore
- 60. articles.describe()! # ! !text section! # count 1405 1405! # unique 1397 10! ! fig = plt.figure()! # histogram of section counts! articles['section'].value_counts().plot(kind='bar') Questions? tweet @zipfianacademy #pydata Explore
- 61. Questions? tweet @zipfianacademy#pydata Explore
- 62. error with NYTAPI Questions? tweet @zipfianacademy #pydata Explore
- 63. api_key='xxxxxxxxxxxxx'! ! ! ! url = 'http://api.nytimes.com/svc/search/v2/ articlesearch.json?fq=section_name.contains:("Arts" "BusinessDay" "Opinion" "Sports" "U.S." "World")&sort=newest&api-key=' + api_key! ! ! ! # make an API request! api = requests.get(url)! Questions? tweet @zipfianacademy #pydata Explore error with NYT API
- 64. Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata Vectorize
- 65. Tokenize article textand create feature vectors with NLTK Questions? tweet @zipfianacademy #pydata Vectorize
- 66. Vectorize wnl = nltk.WordNetLemmatizer()! ! deftokenize_and_normalize(chunks):! words = [ tokenize.word_tokenize(sent) for sent in tokenize.sent_tokenize("".join(chunks)) ]! flatten = [ inner for sublist in words for inner in sublist ]! stripped = [] ! ! for word in flatten: ! if word not in stopwords.words('english'):! try:! stripped.append(word.encode('latin-1').decode('utf8').lower())! except:! print "Cannot encode: " + word! ! no_punks = [ word for word in stripped if len(word) > 1 ] ! return [wnl.lemmatize(t) for t in no_punks]! Questions? tweet @zipfianacademy #pydata
- 67. Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata Train
- 68. Train and scorea model with scikit-learn Questions? tweet @zipfianacademy #pydata Train
- 69. # cross validate! fromsklearn.cross_validation import train_test_split! ! xtrain, xtest, ytrain, ytest = ! ! ! ! ! ! ! ! train_test_split(X, labels, test_size=0.3)! ! # train a model! alpha = 1! multi_bayes = MultinomialNB(alpha=alpha)! ! multi_bayes.fit(xtrain, ytrain)! multi_bayes.score(xtest, ytest) Questions? tweet @zipfianacademy #pydata Train
- 70. Gotchas! • Model onlyexists locally on Laptop • Not Automated for realtime prediction Questions? tweet @zipfianacademy #pydata Train
- 71.
- 72. Iteration 2: • Exposeyour model • Automate your processes Questions? tweet @zipfianacademy #pydata Exposé
- 73. Getting that model offyour lap(top) Questions? tweet @zipfianacademy #pydata Exposé
- 74. Source: http://pixel.nymag.com/imgs/daily/vulture/2012/03/09/09_joan-taylor.o.jpg/ a_560x0.jpg Questions? tweet@zipfianacademy #pydata Exposé
- 75. A model isjust a function Questions? tweet @zipfianacademy #pydata Exposé
- 76. Inputs... Questions? tweet @zipfianacademy#pydata Exposé
- 77. Outputs... Questions? tweet @zipfianacademy#pydata Exposé
- 78. Serialize your modelwith pickle (or cPickle or joblib) Questions? tweet @zipfianacademy #pydata Persistence
- 79.
- 80. Persistence SerDes • Disk • Database •Memory Questions? tweet @zipfianacademy #pydata
- 81. Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata Exposé
- 82. Deploy your Modelto yHat Questions? tweet @zipfianacademy #pydata Exposé
- 83. class DocumentClassifier(YhatModel):! @preprocess(in_type=dict, out_type=dict)! defexecute(self, data):! featureBody = vectorizer.transform([data['content']])! result = multi_bayes.predict(featureBody)! list_res = result.tolist()! return {"section_name": list_res}! ! clf = DocumentClassifier()! yh = Yhat("jonathan@zipfianacademy.com", “xxxxxx",! ! ! ! ! ! ! ! ! ! ! ! ! ! "http://cloud.yhathq.com/")! yh.deploy("documentClassifier", DocumentClassifier, globals()) Questions? tweet @zipfianacademy #pydata Exposé
- 84. Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation At Scale Flask yHat scrapy Hadoop Streaming (w/ BeautifulSoup4) mrjob or Mortar (w/ Python UDF) Snakebite (HDFS) MLlib (pySpark) requests BeautifulSoup4 pandas pymongo Flask (on Heroku) yHat Locally scikit-learn/NLTK Questions? tweet @zipfianacademy #pydata Present
- 85. Create a Flaskapplication to expose your model on the web Questions? tweet @zipfianacademy #pydata Present
- 86. yh = Yhat("<USERNAME>","<API KEY>", "http://cloud.yhathq.com/") ! @app.route('/') def index(): return app.send_static_file('index.html') ! @app.route('/predict', methods=['POST']) def predict(): article = request.form['article'] results = yh.predict("documentClf", { 'content': article }) return jsonify({"results": results}) Questions? tweet @zipfianacademy #pydata Present
- 87. Pipeline Only Data shouldFlow Questions? tweet @zipfianacademy #pydata
- 88. Data Remember to Remember (Lineage) Acquisition Parse Storage Transform/Explore Vectorization Train Model Expose Presentation Questions?tweet @zipfianacademy #pydata Pipeline
- 89. Immutable append onlyset of Raw Data Computation is a view on data *Lambda Architecture by Nathan MarzQuestions? tweet @zipfianacademy #pydata Pipeline
- 90. Functional Data Science •Modularity • Define interfaces • Separate data from computation • Data Lineage Functional Questions? tweet @zipfianacademy #pydata
- 91. Need Robust andFlexible Pipeline! Questions? tweet @zipfianacademy #pydata Pipeline
- 92. Whatever you do,DO NOT cross the streams Questions? tweet @zipfianacademy #pydata Pipeline
- 93.
- 94. Gotchas! • Only havea static subset of articles • Pipeline not automated for re-training Questions? tweet @zipfianacademy #pydata Gotchas
- 95. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 96. Iteration 3: Source: http://vninja.net/wordpress/wp-content/uploads/2013/03/KCaAutomate.pngQuestions?tweet @zipfianacademy #pydata Iterate
- 97. NYT API MongoDB cron Feature Matrixscikit-learn Web App Model Deploy yHat Heroku POST Predict Predicted Section Where we are NLTK scikit-learn Questions?tweet @zipfianacademy #pydata Amazon EC2
- 98. testing Start small (data) andfast (development) testing Increase size of data set Optimize and productionize PROFIT! $$$ Questions? tweet @zipfianacademy #pydata How to Scale
- 99. How to Scale testing Developlocally testing Distribute computation (run on cluster) Tune parameters PROFIT! $$$ Questions? tweet @zipfianacademy #pydata Can also use a streaming algorithm or single machine disk based “medium data” technologies (i.e. database or memory mapped files)
- 100. Products If you buildit... Questions? tweet @zipfianacademy #pydata
- 101.
- 102. Today • whoami • NwsRdr (News Reader) • The What,Why, and How of Data Products • Data Engineering • Building a Pipeline • Productionizing the Products • Q&A Questions? tweet @zipfianacademy #pydata
- 103. Q & A Q&A Questions?tweet @zipfianacademy #pydata
- 104. Zipfian Academy @ZipfianAcademy Data Science &Data Engineering 12-week Bootcamp (May 12th & Sep 8th) Weekend Workshops http://zipfianacademy.com/apply http://zipfianacademy.com/workshops Next: InteractiveVisualizations w/ d3.js (June 7th) Questions? tweet @zipfianacademy #pydata
- 105. Thank You! Jonathan Dinu Co-Founder,Zipfian Academy jonathan@zipfianacademy.com @clearspandex @ZipfianAcademy http://zipfianacademy.com Questions? tweet @zipfianacademy #pydata
- 106.
- 107. Data Sources Obtain (ranked byease of use) 1. DaaS -- Data as a service 2. Bulk Download 3. APIs 4. Web Scraping Questions? tweet @zipfianacademy #pydata
- 108. DaaS (Data as aService) •Time Series/Numeric: Quandl • Financial Modeling: Quantopian • Email Contextualization: Rapleaf • Location and POI: Factual Data Sources Questions? tweet @zipfianacademy #pydata
- 109. Bulk Download (just likethe good ol’ days) • File Transfer Protocol (FTP): CDC •Amazon Web Services: Public Datasets • Infochimps: Data Marketplace •Academia: UCI Machine Learning Repository Data Sources Questions? tweet @zipfianacademy #pydata
- 110. APIs (if it’s notRESTed, I’m not buying) • Geographic: Foursquare • Social: Facebook •Audio: Rdio • Content:Tumblr • Realtime:Twitter • Hidden:Yahoo Finance Data Sources Questions? tweet @zipfianacademy #pydata
- 111. Web Scraping 1. wgetand curl 2. Web Spider/Crawler 3. API scraping 4. Manual Download (DIY for life) Data Sources Questions? tweet @zipfianacademy #pydata
- 112. • DelimitedValues • TSV •CSV • WSV • JSON • XML • Ad Hoc Formats (avoid these if you can) Data Formats Questions? tweet @zipfianacademy #pydata
- 113. • JSON ismade up of hash tables and arrays • Hash tables: { “foo” : 1, “bar” : 2, baz : “3” } • Arrays: [1, 2, 3] • Arrays of arrays: [[1, 2, 3], [‘foo’, ‘bar’, ‘baz’]] • Array of hashes: [{‘foo’:1, ‘bar’:2}, {‘baz’:3}] • Hashes of hashes: {‘foo’: {‘bar’: 2, ‘baz’: 3}} Questions? tweet @zipfianacademy #pydata Data Formats
- 114. {"widget": {! "debug": "on",! "window":{! "title": "Sample Konfabulator Widget",! "name": "main_window",! "width": 500,! "height": 500! },! "image": { ! "src": "Images/Sun.png",! "name": "sun1",! "hOffset": 250,! "vOffset": 250,! "alignment": "center"! },! "text": {! "data": "Click Here",! "size": 36,! "style": "bold",! "name": "text1",! "hOffset": 250,! "vOffset": 100,! "alignment": "center",! "onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"! }! }} ! Questions? tweet @zipfianacademy #pydata Data Formats
- 115. • XML isa recursive self-describing container <container> <item>Foo</item> <item>Bar</item> <container> <item attr=”SomethingAboutBaz”>Baz</item> </container> </item> <container> Questions? tweet @zipfianacademy #pydata Data Formats
- 116. <widget>! <debug>on</debug>! <window title="Sample KonfabulatorWidget">! <name>main_window</name>! <width>500</width>! <height>500</height>! </window>! <image src="Images/Sun.png" name="sun1">! <hOffset>250</hOffset>! <vOffset>250</vOffset>! <alignment>center</alignment>! </image>! <text data="Click Here" size="36" style="bold">! <name>text1</name>! <hOffset>250</hOffset>! <vOffset>100</vOffset>! <alignment>center</alignment>! <onMouseUp>! sun1.opacity = (sun1.opacity / 100) * 90;! </onMouseUp>! </text>! </widget>! Questions? tweet @zipfianacademy #pydata Data Formats
- 117. • Ad hocdata formats • Fixed-width (Census data) • Graph Edgelists • Voting records • etc. Questions? tweet @zipfianacademy #pydata Data Formats
- 118. • 7-5-5 format •Samfoo bar! •Roger baz 6! •Jane 314 99 Questions? tweet @zipfianacademy #pydata Data Formats
- 119. • Directed GraphFormat 1 2! 1 3! 1 4! 2 3! 4 4 Questions? tweet @zipfianacademy #pydata Data Formats
- 120. • Directed GraphFormat 1 2! 1 3! 1 4! 2 3! 4 4 Questions? tweet @zipfianacademy #pydata Data Formats
- 121. Programming languages like Python,Ruby, and R have built in parsers for data formats such as JSON and CSV. For other esoteric formats you will probably have to write your own Questions? tweet @zipfianacademy #pydata Data Formats