In [1]:
# Shortly before the start of 2019 I decided to start taking my health a bit more seriously
# and educate myself on the subject.
# Luckily I found a good podcast just for that. I decided to leverage my python skills to write a little tool
# to download the episodes.
In [2]:
import requests # library for html requests
from bs4 import BeautifulSoup # parsing rss feed
import os # handling folders and files
In [3]:
# get the feed file, prepare for parsing
url = 'http://themodelhealthshow.libsyn.com/rss'
rss = requests.get(url)
rss_soup = BeautifulSoup(rss.text, "lxml")
In [4]:
# Let's inspect the feed text to see what we're looking for:
# break it into lines and indent for easier search
text = rss.text
text_lines = text.split('\n')
pretty_print = rss_soup.prettify().split('\n')
In [5]:
# First 150 lines:
first_lines = pretty_print[:150]
for line in first_lines :
print(line)
In [6]:
# Last 150 lines
last_lines = pretty_print[-150:]
for line in last_lines :
print(line)
In [7]:
# After a bit of eyeballing we notice:
# - the meat of the feed, the episode information, is contained in the <item> blocks
# - episodes titles are kept in the <title> tags,
# - and the mp3 links are in the url attributes of enclosure tags.
# That's all we need.
In [8]:
list_of_titles = []
list_of_urls = []
items = rss_soup.find_all(name = 'item')
for item in items :
list_of_titles.append(item.title.text)
list_of_titles.reverse()
enclosures = rss_soup.find_all(name='enclosure')
for item in enclosures :
list_of_urls.append(item.get('url'))
list_of_urls.reverse()
In [9]:
# Inspecting the haul - from earlier cursory look at the feed we know
# the title and links to the first and last episodes:
print(str(list_of_titles[0]) + ' ' +str(list_of_urls[0]))
print(str(list_of_titles[-1])+ ' ' +str(list_of_urls[-1]))
In [10]:
# Now we make a folder to save the podcast to:
dir = "D:/Podcasts/TMHS/"
if not os.path.exists(dir) :
os.makedirs(dir)
In [11]:
# Before attempting to download anything, let's see if there are
# any symbols in the filenames that OS wouldn't play well with:
for title in list_of_titles[:5] :
print(title)
print()
for title in list_of_titles[-5:] :
print(title)
In [12]:
proper_titles = []
for title in list_of_titles :
if (title.find(' ')) : title = title.replace(' ', ' ')
if (title.find('"')) : title = title.replace('"', '\'')
if (title.find(':')) : title = title.replace(':', ' -')
if (title.find('\?')) : title = title.replace('\?', '')
if (title.find('?')) : title = title.replace('?', '')
if (title.find('&')) : title = title.replace('&', 'and')
if (title.find('\&')) : title = title.replace('\&', 'and')
proper_titles.append(title)
In [13]:
# Let's see how it looks now:
for title in proper_titles[:5] :
print(title)
print()
for title in proper_titles[-5:] :
print(title)
In [14]:
# We also noticed that the extension(which ain't always .mp3 in the wild)
# is trapped between a comma and a question mark. Let's capture it.
list_of_ext = []
for link in list_of_urls :
list_of_ext.append(link.split('.')[-1].split('?')[0])
In [15]:
# Now, to make a function:
def download_episode(filename, url) :
r = requests.get(url)
with open(filename, 'wb') as fout :
fout.write(r.content)
In [16]:
# Let's check all the components needed to grab the first episode...
title = proper_titles[0]
ext = list_of_ext[0]
url = list_of_urls[0]
filename = dir+title+'.'+ext
print(filename)
print(url)
In [17]:
# and test it:
download_episode(filename, url)
In [18]:
# And now, to get a batch that would last me some time, and
# we'll speed the download up a little bit with processes:
from multiprocessing.pool import ThreadPool
In [38]:
ep_count = 30
filenames = []
for i in range(ep_count) :
filenames.append(dir+proper_titles[i]+'.'+list_of_ext[i])
urls = []
for i in range(ep_count) :
urls.append(list_of_urls[i])
arguments = [(filenames[i], urls[i]) for i in range(ep_count)]
ThreadPool(8).starmap(download_episode, arguments)
Out[38]:
In [39]:
# And as a final touch, let's introduce a function that won't download episodes
# we already had, so that we could just launch it every week to grab only the fresh ones.
def download_episode_smart(filename, url) :
if(os.path.exists(filename)) :
print("Already got " + filename + "\n")
else :
print("Downloading " + filename + "\n")
r = requests.get(url)
with open(filename, 'wb') as fout :
fout.write(r.content)