Python: Picking random events for a game: developing the algrothm

This video walks though developing code to pick random events where the
events have different probabilities.



This could be used in games,
simulations, or picking random visual elements.



It will show two functional programming approaches and one Object
Oriented approach

Web Scrape YouTube channel for video info and a table of contents using Python 3.6 and Beautiful Soup (V1.1)

THIS TECHNIQUE NO LONGER WORKS!

I will post an update in the future with code that using the Goggle API.

I have a YouTube channel for my videos to teach programming and needed to create a nice table of contents for a web page.

YouTube always list the videos and playlists with a lot of graphics and thumbnail images which look nice, but it does not fit much on a page for reference.

I tried to use write a program using their API, but decided it was better to try 'screen scraping' or 'web scraping'. This is a technique that is used to get information out of web pages.

I ended up writing a python program. Since many sites do not like scrapping, I checked the User Terms Contract for YouTube, and it is OK as long as it is as slow as a human.

I walk through version 1.0 of the code at my YouTube video @ https://youtu.be/NvxAhUkVgHU

Also please subscribe to my YouTube channel

I wanted a program that would output a CSV file to import to a spreadsheet or DB and a HTML file providing a table of contents to all the videos on the channel that I could copy to my blog.

I decided to pull the following data for the CSV file:

• channel name
• section name 
• playlist name 
• video title
• video link
• video time length
• number of views
• publication date
• number of likes
• number of dislikes
• video description

I have embedded a gist here that you can see and get the code from:

	from bs4 import BeautifulSoup
	import re
	import requests
	import csv
	import time
	import json
	# version 1.1 added handling of youtube.com/channel/
	# to already handling of youtube.com/user based channels
	# NOTE: url gets to youtube are throttled to 3 seconds between requests
	# this is an ad hoc attempt to look like a human to youtube
	# so youtube does not start limiting access
	wait_between_requests = 3
	""" scrape youtube channel to build table of contents html file and
	csv of video information for excel file
	note this code has a slow down delay to meet youtube terms of use
	"""
	# set youtube channel name here
	channel_name = 'gjenkinslbcc'
	youtube_base = 'https://www.youtube.com/'
	# others to try
	# gotreehouse howgrowvideo gjenkinslbcc howgrowvideo
	# by channel name:
	# UCu8YylsPiu9XfaQC74Hr_Gw UCn34N9fj3x92kOGnQQdHkAQ
	# UCH4aPBlmmW1Vgs0ykktCMUg
	parent_folder = '' # users or channel or empty
	def get_soup(url):
	"""open url and return BeautifulSoup object,
	or None if site does not exist"""
	result = requests.get(url)
	if result.status_code != 200: return None
	time.sleep(wait_between_requests) # slow down to human speed
	return BeautifulSoup(result.text, 'html.parser')
	def channel_section_links(channel_name):
	"""list of
	{ 'title': <section title>,
	'link': <url to section play lists>
	}"""
	global parent_folder
	soup = get_soup(f'{youtube_base}/user/{channel_name}/playlists')
	if soup is None or 'This channel does not exist.' in soup.text:
	url = f'{youtube_base}/channel/{channel_name}/playlists'
	soup = get_soup(url)
	if soup is None or 'This channel does not exist.' in soup.text:
	raise ValueError(
	'The channel does not exists: ' + channel_name)
	parent_folder = 'channel/'
	play_list_atags = \
	soup.find_all('a',
	{'href': re.compile(f'{channel_name}/playlists')})
	# filter out non user play lists next
	elements = [{'title': x.text.strip(),
	'link': fix_url(x['href'])} for x in play_list_atags
	if x.span and
	('shelf_id=0' not in x['href'])]
	# no sections, make up no sections section with default link
	if len(elements) == 0:
	url = f'{youtube_base}{parent_folder}{channel_name}/playlists'
	elements = [ {'title': 'no sections', 'link': url}]
	# i.e. https://youtube.com/gotreehouse/playlists
	return elements
	def fix_url(url): # correct relative urls back to absolute urls
	if url[0] == '/':
	return youtube_base + url
	else:
	return url
	def get_playlists(section):
	"""returns list of list of
	{ 'title': <playlist tile>, <link to all playlist videos> }"""
	global parent_folder
	print(f" getting playlists for section: {section['title']}")
	soup = get_soup(section['link'])
	if soup is None: # no playlist, create dummy with default link
	url = f'{youtube_base}{parent_folder}{channel_name}/videos'
	return [
	{'title': 'No Playlists', 'link':url }]
	atags = soup('a', class_='yt-uix-tile-link')
	playlists = []
	for a in atags: # find title and link
	title = a.text
	if title != 'Liked videos': # skip these
	url = fix_url(a['href'])
	playlists.append({'title': title, 'link': url})
	if not playlists: # no playlists
	url = f'{youtube_base}/{parent_folder}{channel_name}/videos'
	return [{'title': 'No Playlists', 'link': url}]
	return playlists
	def parse_video(vurl):
	# return dict of
	# title, link, views, publication_date,
	# description, short_link, likes, dislikes
	d = {'link': vurl, 'views': None, 'short_link': vurl,
	'likes': None, 'dislikes': None}
	# now get video page and pull information from it
	vsoup = get_soup(vurl)
	o = vsoup.find('title')
	vtitle = o.text.strip()
	xending = ' - YouTube'
	d['title'] = vtitle[:-len(xending)] \
	if vtitle.endswith(xending) else vtitle
	print(f" processing video '{d['title']}'" )
	# o is used in the code following to
	# catch missing data targets for scrapping
	o = vsoup.find('div', class_='watch-view-count')
	if o:
	views = o.text
	d['views'] = ''.join(c for c in views if c in '0123456789')
	o = vsoup.find('strong', class_='watch-time-text')
	d['publication_date'] = \
	o.text[len('Published on ') - 1:] if o else ''
	o = vsoup.find('div', id='watch-description-text')
	d['description'] = o.text if o else ''
	o = vsoup.find('meta', itemprop='videoId')
	if o:
	vid = o['content']
	d['short_link'] = f'https://youtu.be/{vid}'
	o = vsoup.find('button',
	class_='like-button-renderer-like-button')
	if o:
	o = o.find('span', class_='yt-uix-button-content')
	d['likes'] = o.text if o else ''
	o = vsoup.find('button',
	class_='like-button-renderer-dislike-button')
	if o:
	o = o.find('span', class_='yt-uix-button-content')
	d['dislikes'] = o.text if o else ''
	return d
	def add_videos(playlist):
	"""find videos in playlist[link]
	and add their info as playlist[videos] as list"""
	surl = playlist['link']
	soup = get_soup(surl)
	print(f" getting videos for playlist: {playlist['title']}")
	videos = []
	# items are list of video a links from list
	items = soup('a', class_='yt-uix-tile-link')
	# note first part of look get info from playlist page item,
	# and the the last part opens the video and gets more details
	if len(items) > 0:
	for i in items:
	d = dict()
	vurl = fix_url(i['href'])
	t = i.find_next('span', {'aria-label': True})
	d['time'] = t.text if t else 'NA'
	d.update(parse_video(vurl))
	videos.append(d)
	else: # must be only one video
	d = {'time': 'NA'}
	d.update(parse_video(surl))
	videos.append(d)
	# add new key to this playlist of list of video infos
	playlist['videos'] = videos
	print()
	def tag(t,c):
	return f'<{t}>{c}</{t}>' # return html tag with content
	def link(text, url): # return a tag with content and link
	return f'<a href="{url}">{text}</a>'
	def html_out(channel, sections):
	"""create and write channel_name.html file"""
	title = f'YouTube Channel {channel}'
	f = open(f'{channel}.html','w')
	template = ('<!doctype html>\n<html lang="en">\n<head>\n'
	'<meta charset="utf-8">'
	'<title>{}</title>\n</head>\n'
	'<body>\n{}\n</body>\n</html>')
	parts = list()
	parts.append(tag('h1', title))
	for s in sections:
	parts.append(tag('h2',link(s['title'], s['link'])))
	for pl in s['playlists']:
	parts.append(tag('h3', link(pl['title'], pl['link'])))
	if len(pl) == 0:
	parts.append('<p>Empty Playlist</p>')
	else:
	parts.append('<ol>')
	for v in pl['videos']:
	t = '' if v['time'] == 'NA' else f" ({v['time']})"
	parts.append(tag('li', link(v['title'],
	v['short_link']) + t))
	parts.append('</ol>')
	f.write(template.format(channel, '\n'.join(parts)))
	f.close()
	def csv_out(channel, sections):
	""" create and output channel_name.csv
	file for import into a spreadsheet or DB"""
	headers = ('channel,section,playlist,video,'
	'link,time,views,publication date,'
	'likes,dislikes,description').split(',')
	with open(f'{channel}.csv', 'w') as csv_file:
	csvf = csv.writer(csv_file, delimiter=',')
	csvf.writerow(headers)
	for section in sections:
	for playlist in section['playlists']:
	for video in playlist['videos']:
	v = video
	line = [channel,
	section['title'],
	playlist['title'],
	v['title']]
	line.extend([v['short_link'],
	v['time'], v['views'],
	v['publication_date'],
	v['likes'], v['dislikes'],
	v['description']])
	csvf.writerow(line)
	def process_channel(channel_name):
	sections = channel_section_links(channel_name)
	for section in sections:
	section['playlists'] = get_playlists(section)
	for playlist in section['playlists']:
	add_videos(playlist)
	return sections
	if __name__ == '__main__':
	# find channel name by going to channel
	# and picking last element from channel url
	# for example my channel url is:
	# https://www.youtube.com/user/gjenkinslbcc
	# my channel name is gjenkinslbcc in this url
	# this is set near top of this file
	# if the channel is of the form:
	# https://www.youtube.com/channel/xyz then supply xyz
	print(f'finding sections for youtube.com {channel_name}')
	sections = process_channel(channel_name)
	# save sections structure to json file
	with open(f'{channel_name}.json','w') as f:
	f.write(json.dumps(sections, sort_keys=True, indent=4))
	html_out(channel_name, sections) # create web page of channel links
	# create a csv file of video info for import into spreadsheet
	csv_out(channel_name, sections)
	print(f"Program Complete,\n '{channel_name}.html' and"
	f" '{channel_name}.csv' have been"
	f" written to current directory")
	# 3456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
	# 1 2 3 4 5 6 7 8 9

view raw youtube_channel_scrape.py hosted with ❤ by GitHub

You need python 3.6 or greater and just need to change the channel name near the top of the program.
Then run it to create the CSV and HTML file in the current run directory.

If you only need the HTML, json, or CSV file, see the if __name__ == '__main__' section and write an driver py program with imports like run_json.py example.