# Continuing my new-found obsession with Magic, I found a collection of cards I was interested in playing with: Slivers.
# I'm gonna see how much it would set me back to get my hands on every card of that type.

import requests             # handling http requests
from scrapy import Selector # getting to the important bits of html pages

url = 'http://gatherer.wizards.com/Pages/Search/Default.aspx?name=+[sliver]'
r = requests.get(url)
sel = Selector(text=r.text)
titles = sel.xpath('//span[@class="cardTitle"]/a/text()').extract()

r = requests.get(url)
sel = Selector(text=r.text)

titles = sel.xpath('//span[@class="cardTitle"]/a/text()').extract()

# How many cards are there?
len(titles)

98

# Whoa. 98 of them!
# We need somewhere to store prospective prices. A dictionary with lists for values and
#  card titles for keys will do just fine.
dic = {}

# Now we'll go to a price database, mtggoldfish.com, and see about extracting prices.

# BUT, before that, let's save the images. Just in case :)

# Here's the image links.
images = sel.xpath('//table/tr/td/a/img/@src').extract()
print(images[0])

../../Handlers/Image.ashx?multiverseid=397564&type=card

images[0][5:]

'/Handlers/Image.ashx?multiverseid=397564&type=card'

# To get proper urls we could download images from, we'll need to append website's core address:
url_main = 'http://gatherer.wizards.com'
image_urls = []
for image in images:
    image_urls.append(url_main + image[5:])
print(image_urls[0])

http://gatherer.wizards.com/Handlers/Image.ashx?multiverseid=397564&type=card

# And now, to get them:
import os
dir = "./sliver/"
if not os.path.exists(dir) :
    os.makedirs(dir)
for i in range(len(image_urls)) :
    r = requests.get(image_urls[i])
    open(dir+titles[i]+'.png', 'wb').write(r.content)

# Splendid. Now, back on track. Time to get the prices.

query_prefix = 'https://www.mtggoldfish.com/q?utf8=✓&query_string='
url_prefix = 'https://www.mtggoldfish.com/'

for title in titles :
    dic[title] = []
    query = query_prefix+title
    r = requests.get(query)
    sel = Selector(text=r.text)
    
    # If we're not at search results, we're on the card's page already.
    page_title = sel.xpath('//title/text()').extract_first()
    if(page_title != 'Search Results') :
        price = sel.xpath('//div[@class="price-box paper"]/div[@class="price-box-price"]/text()').extract_first()
        if(isinstance(price, str)) :
            dic[title].append(float(price.replace(',','')))
    else :
        # If we're at the results page, we need to get only the cards we're looking for.
        links = sel.xpath('//tbody//tr/td/a[contains(text(), title)]/@href').extract()
        names = sel.xpath('//tbody//tr/td/a[contains(text(), title)]/text()').extract()
        
        urls = []
        
        for i in range(len(names)) :
            if(names[i] == title) :
                urls.append(links[i])
        
        for u in urls :
            r = requests.get(url_prefix+u+'#online')
            sel = Selector(text=r.text)
            price = sel.xpath('//div[@class="price-box paper"]/div[@class="price-box-price"]/text()').extract_first()
            if(isinstance(price, str)) :
                dic[title].append(float(price.replace(',','')))

len(dic)

98

# Let's see if we got all the cards and print the results:
print(len(dic.keys()))
for k, v in dic.items() :
    print(str(k) + str(v))

98
Acidic Sliver[0.58, 0.34]
Armor Sliver[0.3, 0.25]
Barbed Sliver[0.31, 0.25]
Basal Sliver[0.25]
Battering Sliver[0.22]
Battle Sliver[0.26]
Belligerent Sliver[0.28]
Blade Sliver[0.48]
Blur Sliver[0.24]
Bonescythe Sliver[4.24, 4.4, 17.99]
Bonesplitter Sliver[0.24]
Brood Sliver[6.5, 5.86, 32.99]
Cautery Sliver[0.3]
Clot Sliver[0.57, 0.21]
Constricting Sliver[0.34]
Crypt Sliver[0.41]
Crystalline Sliver[5.8, 56.91, 4.4]
Darkheart Sliver[2.0]
Dementia Sliver[0.25]
Diffusion Sliver[4.0]
Dormant Sliver[1.31]
Essence Sliver[2.37, 5.19, 2.49, 6.37]
Firewake Sliver[0.3]
Frenetic Sliver[0.6]
Frenzy Sliver[0.22, 0.58, 0.39]
Fungus Sliver[1.69, 1.87, 3.93]
Fury Sliver[0.68, 0.35, 1.46]
Galerider Sliver[11.99]
Gemhide Sliver[1.6, 1.15, 19.25]
Ghostflame Sliver[0.41]
Groundshaker Sliver[0.17]
Harmonic Sliver[3.34]
Heart Sliver[1.5, 0.42]
Hibernation Sliver[2.8, 0.79]
Homing Sliver[0.39, 19.0, 1.59]
Horned Sliver[3.8]
Hunter Sliver[0.23]
Leeching Sliver[0.39]
Lymph Sliver[0.22]
Magma Sliver[5.0]
Manaweft Sliver[2.47]
Megantic Sliver[0.73, 1.19, 0.71]
Mesmeric Sliver[0.23]
Metallic Sliver[0.37, 0.25]
Might Sliver[1.16, 0.81, 4.19]
Mindlash Sliver[0.19]
Mindwhip Sliver[0.29]
Mistform Sliver[0.19]
Mnemonic Sliver[0.28]
Muscle Sliver[1.31, 20.15, 1.0]
Necrotic Sliver[5.49, 4.79, 9.22]
Opaline Sliver[0.51]
Plague Sliver[0.54]
Plated Sliver[0.35]
Poultice Sliver[0.2]
Predatory Sliver[0.41]
Psionic Sliver[3.74]
Pulmonic Sliver[5.94]
Quick Sliver[1.19, 0.26, 1.56]
Quilled Sliver[0.29]
Reflex Sliver[0.21]
Root Sliver[1.99]
Screeching Sliver[0.24]
Sedge Sliver[18.81]
Sentinel Sliver[0.49]
Shadow Sliver[0.25]
Shifting Sliver[1.76]
Sidewinder Sliver[1.18]
Sinew Sliver[4.0]
Sliver Construct[0.19]
Sliver Hive[18.0, 38.92]
Sliver Hivelord[39.99]
Sliver Legion[120.0]
Sliver Overlord[41.4, 44.99, 95.9]
Sliver Queen[201.3]
Sliversmith[0.33]
Spectral Sliver[0.29, 0.25, 1.02]
Spined Sliver[0.5, 0.28, 0.3, 1.49]
Spinneret Sliver[1.0]
Spitting Sliver[0.19]
Steelform Sliver[0.25]
Striking Sliver[0.25]
Synapse Sliver[19.99]
Synchronous Sliver[0.2]
Syphon Sliver[3.5]
Talon Sliver[0.33]
Telekinetic Sliver[1.18]
Thorncaster Sliver[1.0]
Toxin Sliver[5.59]
Two-Headed Sliver[0.32]
Vampiric Sliver[0.49]
Venom Sliver[1.54]
Venser's Sliver[0.18]
Victual Sliver[0.54, 0.35]
Virulent Sliver[0.9, 4.83, 1.16]
Ward Sliver[0.96]
Watcher Sliver[0.21]
Winged Sliver[1.91, 0.38]

# Time to calculate the smallest mount of cash Slivers might claim,
#  as you're allowed to have up to 4 specimens of the same card...
sum = 0
for k in dic.keys() :
    sum += min(dic[k])
sum = sum * 4
print('{:6.2f}'.format(sum))

2291.56

# ...you know what, I think I'll stick to cheaper hobbies.