#!/usr/bin/python
import json

path = '{}Dropbox/bmi/reading_circle/pydata-book-master/ch02/usagov_bitly_data2012-03-16-1331923249.txt'.format('/Users/so')

records = [json.loads(line) for line in open(path)]

##p16
#print records[0]
#print records[1]

##p17
#print records[0]['tz']

time_zones = [rec['tz'] for rec in records if 'tz' in rec]

##p18

#print time_zones[:10]


def get_counts(sequence):
  counts = {}
  for x in sequence:
      if x in counts:
          counts[x] += 1
      else:
          counts[x] = 1
  return counts

counts = get_counts(time_zones)
#print counts

from collections import defaultdict

def get_counts2(sequence):
  counts = defaultdict(int)
  for x in sequence:
      counts[x] += 1
  return counts

##p19
counts2 = get_counts2(time_zones)
#print counts2
#print counts2['America/New_York']
#print len(time_zones)

def top_counts(count_dict, n=10):
  value_key_pairs = [(count, tz) for tz, count in count_dict.items()]
  value_key_pairs.sort()
  return value_key_pairs[-n:]

out_top_counts = top_counts(counts)
#print out_top_counts

from collections import Counter

counts = Counter(time_zones)
#print counts.most_common(10)

##p20
from pandas import DataFrame, Series
import pandas as pd; import numpy as np

frame = DataFrame(records)
#print frame

##p21
tz_counts = frame['tz'].value_counts()
#print tz_counts[:10]

##p22
clean_tz = frame['tz'].fillna('Missing')
clean_tz[clean_tz == ''] = 'Unknown'
tz_counts = clean_tz.value_counts()

#print tz_counts[:10]

import matplotlib.pyplot as plt

tz_counts[:10].plot(kind='barh', rot=0)
#plt.show()

#print frame['a'][1]
#print frame['a'][50]
#print frame['a'][51]

##p23
results = Series([x.split()[0] for x in frame.a.dropna()])
#print results[:5]
#print results.value_counts()[:8]

cframe = frame[frame.a.notnull()]

##p24
operating_system = np.where(cframe['a'].str.contains('Windows'), 'Windows', 'Not Windows')
#Function: where(condition, x, y)

#print operating_system[:5]

by_tz_os = cframe.groupby(['tz', operating_system])
agg_counts = by_tz_os.size().unstack().fillna(0)

#print agg_counts[:10]

##p25
indexer = agg_counts.sum(1).argsort()

#print indexer[:10]

##p26
count_subset = agg_counts.take(indexer)[-10:]

#print count_subset

count_subset.plot(kind='barh', stacked=True)

#plt.show()

normed_subset = count_subset.div(count_subset.sum(1), axis=0)
normed_subset.plot(kind='barh', stacked=True)

#plt.show()

#!/usr/bin/python
import pandas as pd

##p28
inpath = "{}Dropbox/bmi/reading_circle/pydata-book-master/ch02/movielens/".format('/Users/so/')

unames = ['user_id', 'gender', 'age', 'occupation', 'zip']
users = pd.read_table(inpath+"users.dat", sep='::', header=None, names=unames)

rnames = ['user_id', 'movie_id', 'rating', 'timestamp']
ratings = pd.read_table(inpath+'ratings.dat', sep='::', header=None, names=rnames)

mnames = ['movie_id', 'title', 'genres']
movies = pd.read_table(inpath+'movies.dat', sep='::', header=None, names=mnames)

#print users[:5]

#print ratings[:5]

##p29
#print movies[:5]

#print ratings

data = pd.merge(pd.merge(ratings, users), movies)
#print data

##p30
mean_ratings = data.pivot_table('rating', rows='title', cols='gender', aggfunc='mean')
#print mean_ratings[:5]

##p31
ratings_by_title = data.groupby('title').size()
#print ratings_by_title[:10]

active_titles = ratings_by_title.index[ratings_by_title >= 250]
#print active_titles

mean_ratings = mean_ratings.ix[active_titles]
#print mean_ratings

##p32
top_female_ratings = mean_ratings.sort_index(by='F', ascending=False)
#print top_female_ratings[:10]

mean_ratings['diff'] = mean_ratings['M'] - mean_ratings['F']
sorted_by_diff = mean_ratings.sort_index(by='diff')
#print sorted_by_diff[:15]

#print sorted_by_diff[::-1][:15]

rating_std_by_title = data.groupby('title')['rating'].std()
rating_std_by_title = rating_std_by_title.ix[active_titles]
print rating_std_by_title.order(ascending=False)[:10]

#!/usr/bin/python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

##p34
inpath = "/Users/so/Dropbox/bmi/reading_circle/pydata-book-master/ch02/"

#? unclear
#names = inpath + "ch02/names/"
#print names.head(10)

##p35
names1880 = pd.read_csv(inpath+'names/yob1880.txt', names=['name', 'sex', 'births'])

#print names1880

##p36

#print names1880.groupby('sex').births.sum()

years = range(1880, 2011)

pieces = []
columns = ['name', 'sex', 'births']

for year in years:
  path = inpath+'names/yob%d.txt' % year
  frame = pd.read_csv(path, names=columns)
  
  frame['year'] = year
  pieces.append(frame)
names = pd.concat(pieces, ignore_index=True)

#p37

#print names

total_births = names.pivot_table('births', rows='year', cols='sex', aggfunc=sum)

#print total_births.tail()

def add_prop(group):
  births = group.births.astype(float)

  group['prop'] = births / births.sum()
  
  ##p38
  return group
names = names.groupby(['year', 'sex']).apply(add_prop)

#print names

##p39
#print np.allclose(names.groupby(['year', 'sex']).prop.sum(), 1)

def get_top1000(group):
  return group.sort_index(by='births', ascending=False)[:1000]
grouped = names.groupby(['year', 'sex'])
top1000 = grouped.apply(get_top1000)

#print top1000

pieces = []
for year, group in names.groupby(['year', 'sex']):
  pieces.append(group.sort_index(by='births', ascending=False)[:1000])
top1000 = pd.concat(pieces, ignore_index=True)

#print top1000

boys = top1000[top1000.sex == 'M']
girls = top1000[top1000.sex == 'F']

##p40
total_births = top1000.pivot_table('births', rows='year', cols='name', aggfunc=sum)

#print total_births

subset = total_births[['John', 'Harry', 'Mary', 'Marilyn']]

subset.plot(subplots=True, figsize=(12, 10), grid=False, title="Number of births per year")
#plt.show()

##p41
table = top1000.pivot_table('prop', rows='year', cols='sex', aggfunc=sum)
table.plot(title='Sum of table1000.prop by year and sex', yticks=np.linspace(0, 1.2, 13), xticks=range(1880, 2020, 10))
#plt.show()

df = boys[boys.year == 2010]
#print df

#p42
prop_cumsum = df.sort_index(by='prop', ascending=False).prop.cumsum()

#print prop_cumsum[:10]

#ERROR, object type is different
#print np.searchsorted(prop_cumsum, 0.5)
#print prop_cumsum.searchsorted(0.5)

df = boys[boys.year == 1900]
in1900 = df.sort_index(by='prop', ascending=False).prop.cumsum()
#again ERROR!
#print in1900.searchsorted(0.5) + 1

#p43
#def get_quantile_count(group, q=0.5):
#    group = group.sort_index(by='prop', ascending=False)
#    return group.prop.cumsum().searchsorted(q) + 1

#diversity = top1000.groupby(['year', 'sex']).apply(get_quantile_count)
#diversity = diversity.unstack('sex')

#diversity.head()

##p44
get_last_letter = lambda x: x[-1]
last_letters = names.name.map(get_last_letter)
last_letters.name = 'last_letter'

table = names.pivot_table('births', rows=last_letters, cols=['sex', 'year'], aggfunc=sum)

#print last_letters.head(5)

#print last_letters.tail(5)

##p45
subtable = table.reindex(columns=[1910, 1960, 2010], level='year')

#print subtable.head()

#print subtable.sum()

letter_prop = subtable / subtable.sum().astype(float)

fig, axes = plt.subplots(2, 1, figsize=(10, 8))
letter_prop['M'].plot(kind='bar', rot=0, ax=axes[0], title='Male')
letter_prop['F'].plot(kind='bar', rot=0, ax=axes[1], title='Female', legend=False)

#plt.show()

letter_prop = table / table.sum().astype(float)
dny_ts = letter_prop.ix[['d', 'n', 'y'], 'M'].T

#print dny_ts.head()

dny_ts.plot(style={'d':'-.', 'n':'-', 'y':':'})
#plt.show()

##p47
all_names = top1000.name.unique()
mask = np.array(['lesl' in x.lower() for x in all_names])

lesley_like = all_names[mask]

#print lesley_like

filtered = top1000[top1000.name.isin(lesley_like)]
#print filtered.groupby('name').births.sum()

##p48
#print filtered.head(10)
#print filtered.tail(10)

##p49
table = filtered.pivot_table('births', rows='year', cols='sex', aggfunc='sum')
#print table.head(5)
#print table.tail(5)

table = table.div(table.sum(1), axis=0)

#print table.tail()

table.plot(style={'M': 'k-', 'F': 'k--'})
#plt.show()