Speculative fiction mining

import pandas as pd
import numpy as np

def genres_split(s):
    return frozenset(s.split(','))

def year_converter(s):
    try:
        years = int(s)
        if 1900 < years < 2017:
            return years
    except ValueError:
        return np.nan
    
filename='/home/nik/workspace/fiction-miner/fantlab_annot.csv'
df = pd.read_csv(filename, converters={"genres": genres_split, "year": year_converter}, index_col='id')
df.head(3)

main_genres_dict = {
    'Фантастика': 'Science fiction',
    'Фэнтези': 'Fantasy',
    'Магический реализм': 'Magic realism',
    'Мистика': 'Mystic fiction',
    'Хоррор/Ужасы': 'Horror fiction',
    'Сказка/Притча': 'Fairytale / parable',
    'Детектив': 'Detective fiction',
    'Боевик': 'Action fiction',
    'Триллер': 'Thriller',
    'Любовный роман': 'Romance novel',
    'Историческая проза': 'Historical fiction',
    'Сюрреализм': 'Surrealism',
    'Постмодернизм': 'Postmodernism',
    'Реализм': 'Realism',
    'Психоделика': 'Psyhodelic',
    }
scifi_subgenres = {
    '«Твёрдая» НФ': 'Hard SF',
    'Гуманитарная («мягкая») НФ': 'Soft SF',
    'Космоопера': 'Spaceopera',
    'Киберпанк': 'Cyberpunk',
    'Планетарная фантастика': 'Planet SF',
    'Таймпанк': 'Timepunk',
    'Хроноопера': 'Chronoopera',
    'Постапокалиптика': 'Postap',
    'Роман-катастрофа': 'Disaster novel',
    'Утопия': 'Utopia',
    'Антиутопия': 'Antiutopia',
    }
fantasy_subgenres = {
    'Эпическое фэнтези': 'Epic Fantasy',
    'Героическое фэнтези': 'Heroic Fantasy',
    'Городское фэнтези': 'Urban Fantasy',
    'Dark Fantasy': 'Dark Fantasy',
    'Технофэнтези': 'Technofantasy',
    'Science Fantasy': 'Science Fantasy',
    '«Дотолкиновское» фэнтези': 'Pretolkien Fantasy',
    '«Классическое» фэнтези': 'Classic Fantasy',
    'Артуриана': 'Arhturiana',
    'Анималистическое': 'Animalistic',
    'Мифологическое': 'Mythological Fantasy',
    }
# genres_dict = {**main_genres_dict, **scifi_subgenres, fantasy_subgenres}
# genres_set = frozenset.union(*df.genres)
def encode_genres(df, genres_dict):
    genres_set = genres_dict.keys()
    for genre in genres_set:
        genre_lab = genres_dict[genre]
        #taken from http://datascience.stackexchange.com/a/11799
        df[genre_lab] = df.apply(lambda x: int(genre in x['genres']), axis='columns')

        df[genre_lab+'_rat'] = df.apply(lambda x: x['rating'] if x[genre_lab]==1 else np.nan, axis=1)
        df[genre_lab+'_rat_count'] = df.apply(lambda x: x['rating_count'] if x[genre_lab]==1 else np.nan, axis=1)
        print(genre_lab + ": \t\t" + str(len(df[df[genre_lab] == 1])))
print('  Main genres')
encode_genres(df, main_genres_dict)
print('  SciFi subgenres')
encode_genres(df, scifi_subgenres)
print('  Fantasy subgenres')
encode_genres(df, fantasy_subgenres)
df.head(3)

import re
def containsEn(s):
    return bool(re.search('[a-zA-Z]', s))
print(containsEn('Майкл (Michael)'))
print(containsEn('Петр (псевдоним Ивана)'))

df_en = df[df.apply(lambda x: containsEn(x['author_name']), axis=1)]
df_rus = df[df.apply(lambda x: not containsEn(x['author_name']), axis=1)]
print(df.shape)
print(df_en.shape)

def read_in_set(filename, prefix='/home/nik/workspace/fiction-miner/names/'):
    with open(prefix + filename) as f:
        content = f.readlines()
    res = set([x.strip() for x in content if len(x.strip()) > 1])
    return res
# taken from GATE 8 gazeteers and http://www.sonnik-online.net/imena/_mujskie.html
names_female_ru = read_in_set('first_names_female.lst')
names_male_ru = read_in_set('first_names_male.lst')
names_female_en = read_in_set('person_female_ext.lst')
names_male_en = read_in_set('person_male_ext.lst')
names_female = set.union(names_female_ru, names_female_en)
names_male = set.union(names_male_ru, names_male_en)
# since my list contains forms of male names like Alexandra, lets filter them out
names_male = names_male.difference(names_female)

import string
def check_gender(s, names_set):
    translator = str.maketrans('', '', string.punctuation)
    words_in_s = set(s.translate(translator).split())
    return bool(words_in_s.intersection(names_set))

def isMale(s):
    return check_gender(s, names_male)

def isFemale(s):
    return check_gender(s, names_female)

print(isMale('Вера Петрова'))
print(isMale('Владимиp Ильин')) #incorrect layout
print(isMale('Владимир Ильин'))
print(isMale('Александра Олайва (Alexandra Oliva)')) #because of Oliva
print(isFemale('Алексей В. Андреев'))
print(isFemale('Вера Петрова'))
print(isMale('Алан Глинн (Alan Glynn)'))
print(isMale('Дэвид С. Гарнетт (David S. Garnett)'))
print(isFemale('John Doe'))

df_female = df[df.apply(lambda x: isFemale(x['author_name']), axis=1)]
print('females: ' + str(df_female.shape))
df_male = df[df.apply(lambda x: isMale(x['author_name']), axis=1)]
print('males: ' + str(df_male.shape))

df_unknown = df[df.apply(lambda x: not isMale(x['author_name']) and not isFemale(x['author_name']), axis=1)]
print('undefined: ' + str(df_unknown.shape))
print(np.unique(df_unknown['author_name'])[:5])
df_both = df[df.apply(lambda x: isMale(x['author_name']) and isFemale(x['author_name']), axis=1)]
print('both: ' + str(df_both.shape))
print(np.unique(df_both['author_name'])[:10])

import matplotlib.pyplot as plt
%matplotlib notebook
# %matplotlib inline
from bokeh.palettes import colorblind, d3, brewer

def draw_plot(df2plot, genres_to_show, title, roll_window=2):
    fig, ax = plt.subplots(figsize=(11, 6))
    stackplot(df2plot, genres_to_show, ax, title, roll_window)

def stackplot(df2plot, genres_to_show, ax, title, roll_window):
    ax.set_title(title)
    palette = d3['Category10'][len(genres_to_show)]
    x = np.unique(df2plot['year'].dropna().values)
    y_all = []
    for i, genre in enumerate(genres_to_show):
        y = df2plot.groupby('year')[genre].sum()
        y_avg = pd.Series.rolling(y, center=False,window=roll_window).mean()
        y_all.append(y_avg)
    polys = ax.stackplot(x, y_all, labels=genres_to_show, linewidth=0, colors=palette, baseline='wiggle')
    ax.set_xticks(np.arange(1900,2020,10))
    ax.grid()
    ax.set_axis_bgcolor((0.8,0.8,0.8))
    ax.legend(loc='lower left', prop={'size':13})
    return ax

top_genres = [  #i.e those having at least 1000 items
    'Science fiction',
    'Fantasy',
    'Mystic fiction',
    'Horror fiction',
    'Fairytale / parable', 
    'Detective fiction', 
    'Realism', 
]
draw_plot(df, top_genres, 'Genres development over years', 2)

sf_subgenres = [
    'Hard SF',
    'Soft SF',
    'Spaceopera',
    'Cyberpunk',
    'Planet SF',
    'Timepunk',
    'Chronoopera',
    'Postap',
#     'Disaster novel',
    'Utopia',
    'Antiutopia',
]
draw_plot(df, sf_subgenres, 'Science fiction subgenres development over years')

fantasy_subgenres = [
    'Epic Fantasy',
    'Heroic Fantasy',
    'Urban Fantasy',
    'Dark Fantasy',
    'Technofantasy',
    'Science Fantasy',
#     'Pretolkien Fantasy',
    'Classic Fantasy',
#     'Arhturiana',
#     'Animalistic',
    'Mythological Fantasy',
]
draw_plot(df, fantasy_subgenres, 'Fantasy subgenres development over years')

#draw_plot(df_rus, top_genres, 'Genres development over years (Russian authors)', 2)

def compare2plots_row(df2plot1, df2plot2, genres_to_show, title1, title2, roll_window=2):
    fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(11, 10))
    stackplot(df2plot1, genres_to_show, axes[0], title1, roll_window)
    stackplot(df2plot2, genres_to_show, axes[1], title2, roll_window)

compare2plots_row(df_rus, df_en, top_genres, 'Genres (Russian authors)', 'Genres (Foreign authors)')

compare2plots_row(df_rus, df_en, sf_subgenres, 'SciFi subgenres (Russian authors)', 'SciFi subgenres (Foreign authors)') 
compare2plots_row(df_rus, df_en, fantasy_subgenres, 'Fantasy subgenres (Russian authors)', 'Fantasy subgenres (Foreign authors)') 

compare2plots_row(df_female, df_male, top_genres, 'Genres (Female authors)', 'Genres (Male authors)')

compare2plots_row(df_female, df_male, sf_subgenres, 'SciFi subgenres (Female authors)', 'SciFi subgenres (Male authors)') 
compare2plots_row(df_female, df_male, fantasy_subgenres, 'Fantasy subgenres (Female authors)', 'Fantasy subgenres (Male authors)') 

def lineplot(df2plot, genres_to_show, title, roll_window=3, col_suffix='_rat'):
    fig, ax = plt.subplots(figsize=(12, 6))
    ax.set_title(title)
    x = np.unique(df2plot['year'].dropna().values)
    for genre in genres_to_show:
        y = df2plot.groupby('year')[genre+col_suffix].mean()
        y_avg = pd.Series.rolling(y, center=False,window=roll_window).mean()
        ax.plot(x, y_avg, label=genre)
    # plt.plot(x, y, label='overall')
    ax.set_xticks(np.arange(1900,2020,10))
    ax.grid()
    ax.set_axis_bgcolor((0.8,0.8,0.8))
    ax.legend(loc='lower left',prop={'size':13})

lineplot(df, top_genres, 'Rating by genres', 5)

lineplot(df, sf_subgenres, 'Rating by SciFi subgenres', 5)
lineplot(df, fantasy_subgenres, 'Rating by Fantasy subgenres', 5)

lineplot(df, top_genres, 'Popularity (rating counts) by genres (without smoothing)', 1, '_rat_count')

def add_line(df2plot, ax, label,roll_window):
    x = np.unique(df2plot['year'].dropna().values)
    y = df2plot.groupby('year')['rating'].mean()
    y_avg = pd.Series.rolling(y, center=False,window=roll_window).mean()
    ax.plot(x, y_avg, label=label)

def lineplot_compare(df2plot1, df2plot2, label1, label2, title, roll_window):
    fig, ax = plt.subplots()
    ax.set_title(title)
    add_line(df2plot1, ax, label1,roll_window)
    add_line(df2plot2, ax, label2,roll_window)
    ax.set_xticks(np.arange(1900,2020,10))
    ax.grid()
    ax.set_axis_bgcolor((0.8,0.8,0.8))
    ax.legend(loc='upper left',prop={'size':13})

lineplot_compare(df_rus, df_en, 'Russian', 'Foreign', 'Compare rating of Russian and Foreign authors', 3)

lineplot_compare(df_female, df_male, 'Female', 'Male', 'Compare rating of Female and Male authors', 3)