{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "unable to import 'smart_open.gcs', disabling that module\n" ] } ], "source": [ "\n", "import sys\n", "import os\n", "\n", "from sklearn.linear_model import SGDClassifier, LinearRegression, Lasso, Ridge\n", "from sklearn.utils import shuffle\n", "from sklearn.decomposition import PCA\n", "import seaborn as sn\n", "import random\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "from collections import defaultdict\n", "from sklearn.manifold import TSNE\n", "import tqdm\n", "import copy\n", "from sklearn.svm import LinearSVC \n", "\n", "from sklearn.cross_decomposition import PLSRegression\n", "from sklearn.decomposition import TruncatedSVD\n", "import torch\n", "from sklearn.linear_model import SGDClassifier\n", "\n", "from sklearn.svm import LinearSVC\n", "\n", "import sklearn\n", "from sklearn.linear_model import LogisticRegression\n", "import random\n", "import pickle\n", "import matplotlib.pyplot as plt\n", "from sklearn import cluster\n", "from sklearn import neural_network\n", "from gensim.models.keyedvectors import Word2VecKeyedVectors\n", "from gensim.models import KeyedVectors\n", "import numpy as np\n", "import warnings\n", "import argparse\n", "from sklearn.neural_network import MLPClassifier\n", "from collections import defaultdict\n", "import scipy\n", "from scipy import stats\n", "from scipy.stats import pearsonr\n", "import pandas as pd\n", "from collections import Counter" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def load_bios(group):\n", " \n", " with open(\"bios_data/{}.pickle\".format(group), \"rb\") as f:\n", " bios_data = pickle.load(f)\n", " z = np.array([1 if d[\"g\"]==\"f\" else 0 for d in bios_data]) # gender labels\n", " professions = np.array([d[\"p\"] for d in bios_data]) # profession labels\n", " txts = [d[\"hard_text_untokenized\"] for d in bios_data] # biographies without gendered pronouns/names\n", " \n", " return z,txts,professions,bios_data\n", "\n", "def load_bios_representations(group, finetune_mode, seed=0):\n", " \n", " if finetune_mode == \"freezed\": # only 1 random seed for the pretrained bert\n", " X = np.load(\"encodings//{}/{}_cls.npy\".format(finetune_mode,group))\n", " else:\n", " X = np.load(\"encodings/{}/{}_{}_cls.npy\".format(finetune_mode, group, seed))\n", " \n", " # perform PCA - as was done in training\n", " with open(\"pca/pca_{}_{}.pickle\".format(finetune_mode, seed), \"rb\") as f:\n", " pca = pickle.load(f)\n", " X = pca.transform(X)\n", " \n", " # return transformed X\n", " return X\n", "\n", "def load_projections(proj_type, finetune_mode, seed=0):\n", " with open(\"interim/{}/{}/run={}/Ps_{}.pickle\".format(finetune_mode,proj_type,seed,proj_type), \"rb\") as f:\n", " rank2P = pickle.load(f)\n", " return rank2P" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "z_train,txts_train,professions_train,train = load_bios(\"train\")\n", "z_test,txts_test,professions_test,test = load_bios(\"test\")\n", "z_dev,txts_dev,professions_dev,dev = load_bios(\"dev\")\n", "\n", "\n", "if os.path.exists(\"analysis/\") and os.path.exists(\"analysis/mode2x.pickle\") and os.path.exists(\"analysis/mode2p.pickle\"):\n", " with open(\"analysis/mode2p.pickle\", \"rb\") as f:\n", " mode2p = pickle.load(f)\n", " with open(\"analysis/mode2x.pickle\", \"rb\") as f:\n", " mode2x = pickle.load(f)\n", "\n", "else:\n", " if not os.path.exists(\"analysis\"): os.mkdir(\"analysis\")\n", " mode2x = defaultdict(dict)\n", " mode2p = defaultdict(dict)\n", "\n", " for mode in [\"freezed\", \"linear-adv\", \"mlp-adv\", \"no-adv\"]:\n", " \n", " for group in [\"train\", \"dev\", \"test\"]:\n", " mode2x[mode][group] = {} \n", " num_seeds = 1 if mode == \"freezed\" else 5\n", " \n", " for seed in range(num_seeds):\n", " print(mode, group, seed) \n", " X = load_bios_representations(group, mode, seed=seed)\n", " \n", " mode2x[mode][group][seed] = X\n", " mode2p[mode][seed] = {} \n", " for projtype in [\"rlace\", \"inlp\"]:\n", " rank2P = load_projections(projtype, mode, seed=seed)\n", " mode2p[mode][seed][projtype] = rank2P\n", " \n", " \n", " with open(\"analysis/mode2x.pickle\", \"wb\") as f:\n", " pickle.dump(mode2x, f)\n", " with open(\"analysis/mode2p.pickle\", \"wb\") as f:\n", " pickle.dump(mode2p, f)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# with open(\"analysis/mode2x.pickle\", \"rb\") as f:\n", "# mode2x = pickle.load(f)\n", "# with open(\"analysis/mode2p.pickle\", \"rb\") as f:\n", "# mode2p = pickle.load(f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## finetune profession and gender classifiers" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def train_classifier(X,y,X_dev, y_dev, X_test,y_test):\n", " random.seed(0)\n", " np.random.seed(0)\n", "\n", "# clf = SGDClassifier(loss=\"log\", fit_intercept=True, max_iter=3, tol = 0.1*1e-3,n_iter_no_change=1,\n", "# n_jobs=32,alpha=1e-4)\n", " clf = LogisticRegression(warm_start = True, penalty = 'l2',\n", " solver = \"saga\", multi_class = 'multinomial', fit_intercept = True,\n", " verbose = 5, n_jobs = 64, random_state = 1, max_iter = 10)\n", " \n", " clf.fit(X,y)\n", " score_dev = clf.score(X_dev,y_dev)\n", " score_test = clf.score(X_test, y_test)\n", " \n", " return clf, score_dev, score_test\n", "\n", "random.seed(0)\n", "np.random.seed(0)\n", "\n", "prof_clfs = defaultdict(dict)\n", "gender_clfs = defaultdict(dict)\n", "\n", "for mode in [\"freezed\", \"linear-adv\", \"mlp-adv\", \"no-adv\"]:\n", " num_seeds = 1 if mode == \"freezed\" else 5\n", " for seed in range(num_seeds):\n", " prof_clfs[mode][seed] = dict()\n", " gender_clfs[mode][seed] = dict()\n", " print(\"============================\")\n", " print(\"mode:\", mode, \"seed:\", seed)\n", " for do_projection in [False, True]:\n", " \n", " \n", " \n", " X_train = mode2x[mode][\"train\"][seed]\n", " X_dev = mode2x[mode][\"dev\"][seed]\n", " X_test = mode2x[mode][\"test\"][seed]\n", " \n", " if not do_projection:\n", " prof_clf,prof_score_dev,prof_score_test = train_classifier(X_train, professions_train, X_dev, professions_dev, X_test, professions_test)\n", " prof_clfs[mode][seed][do_projection] = {\"clf\": prof_clf, \"dev_score\": prof_score_dev, \"test_score\": prof_score_test}\n", " \n", " gender_clf,gender_score_dev,gender_score_test = train_classifier(X_train, z_train, X_dev, z_dev, X_test, z_test)\n", " gender_clfs[mode][seed][do_projection] = {\"clf\": gender_clf, \"dev_score\": gender_score_dev, \"test_score\": gender_score_test}\n", " else:\n", " if mode in [\"linear-adv\", \"mlp-adv\"]: continue\n", " \n", " prof_clfs[mode][seed][do_projection] = defaultdict(dict)\n", " gender_clfs[mode][seed][do_projection] = defaultdict(dict)\n", " \n", " for projtype in [\"rlace\", \"inlp\"]:\n", " for rank in [1, 4, 8, 16, 32, 50, 64, 100]:\n", " print(projtype, rank)\n", " P = mode2p[mode][seed][projtype][0][rank if projtype == \"rlace\" else rank-1]\n", " prof_clf,prof_score_dev,prof_score_test = train_classifier(X_train@P, professions_train, X_dev@P, professions_dev, X_test@P, professions_test)\n", " prof_clfs[mode][seed][do_projection][projtype][rank] = {\"clf\": prof_clf, \"dev_score\": prof_score_dev, \"test_score\": prof_score_test} \n", " \n", " gender_clf,gender_score_dev,gender_score_test = train_classifier(X_train@P, z_train, X_dev@P, z_dev, X_test@P, z_test)\n", " gender_clfs[mode][seed][do_projection][projtype][rank] = {\"clf\": gender_clf, \"dev_score\": gender_score_dev, \"test_score\": gender_score_test}\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "with open(\"analysis/gender_clfs.pickle\", \"wb\") as f:\n", " pickle.dump(gender_clfs, f)\n", " \n", "with open(\"analysis/prof_clfs.pickle\", \"wb\") as f:\n", " pickle.dump(prof_clfs, f)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "with open(\"analysis/gender_clfs.pickle\", \"rb\") as f:\n", " gender_clfs = pickle.load(f)\n", " \n", "with open(\"analysis/prof_clfs.pickle\", \"rb\") as f:\n", " prof_clfs = pickle.load(f)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## record clf accuracy vs rank" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def create_df(clf_dict, mode, projtype, do_projection):\n", " d = defaultdict(dict) if do_projection else dict()\n", " num_seeds = 1 if mode == \"freezed\" else 5\n", " idx = list(range(num_seeds))\n", " ranks = [1, 4, 8, 16, 32, 50, 64, 100]\n", " \n", " for seed in range(num_seeds):\n", " if not do_projection:\n", " d[seed] = clf_dict[mode][seed][do_projection][\"test_score\"]\n", " else: \n", " \n", " for rank in ranks:\n", " d[seed][rank] = clf_dict[mode][seed][do_projection][projtype][rank][\"test_score\"]\n", " \n", " try:\n", " df = pd.DataFrame(d)\n", " except:\n", " df = pd.DataFrame({k:[v] for k,v in d.items()}, index = range(len(ranks)))#, index = ranks)\n", " df['avg'] = df.mean(numeric_only=True, axis=1)\n", " df[\"std\"] = df.std(numeric_only=True, axis=1)\n", " df.rename_axis(\"rank\", inplace=True)\n", " df.reset_index(inplace=True)\n", " return df\n", "\n", "def get_maj(Y):\n", " \n", " from collections import Counter\n", " c = Counter(Y)\n", " p,q = list(c.values())\n", " return max(p/(p+q), 1 - p/(p+q))\n", "\n", "def plot(df_rlace, df_inlp, xlabel, ylabel, filename, baseline=None, baseline_label=None):\n", "\n", " sn.set()\n", "\n", " fig, ax = plt.subplots()\n", " plt.rcParams['font.family'] = 'Serif'\n", "\n", " df_rlace.plot('rank', 'avg', yerr='std', ax=ax, label=\"RLACE (ours)\", marker=\"*\")\n", " df_inlp.plot('rank', 'avg', yerr='std', ax=ax, label=\"INLP\", marker=\"*\")\n", "\n", " \n", " plt.ylabel(ylabel, fontsize=18, fontname=\"Serif\")\n", " plt.xlabel(xlabel, fontsize=18, fontname = \"Serif\")\n", " if baseline:\n", " ax.axhline(baseline, label=baseline_label, color = \"black\", linestyle=\"--\")\n", "\n", " plt.legend(fontsize=18)\n", " #ax.yaxis.grid(color='gray', linestyle=\"-\")\n", " #ax.xaxis.grid(color='gray', linestyle='-')\n", " plt.yticks(fontsize=18)\n", " #plt.xticks(range(1,21,2), fontsize=18)\n", " plt.subplots_adjust(bottom=0.17)\n", " plt.subplots_adjust(left=0.15)\n", " ax.figure.savefig(\"analysis/analysis-results/{}\".format(filename), dpi = 700) \n", "\n", " plt.show()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "mode2proj2profdf = defaultdict(dict)\n", "mode2proj2genderdf = defaultdict(dict)\n", "\n", "for mode in [\"no-adv\", \"mlp-adv\", \"freezed\", \"linear-adv\"]:\n", " for projtype in [\"rlace\", \"inlp\", \"none\"]:\n", " if (projtype != \"none\") and mode in [\"mlp-adv\", \"linear-adv\"]: continue\n", " \n", " df = create_df(prof_clfs, mode, projtype, True if projtype!=\"none\" else False)\n", " mode2proj2profdf[mode][projtype] = df\n", " df = create_df(gender_clfs, mode, projtype, True if projtype!=\"none\" else False)\n", " mode2proj2genderdf[mode][projtype] = df \n", " " ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEQCAYAAACnaJNPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd1xV9f/A8de5g62ADCe5SlFx79QcuEDFlZuyLJuK1dfKLJvmSi1HFuZGzZkK7r1RMy0HYo40N4qgjMvdvz/4cROB6wEuXsbn+Xh8H98453POeX8Aed9zzufz/khms9mMIAiCIMiksHcAgiAIQtEiEocgCIKQKyJxCIIgCLkiEocgCIKQKyJxCIIgCLkiEocgCIKQKyJxCIIgCLkiO3H07NmzIOMQBEEQigjZiePixYt88sknHD9+vCDjEQRBEAo5ldyGbm5u1KpVi/Hjx6PRaOjbty+9evXC19e3IOMTBEEQChlJbsmRqVOnMnr0aADOnj3L2rVr2bJlC/Xr1+fFF1+kffv2KJXKAg1WEARBsD/ZiSM7Op2O7du3s3DhQm7fvk3Pnj3p27cv1atXt2WMgiAIQiEiO3FotVocHR0tX5tMJvbt28eaNWvYt28fBoOBcuXKoVAo8PHx4ZVXXiEoKKjAAhcEQRDsQ3biaN26NQcPHuSff/5h7dq1rF+/nvj4eFQqFR06dKBv3760adMGSZKIjo5m7ty5VK9enc8++6yg+yAIgiA8RbITR8OGDfH39+fPP//EbDbj7+9Pnz59CAkJwcPDI0t7g8FA+/btOXDggM2DFgRBEOxH9qgqjUbD5cuXGTx4MH379qV27do5tjUajURFRZGWlmaTIAVBEITCQ3bi8PDw4MCBA6jV6ie21Wq1HDt2jJdffjlfwQmCIAiFj+xHVcePH6dJkyYFHY8gCIJQyMmeOV6+fHkmTpzIxIkT+eeffyzb4+Pj+fDDD7lw4UKBBCgIgiAULrIfVa1evZply5bRp08fXFxcLNudnZ0xmUwMGjSIBQsWUK9evQIJ9GlJSEjBZJI3tcXLy434+OQCjqjwKEn9LUl9hZLV35LUV8hbfxUKCU9P1xz3y04c+/bt44cffqBjx46Ztru4uDBt2jQiIyOZMWMG8+fPz1WAhY3JZJadODLalyQlqb8lqa9QsvpbkvoKtu+v7EdV9+7dy5I0HhUSEiIeVwmCIJQAshOHwWDAaDRa3W8wGGwSlCAIglB4yU4czz33HHPnzs1x/4IFC3j22WdtEpQgCIJQeMl+x/HWW2/x+uuvs2fPHlq3bo2vry96vZ67d++yZ88eLl++zLx58woyVkEQBKEQkJ04nn/+eSZPnsw333zDqVOnkCQJALPZTOnSpZk8eTItW7YssEAFQRCEwkF24gDo0aMHgYGBHDx4kKtXr2I2m6latSqtWrXKNERXEARBKL7ytR7H41JSUnB1zXnsb1EQH58se+iaj08p7t5NIjVqIgAuPT4pyNDsLqO/xY1eryMpKRGDQYfJlD4ARKFQYDKZ7BzZ01OS+luS+gqZ+6tUqnBz88DZ2frfaYVCwsvLLcf9ubrjeJIuXbpw8OBBW55SEAqURpNCUlICbm7uODqWQaFQIkkSKpUCg6Hk/HEpSf0tSX2F//prNpvR63UkJt4FeGLysHrO3DROTk5mw4YNXLx4MdvKt8nJJWc2plA8JCc/wMPDGwcHJ3uHIggFSpIkHBwc8fDw4cGDe08ncVy7do0hQ4YQFxdnCeLxp1wZL8xLGrPRgCnhJqbURBQuWdcmEQovo1GPWu345IaCUEyo1Q4Yjfmbcyd7Hse0adOoWrUqmzdvJiYmBi8vL2JjY4mNjWX37t20b9+eWbNm5SuYosqUfA/0GnR/bLB3KEIelNQPPELJZIvfd9l3HMeOHSMyMhJvb+8s+ypUqMCkSZN48803rZYlKW7+mTwQs0Fv+Vp/bg/6c3tAqabUa7/YMTJBEISCI/uOQ5KkLEnj0ZEJ7u7u3Lp1y3aRFQF+7/yEqnqL/zYo1aiebYHroO/sF5QgCEIBk5043NzcMiUGHx8fjh07Zvn6xIkTVmtZFUeqUp5IDs7/bTDqkdTO4j2HIAjFmuxHVc2aNeONN97ggw8+oH379gQGBjJy5EhCQkIAiIyM5IUXXiiwQAsrs+YBuLiDQQ8GLaaUBHuHJAjZunXrJm+++SpabRopKSm4u7uzdu0mnJxyHlF25MhhRo8OQ6VSUbq0OwMHhjJ48Eu5uu6IEW8AMHt2zrXu8kOn0/Hqq4OpXTuATz/9skCuIWQm+45jyJAhtG7dmuvXrwPwyiuvULlyZZYtW8ayZcuoUKECH374YYEFWlg5dw5D6V4ORSlvMBlRVaxt75AEIVvly1cgMnIbo0aNBuDBgwds2mR9QMfy5UsACAioR2TktlwnjfTrJPLgQWLuA5bJaDTy8OHDbK/RunUTvv32ywK7dkkl+47D398ff39/y9dubm6sWbOG2NhY1Go1VatWRaGQnYeKHcnBGWW5GujObEddJxBJobR3SIKQo2eeqcytWzdZsWI5vXq9iEqV9d9ubOw5Hjx4kO9rLVy4PN/nsMbZ2Zk1a6JQqWw6n1mwQvZf+sDAQAIDA/ntt98ybff396d69eolOmlkUNfrgjnpHoZ//rB3KIJgVZkyXnTq1JVbt26wZ8/ObNssW7aYgQOH5PtaKpWqwP+oOzo6olSKD2tPi+yf5o0bN/j6669L1HDb3FI90xCpdFl0p7aiqtZUzA8QCrWBA0PZsmUjy5dH0LVrUKZ9N25c59y5s3zxxfgsj3qMRiNr165i377d3Lx5A40mFV/fsvTu3Y/evV+0tMt4p5KcnIxOp+XgweOZzhMbe44FC8I5dy4Gs9lM2bLlGDBgCJ07d7W0+eCDkZw7d5akpIeMGTOO+Ph7bNu2mRs3rmM0Gpk/fykffjgqyzU2bYokPPxHAHbt2sHRo9EAtGnTlvv34zl6NBqdToenZxn69OnHq68OJyEhgaFDB5KU9BBHR0dGjRpNUFB3m32/ixPZtwkVKlSgf//+lC5duiDjKZJcenyCS49PkBQKHOp2xnT3MsY7YhldoXCrVq06zZu35O+/Yzl27Gimfb/+upQ+ffple6eg1WqZOXMaL7zQjrVrN7J5825efnkYP/zwHcuWLba0y3inEhjYKcs5Tpw4zjvvvEbZsuVZsyaSqKjtvPjiAMaP/5zFi+db2k2fPotvv50CwOrVK/D0LMOSJStZvHgFSqWS0qVLZ3uNbt1CiIzcBkBgYCciI7cRGbmNDz8cy3fffc9rr70JwLhxX/Pqq8MB8PT05LffNlGqVGnmz18qkoYVsu84mjdvzsmTJ2nYsGGObTp37sz27dttElhRpa7ZGu3x39Cf2oqqXA17hyPkU69ewVm2hYT0Ztiw4aSmpjJ48ItZ9g8cOISBA4cQHx/Pa69lfZn8yiuv0atXX27cuM67776RZf/bb4+kS5cgLl68wOjRozLtW79+cz56k9WgQS9x5Mhhli5dTKNGTQFITExk//49LF++NttjlEoFzz/fmgED/nuM1bFjFw4c2Muvvy5l8OCXrd5tm0wmpkyZgJOTMyNHvo+DgwMAQUHd2bNnJwsWzKVjxy5UrFgp03E+Pj6EhPQGoEqVqvzvf2Py/EG2a9duzJ07h02bNtCs2X9zsQ4dOkCVKlWzXFvITPYdx9ChQ/nhhx9YsGABp06d4vr169y8eTPT/0SRQ5BUjjjU7oDhyklMD27bOxxBsKpx46bUqOHPsWNHuHDhPABr1qygc+cg3NyyL6vt6OjElCk/ZNnu51eZxMQEEhLuW73m33+f5/r1f2natJklaWRo1eoFjEYje/fuynJc/fqNMn0dEtIbV9ecS39b4+XlTYsWz3PgwD4ePnxo2b5pUyTduoXk6Zwliew7jl69egFkmvSXXyaTiSVLlrBixQpu3LhBmTJlCAoKIiwsTNbCUHq9nvnz57NhwwauXbuGq6srzZo147333qN69eo2izO31HUC0f21Bd3p7Ti1ftlucQj5Z+0TvouLi9X9Xl5eVvdXrFjJ6v5nn33O5ncY2Rk0KJSvvvqMZcuWMGbMOKKi1jF37mKrx5w4cZyVK5dx+fJl0tI0SJKERpMKpD/KsubGjWtA+h/vx3l7+wBYhv0/ytPTU1Z/5AoODuHQoQPs2LGFvn0HEB9/j7NnT/HNNxNtep3iSHbicHZ25rXXXstxv9lsZuHChbm6+IQJE4iIiKBTp04MGzaMS5cuERERQUxMDIsWLbI6UstsNvPOO++wf/9+AgMDCQ0NJSEhgeXLlzNgwABWrFjBs88+m6t4bEXh4oH6uZbozx/EsUkfJKe8fSoShKehffuOhIf/yJ49O/Hy8qJx42aULVsux/YHD+7jk09G06lTV+bOXYinZxkA5s8PZ+HCJ9dos7Z2nLV9th652apVGzw9y7BpUyR9+w5gy5aNtG/fEUdHUWL/SWQnDldXV0aMGGG1TWRkpOwLX7hwgaVLl9K5c+dMVXUrVarE+PHj2bRpEz169Mjx+F27drF//34GDBjA119/bdnes2dPunfvzvjx41m0aJHseGxNXbcr+vMH0MXsxrGRuPUVCi+VSsWAAYOYMWM6K1cuZ+HCZVbbb9oUidlsZuTI9y1JIzf8/CoDcO/evSz74uPv/X8bv1yfN7dUKhWdOwexcuUy/v47ls2boxg37usnHyjIf8exc2f2Y70ftWPHDtkX3rhxI2azmaFDh2ba3r9/f5ydnZ+YhI4cOQJAnz59Mm338/OjSZMmREdHc/PmTdnx2JqyTEWUfnXRn92J2aAjNWqiZYlZQShsevbsg5tbKZo0acZzz9W02latzngvkfkF+J078t7pPfdcDSpVeobjx4+h0+ky7Tt0aD9KpZK2bTvIjt0aJycnSw29hIT7TJ8+mdTUVMv+7t17AjB16iTUajW1atWxyXWLO9mJw1o9mwzh4eGyL3zmzBkUCgX16tXLtN3R0RF/f39Onz5t9fiMX7js4srY9tdff8mOpyA41O2KWfMQw8Ujdo1DEJ7ExcWFTZt2Mn367Ce2bd8+EIA5c2ag0WgAiI4+yM6d22RdS6FQ8PHHn5KWpmHWrO/RarWYzWa2bNnIkSOHGTbsDZuNaqpcuSr//nsVk8nE0aPR7Ny5HWfn/wqTVq1ajVq16hATc0a8FM8Fmz40jIiIkN02Li4OT0/PLKMqAMqWLUtCQkKWTyOPeu6554D/7jwyaDQaS8K4fdu+o5qUFWujKOOH7vRWq89uBeFpSEtLIySkCzNmTOXMmVOEhHRh06b/7uyVSmWm9wiTJ48nJKQLgKV9ZOQ62rfvyJgx44iNjSEkpAuhof3Zv38v3bqlf3ofPvxl5s372WosDRs2Zs6cedy+fZO+fbvTo0cn1qxZyaeffsnQof+9S/3mm3F8+ulHAMyYMZWQkC5cuPC3Zf/587GEhHRh1670px0hIV0yzQN5773R6PV6QkI6s3jxfMaO/SLLUOFu3UJQq9V07px16LWQPdnvOGrVqmXTC2s0mmyTBqTfdUD6L3pObUJCQvjpp5+YOXMmLi4utGzZkoSEBGbNmkViYqLlGrnl5ZW7F9k+PqWs7k9q1Yu7UbNQ+zyD0rnUE9sXdkU9/sfFxSmyrdME5Li9qHJzc2Hz5pwfJz/e308//TzHtr169aZXr95Ztn/88SdZtkmShCRJWc4fEBDA999bXzX0q6++tbq/Tp3aVvvUsGFDli1bmWX7o7E4Oqpp06Yt3t65f19TVDz+vVcoFPn6tyw7cTg4OBAcnDkjm0wm4uPjOXv2LF5eXtStW1f2hZ2dnYmPj892X8ZwPmuPx9zd3Vm4cCEff/wx48aNs2xv2rQpr7/+Oj/99FOO49CtiY9PxmSSd3fg41OKu3eTrLYx+9ZHcvFAn3gXfeJd7ly9VmTX65DT36LGZDJhMJiybFepFNluL65s2d/t27dSqVIlatcOANInFPr6li0030+FApKSki1zQDZujCI09JVCE5+tZfezNZlMVv8tKxSS1Q/RshNHqVKlmDgx+5e7er2e6dOn06RJE7mnw9fXl4sXL6LT6bLcVdy5cyfHx1iPqlmzJuvXr+fq1avExcXh6+tL5cqVmTIlvURBtWrVZMdTUCSlCnVAJ3THVgOg+2MDTm2GPuEoQSi6zp07y6FD+/n882+4ffsWJ0/+kamGlb3dvn2bUaPeYcmSlZw+/RcPHiTSvHlLe4dVpMhOHNZGVanVat577z0GDRpEYGCgrPMFBARw8OBBTp06lSnhaLVaYmNjc5WEKleuTOXKlS1fHzhwADc3Nxo1amTlqKcjaf5wMIp1yYWSo3nzlsyb9zM9e3bFycmJ3r37WmpDFQaOjg7//96jC97e3nz22VeiuncuyU4cTxpVJUkS165dk33h4OBgwsPDWbx4caYksWrVKjQaTaY5HHFxcSQlJVGhQoVMIyKyExERwd9//82IESNkzT4vaK6DvkMbvQLDpf9/ia9Uo6raGMcWA+0bmCAUkBYtnqdFi+ftHUaOvLy8WbMmyt5hFGmyE0dOcyKMRiO3bt0iIiKCSpXkD6GrWbMmQ4YMYenSpYwYMYK2bdtaZo43a9YsU+KYPn0669atY8mSJTRv3tyyffjw4fj5+VG9enUkSeLQoUPs3LmTdu3a8dZbb8mOpSApXDyyrEuOWJdcEIQiTHbi6NChg9WKl2q1mp9/tj4E73Fjx46lYsWKrFy5kr179+Lp6UloaChhYWGybh0bNGjAli1bWLduHZD+TuPzzz9n4MCBhWpRF8u65JICUhIwxV+1d0iCIAh5JpllTjBo2LBhtrWqlEolZcuWpVWrVpQtW9bmAT5tth5VlSE1amL6XA5tMmajAdd+3yIp1fkJ9akrjqOqbt++SrlylbNsF6Oqiq+S1FfIvr85/d5nsNmoKm9v7yfWqhKskyQJhxaD0GyZhv7MThzqBz35IEEQhEJG9lCC3NShEnKm8quL8pn6aE9EYtI8fPIBgiAIhYzsxJGSksKuXbvYtWsX9+//t1BLUlISq1evztMs7ZLKqcVAMOjQ/f6bvUMRBEHINdmJY/369bz77rtMnTqVu3fvWrYbjUbmzJlDv379iIuLK5Agi4OMdckBFB7lUdcJRH9+H8b4f+0cmVDQJi87weRlJ+wdhiDYjOzEsWXLFt588022bNlCzZr/lV328PBg9+7dtGjRgunTpxdIkMWRY+Oe4OCCNvpXUQBREAqZlJSiswx2amrKU7+m7Jfj165dy3GFP0mSGD16NEFB4mWvXJKjK45NeqM9tBTD1ROoqzS2d0hCMZaWlkb//j3RatNISUnB1dXVstJdSkoyZcp40bBhY4YPfxsfH1/LcQsX/sK6dWtITEwAwMPDk969X+TVV4fLum58/D369OlG3br1mT177hPb//vvFX79dRknTvyORqNBq02jXLny1K5dl5YtW9GyZSvU6vTRiKGh/YmPv0dS0kOcnZ1xds464VejSaVdu0A+/fRLWfGaTCbmzw/n/v37fPzxp7KOsbd9+/awf/8ePv30qzzV58sL2YnDYDBYfmDZcXJysloGXchKXas9+pjdaI+sROVXr8gNzxWKDicnJyIjt7F5cxQTJnzFqFGjCQ5On2RrNBo5dGgfX3zxKUePRhMRsZLSpd0BePXV4bz66nBefDG9bW5nXO/cuQ2j0chff53k9u3blCuX85K0mzZFMm3aJEJDX2Hu3EW4u3ug0+n466+TzJ79PVFR62jXLpDx4ycDsHTpKk6cOE5Y2FsMHBiabVmT+fPDuX37lux4v//+Oy5cOM/06dar9hYmQUHdiYk5y/vvv8vs2eFPZelb2Y+qvL292bdvX4779+/fT5kyxbcscUGQFEocWw7G/DAO/Rkxak2wD6VSSYcOHenUqSvx8ffYsmWjzc69bdsWqld/FrPZzI4dW3Nsd+TIYSZN+obXXnuTYcPewN09vbKCg4MDTZs2Z/r02Xh6lsFoNNgstsdFRx9kw4a1fPTRp7i4uBbYdQrCyJHvc+/eXRYtmv/kxjYgO3H079+fUaNGMXXqVI4cOcLly5c5f/48Bw8eZPz48YwaNYpBgwYVZKzFkqpSwH/Dc1Mf2DscoQTLWDL2339tU9ng6tUr3Lp1kzFj0pc92L59c7btjEYj338/BU/PMgwYMCTbNl5e3nTunPtH4a++OpxPPsl5XZFHLVjwCy1atKJateq5vo69OTg40Ldvf1av/pXk5IJ/PyP7UdWQIUM4f/488+bNY/78zFnNbDYzcOBABg8ebPMASwKnFgNJWf0ZuuPrcHrhFQDL+uQZI7GEostgNHEzPoUHyVrc3RztHU6OzOb02cUZn/bza/v2LbRrF0itWnWoVq06ly9f4sKF81nWND958g9u3LhO167dUKly/pMUGvoKDx4kyrr2iRPHmTDhK9mP1m7cuM65c2cJC/sg2/3x8feYN+9njhw5jMFgwNXVlc6dgwgNfcWy/MM334zj8OFDJCU9ZOzYLyyPAj/4YCRnz54iJSWFmTN/plGjJpbt586dJSnpIWPGjCM+/h7btm3mxo3rGI1GVq+OxNe3LL/+GsGWLRtJTk5CqVRRtWp12rcPtKyXnqFJk+b8/PNsDhzYS1BQd1n9zqtc1RL++uuvWbp0KYMHD6Z169a0atWKIUOGsGzZMr788ssCCrH4E8Nzi7d7D9LQaI1EHvrH3qFY9fff51EoFLRr18Em59uxYyudO3cFoFOn9LuF7duzPq46fTp9qeeqVa2vn+Pp6UmVKlVtEtvjTp36E4BnnqmSZV9Cwn3eeOMVrly5zC+/LCEqajtffvkt69atYcyY/2EypSfcceO+4dtvp2Q5fvr0WYwaNTrb7RntV69egadnGZYsWcnixSsstfYWLZrH2rWrmDhxGhs2bGPZsjX4+PgwadI3Wc6X8b3566+Tefsm5ILsO44MTZo0ydVaGYI8jo17YrhwGO3h5Th3/9je4QjAodO3OHhK/ovVx/19LZFHB1rvOXmTPSdvIgE1/PL2qb51vfK0qls+zzFlR6tNY/v23cTGxvDNN5OoUcM/3+c8ffovDAYDDRqkr4nTqVNX5s79kZ07t/H22yMzFTGNj78HgJtb3pcyXbFiKRs2/DehVq/X4+oq/z3FtWvpH9i8vLyy7AsP/5E7d27z7bdT8Pb2BsDfvzaDBr3EnDkz2Lp1k+XuIq98fHwICUlfirdKlar8739jKF26NAcP7qN27To880x6XSlnZ2feeWeUJdE9ysnJCVdXV0tfCpLsOw69Xk9sbCyxsbGkpqZatqekpBAdHV0gwZUkkqMrDk16Y7wVi+GqmCxWHFSrUJpSLv+NlJMkKOWiplqF0naMKt2MGVMJCelCcHAgHTu2YerUyQwe/DJt29rmbmP79q106tTVUlG7XLly1K/fkLt34zhx4ni2x1irvv0kAweGEhm5zfK/7D75W/PwYfr7xceH9JpMJvbu3UWZMl74+9fOtK916xcA2L07/wNb6tfPvOhcSEhvXF3d8PQsQ3T0YSIj16HVpgFQunRpli9fm+15nJycZT/Oyw/ZdxwbN27kk08+wc3NjQULFlCvXj0gfXz4W2+9RYMGDfjxxx+f2jji4khdq51leK7k4o4kiVXJ7KlV3fx/ul+yNZa9f6bfZWCGJjV9eKlL/j/R59ejw3H/+uskY8Z8wMSJX+Pn9wx169bP17kNBgN79uxgxozMyyx06tSVP/88wfbtW2jSpJllu5dX+qf4pCT71W7T69NX6Xx8OYfExASSk5N57rkKWY7x9vYB4Pr16/m+vqenZ7bbR40azaeffsSUKd8ya9b3tGjxPMHBPWjR4vlsE61SqbT0pSDJ/su0ceNGevfuzeHDhy1JA9Jv7Q4dOoSnpyczZswokCBLivThuYMwP4zDlBSP8d6/mFIL/tODUHAepupwd3XgmbKlaNewIg9SCt9cp/r1G/Luu6Mwm80sXDgv3+c7ejSapKQk3n//XUJCulj+98svPyFJEvv27Uar1VraZySqK1ds9w6oUaMmuZpzkrHCqcGQebivrao6ZLwHyUlO6w9VrlyFiIiVTJ8+m/btAzly5DAffjiKsWM/zDY2g8HwxNVabUF24rh8+TLjxo2zjCB4lJubG19//TW7d++2aXAlUcbwXFLug16D7o8N9g5JyIcRfepRrowLTg5KXupSkxF96j35IDvo0aMnlSr5cexYNJcuXczVsUlJScyfH275evv2LYwY8X6mR0eRkdvYuHEHbdt2ICUlhYMH91vaN2zYmEqVnuHo0egsf7gfNWXKt4wZk/2op/zKuHtISsq83oynZxnc3Ny4d+9elmPu3Uuv2efn52fZplSmP8QxGo2Z2iYkJOQpLqPRiCRJNGvWgrFjv2Ddus20bv0CBw7s5Y8/fs/SPjk5CW9v32zOZFu5esdhbQ3v0qVLZ/oUIeRN0vzhGP/9y/K1/twekua+QtJ8eSUeBCEvlEolQ4emL9T2668RuTo2OTmJhQt/ASA1NZXo6EMEBnbKtm2nTl0A2LFjS6Zr/+9/H/PgQSKrV6/I9rhz586yceMGmjZtnu3+nGzZspFRo95+YruqVdPnbty9eyfT9vRRZoEkJNwnNjYm076M5Nehw3999fFJT0BxcZnPc/bsqVzFnWHAgF6cO3fW8rWbmxs9eqS/RE9OzpzkEhIS0Ol0T2UeiuzE4ebmxqlTOXf+9OnTuRrFIGTPddB3qKq3+G+DUo3q2Ra4DvrOfkEJJULnzkFUquTHzp3bsvzhk2vfvt0EBNSlTJmso5MAWrRohaurK0eOHM70Erdp0+aMGTOOefN+YtGieZaX1Tqdjp07t/HRR+/TtWs3+vTpn6t4zGZzlk//2WnQoBEqlYqLFy9k2ffmm+9Stmw5ZsyYahkBFhsbw6+/RtCsWQu6dAm2tK1QoSLVqlVn+/YtxMXdwWQysWPHVu7cuZ2ruB81f3645Y4lOTmZzZsj8fDwpFGjppnaXbx4HiDXyTUvZC8dO2fOHJYuXcrIkSNp3bo1ZcuWRafTcffuXfbu3cv8+fMJDQ3lrbfeKuiYC1RBLR2bG2kHFqM/t9aIhC8AACAASURBVMfytbpWe5zaDLX5dXJLLB2bNxkl1T8e0ugJLQtOTkUOv/12CnXr1rf0N6OWlZubG25upejYsQurVi231KHL7lG12WxGr9fTvn1HoqMPIkkS7u4eWd4xnD8fy4cfjuLhwwcYDAZKl3anX7+BmQom/vvvFZYvj+CPP35Hq9WiUqnw86tM7959adu2Q6YXwr17B5OQcB+DwYBSqbTMfXiU0WjMUmAxp5/tl19+yuXLF1m8eEWWF8/37t1j3ryfOHLkMEajERcXFzp3DuKll17N8j25ceM6U6dO5Ny5s7i7exAU1B1v7/S5F6VKlSYgoC7ffTcj04TBjJ/HtGmzeO65GpZzHTy4ny1bovj77/PodFpUKjX16jVg2LA38PN7JtN1p0z5lt9/P8rKleszvTMpiKVjZScOvV7Pu+++y/79+7N8U81mM23btuXHH3+0OvOzKCgMiUOzfSaGuEtgBjQPUFTwx7X7GJtfJ7dE4sibwpA4nqQkrcOdU19v3LjO0KEDGTv2Szp06GiHyPLu9u3bDBrUm3HjvskSu13XHFer1YSHhxMZGcmWLVv4999/MZvNVKlShaCgIHr06JGvcdjCf5w7h5EaNRGzyYjJZEBSFO1kLAhFQcWKlfjii/FMmTKBcuXKUbt2gL1DkiUhIYGPPhrFgAFDnlrCy9VfJEmS6NmzJz179sx2/7p16+jdu7dNAhP+f3hug+5oj67EcDMWVQX7j/8Xcq8w32kImbVp044yZbyJjj5UZBLH8eNHGTz4Zbp27fbUrmmzGWZGo5GpU6fa6nTC/1PXCURy9UT7+xqxUqAgPAV16gRku7ZHYdWpU9enmjTABonj9OnTTJgwgbZt23L//n1bxCQ8QlI54NCoJ6Y7FzH+m7U+jSAIwtOWp4fn165dIyoqiqioKK5cuYLZbEahUOQ4+1HIH3XN1uj+2oL297Uon6kvSpEIgmBXshNHYmIimzdvJjIykr/+Sp+gZjabCQgIoHv37gQFBdG3b98CC7SkeXQdDkmhwrFJb9J2/4zh4hHUzz1vx8gEQSjprCYOrVbLrl27iIyM5ODBgxiNRsxmMxUqVKBHjx6sWrWKNWvWWNp/9dVXBR5wSaWq3gzFX5vRHl+HqlozJKUYaSUIgn3k+Nfnk08+YceOHaSkpGA2m3F3d6dr16706NHDsh7H2rWZS/sGBgYWbLQlmCQpcGz6Ipqt09Gf349DbduUvxYEQcitHBPHunXrAKhZsybvvfcebdq0KfKT+4o6pV9dlOVqoDsRibpGKyRV4V2GVBCE4ivHt6yrVq0iNDSU+Ph4pk+fzoIFC7h1K++roQn5J0kSDs36YU5NRHdmp73DEQShhMrxFqJevXrUq1ePsWPHcujQIaKioujevTv+/v707NmTLl265PviJpOJJUuWsGLFCm7cuEGZMmUICgoiLCzMaiXeDGazmY0bN7Js2TL++ecfdDodFSpUICgoiFdeeaVYLiqlKvccymfqo/tzEw612iE5isKShV1q1EQg84AHQSjKnjiuU6FQ0KZNG6ZMmcKhQ4cYNGgQu3btol27diQlJbF3715LDf3Q0NBcXXzChAlMnDiRZ599lnHjxtG1a1ciIiJ46623nrjwCcAPP/zA6NGjcXR0ZMSIEXz00UfUqFGDWbNmMWzYsGI7Yc6xaV/QpaL7a8uTGwuCINhYrl5aODk50b17d7p37879+/fZvHkzP//8M2PGjKFLly6cP39e9rkuXLjA0qVL6dy5M7NmzbJsr1SpEuPHj2fTpk306JHzAvAGg4HFixdTp04dFi5caJlDMmjQIJRKJVFRUcTGxlKrVq3cdLFIUHo9g6p6C3RntqMO6IjCxcPeIQlFQE4VcmfM+IlRo962bA8O7sHYsV9kOf6HH6aye/cOEhMTcHV1Q61Ws27dZk6cOM4333xOcnIyOp0WDw8PatSoxfTps7KJAjZtiiQ8/EdLldyMEuxabRoKhZK6devz2mtvUrOmKLFTWOV5JlmZMmUIDQ1lxYoVrF69Gh8fH0vpZTk2btyI2Wxm6NDM5cL79++Ps7MzkZGRVo83GAykpaXh7e2dZeKhr2/6CljOzs6y4ylqHJv0BqMR3Qn5y2MKJZuTkxORkdsYNWo0kL6edWTkNqpWrcbmzTss2zdvjmLv3l1Zjn/vvfT2vr5l+fbbKURGbkOpVNK0aXMiI7dZFm/65ZclOSYNgG7dQoiM3EZAQPpqiBkrBG7bto+ZM3/m2rWrvP32MM6ePWPrb4FgIzaZguzn58eIESNy9en+zJkzKBSKTOuXAzg6OuLv78/p06etHu/k5ETTpk05cOAAc+fO5erVq1y/fp3ffvuNX3/9lZCQEKpUqZKX7hQJCveyqP3boI/di+nhXXuHIxQT3t4+qFQqpkyZwN27cU/9+s89V4NRo0aj0+mYO3fOU7++II9Nx9euWJH9so/ZiYuLw9PTM9uFYcqWLcvJkyfR6XTZ7s8wdepUPv74Y6ZNm8a0adOA9JFHb731FqNGjcp9B8BqDfrs+PiUytN1bMHQcTDXLhxGOrsRn5Cwp3JNe/a3IMTFKVCpsv/8lNP23MpYbsBW58svhUKy/P+jMSkUEn5+zzBw4GBmz57BhAlfMXPmnGyXS1Aqs37fMtplty87OX1f6tevD0BMzOkC+54Vlp/F0/J4fxUKRb7+LdttYoZGo8kxKTg6ps9PSEtLs5o41Go1fn5+lC1bljZt2iBJEtu2beOnn37C0dGRt99+8lrDjysMCznJ54CqdiDJp7ZiqtkJZZlKBXo1+/fX9kwmU7aL+thyYSOTQY8p4Sa6h/cLxfuojN9vk8ls6aNKpcBkMmM2mxkwIJSjR4/w++9HWbYsgoEDsw56MRqzft8yBqNkty87Ge0fb6vXG/7/v6QCWVyqJC1aBdn312QyWf23/KSFnOyWdp2dnXN8J6LVaoH0x1E50Wg0DBo0iOTkZCZPnkz37t3p1q0bM2fOJDg4mJkzZ3L58uUCib0wcWzQDdRO6H5f++TGgl2Yku+BXoPujw32DkUWSZL47LOv8PDwZO7cOdmuw12QTp8+BUDduvWf6nUF+ex2x+Hr68vFixezfRx1586dHB9jZdi2bRtXrlzhgw8+yLKva9eubN68mT/++INq1arZPPbCRHJyw6F+ELrjv2G8cxFl2WftHVKxof/7EPrz+/N8vPHW36Sv//v/5zu35//XkpdQlq+R43HWqGu+gLpGqzzHJJeXlzdjx37BRx+9x1dffcq8eRGWJwEFxWg0cu7cWWbMmIqzszNvvfVugV5PyDu73XEEBARgMpk4depUpu1arZbY2FgCAqyvvnXnzh0g/ZftcRnbsttXHDnU7YzkXBrt72uL7dyVokjhWw2cHn2OLIFTqfTtRcDzz7emX79B/PPPZX76KedRUvkVEtKFkJAu9OjRmXHjxlCrVm3mzl1MjRpiOG5hZbc7juDgYMLDw1m8eLGlaCKklzrRaDSZ5nDExcWRlJREhQoVLENsq1evDsD69esJDg7OdO6MOlt169Yt6G4UCpLaCYeGPdAeXobxxllUlYrGkpeFnbpGq3x/uk87sNhylwGgrtoEpzZDrR9UiLz99khOnvyDtWtX0rJlK5o3b2nza0RGbrP5OYWCZdM7jqCgINlta9asyZAhQ9i+fTsjRoxg9erVTJo0iUmTJtGsWbNMiWP69OkEBwdnujtp37499erVY9++fQwZMoTFixezaNEihgwZwv79++natSt16tSxZfcKNXWtdkhuXmiPiSVmCxOz5gG4uKPwfgZ17fbpXxchDg4OfPXVBBwdHZkw4UsSExPtHZJQCOTqjuPevXscOHCAuLg49Hp9lv0Zj4/kGjt2LBUrVmTlypXs3bsXT09PQkNDCQsLe+JqgkqlkoULFzJ37ly2b9/Od999hyRJVKlShdGjR/Pqq6/mKpaiTlKq0xd72jsPwz/HUVdrau+QBMC5c5ilVpVT65ftHE3eVK5chbCw/zFlyrdMmvR1ns+zatVy2rRpR/nyFWwYnWAPshNHdHQ0b7/9NmlpaTm2yW68tzVKpZJhw4YxbNgwq+0y7kQe5+bmxgcffJDtC/KSSPXs8yj+2ozu97WoqjRCUijtHZJQTISE9ObYsSPs3bsr1//OM6xa9SvPPltDJI5iQHbimDJlCi1btuTll1+mfPnyqNXqTPvNZjP9+vWzeYCCfJJCgUOTvqTtmIXh70Oo/V+wd0hCMfLxx59x7txZ7ty5be9QBDuTnTiuXbvGypUrrQ6RfbzulPD0qao0QuFTDe0f61E92wJJlfPPSyhZHi1yCDBjxlR+/nl2piKHWq2WkJAu9O8/iNDQVzIdX6pUKT7/fDxhYW9m2h4dfYhPP/3QUiV78OC+2d6VZMzberTIIaSPqgJYsybK6t8XofCQzDLfpPbq1YtVq1ZZ/cFqtdoCH+td0IrWzPHsGW7EoNk0BceWg3Com/91UzIU1v7mx+3bVylXrnKW7bacXVwU1uMoSbOpS1JfIfv+5vR7n8FmM8fffvttvv/+e8uniuyINccLB1XF2igr1kZ3ciNmncbe4QiCUMzIflS1d+9eTp8+TWRkJAEBAXh6ema5HU1OTrZ5gELeODZ9kdT1X6M7vR3Hxj3tHU6JVpjvNAQhL2QnjoxJdQD79u3Ltk1eR1sItqf0rYaqSmN0p7agrtMBhVPxqmorCIL9yE4c3t7eHDx40Gqb1q1b5zsgwXYcmvbBcPUEuj834dRioL3DEQShmJD9jmPQoEFPbPPuu6IoWWGi9KyI6rnn0Z/diSn5vr3DEQShmJCdOOQkBTnJRXi6HBv3ArMZ3YmiUdJbEITCL9dFDtevX8+WLVu4evUqAFWqVCEoKIiePcUL2MJIUcoHda326GN241AvCIVHOXuHVOiYzWbxfk4oMWxRy0524tDpdLz77rscOHAg0/YrV66wb98+Nm7cyJw5c7LMKBfsz6FhD/TnD6A9/hvOHd+xdziFilKpRq/X4uCQ86JhglCc6PU6lMr8FUaX/ahq7ty5xMTE8NFHH7F161ZOnDjBiRMn2Lp1K6NHjyYmJoa5c+fmKxihYChc3HGo2xnD5WMY710B0ielZUxMK8nc3NxJTLxHSkoSRqNBVBYWii2z2YxOpyUx8S5ubvlbwlh22omKimLu3LlZSpVXqVKF1157jWbNmjF69GjxgryQcqgfhC5mN9rf1+IS9D97h1NoODu7olKpSU5OJCXlASZT+uJfCoUCk6nkzC4uSf0tSX2FzP1VKlWUKuWJs7Nrvs4pO3GkpKRYXd+ibt26pKSk5CsYoeBIDi44NuiG9ugqDLfO2zucQkWtdsDT0zfTtuJYXsWaktTfktRXKJj+yn5UpVQqiY+Pz3H/vXv3nriGhmBf6jodkVw80B5bLR7JCIKQZ7L/0rdp04aRI0cSExOTZd/Zs2cJCwujbdu2Ng1OsC1J5YBDo56Y7lzErHmI8d6/mFLFim6CIOSO7EdV7733Hv369aNv3754eXnh65t+ax8XF0d8fDzly5dn5syZBRaoYBtq/zboTm3F/DAOzCZ0f2woUmtgC4Jgf7LvOLy9vVm7di19+vRBq9USExNDTEwMWq2Wvn37smbNGry9vQsyVsEGkhe+jfnhHTCnvyzTn9tD0txXSJo/3M6RCYJQVMhej+NRZrOZ+/fvYzab8fLyKlaTp4rDehzWmFIT0Ub/iuHS0fQNCiWqak1xbDEQhYv1IXpFsb95VZL6CiWrvyWpr5C3/tpsPY5HSZKEl5cX3t7emZJGeHh4Xk4nPEUKFw8kB5f/NpiMmB7cQXJ2t19QgiAUKTYdBhUREWHL0wkFxKx5AC7uSF6VkUqXxXT3H7TRyzGbS87YdkEQ8i7Hl+OffPIJiYmJzJkzB0mSqFWr1tOMSyhAzp3DLLPGnft8gTZ6Bfoz2zGnJePU7jUkRf7KEQiCULzl+Bfizz//JCkpCb1ej4ODAw4ODgQHB+d4IrPZzNatWwskSKHgSJICx5aDkJxLoft9LRptCs4d30VSF+214wVBKDg5Jo4NGzZgNBpxcHAAoFSpUkycaL220ZMWehIKJ0mScGzYA8mpFNqDi0nd/B0uXd5Dcsr55ZggCCVXju84HBwccHZ2tnwtp4ChKHJYtDnUaodT4DuY7l4hNWoippQEe4ckCEIhJPvl+OPl1B81efJk+vTpg8FgsElQQsFz6fEJLj0+ybJdXa0pzkEfYEqOJ3XDeEyJt+0QnSAIhZnsxGFtxNSLL75IYGAgn3/+uU2CEuxLVbE2Lt3HgEFHauS3GO9esXdIgiAUIjYZjlu9enXeffdd7t69a4vTCYWA0qcKLiGfgsqB1I2TMNw8Z++QBEEoJKyOu5w9e7blv1NTU/nxxx+zrapqNBq5dOkSbm7iZWpxovAoh0vPz9Bsnopm8zRSHN8Hr5xL6wuCUDJYLTni7+//X0NJslqKu0yZMowfP54OHTrIvrjJZGLJkiWsWLGCGzduUKZMGYKCgggLC8PFxcXqsUePHuXll1+22mb58uU0btxYdjxQ/EuO5IU5LZnUbT9giruMY+uXcajVzt4hFbiS8rPNUJL6W5L6CgVTcsTqHceuXbuA9Dka/fr1Y82aNdm2c3Z2pkyZMrkKDGDChAlERETQqVMnhg0bxqVLl4iIiCAmJoZFixZZXd+jevXqTJkyJct2nU7H559/jqenJ/Xq1ct1TEJWkpMbLsEfYtz/M5oDizCnJeHQoHuxqlEmCIJ8VhNHxYoVLf8dGhqa6ev8unDhAkuXLqVz587MmjXLsr1SpUqMHz+eTZs20aNHjxyP9/b2pmfPnlm2b9y4EZPJRM+ePVGr1TaLt6ST1I6U6zeGa2t+QPf7WsyaJBxbDkSSxOJdglDSyP5Xb+u1xDdu3IjZbGbo0MxrQfTv3x9nZ2ciIyPzdN7Vq1cD0K9fv3zHKGQmKVU4tR+OOqAT+jPbSdvzC2aTGIItCCWN7MRx4cIFRowYwYgRI4iNjbVsv3PnDoMHDyY6OjpXFz5z5gwKhSLL4yRHR0f8/f05ffp0rs4HcO3aNY4ePUrjxo2pVq1aro8Xniy9RMlgHJr0wXAxGs32WZgNWnuHJQjCUyS7mt3q1as5duwYr7/+OuXKlbNs9/DwoGnTpoSFhfH999/TunVrWeeLi4vD09PTUtLkUWXLluXkyZPodLps9+dk7dq1lvcxeWXthVB2fHxK5flaRZGlv12G8NDHh3tb5qLfPp1y/ceidC5eo+pK7M+2BChJfQXb91d24jh8+DA//fRTllFKjo6OvP/++zRr1ozZs2fLThwajSbHpODomF5gLy0tTXbiMBqNrFu3Djc3N7p27SrrmOyIUVU5y9Jfv5Y4dVSRtjucaws/xTn4fyhcPe0XoA2V+J9tMVaS+gp2XsgpMTHR6tDWVq1acf36ddmBOTs7o9Ppst2n1aY/+nBycpJ9voMHD3L79m26deuWqcaWULD+K1FyL71EyQNRokQQijvZicNkMuX4hx7S/9gbjUbZF/b19SUhISHbc965cyfHx1g5yRgqLF6KP33pJUo+Ti9RsuFbjPeu2DskQRAKkOzEUa9ePSZMmJBtctDr9UyZMiVX8yYCAgIwmUycOnUq03atVktsbCwBAQGyzxUfH8+ePXuoWbMmdevWlX2cYDtKn6r/lSiJEiVKBKE4k/2OY8SIEQwZMoTdu3fTsmVLfH190ev13L17l0OHDqHRaFi+fLnsCwcHBxMeHs7ixYtp0qSJZfuqVavQaDSZ5nDExcWRlJREhQoVsn0MtX79evR6vbjbsLP/SpR8h2bzNJwC30ZdNXcz9wVBKPxkJ46AgADmzp3L559/zoYNGzLtq1y5MrNmzaJOHfl1jGrWrMmQIUNYunQpI0aMoG3btpaZ482aNcuUOKZPn866detYsmQJzZs3z3KutWvX4ujoSEhIiOzrCwVD4eqJS4+xpG79nrSdszG3eQUH/7b2DksQBBvK1eLSzZs3Z+vWrcTExHD16lUAqlSpQq1atfJUfmLs2LFUrFiRlStXsnfvXjw9PQkNDSUsLMxquZFHnThxgkuXLtG9e3fc3d1zHYNge5KTGy7dPkKzczba/Qsxa5JwaNBNlCgRhGLCapHDkkgMx81ZbvtrNhpI2zcPw8UjqOt2wbHFgCJTokT8bIuvktRXsEORw8elpaWxYsUKDhw4wIMHD1izZg2xsbEcOXKEF198UZRVFzJJL1HyBlqnUuhPb8OcloRT22FIilz92gmCUMjI/hecmJhIaGgoFy9eBLCUPXdwcGDt2rWsXr2axYsX4+3tXTCRCkVSRokSyakUuuO/odGm4NzxHSSVo71DEwQhj2Q/N5g5cyYmk4nw8HAOHz5sSRzVqlUjMjLSMnNcEB4nSRKOjUJwbP0yxn9Podk0FbM2xd5hCYKQR7ITx549e/jxxx9p27ZtlrU3JEli9OjRHDp0yOYBCsWHQ+0OOHV8G+Pdy6RGTcSUkmDvkARByAPZiSM1NZWqVavmuN/V1RWNRmOToITiS12tGc5dP8CUdI/UyG9FiRJBKIJkJw5nZ2euXLmS4/6zZ89aihMKgjWqSnXSS5TotaRGTsB476q9QxIEIRdkJ44uXbrwzjvvEB0djcHw3+I9Go2GyMhIRo4cSVBQUIEEKRQ/6SVKxoJSTWrURFGiRBCKENnzOJKTk3nppZeIjY1FrVZjNBpxdXUlKSl9fHCdOnVYsmSJ5aV5USXmceSsIPprSr6PZstUTA/j0kuUVCkcJUrEz7b4Kkl9BTuXVXdzc2PFihWMHDmSypUro1Kp0Gq11KhRg/fee4/ly5cX+aQhPH0KtzK49BiLwqsyaTtmo4vdZ++QBEF4AjFz/DHijiNnBdlfs16LZscsjNfP4NCsHw71g+1aokT8bIuvktRXsPMdhyAUJEntiHOX91BVb4Hu2Gq0R1diNpvsHZYgCNnIceb433//TVpammWNjfXr18s6oSRJlC9fngYNGuRqISZBkJQqnDq8gdbJDf2prZg1STi1fTVLiZLUqIkAuPT4xB5hCkKJl2PieOmll0hJSeH48eM4OTkxZswYJElCzpMtSZKoVKkS4eHhVKtWzaYBC8WbJClwfH4IknMpdMfXodEmixIlglDI5Jg4Ro8eTVJSkmXdb3d3d2bNmiXrpAkJCaxbt44JEyYwb94820QqlBjpJUp6IjmVQnswAs3maTh3GYXk6Grv0ARBwErieHw1vcaNG9OsWTPZJw4MDKR9+/Z5j0wo8Rxqd0BydCNtTzipURNxDh6NwsXD3mEJQokn++X4nDlzcnXi+Pj4XAcjCI9TV///EiUP75K64VtMD+7YOyRBKPFyvTDCtWvX2L59e6YVADt16oSfn5+lzaJFi/jpp59o2bKl7SIVSqyMEiWard+TGvktOJfCnHQfU2qiuAMRBDvI1TyO8PBwZs2ahdFozPSSXKVSERYWxhtvvAHArVu3ePDgAWXLlsXT09P2URcgMY8jZ/burzHxZnpJ9pT7AKhrtcepzdACuZa9+/q0laT+lqS+gp1XANy4cSMzZszghRdeoEOHDvj6+mI2m7l79y67du1ixowZVKhQge7du1O+fHnKly+fq0AF4UlS134BRr3la/25PejP7QGlmlKv/WLHyAShZJGdOBYtWsSECRPo1atXln39+/dn/fr1LFq0iO7du9s0QEHI4DroO7TRKzBcOgo8clfoXBrtnxtR12gtHl0JwlMg++X4rVu3sk0aGXr16sXNmzdtEpQgZEfh4oHk4Ex60pAACWWFWihLeaM7toaUZf9Ds30WhmunMJvErHNBKCi5ejmu0+lynA2u0+lsEpAgWGPWPAAXdxQuHih9q2NOTcS5cximxFvoYvdh+PsQhit/ILl5ofZ/AXXNF1C4Fq33bIJQ2Mm+46hbty5TpkxBr9dn2afX65k8ebKlPIkgFBTnzmEo3cshqZ1wav0yzp3DAFB4lMepxUBch0zHKfAdFO7l0B1fR8ryD0jd+gOGqycxm4x2jl4QigfZdxwjRoxg8ODBbNy4kVatWuHj4wNAXFwchw8fRqPRsHz58gILVBDkkJRq1NWboa7eDNPDOPSx+9GfP4Dm3z+RXD1R12yTfhdSytveoQpCkSU7cQQEBPDLL7/w+eefs2nTpkz7KleuzKxZs6hTp47NAxSEvFKU9sWx2Ys4NOmF4epf6GP3oTsRhe5EFEq/ANT+bVFVbpCliKIgCNbl6l9M8+bN2bp1KzExMZkmANaqVcuuaycIgjWSQoW6amPUVRtjSrqH/vwB9OcPkLZjNpJz6fS7EP+2KEr72jtUQSgSZE8AfPnllwEIDQ2lc+fOBRqUPYkJgDkrTv01m0wYr59Cf24fhn//ArMJZcXa6XchVRrhW65MsemrHMXpZ/skJamvYOcJgMeOHePNN98kICAgVwEIQmEkKRSonmmA6pkGmFIS0u9CYveRtusnJKdSxNdvj6lySxQeYiKrIDxOduLw9fXl/fffL8hYBMEuFK6eODYKwaFhd4w3YtCf28uD3zfB0UiU5WqgrtUOVdUmSCqxMJkgQC6G49avX59Lly5ZbZPxOEsuk8nEokWL6Nq1K3Xr1qVt27ZMmjSJ1NRU2ecwGAwsWbKE3r1706BBAxo3bkzv3r1ZsWJFrmIRBElSoKoUgHOnETwzci4OzfphSn1A2p65JC99j7TDyzDev27vMAXB7mTfcXz22Wd8/fXXhISE0KRJE7y8vLK0uXz5cq4uPmHCBCIiIujUqRPDhg3j0qVLREREEBMTw6JFi1AorOc1nU7H22+/zdGjR+nRowcDBw7EYDBw9epVMYtdyBeVmweODbrhUD8I463z6M/tRR+zB/2ZHSh8q+NQqx2qas2Q1GJlQqHkkZ042rVrB8Du3bttcuELFy6wdOlSOnfunGllwUqVKjF+/Hg2Bl62fQAAIABJREFUbdpEjx49rJ5jzpw5REdHs2DBAlq0aGGTuAThUZKkQFWhFqoKtTClJWH4+zD62L2k7ZsPh5ejfrYF6lrtUHpXtneogvDUyE4cDg4OBAcH57jfbDazdetW2RfeuHEjZrOZoUMzl8Xu378/06ZNIzIy0mriSE1NZcmSJQQGBtKiRQvMZjMpKSm4ueU8EkAQ8kPhVAqHel1Q1+2M8c4F9Of2of/7IPpze1B4V0Ht3xb1sy3+v56WIBRfshNHqVKlmDhxotU2Bw8elH3hM2fOoFAospQpcXR0xN/fn9OnT1s9/vjx46SkpFCnTh3Gjx/P2rVrSU1NxdPTk/79+xMWFoZKJSZ2CbYnSRKqcjVQlauB+fnB6C9Eo4/di/bgYrRHVqCu3hx1rXYofKqK+U1CsST7L2tERMQT2zw+o9yauLg4PD09sy2aWLZsWU6ePGm1qOI///wDwOLFi1Gr1Xz44Yd4eHgQFRVFeHg4d+7cYfLkybLjEYS8kBxdcQjoiLpOIKa7l9PvQi4dQX9+P4oyful3Ic+1RHJ0tXeogmAzT0wcJ06c4M8//0ShUJCUlGS1kKG7u7vsC2s0mhyTgqNj+gvHtLS0HNukpKQA8ODBA6KioqhevToAwcHBvPTSS6xfv57hw4fz7LPPyo4JsDrpJTs+PqVy1b6oK0n9zXVffRtAnQaYtMNJPnuQhyd3oj28FN2xVbjWep7SDTvhWKlmob0LET/b4svW/c0xcRgMBt5//3127tyZaXuPHj2YPHlyvn/5nZ2diY+Pz3afVqsFwMnJKcfjM/bVr1/fkjQy9OrVi2PHjnHs2LFcJw4xczxnJam/+e6rX0sc/VqiuncF/bl9JMdGk3x6LwqPCul3ITVaITkVnvdx4mdbfD3VmePz5s1jx44d1K9fn/r162MymTh58iRRUVHUrVuXl156KVeBPM7X15eLFy9m+zjqzp07OT7GylCuXDkAS5XeR2Vse/jwYb5iFIT8UnpXQdmmCo4tBmC4dAxd7F60R35Fe2w1qqpNUNdqi7K8P5IkkbL+G0wJN3EdMFGsZCgUajkmjg0bNjBixAhGjBiRafvUqVNZt25dvhNHQEAABw8e5NSpUzRp0sSyXavVEhsbm2lbdurWrQvA7du3s+y7c+cOQLZzTQTBHiS1U/rCUv4vYLx/Lf1dyIXDGC4dQXIvi7pmW0wP74Jeg+6PDTi1GfrkkwqCneQ4w+7mzZu88cYbWbYPHz6cGzdu5PvCwcHBSJLE4sWLM21ftWoVGo0m01DcuLg4Ll26hEajsWzz8/OjUaNGnDp1irNnz1q2G41GVq1ahUqlolWrVvmOUxBsTVnGD6dWobiF/oBTu+GYH8ShO7YK0tLvkPXn9pA09xWS5g+3c6SCkL0cE0epUqWyfVTk7u6OWq3O9pjw8HDZF65ZsyZDhgxh+/btjBgxgtWrVzNp0iQmTZpEs2bNMiWO6dOnExwczKlTpzKdY9y4cTg7O/Pqq68ya9YsIiIiCA0N5dSpU7z55ptUqFBBdjyC8LRJKgfUNVrhGvo9Sr/6WRuoHdHsDkd//gCm5PtPP0BByIFNJzpERETw5ptvym4/duxYKlasyMqVK9m7dy+enp6EhoYSFhb2xHIjALVr1+bXX3/lhx9+YPHixWi1WqpXr87EiRPp06dPfroiCE+NwsUDhVsZ0he2TR90oixfE8nVA+ONsxguRqe3cy+HslIdlBXroKrgj+TgYreYhZItx/U4GjZsyOuvv052uxcsWMBrr72WZd+CBQs4ceJEwUT6lIhRVTkrSf192n3VbJ+JIe4SChcPlL7VMacm4tw5DLPZjCnhOsbrZzHciMF4KxYMOpAkFD7VUFWszf+1d97hUVT7/3/NtiSbhDQSShCCeBOqhmKChBohCNIEpCi5KrEhTS4WUEG/ggLeyyNIE/whKCIoUqQpWCiC0hSFEFAJUiRASEgvm92d8/tjkyXLJmE3BIFwXs+zz5Qz58znMzN73nPKnKMNbYa21l0o2sq/B8p7W325Hr2qyhWOxo0bu2cdti9qjx496na8mwkpHOVzO/l7s/oqrBasqcm2ksjfR1AvngAhQGdAW6exXUg0gfXc6jJ/s/p7PbidfIV/uDuun5+fw+CDV0MIwZgxY9wyTiKRuIei1aGrE4GuTgQebfojivKxpBzDevYI1rNJmPbYphNQvGqgDW2KLrQZ2tCmaHxkD0NJ1VGucAQHBxMVFeVWYmV9UyGRSK4fisGIPqwV+rBWAKi56VjPJtmqtc4ewXJ8j+04v9r20oiubuNrGgIlf4NtzDpj74nX7oDklqRc4di4caPbiVUmjkQiqTo0PkFoIjqgj+hQ3D5y1iYgZ5Mw/7Ebc9L3xe0jDe2lEW0t90ZXkEgq3Zr21Vdf0aNHj6q0RSKRVCGKoqANrIc2sB6GFt1LtY8kYTl7hKJfN8HBDaAzYK3fDDUkvFT7iMuTg0puQyotHG+99ZYUDonkFsKxfeQhRFEB1pRjWM4ewXLhGOYTB23Hefpebh+p18ypfURYLagZKaj5mXJolNsUOWGFRHKbohi80IW1RBfWkuBgXy6cPG0rjfydaFsm77Ud51fLXq2lq9sENTdNDo1ymyOFQyKRAKDxDkATHoM+PKa4fSSluH3kCOY/f7S1j5TCfHQb5qPbQKPFe8h/Ubz9ZRXXbYIUjtuUGcttH2q+/GirG2yJ5GbE1j4SijYwFEOLOIRqwXL6EKb9qxEZV4xVp1rJ+/Q/oNGh+Aah8Q1G41MTpUZN27pvMIpvTRRP35t2LhKJe1RaOL744ouqtEMikdzEKBod+rBWWM8cxpxxlpKhUXSNotCHt0fNuYjISUPNSUPNuYgl7RSi8IqPznQedhHR1ChDXORc7bcMLgvHwoULHcahKpkPA2DGjBns3buXN954o8IZAiU3DxarSkp6Hlm5Jvx8PG60OZJbBFGQBUY/h6FRdHe0KPvYogLU3DREtk1MSouL+dwxMBc6RvDwLhaRmjZxKRGUGjVtIqMrf34eyT+LW3OOlzeA4cCBA/Hx8WHy5MmsW7euyoyTXD/SsgopMFlZv/sv4ru7P7yM5PbEK26M/QNAz/b/rvBYxeCFNvAOCLzDKUwIAaY8u6Co2WmI4nXrpb8Rp38Fq8UxPaN/KUG5Qlx8AlE02qpzVFIhVdLG0ahRI0aOHMmnn35aFcndcrz18QFS0vN4+6m2N/3b+zP/3Y7Zqtq3tx1MYdvBFPRaDQtf7HzjDJPcViiKAp4+aD190AY3dAoXQkXkZ6HmXBaUEnGxnv8DS/Ie2xhd9gQ1KD6BpUTliqXRTzbcVyEVCsfcuXPt6/n5+cybN6/M0XKtVivJycn4+Nw8cyj/k9yMb+8Wq8rFzAJS0vI5fynPvtRooHj8bgfMVpWR7+7Ax0uPj5cBX6O+eL34Z9QTWqsGqtmCj9GAr5ceby8dWheGv5dI3EVRNCjeAWi8A6D2v5zChWpB5GZcUQV2ETUnDcvpQ7YqtdJodcVtKsGkBdelSO9nFxWNbzB4eFfLhvvMXBMzP/+NJ3s2rtKX2nJHxwXHEXIVRSlTNEoIDAxk6tSpxMbGVplxNwJ3Rsd95n/bMVtUp/1lvb27WyrJzDXx/peJjOjbvMLjC0wWzl/K51x6HufS84t/eaRmFGAt5UeArwd1gozUCfTmdGoOf/59+Y8VXs+Ppg0DyS0w2375ZnKKl7kFZkzmMpSmGG9PnV1YfL0M9vUSwfEtte1rNGD01KG5xj/oP1HCkyOolo+rz+aNRFiKymlfuQi56aiFuY4R9J6OpZUaV5RY9J43xpFrZNmWY2z/NYXOkXXdeqmt9LDqgH2KWCEEDz/8cLk9qby8vAgMDHTZqJsZd4RD66Fn/qpf2Zt0wWG/v4+BpmGBNK4fQOMG/tT082LcnF1k5RXRpaVrN3DZlmNsP5hC55Z1GRYXQXZeESnp+ZxPzyMl/bJQZOSY7HE0ikJIgBd1gozUrelN7cDLSy+Py4XLuWsOkXw2G38fD+6sW4OsPBOj+pffqaHIbCW3wIze08DplEy7oOTkF5FXYCGnoMhBcHLyzViszoIKoCjg7al3KNH4GvV4ezkKj2+ppZeHzuFt0N1rWRluhHDcyC7S7vhb+tm8WUrY7hAc7Evq2Qv2HmBlioulyDGShw/41ER41wSfIFTvmqjGIFRjIFavAIRGj6oKVCFQVYEQ2NaFQKi2dVEcpgpbnmo7tlSYoII0isNLrV9OwznNr/eeoqxszNUq6WsSjtLMmzePkSNHunLoLY2783HMXLaf7b+mUJKt3VXPDz9vA8dOZ5JbYC43rkaj8FCHhojiG17ykGz88RTqVW6Jh0FLnUCjrQQR5F38MxIS4IVO61rVUWUyKVczFyEERWbVSVBKlnkFJdu28JIwaznXXatR8PbSk51XVGa4RlHo1a4Bep0GrUZjW2oV9NrSSw16rQadVkGn1RT/bOtXHqPVKtSuVYO0tNwyz3e9+KfbyqyqisUiMFtV/PyNXLiQjdmqYrEKLFYVs0W1bVtULFaV9788UuY90mgUBnS683IGVpL5lWRmxduidMZ4RZiqgkA4pWHPPMtM44rtctO4nCkLIRAoWK1qmWnY/noCH6WQIE0ugdpc21KTS1DxeoAmD51y+cVIFZAtjKRbfUhXfbik+jisZ6pGRPmzdFcpimL7PygK9usAYNBpaBUezODYu1x6tio9H8eV3A6iURmy84vw8zY4vL0/91ALVCFIuZjHL39eZMfBs2TkOmZ6qipYvePEVdNXFAj296Jd89o0qutHnSAjAb4eN3V9rKIoeBi0eBi8qOnnWt98IQSFRdZSVWRF5BSXbEp+GdmF/HU+h5z8y4KsUQAE63efrGIfKBYh5bIYaZRS4qSg1WrQaRR0Og06jca2LEOYStZLxOny/tLhGs5fyqfAZOXDzUfpeE/d4ky7OBMvzsBtGfvlDN/isF9gLs7kS2f4Zquwr5cWBNdeGa+OqgpWbUt2vobYMiBFUdBobM+FRlHQKLb9JRlcyfqV20rxsUpJeKk0tBoFvUZz1TSd01AweukxmSxOaWo0VGhHpkYhW1E4pQgM5mw8izLxLMrEo+gSBlMGtQszqG+6hM70FwqXL65QNFg8A7B6BWI1BmE1BiGMQVi9gxDeQSgeNdBoNfZM33ZeLp9bc4UPCo777euOecLHX9uqqRQFzBYVLw9tlb2QuCwcSUlJLFu2jJCQEMaNGwfYRsh95513yM7O5sEHH2Ty5MnodLfXx+jlVfFoFIV6IT7UC/EhM8fkUCppf3cdHukWbr/pSskS29J+wwEENAsLoE+Mc8+T6oSiKHh56PDy0BHiX77YOFwboFNx3a0qBNaSzNOqYrUvbRmlVRX2TNaiCsdlSQZc/KZtsaoYPPRk5xQ67HM85vJ+U4H5iuOuOMai4k4effjEJQ6fuFTBtaK49GQTK32JAOmKS1XFSy+DziZYusviZAu/vK8knQB/LwoKihzT1V0OL0ln/e6T7E26YL/+MS3qMLTrv8rIbLlpX26udzWksFoQuen2xnqRk4a+pCosPQnxd7ZjBK3BsXtxjZK2leL2lUrOnZKdX0SHgPN0CrnEbmM3svJMV4/kIi7n8itXruTAgQOMGDECgDNnzvDSSy/h7e1NTEwMW7dupX79+jz55JNVZtyN4Omnn+D8+fP27T59HmL48KfIz8/nkUcGOhyr12sZMGAIQ4Y8Snp6OgkJ8U7pPf54AtnqvzDqrPzyzULCmtzHd5f+4MtF4wEYMWI03bv34PjxP3nhhbEA+IT3QS3Kw5R6iKhuj5GVV8Thw4eYNGmCU/qvvPI6UVHR7Nu3l7ff/j+n8ClTptOixd3s2LGNd9/9r1P48ejZ3HXXv9iy5SsWLHCe8XHevEWEhtZj3brVLF++FPMVDeWLFy8jKCiIlSuXs3Llcqf4n376BUajkQ8//ID169c6ha9bt7n4PO/xzTdfO4R5enqycuUaAGbOnMEPP+zAJ7wP2ZmZpJ06QERkF7LyagLw9lv/x4ED+xzi16lTlwUL/h8Ar732MomJhx3CGzW6i5kz3wNg/PgxJCcft4fp9VoiIpoydeoMAEaMeJJz51Ic4rdpE8Vrr70BwBNPDCMjwzGz79ChE+NfeBmAIUMGUFhkRlG0oNGiKFpiOnRhyNB4MrILmb5oHTrfO9BodQjVgjnrFK0bejBsyMOYi0yMfG44QrWCaoFiGRoy5NGrPnv9+g3g7Nm/GTnyaafw0s/exInPOd3bceNepFOnLg7PXulns12PBPJNZg7/9nOlnr3//c/1Z2/p0sVO4ZV99vR6LWaz1e1nrzQBAYEsWfIJAFOnvuHWs+ehhZbhDXnh2adQcy6ybeMq9GdPEOx1imCjFm+9Y7VWvgVS88xcLLByMV/lYoEV/9BGDH1qLBrfmgx/KqHsZ2/8y+St+5Kjib/x7aGvySpS+fZjW3i3bg8wcqRtxtZ+/Xo6Xbthwx5jxIjy83KXhWP//v0sXLiQO++8E4DPP/8ci8XC4sWLadasGUePHuXFF1+85YXjejCq/928OvdrCrMvkH/yu6sen/vHevt62zsVOnW6m8OHD11PE28Zcv9Yz98Xit8WU39iVP9bpQpVgGpBYAGrLev31BQRWtOb0JreqIVZKH4NUK1mFI0W1ZSNn6EGwf5e5OcLhKXwqmf4Jyj9bHaI8CAq6m727dt7Ay269TBZ4WKRFl2DSAC2LtlAcvLf9nCjTuG+FhE8Gz8UkZPGjxs/xyhU6nrruCdYi0GrAKnkr3oVgKmttaTmBRQLi5WLBVZqa/NQs1NRc9L4V7A3/f6l4aMjVddm53LjePv27dm1a5d9Oy4ujlq1arFs2TL7vg4dOvDDDz9UmXE3Ancbx2WXzerJP+2ruz3dqhp5b28NhBCIgqzL361kO37DInIvgSi/+zxaPb4JH1z1PFXWOG4wGCgqKsJgMPDbb79x+vRpEhIS7OGqqqKRH4NJJJXinxQJya2LoigoRn8w+pc55a9QrYi8DKwX/6Lo0FeoqcUdcLQGdA1b4dF2SJXY4bJwtGrVitdff53u3bvz3nvv4eXlRc+el+vGVq1aRZ06darEKIlEIpG4j6LRFjey18R6NqlYOBSwmlH0XlU2Y6PLwjFu3DiGDx/O2rVr0Wq1TJo0CV9fX1RVpU+fPiQnJ/Piiy9WiVESiUQiuTZKRjI21KiJCGyAyM+ssrRdFo7Q0FA2bdrE8ePHCQgIoFatWgBoNBomT54MQNOmTavMMIlEIpFUnpKRjDV6HfqrjGTsLm59dKHT6RzGryohKiqqygySSCQSSdVg7D3xunQGcPtrvXXr1vHVV19x6tQpAMLCwujRowd9+/atUsMkEolEcnPisnAUFRUxcuRIp+62J0+eZMeOHWzcuJH58+ej1+ur3EiJRCKR3Dy4LByLFi0iKSmJl156idjYWEJCQgBITU3lu+++48MPP2TRokVyTCuJRCKp5rgsHBs2bGDRokU0a9bMYX9YWBgJCQlERUXxwgsvuCUcqqry8ccfs3LlSs6ePUtgYCA9evRgzJgxGI3Gq8aPj49n3759ZYZ98cUXtGhR9lzIEolEIqk8LgtHXl6ek2iUpkWLFuTl5bl18rfffptly5bRrVs3hg8fTnJyMsuWLSMpKYmlS5e69EFhQEAAEydOdNp/xx3O8xxLJBKJ5NpxWTi0Wi3p6ekEBQWVGZ6WlubWl+N//vknn3zyCXFxccyZc3mAs3r16jF16lQ2bdpE7969r5qO0WiUDfMSiUTyD+JyTt+hQwdGjx5NUlKSU9iRI0cYM2YMnTp1cvnEGzduRAjBY4895rB/0KBBeHl5sX79+nJiOqOqKrm5uRVObSuRSCSSqsHlEsfzzz/Pww8/zIABAwgKCnJoHE9PT6dOnTq89957Lp84MTERjUbD3Xc7jtHj4eFB48aNOXz4cDkxHblw4QItW7aksLAQLy8v2rdvz7hx42jUqJHLtpRGo3FvDgF3j7/VuZ38vZ18hdvL39vJV6j6fM1l4ahZsyarV69m5syZbN261V7y8PX1ZcCAAfznP/9xa97x1NRUAgICMBgMTmG1atXi4MGD9kEVy6NevXq0atWKiIgINBoNv/32G8uXL+enn37i008/JSIiwmV7SggIcG/SlIpGkKyO3E7+3k6+wu3l7+3kK1S9vy4Pq14aIQSXLl1CCEFQUFClZvrq2rUrFouF7du3O4W99NJLfPnll+zfv58aNWq4le6BAweIj4+nbdu2LFmyxG27JBKJRFIxVy1xnDt3jkOHDtmrlWrVqoWiKOU2kruKl5cX6enpZYaZTLYpDj09Pd1Ot02bNrRp04a9e/dSWFhYqTQkEolEUj4VCsf06dNZtmwZqqoCtp5VTz31FGPHjr3mE4eEhHD8+PEyq6MuXLhQbjWWK9SrV499+/aRlZUlhUMikUiqmHJ7Va1YsYKlS5cSEhJCbGwsnTt3JiAggPfff9+tHk/l0bx5c1RV5dAhxylRTSYTx44do3nz5pVO++TJk+h0Ovz9q2bseYlEIpFcplzhWLlyJYMGDeKbb75h3rx5LFiwgO+++47evXuzYsWKaz5xz549URSFjz76yGH/559/TkFBgcM3HKmpqSQnJ1NQUGDfl5OTg9XqPEXi9u3b+eWXX2jXrh0eHh7XbKdEIpFIHCm3cTwyMpLdu3fj7e3YyygtLY1evXqxZ8+eaz75lClT+OSTT+jWrRudOnWyfzneqlUrPvroI/sHhRMmTGDt2rV8/PHHREdHA/Dtt98ybdo0unTpwh133IFOp+PQoUOsX78ePz8/VqxYQcOGDa/ZRolEIpE4Um4bh9FodBINsHXL1Wq1Zcb58ssv3fqK+5VXXiE0NJTPPvuM7du3ExAQwLBhwxgzZsxVv0Jv2LAhzZo1Y/v27aSnp2M2m6lduzZDhgzh2WeftU80JZFIJJKqpdwSR/v27dm1a1eZkcoLqyiORCKRSKoH5ZY4CgsLWbduXZlhJpOpzLCSbrQSiUQiqb6UW+Jo3LhxuR/2CSGcwkr2HT16tOqtvMn466+/mDBhApmZmfj7+zNjxgzCwsJutFlVRkZGBi+99BKnT5/GYDDQoEED3nzzTQIDA6u173PnzmXOnDls2LCB8PDwaumryWTi7bff5qeffsLDw4PIyEimTJlSLX0F2LZtG7Nnz0YIgaqqjB49mri4uGrh74wZM9iyZQtnz561P7NQcf5UZX6LcmjdurVYs2aNy7/Vq1eLNm3alJdctSI+Pl6sW7dOCCHEunXrRHx8/A22qGrJyMgQe/bssW9Pnz5dTJw4UQhRfX1PTEwUCQkJonPnzuL3338XQlRPX6dMmSLeeustoaqqEEKIixcvCiGqp6+qqoo2bdrY7+fRo0dFZGSksFqt1cLf/fv3i5SUFNGlSxe7j0JUfC+ryu9yhSMmJsbtxCoT51YjLS1NtG7dWlgsFiGEEBaLRbRu3Vqkp6ffYMuuH19//bV47LHHqq3vJpNJDBo0SJw+fdr+J6yOvubm5orWrVuL3Nxch/3V0VchbMIRFRUlDhw4IIQQYt++fSIuLq7a+VtaOCryrSr9LreN49tvv3W79FKZOLca586do1atWvaeZVqtlpCQEM6dO+fWII+3CqqqsmLFCmJjY6ut77Nnz6ZPnz4Ok39VR1/PnDmDv78/c+fOZe/evXh7ezN27Fg8PT2rna8AiqIwa9YsnnvuOYxGI3l5eSxcuLBa3tsSKvJNCFFlfpfb57UyQ3XI4T2qH1OmTMFoNDJs2LAbbcp14eDBgxw+fJhHHnnkRpty3bFYLJw5c4amTZuyZs0aXnjhBUaPHk1+fv6NNu26YLFYWLhwIfPnz2fbtm0sWLCAcePGVVt//0lcn7JPAkCdOnW4cOGC/at1q9VKamoqderUucGWVT0zZszg1KlTzJo1C41GUy19379/PydOnOD+++8nNjaW8+fPk5CQwOnTp6udr3Xr1kWn09GrVy8A7rnnHgICAvD09Kx2vgIcPXqU1NRUWrduDUDr1q3x8vLCw8OjWvoLFedPVfn/lcLhJkFBQTRp0oSNGzcCtpkMmzRpcssXca/k3XffJTExkXnz5tkHm6yOvj/99NPs2rWL77//nu+//57atWuzePFievbsWe18DQwMJDo6mt27dwO2Hjbp6emEhYVVO18Bateuzfnz5zlx4gQAycnJpKWl0aBBg2rpL1T8H63K/2+l5uO43UlOTmbChAlkZ2dTo0YNZsyYwZ133nmjzaoy/vzzT3r16kVYWJi9+rFevXrMmzev2vseGxvL+++/T3h4eLX09cyZM7zyyitkZmai0+l4/vnn7cP9VDdfAdavX88HH3xg/3xgzJgxdO3atVr4O3XqVLZu3UpaWhoBAQH4+/uzadOmCn2rKr+lcEgkEonELWRVlUQikUjcQgqHRCKRSNxCCodEIpFI3EIKh0QikUjcQgqHRCKRSNxCCoekTAoKCoiJiaF169ZERETQunVrYmJiaNu2LR06dOCZZ55h1apVFBUVlRl/woQJ9OjRo9zwW5Vjx44RHR3NqlWrbrQpkipgxIgRREVFERERwZo1a260ObcMUjgkZeLl5cXu3bt59dVXAXj11VfZvXs3e/bs4csvv6Rz587MmjWLvn37kpyc7BQ/IyODrKwsLBbLP236daWwsJDc3Fyys7NvtCmSKmDBggXMmTPnRptxy1HuIIcSSXkEBgYydOhQOnbsyODBg3niiSdYvXo1wcHB9mMWLFiA2WzGw8PjBlpa9URGRrJ//36MRuONNkUiuWHIEoek0oSGhjJx4kQuXLjAu+++6xCm0WiqnWiUIEVDcrsjhUNyTXTv3h1fX1/Wr19PYWEhYBu2o6RtZO/evcDlNpOWLVsSERFBUlIS48ePp127drRv355FixYBkJiYyNChQ2nTpg1bq8+NAAAMLElEQVQDBw4kMTHR6ZxCCJYtW0avXr2Iioqibdu2jBw5kt9//91+zI8//khMTAzNmzcnNjaW33//nfj4eKKjo+natStLly51Svfvv/9m/PjxdOnShZiYGLp3787kyZPtNnzyySfExMTQpEkT4uPjneInJibyzDPP0K5dO+677z769+/Phg0bHI5JSEiw16l/8cUXzJ07l27dutGmTRsSEhI4c+aMU7qrVq2ib9++tG/fno4dOxIfH8+yZcsqbD86ceIEMTEx3H333URERHD+/HlGjRpFhw4diIiIcLD/0KFDPPnkk0RFRREVFcVDDz3k1Ibz4IMP2u/pzp07efPNN+nQoQPR0dFMmzYNq9XK6dOnGT58ONHR0fTu3ZsffvihTNs2bdrEwIEDue+++7jvvvt46qmnOHTokMM1at68OREREcTExLB48WL7/Sm5p/feey+bN28GXHseSt+jYcOG0bJlS2JjY5k6dar9uZW4QRXMIyKpxqxevVqEh4eL1atXl3tMfHy8CA8PF/v373eKV3omQSGEeO+990R4eLhISEgQx48fF6qqiiVLlojw8HCxfPly8dZbb4mCggKRk5MjBg0aJDp16iTMZrNDGpMmTRKRkZFi27ZtQlVVkZmZKUaOHCkiIyPFsWPHHI4dNmyYiIqKEqNHjxbp6enCarWK999/X4SHh4utW7fajysqKhJxcXHi5ZdfFvn5+UIIIY4fPy7uv/9+8fLLLzuk2aVLFzFs2DCHfXv27BHNmzcXb7zxhigoKBCqqoq1a9eKxo0bi/nz5zsdGx4eLvr06SM2bNggrFarSElJEbGxsaJ3795O1z8yMlL8+uuvdjtnz54twsPDxZkzZ8q9JyW8/PLLIjw8XDz77LPizz//FEIIMXPmTLv9JXZPnjxZ5OfnC7PZbLd79uzZTraEh4eLwYMHi59//lkIIcSWLVtEeHi4mD9/vnj99ddFdna2MJlM4rnnnhORkZEiMzPTIY2FCxeKxo0bizVr1ghVVUVBQYF44403RPPmzcWPP/5oP+7dd98V4eHh4pdffnGIbzKZRHR0tDh37px9n6vPQ3JysmjZsqWIj48Xly5dElarVWzevFn06dPnqs+4xBFZ4pBcMzVr1gQgNTXV5TidO3emUaNGKIrCI488gl6vZ8aMGYwYMQJPT098fHx4+OGHOXfunMPb6IEDB/jss8/497//TefOnVEUBT8/P6ZMmYLVanWqMgPIzMxkxIgRBAYGotFoeOKJJ9DpdA4TjyUnJ3Py5Eni4uLw8vICoFGjRjz77LMObTdloaoqkydPxmg0MnHiRDw9PVEUhX79+tGxY0fmzJnD6dOnneKFhobSq1cv+5D1vXv35vfff3codXz33Xc0bNiQe+65BwC9Xs+YMWNo1qwZer3e5evdu3dv7rrrLgAGDx7Mo48+arc7ODiY1157DS8vL3Q6Hf369SMuLo5FixZx8eJFp7RatmxJq1atAIiLiyM0NJQ5c+bwyCOP4Ovri8FgYNiwYeTn57Nz5057vL///ptZs2bRsWNHHnroIRRFwdPTk4kTJ2I0Gpk0aZJ9yO/+/fujKApr1651OPe3335Ls2bNqF27NuDe8zBr1izy8vJ45ZVXCAgIQKPR0KNHD5o3b+7ydZTYkMIhuWZUVQWwj0DqCnfffbd93WAw4O/vT2hoKAEBAfb9JZnD+fPn7fu++uorAGJiYhzSCwgI4I477uCnn35y6snl6elJkyZNHM4XEBDgIHT+/v5otVrmzJnDr7/+at8/cOBAxo8fX6EvSUlJnDx5knbt2tmHoC8hNjYWq9XKli1bnOK1bNnSYbtkXoTSdgUFBZGUlMSiRYscenKtWbOGWrVqVWhXae699177emhoKA888IDd7ujoaCcRatGiBWazmT179jil1aJFC4ftkJAQDAYD4eHh9n1l3butW7ditVrp3LmzQ3yDwUD79u05c+YMR44cAaB+/fq0adOGzZs3O1QlrV69mv79+9u3XX0erFYrP/zwA8HBwTRu3Njh2OjoaCcfJRUje1VJrpm0tDSAq76Zl6a0QMDlzLw0JZlZ6Yzj1KlTgG147JIpMEsoLCxEo9GQlZVFUFBQuecqOV9pgalduzaTJk1ixowZDB48mPr169O9e3cGDx7sMKVsWZSUJsryPyQkxOGY0pTnb2m7Ro0axR9//MHMmTOZM2cO7dq1o2/fvsTFxaHTuf73LX09rrT766+/digZlNhgNBrt9/Zqdvv7+5fpS1n3rqLrdOrUKftLRf/+/Zk4cSJbt26lT58+nD9/nsTERObPn++U5tWeB1VVyc/Pp0GDBk7ndue5ldiQwiG5JoqKikhKSsJgMDi9iVZEWaUTjcb1AvCSJUscShEV4Wq6Q4cOpWfPnmzevJmNGzfywQcfsHTpUt555x169uxZbjxRwcwEFYW5YldISAgrV67kl19+YdOmTWzcuJHt27cTGRnJRx995PJ0zRWda+DAgfbvdSqbliu+uHudHnjgAaZMmcLatWvp06cPa9eu5YEHHiizt97VnoeSUpw7pWJJ+ciqKsk1sWnTJvLy8ujbt+8/0v02LCwMgAsXLjiFpaWl2XtxuYsQAqvVip+fH0OHDmX58uV88cUXeHt7M2PGDJdsKquNp6SNoKw3XVewWq0IIWjVqhWTJk1i586dDB48mF9//dWpx5a7VHQtAfbs2cOlS5eu6RyladiwIeD6dTIajTzwwAPs2bOHlJQU1q5dy4ABAxziufo8BAUFYTQay2yzKWufpGKkcEgqzenTp3nnnXeoVasWY8eO/UfOWfLmv3XrVqewuXPnsmTJkkqlu2/fPvr06eOwr0WLFkRHR1/1K/EmTZoQFhbGTz/95NRF9vvvv0er1RIXF1cpux5//HG+/vpr+7aHhwfDhg0DICcnp1JpltCkSRMaNmzI7t27ycvLcwg7dOgQCQkJ9sbqqqBbt25otVp27NjhsL+oqIhdu3ZRv359p4bq/v37o6oqkyZNQq/XO7SNgevPg1arpWPHjly8eJFjx445HLdv375r9u12QwqHxG0uXbrE8uXLGTRoEH5+fixZsuQfqydu1aoVjz76KOvXr2fTpk2oqorVamX16tVs2LCBMWPGVDrt48ePs2LFCntmefToUfbu3cuDDz5YYTyNRsOUKVPIz89n2rRpmEwmhBCsW7eOnTt3Mnr0aOrXr19puz744APOnj0LgMlkYuXKlXh6enL//fdXOk2wVdtMnToVs9nM66+/bhfI48ePM2HCBBISEqr0vtarV4/nn3+enTt3sm7dOoQQmEwmpk+fTl5eHm+++aZTlde9995L/fr12bVrl0OjeAnuPA9jx47FaDQybdo0MjIyUFWVLVu22Odgl7iOnDpWUiYFBQV07drVPjaTj48Pnp6eWK1WdDodTZo0IS4ujr59+5bZkygrK4vc3Fz8/PyIjIxk0aJFdO3alfT0dPLz8wkICGD48OG0atWKsWPHcunSJbRaLX5+fnz22Wd8+OGHbNy4kaysLHx8fGjatCnLli0DbNVKK1euZMWKFVy4cAGj0UhERASjRo2yv7EeO3aMhIQEsrKysFqtBAYG8t///heNRsP48eMdzrd06VLq1KnDypUr+eabb0hJSUEIQY0aNejXrx+PP/44BoOBTz75hAULFjjEnTlzJm3btgXg8OHDzJkzh8OHDyOEoG7dujz22GP07dvXfm1efPFFduzYYfcrMjKSxYsX8/TTT/Pzzz/br9n999/PtGnTOHDgAJ9//jkHDx6koKAAjUZDs2bNGDVqFM2aNSv3/mVmZvLggw+Sk5ODyWSiZs2a3HnnnfZrWJqkpCTee+89Dh48iF6vJygoiCFDhjBkyBB7m0B8fDxJSUl2+3r16sXw4cMZPHiwwzWePXs2v/zyCx9++CEZGRkYjUaCgoIcuj5v3LiRJUuW2K9z8+bNGTNmjFNpooT58+czb948tm/fXqaQufI8lJCYmMj06dM5cuQIfn5+3HfffcTGxjJq1Ch8fHyoXbs2mzZtKve6SmxI4ZBIJBKJW8iqKolEIpG4hRQOiUQikbiFFA6JRCKRuIUUDolEIpG4hRQOiUQikbiFFA6JRCKRuIUUDolEIpG4hRQOiUQikbiFFA6JRCKRuIUUDolEIpG4xf8HaIU9MHCqBlwAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plot(mode2proj2genderdf[\"no-adv\"][\"rlace\"], mode2proj2genderdf[\"no-adv\"][\"inlp\"], \"Dimensions removed\", \"Post-Projection Accuracy\", \"gender-finetuned.pdf\", \n", " get_maj(z_test),\n", " baseline_label=\"Majority\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "plot(mode2proj2genderdf[\"freezed\"][\"rlace\"], mode2proj2genderdf[\"freezed\"][\"inlp\"], \"Dimensions removed\", \"Post-Projection Accuracy\", \"gender-freezed.pdf\", \n", " get_maj(z_test),\n", " baseline_label=\"Majority\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## TPR" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def get_TPR(y_pred, y_true, p2i, i2p, gender):\n", " \n", " scores = defaultdict(Counter)\n", " prof_count_total = defaultdict(Counter)\n", " i2g = {1: \"f\", 0: \"m\"}\n", " \n", " for y_hat, y, g in zip(y_pred, y_true, gender):\n", " if g == \"m\":\n", " g = 0\n", " if g == \"f\":\n", " g = 1\n", " \n", " if y == y_hat:\n", " \n", " scores[i2p[y]][i2g[g]] += 1\n", " \n", " prof_count_total[i2p[y]][i2g[g]] += 1\n", " \n", " tprs = defaultdict(dict)\n", " tprs_change = dict()\n", " tprs_ratio = []\n", " \n", " for profession, scores_dict in scores.items():\n", " if profession == \"model\": continue\n", " \n", " good_m, good_f = scores_dict[\"m\"], scores_dict[\"f\"]\n", " prof_total_f = prof_count_total[profession][\"f\"]\n", " prof_total_m = prof_count_total[profession][\"m\"]\n", " \n", " tpr_m = (good_m) / prof_total_m\n", " tpr_f = (good_f) / prof_total_f\n", " \n", " tprs[profession][\"m\"] = tpr_m\n", " tprs[profession][\"f\"] = tpr_f\n", " tprs_ratio.append(0)\n", " tprs_change[profession] = tpr_f - tpr_m\n", " \n", " return tprs, tprs_change, np.mean(np.abs(tprs_ratio))\n", "\n", "def get_FPR2(y_pred, y_true, p2i, i2p, y_gender):\n", " \n", " fp = defaultdict(Counter)\n", " neg_count_total = defaultdict(Counter)\n", " pos_count_total = defaultdict(Counter) \n", "\n", "\n", " label_set = set(y_true)\n", " # count false positive per gender & class\n", " \n", " for y_hat, y, g in zip(y_pred, y_true, y_gender):\n", " \n", " if y != y_hat:\n", " \n", " fp[y_hat][g] += 1 # count false positives for y_hat\n", " \n", " # count total falses per gender (conditioned on class)\n", " \n", " total_prof_g = defaultdict(Counter)\n", " \n", " # collect POSITIVES for each profession and gender\n", " \n", " for y,g in zip(y_true, y_gender):\n", " total_prof_g[y][g] += 1\n", " \n", " total_m = sum([total_prof_g[y][\"m\"] for y in label_set])\n", " total_f = sum([total_prof_g[y][\"f\"] for y in label_set])\n", " \n", " # calculate NEGATIVES for each profession and gender\n", " \n", " total_false_prof_g = defaultdict(Counter) \n", " for y in label_set:\n", " total_false_prof_g[y][\"m\"] = total_m - total_prof_g[y][\"m\"]\n", " total_false_prof_g[y][\"f\"] = total_f - total_prof_g[y][\"f\"]\n", " \n", " fprs = defaultdict(dict)\n", " fprs_diff = dict()\n", " \n", " for profession, false_pred_dict in fp.items():\n", "\n", " false_male, false_female = false_pred_dict[\"m\"], false_pred_dict[\"f\"]\n", " prof_total_false_for_male = total_false_prof_g[profession][\"m\"]\n", " prof_total_false_for_female = total_false_prof_g[profession][\"f\"]\n", " \n", " ftr_m = false_male/prof_total_false_for_male\n", " ftr_f = false_female/prof_total_false_for_female\n", " fprs[i2p[profession]][\"m\"] = ftr_m\n", " fprs[i2p[profession]][\"f\"] = ftr_f\n", " fprs_diff[i2p[profession]] = ftr_m - ftr_f\n", " \n", " return fprs, fprs_diff\n", " \n", "\n", " \n", "def similarity_vs_tpr(tprs, word2vec, title, measure, prof2fem, plot=False):\n", " \n", " professions = list(tprs.keys())\n", " #\n", " \"\"\" \n", " sims = dict()\n", " gender_direction = word2vec[\"he\"] - word2vec[\"she\"]\n", " \n", " for p in professions:\n", " sim = word2vec.cosine_similarities(word2vec[p], [gender_direction])[0]\n", " sims[p] = sim\n", " \"\"\"\n", " tpr_lst = [tprs[p] for p in professions]\n", " sim_lst = [prof2fem[p] for p in professions]\n", "\n", " #professions = [p.replace(\"_\", \" \") for p in professions if p in word2vec]\n", " \n", " plt.plot(sim_lst, tpr_lst, marker = \"o\", linestyle = \"none\")\n", " plt.xlabel(\"% women\", fontsize = 25)\n", " #plt.ylabel(r'$GAP_{female,y}^{TPR}$', fontsize = 20)\n", " plt.ylabel('TPR-Gap', fontsize = 25)\n", " for p in professions:\n", " x,y = prof2fem[p], tprs[p]\n", " plt.annotate(p , (x,y), size = 7, color = \"red\")\n", " plt.ylim(-0.4, 0.55)\n", " plt.xticks(fontsize=20)\n", " plt.yticks(fontsize=20)\n", "\n", " plt.subplots_adjust(bottom=0.21)\n", " plt.subplots_adjust(left=0.21)\n", " z = np.polyfit(sim_lst, tpr_lst, 1)\n", " p = np.poly1d(z)\n", " plt.plot(sim_lst,p(sim_lst),\"r--\")\n", " \n", " if plot:\n", " plt.savefig(\"analysis-results/{}_vs_bias_{}_bert.pdf\".format(measure, title), dpi = 1000)\n", " print(\"Correlation: {}; p-value: {}\".format(*pearsonr(sim_lst, tpr_lst)))\n", " plt.show()\n", " \n", " return pearsonr(sim_lst, tpr_lst)\n", "\n", "def rms_diff(tpr_diff):\n", "\n", " return np.sqrt((1/(len(tpr_diff)-1)) * np.sum(tpr_diff**2))\n", " \n", "def save_vecs_and_words(vecs, words):\n", " def to_string(arr):\n", " return \"\\t\".join([str(x) for x in arr])\n", " \n", " with open(\"vecs.txt\", \"w\") as f:\n", " for v in vecs:\n", " assert len(v) == 300\n", " f.write(to_string(v) + \"\\n\")\n", " \n", " with open(\"labels.txt\", \"w\") as f:\n", " f.write(\"Profession\\n\")\n", " for w in words:\n", " f.write(w + \"\\n\")\n", "\n", "def count_profs_and_gender(data):\n", " \n", " counter = defaultdict(Counter)\n", " for entry in data:\n", " gender, prof = entry[\"g\"], entry[\"p\"]\n", " counter[prof][gender] += 1\n", " \n", " return counter\n", "\n", "def eval_tpr_rms(P, clf, X, y_gender,train,profs):\n", "\n", " prof2fem = {}\n", " counter = count_profs_and_gender(train)\n", " f,m=0.,0.\n", " for k, values in counter.items():\n", " f += values['f']\n", " m += values['m']\n", " prof2fem[k] = values['f']/(values['f'] + values['m'])\n", " i2p = {w:w for w in set(profs.tolist())}\n", " p2i = i2p\n", "\n", " y_pred_after = clf.predict((P.dot(X.T)).T)\n", " tprs, tprs_change, mean_ratio_after = get_TPR(y_pred_after, profs, p2i, i2p, y_gender)\n", " #similarity_vs_tpr(tprs_change_after, None, \"after\", \"TPR\", prof2fem) \n", " \n", " change_vals = np.array(list((tprs_change.values())))\n", "\n", " return rms_diff(change_vals)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "mode2proj2seed2tpr = defaultdict(dict)\n", "mode2proj2seed2corr = defaultdict(dict)\n", "mode2proj2seed2profacc = defaultdict(dict)\n", "mode2proj2seed2genderacc = defaultdict(dict)\n", "\n", "for mode in [\"freezed\", \"no-adv\", \"mlp-adv\", \"linear-adv\"]:\n", " for projtype in [\"rlace\", \"inlp\", \"none\"]:\n", " if projtype!=\"none\" and mode in [\"mlp-adv\", \"linear-adv\"]: continue\n", " mode2proj2seed2tpr[mode][projtype] = defaultdict(dict)\n", " mode2proj2seed2corr[mode][projtype] = defaultdict(dict)\n", " mode2proj2seed2profacc[mode][projtype] = defaultdict(dict)\n", " mode2proj2seed2genderacc[mode][projtype] = defaultdict(dict)\n", " \n", " num_seeds = 1 if mode == \"freezed\" else 5\n", " for seed in range(num_seeds):\n", " for rank in [1,50,100]:\n", " if projtype != \"none\":\n", " clf = prof_clfs[mode][seed][True][projtype][rank][\"clf\"]\n", " P = mode2p[mode][seed][projtype][0][rank if projtype!=\"inlp\" else rank-1]\n", " gender_clf_acc = gender_clfs[mode][seed][True][projtype][rank][\"test_score\"]\n", " else:\n", " clf = prof_clfs[mode][seed][False][\"clf\"]\n", " gender_clf_acc = gender_clfs[mode][seed][False][\"test_score\"]\n", " P = np.eye(300)\n", " \n", " tpr = eval_tpr_rms(P, clf, mode2x[mode][\"test\"][seed], z_test,test,professions_test)\n", " y_pred = clf.predict(mode2x[mode][\"test\"][seed]@P)\n", " i2p = {w:w for w in set(professions_dev.tolist())}\n", " p2i = i2p\n", " tprs, tprs_change, mean_ratio = get_TPR(y_pred, professions_test, p2i, i2p, z_test)\n", " corr = similarity_vs_tpr(tprs_change, None, \"before\", \"TPR\", prof2fem,plot=False)[0]\n", "\n", " mode2proj2seed2tpr[mode][projtype][seed][rank] = tpr\n", " mode2proj2seed2corr[mode][projtype][seed][rank] = corr\n", " mode2proj2seed2profacc[mode][projtype][seed][rank] = (y_pred==professions_test).mean()\n", " mode2proj2seed2genderacc[mode][projtype][seed][rank] = gender_clf_acc" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ " mode2p[\"freezed\"].keys()" ] }, { "cell_type": "code", "execution_count": 681, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mode: no-adv; proj: none; TPR-gap:\n", " rank 0 1 2 3 4 avg std\n", "0 1 0.119659 0.103819 0.136586 0.121378 0.134014 0.123091 0.011726\n", "1 50 0.119659 0.103819 0.136586 0.121378 0.134014 0.123091 0.011726\n", "2 100 0.119659 0.103819 0.136586 0.121378 0.134014 0.123091 0.011726\n", "---------------\n", "Mode: no-adv; proj: none; Corr:\n", " corr 0 1 2 3 4 avg std\n", "0 1 0.82931 0.829949 0.816137 0.767587 0.810962 0.810789 0.022826\n", "1 50 0.82931 0.829949 0.816137 0.767587 0.810962 0.810789 0.022826\n", "2 100 0.82931 0.829949 0.816137 0.767587 0.810962 0.810789 0.022826\n", "----------------\n", "Mode: no-adv; proj: none; Profession prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.850891 0.852335 0.849884 0.851877 0.851125 0.851222 0.000846\n", "1 50 0.850891 0.852335 0.849884 0.851877 0.851125 0.851222 0.000846\n", "2 100 0.850891 0.852335 0.849884 0.851877 0.851125 0.851222 0.000846\n", "----------------\n", "Mode: no-adv; proj: none; Gender prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.979673 0.950999 0.97042 0.966038 0.977447 0.968915 0.010199\n", "1 50 0.979673 0.950999 0.97042 0.966038 0.977447 0.968915 0.010199\n", "2 100 0.979673 0.950999 0.97042 0.966038 0.977447 0.968915 0.010199\n", "=====================================\n", "Mode: no-adv; proj: rlace; TPR-gap:\n", " rank 0 1 2 3 4 avg std\n", "0 1 0.110860 0.101409 0.129306 0.116696 0.130285 0.117711 0.011011\n", "1 50 0.111851 0.097678 0.127121 0.116460 0.129522 0.116526 0.011474\n", "2 100 0.111221 0.094351 0.130194 0.112013 0.131469 0.115850 0.013770\n", "---------------\n", "Mode: no-adv; proj: rlace; Corr:\n", " corr 0 1 2 3 4 avg std\n", "0 1 0.813383 0.816783 0.800524 0.748158 0.791323 0.794034 0.024682\n", "1 50 0.816766 0.805625 0.810765 0.739430 0.799935 0.794504 0.028094\n", "2 100 0.822013 0.798945 0.800650 0.744381 0.798665 0.792931 0.025813\n", "----------------\n", "Mode: no-adv; proj: rlace; Profession prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.850474 0.851704 0.849803 0.851582 0.851013 0.850915 0.000708\n", "1 50 0.851145 0.852141 0.849752 0.851043 0.850759 0.850968 0.000766\n", "2 100 0.850240 0.851816 0.849162 0.850769 0.850494 0.850496 0.000856\n", "----------------\n", "Mode: no-adv; proj: rlace; Gender prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.551249 0.548107 0.549459 0.532935 0.547974 0.545945 0.006611\n", "1 50 0.533668 0.541792 0.550028 0.528177 0.536240 0.537981 0.007449\n", "2 100 0.539169 0.539311 0.547883 0.543409 0.546795 0.543313 0.003639\n", "=====================================\n", "Mode: no-adv; proj: inlp; TPR-gap:\n", " rank 0 1 2 3 4 avg std\n", "0 1 0.119628 0.104561 0.134804 0.121294 0.133703 0.122798 0.011027\n", "1 50 0.111329 0.097548 0.124686 0.117254 0.127199 0.115603 0.010620\n", "2 100 0.110024 0.097533 0.124994 0.113774 0.122209 0.113707 0.009745\n", "---------------\n", "Mode: no-adv; proj: inlp; Corr:\n", " corr 0 1 2 3 4 avg std\n", "0 1 0.826635 0.833085 0.807683 0.762456 0.810312 0.808034 0.024723\n", "1 50 0.811003 0.814558 0.818097 0.740887 0.784238 0.793757 0.029012\n", "2 100 0.813256 0.819050 0.802136 0.742608 0.810154 0.797441 0.027955\n", "----------------\n", "Mode: no-adv; proj: inlp; Profession prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.850850 0.852233 0.849996 0.851979 0.851033 0.851218 0.000809\n", "1 50 0.850301 0.851908 0.849793 0.851236 0.850454 0.850738 0.000746\n", "2 100 0.849203 0.851053 0.849467 0.850128 0.849549 0.849880 0.000660\n", "----------------\n", "Mode: no-adv; proj: inlp; Gender prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.945386 0.913457 0.942976 0.923625 0.951009 0.935290 0.014289\n", "1 50 0.584682 0.590011 0.590061 0.576039 0.598023 0.587763 0.007246\n", "2 100 0.533668 0.513310 0.540246 0.538264 0.526631 0.530424 0.009754\n", "=====================================\n", "Mode: mlp-adv; proj: none; TPR-gap:\n", " rank 0 1 2 3 4 avg std\n", "0 1 0.128982 0.130225 0.134004 0.121669 0.126006 0.128177 0.004144\n", "1 50 0.128982 0.130225 0.134004 0.121669 0.126006 0.128177 0.004144\n", "2 100 0.128982 0.130225 0.134004 0.121669 0.126006 0.128177 0.004144\n", "---------------\n", "Mode: mlp-adv; proj: none; Corr:\n", " corr 0 1 2 3 4 avg std\n", "0 1 0.818192 0.837786 0.850729 0.862381 0.832336 0.840285 0.015197\n", "1 50 0.818192 0.837786 0.850729 0.862381 0.832336 0.840285 0.015197\n", "2 100 0.818192 0.837786 0.850729 0.862381 0.832336 0.840285 0.015197\n", "----------------\n", "Mode: mlp-adv; proj: none; Profession prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.849223 0.847098 0.849233 0.85022 0.847759 0.848707 0.001124\n", "1 50 0.849223 0.847098 0.849233 0.85022 0.847759 0.848707 0.001124\n", "2 100 0.849223 0.847098 0.849233 0.85022 0.847759 0.848707 0.001124\n", "----------------\n", "Mode: mlp-adv; proj: none; Gender prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.995912 0.996207 0.995831 0.996024 0.994814 0.995758 0.000488\n", "1 50 0.995912 0.996207 0.995831 0.996024 0.994814 0.995758 0.000488\n", "2 100 0.995912 0.996207 0.995831 0.996024 0.994814 0.995758 0.000488\n", "=====================================\n", "Mode: linear-adv; proj: none; TPR-gap:\n", " rank 0 1 2 3 4 avg std\n", "0 1 0.123573 0.131368 0.131643 0.120892 0.117029 0.124901 0.005781\n", "1 50 0.123573 0.131368 0.131643 0.120892 0.117029 0.124901 0.005781\n", "2 100 0.123573 0.131368 0.131643 0.120892 0.117029 0.124901 0.005781\n", "---------------\n", "Mode: linear-adv; proj: none; Corr:\n", " corr 0 1 2 3 4 avg std\n", "0 1 0.806425 0.820909 0.839631 0.838899 0.830614 0.827295 0.012443\n", "1 50 0.806425 0.820909 0.839631 0.838899 0.830614 0.827295 0.012443\n", "2 100 0.806425 0.820909 0.839631 0.838899 0.830614 0.827295 0.012443\n", "----------------\n", "Mode: linear-adv; proj: none; Profession prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.849498 0.846976 0.849833 0.850545 0.849152 0.849201 0.001204\n", "1 50 0.849498 0.846976 0.849833 0.850545 0.849152 0.849201 0.001204\n", "2 100 0.849498 0.846976 0.849833 0.850545 0.849152 0.849201 0.001204\n", "----------------\n", "Mode: linear-adv; proj: none; Gender prediction accuracy:\n", " Acc. 0 1 2 3 4 avg std\n", "0 1 0.992058 0.991448 0.994224 0.991743 0.992252 0.992345 0.000979\n", "1 50 0.992058 0.991448 0.994224 0.991743 0.992252 0.992345 0.000979\n", "2 100 0.992058 0.991448 0.994224 0.991743 0.992252 0.992345 0.000979\n", "=====================================\n", "Mode: freezed; proj: none; TPR-gap:\n", " rank 0 avg std\n", "0 1 0.145584 0.145584 0.0\n", "1 50 0.145584 0.145584 0.0\n", "2 100 0.145584 0.145584 0.0\n", "---------------\n", "Mode: freezed; proj: none; Corr:\n", " corr 0 avg std\n", "0 1 0.813224 0.813224 0.0\n", "1 50 0.813224 0.813224 0.0\n", "2 100 0.813224 0.813224 0.0\n", "----------------\n", "Mode: freezed; proj: none; Profession prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.791436 0.791436 0.0\n", "1 50 0.791436 0.791436 0.0\n", "2 100 0.791436 0.791436 0.0\n", "----------------\n", "Mode: freezed; proj: none; Gender prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.993208 0.993208 0.0\n", "1 50 0.993208 0.993208 0.0\n", "2 100 0.993208 0.993208 0.0\n", "=====================================\n", "Mode: freezed; proj: rlace; TPR-gap:\n", " rank 0 avg std\n", "0 1 0.109742 0.109742 0.0\n", "1 50 0.114757 0.114757 0.0\n", "2 100 0.102170 0.102170 0.0\n", "---------------\n", "Mode: freezed; proj: rlace; Corr:\n", " corr 0 avg std\n", "0 1 0.680319 0.680319 0.0\n", "1 50 0.683391 0.683391 0.0\n", "2 100 0.615245 0.615245 0.0\n", "----------------\n", "Mode: freezed; proj: rlace; Profession prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.788691 0.788691 0.0\n", "1 50 0.782224 0.782224 0.0\n", "2 100 0.772838 0.772838 0.0\n", "----------------\n", "Mode: freezed; proj: rlace; Gender prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.524821 0.524821 0.0\n", "1 50 0.523753 0.523753 0.0\n", "2 100 0.527729 0.527729 0.0\n", "=====================================\n", "Mode: freezed; proj: inlp; TPR-gap:\n", " rank 0 avg std\n", "0 1 0.137030 0.137030 0.0\n", "1 50 0.114868 0.114868 0.0\n", "2 100 0.099702 0.099702 0.0\n", "---------------\n", "Mode: freezed; proj: inlp; Corr:\n", " corr 0 avg std\n", "0 1 0.816106 0.816106 0.0\n", "1 50 0.653213 0.653213 0.0\n", "2 100 0.604284 0.604284 0.0\n", "----------------\n", "Mode: freezed; proj: inlp; Profession prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.790928 0.790928 0.0\n", "1 50 0.768415 0.768415 0.0\n", "2 100 0.719454 0.719454 0.0\n", "----------------\n", "Mode: freezed; proj: inlp; Gender prediction accuracy:\n", " Acc. 0 avg std\n", "0 1 0.989893 0.989893 0.0\n", "1 50 0.528746 0.528746 0.0\n", "2 100 0.532163 0.532163 0.0\n", "=====================================\n" ] } ], "source": [ "for mode in [\"no-adv\", \"mlp-adv\", \"linear-adv\", \"freezed\"]:\n", " for projtype in [\"none\", \"rlace\", \"inlp\"]:\n", " if projtype!=\"none\" and mode in [\"mlp-adv\", \"linear-adv\"]: continue\n", " df = pd.DataFrame(mode2proj2seed2tpr[mode][projtype])\n", " df['avg'] = df.mean(numeric_only=True, axis=1)\n", " df[\"std\"] = df.std(numeric_only=True, axis=1)\n", " df.rename_axis(\"rank\", inplace=True)\n", " df.reset_index(inplace=True)\n", " print(\"Mode: {}; proj: {}; TPR-gap:\".format(mode, projtype))\n", " print(df)\n", " print(\"---------------\")\n", " df = pd.DataFrame(mode2proj2seed2corr[mode][projtype])\n", " df['avg'] = df.mean(numeric_only=True, axis=1)\n", " df[\"std\"] = df.std(numeric_only=True, axis=1)\n", " df.rename_axis(\"corr\", inplace=True)\n", " df.reset_index(inplace=True)\n", " print(\"Mode: {}; proj: {}; Corr:\".format(mode, projtype))\n", " print(df)\n", " print(\"----------------\")\n", " \n", " df = pd.DataFrame(mode2proj2seed2profacc[mode][projtype])\n", " df['avg'] = df.mean(numeric_only=True, axis=1)\n", " df[\"std\"] = df.std(numeric_only=True, axis=1)\n", " df.rename_axis(\"Acc.\", inplace=True)\n", " df.reset_index(inplace=True)\n", " print(\"Mode: {}; proj: {}; Profession prediction accuracy:\".format(mode, projtype))\n", " print(df) \n", " print(\"----------------\")\n", " \n", " df = pd.DataFrame(mode2proj2seed2genderacc[mode][projtype])\n", " df['avg'] = df.mean(numeric_only=True, axis=1)\n", " df[\"std\"] = df.std(numeric_only=True, axis=1)\n", " df.rename_axis(\"Acc.\", inplace=True)\n", " df.reset_index(inplace=True)\n", " print(\"Mode: {}; proj: {}; Gender prediction accuracy:\".format(mode, projtype))\n", " print(df) \n", " print(\"=====================================\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "x=5\n", "print(x)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "seed=0\n", "mode = \"freezed\"\n", "projtype = \"rlace\"\n", "rank=1\n", "\n", "prof2fem = dict()\n", "from collections import Counter\n", "counter = count_profs_and_gender(train)\n", "f,m=0.,0.\n", "for k, values in counter.items():\n", " f += values['f']\n", " m += values['m']\n", " prof2fem[k] = values['f']/(values['f'] + values['m'])\n", " \n", "clf_original = prof_clfs[mode][seed][False][\"clf\"] \n", "clf_finetuned = prof_clfs[mode][seed][True][projtype][rank][\"clf\"] if projtype!=\"none\" else clf_original\n", "P = mode2p[mode][seed][projtype][0][rank if projtype == \"rlace\" else rank-1] if projtype != \"none\" else np.eye(300)\n", "tpr_before = eval_tpr_rms(np.eye(300), clf_original, mode2x[mode][\"test\"][seed], z_test,test, professions_test)\n", "tpr_after = eval_tpr_rms(P, clf_finetuned, mode2x[mode][\"test\"][seed], z_test,test, professions_test)\n", "\n", "#clf = SGDClassifier()\n", "#clf.fit(mode2x[mode][\"train\"][seed]@P, professions_train)\n", "print(\"profession clf score finetuned on projected data\", clf_finetuned.score(mode2x[mode][\"dev\"][seed]@P, professions_dev))\n", "print(\"profession clf score original\", clf_original.score(mode2x[mode][\"dev\"][seed], professions_dev))\n", "print(\"profession clf score original on projected data\", clf_original.score(mode2x[mode][\"dev\"][seed]@P, professions_dev))\n", "\n", "\n", "y_pred = clf_original.predict(mode2x[mode][\"dev\"][seed]@P)\n", "i2p = {w:w for w in set(professions_dev.tolist())}\n", "p2i = i2p\n", "tprs_after, tprs_change_after, mean_ratio_after = get_TPR(y_pred, professions_dev, p2i, i2p, z_dev)\n", "corr_after = similarity_vs_tpr(tprs_change_after, None, \"before\", \"TPR\", prof2fem,plot=False)[0]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "df.rank==32" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\n", "y_pred = clf_original.predict(mode2x[mode][\"dev\"][seed])\n", "i2p = {w:w for w in set(professions_dev.tolist())}\n", "p2i = i2p\n", "tprs_before, tprs_change_before, mean_ratio_before = get_TPR(y_pred, professions_dev, p2i, i2p, z_dev)\n", "corr_before = similarity_vs_tpr(tprs_change_before, None, \"before\", \"TPR\", prof2fem,plot=False)[0]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "tpr_before, tpr_after" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "corr_before, corr_after" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pca = PCA(n_components=2)\n", "X_proj =pca.fit_transform(mode2x[mode][\"test\"][seed])\n", "sn.scatterplot(X_proj[:,0], X_proj[:,1], hue = z_test)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pca = PCA(n_components=2)\n", "P = mode2p[mode][seed][projtype][0][rank if projtype == \"rlace\" else rank-1]\n", "X_proj =pca.fit_transform(mode2x[mode][\"test\"][seed]@P)\n", "sn.scatterplot(X_proj[:,0], X_proj[:,1], hue = z_test)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "clf1 = SGDClassifier()\n", "clf1.fit(mode2x[mode][\"train\"][seed], professions_train)\n", "\n", "clf2 = SGDClassifier()\n", "clf2.fit(mode2x[mode][\"train\"][seed]@P, professions_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds1 = clf1.predict(mode2x[mode][\"dev\"][seed])\n", "preds2 = clf2.predict(mode2x[mode][\"dev\"][seed]@P)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds1" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds2" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "preds2.mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 4 }