from sklearn.metrics import silhouette_score
from sklearn.cluster import KMeans, AgglomerativeClustering
from sklearn.decomposition import PCA
from MulticoreTSNE import MulticoreTSNE as TSNE
import umap
# В основном датафрейме для облегчения последующей кластеризации значения "не голосовал" (2) были приравнены к "воздержался" (0).
for d in df_full.drop(columns=['law_name', 'vote_result']).columns:
    df_full[d] = df_full[d].map(lambda x: 0 if x == 2 else x)
# Транспонирование матрицы для последующей кластеризации
df_t = df_full.transpose().drop(index=['law_name', 'vote_result'])
df_t.iloc[:3, :7]
>>>

X = df_t # Матрица для кластеризации
y = dep_df_cr # Данные о партийной принадлежности
def plot(x1, x2, hue=None, name=None):
    plt.figure(figsize=(10,10))
    sns.scatterplot(x1, x2, hue=hue, palette="bright")
    plt.title(name, fontsize=18)
    plt.show()

pca = PCA(random_state=7)
pca.fit(X)
PCA_transformed = PCA(n_components=2, random_state=7).fit_transform(X)
plot(PCA_transformed[:, 0], PCA_transformed[:, 1], hue=y, name='PCA')

tsne = TSNE(n_jobs=-1, random_state=7)
tsne_transformed = tsne.fit_transform(X)
plot(tsne_transformed[:,0], tsne_transformed[:,1], hue=y, name='t-SNE')

reducer = umap.UMAP(random_state=7)
embedding = reducer.fit_transform(X)
plot(embedding[:, 0], embedding[:, 1], hue=y, name='UMAP')

silhouette = []
for i in range(2,11):
    kmeans = KMeans(n_clusters=i, random_state=7, n_jobs=-1).fit(X)
    labels = kmeans.labels_
    score = silhouette_score(X, labels)
    silhouette.append(score)
plt.plot(range(2,11), silhouette, marker='o')

kmeans_4 = KMeans(n_clusters=4, random_state=7, n_jobs=-1).fit(X)
labels_km_4 = kmeans_4.labels_
plot(tsne_transformed[:,0], tsne_transformed[:,1], hue=labels_km_4, name='t-SNE')

plot(embedding[:, 0], embedding[:, 1], hue=labels_km_4, name='UMAP')

from sklearn.linear_model import LogisticRegressionCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from xgboost.sklearn import XGBClassifier
from lightgbm import LGBMClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import roc_auc_score, roc_curve
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from pymystem3 import Mystem
from joblib import Parallel, delayed
# Распределение голосований
df_full.vote_result.value_counts()
>>>
принят         12770
отклонен        3328
Рейтинговое      151
# Удаление "рейтинговых" голосований
df_lem = df_full.drop(index=df_full[df_full['vote_result'] == 'Рейтинговое'].index)
# Перекодирование таргета в числовую форму
df_lem.vote_result = df_lem.vote_result.map(lambda s: 1 if s=='принят' else 0)
# Список с текстами вопросов голосования
law_list_p = df_lem.law_name.astype('str').tolist()

# Функция лемматизации с объединением текстов в один большой (для увеличения скорости работы pymystem3)
def lemma3(text):
    m = Mystem()
    merged_text = "".join(txt + ' br ' for txt in text)
    doc = ''
    res = []
    for t in m.lemmatize(merged_text):
        if t != '\n':
            if t != 'br':
                doc += t
            else:
                res.append(doc)
                doc = ''
    return res
text_batch = [law_list_p[i: i + 1000] for i in range(0, len(law_list_p), 1000)]
# Лемматизация с использованием всех ядер процессора
law_list_p_clean = Parallel(n_jobs=-1)(delayed(lemma3)(t) for t in text_batch)
# Пересобрание списка с текстами вопросов голосований
law_list_p_clean_list = []
for l in law_list_p_clean:
    for b in l:
        law_list_p_clean_list.append(b)
df_law_clean = pd.DataFrame(law_list_p_clean_list)
# Разбиение датасета на train и test
X_train, X_test, y_train, y_test = train_test_split(
    df_law_clean[0],
    df_lem.vote_result,
    test_size=0.3,
    stratify = df_lem.vote_result,
    random_state = 7
)
# Векторизация текстов
v = TfidfVectorizer(stop_words=stopwords.words('russian'), ngram_range=(1, 2), max_df=0.95, min_df=0.002, norm=None)
X_train_tfid = v.fit_transform(X_train).todense()
X_test_tfid = v.transform(X_test).todense()
models = {}
# Логистическая регрессия
models[0] = LogisticRegressionCV()
# Random Forest
models[1] = RandomForestClassifier(n_estimators=400, min_samples_split=3, random_state=7)
# Градиентный бустинг (sklearn)
models[2] = GradientBoostingClassifier(random_state=7)
# XGB
models[3] = XGBClassifier(n_jobs=-1, random_state=7)
# LightGBM
models[4] = LGBMClassifier(n_jobs=-1, random_state=7)
# kNN
models[5] = KNeighborsClassifier(n_neighbors=3)
for m in models:
    models[m].fit(X_train_tfid, y_train)
model_names = {0: 'log_reg', 1: 'RF_clf', 2: 'GB_clf', 3: 'XGB_clf', 4: 'LGBM_clf', 5: 'knn_clf'}
# Функции вывода метрик классификации
def dataframe_quality_metrics(actual, prediction):
    stats = [
        accuracy_score(actual, prediction),
        precision_score(actual, prediction),
        recall_score(actual, prediction),
        f1_score(actual, prediction)
    ]
    return stats
def metrics_all(models, model_names, y_test):
    measured_quality_metrics = pd.DataFrame({"Test_quality":["Accuracy", "Precision", "Recall", "f1_score"]})
    measured_quality_metrics.set_index("Test_quality")
    y_test_baseline = np.array([1]*len(y_test))
    measured_quality_metrics["baseline"] = dataframe_quality_metrics(y_test_baseline, y_test)
    for m in models:
        measured_quality_metrics[model_names[m]] = dataframe_quality_metrics(models[m].predict(X_test_tfid), y_test)
    return measured_quality_metrics

# Вывод метрик
metrics_all(models=models, model_names=model_names, y_test=y_test)