import os
from google.colab import drive

drive.mount('/content/drive')

session_path = '/content/drive/MyDrive/telethon_sessions/'

os.makedirs(session_path, exist_ok=True)

session_name = 'my_telegram_session'

session_file_path = os.path.join(session_path, session_name)

print(f"Файл сессии будет сохранен по пути: {session_file_path}.session")

!pip install telethon

from telethon.sync import TelegramClient
from telethon.tl.types import Channel

# !!! ЗАМЕНИТЕ НА ВАШИ API ID и API Hash !!!

api_id = 0
api_hash = ''

client = TelegramClient(session_file_path, api_id, api_hash)

try:
    await client.start()
except Exception as e:
    if not client.is_user_authorized():
        phone_number = input('Введите ваш номер телефона (с кодом страны, например +79XXXXXXXXX): ')
        await client.start(phone=phone_number)
        
print(f"Авторизация успешна как: {(await client.get_me()).first_name} {(await client.get_me()).last_name}")

# client.disconnect() # Отключение после парсинга

!pip install numpy==1.26.4 'pandas<1.8'
# ПЕРЕЗАПУСТИТЬ

import pandas as pd
import re
import numpy as np
from matplotlib import pyplot as plt

channel_username = '@m0rtymerr_channel'
safe_channel_name = channel_username.replace('@', '')
output_file_name = f"{safe_channel_name}_posts.json"
output_file_path = os.path.join(session_path, output_file_name)

df=pd.read_json(output_file_path)
df.rename(columns={0:'text'}, inplace = True )
df.head()

Нижний регистр

df["tokens"] = df.text.apply(str.lower)

Удаление эмоджи

!pip install emoji

import emoji
df['tokens'] = df['tokens'].apply(lambda s: emoji.replace_emoji(s, ''))

Удаление цифр

df['tokens'] = df['tokens'].apply(lambda s: re.sub(r"\d+", "", s, flags=re.UNICODE))

Токенизация

!pip install razdel

from razdel import tokenize
def get_tokens(sentence):
  return [_.text.strip() for _ in tokenize(sentence)]

df["tokens"] = df.tokens.apply(get_tokens)

Удаление пунктуации

import string

df["tokens"] = df.tokens.apply(lambda row: [token for token in row if token not in string.punctuation + string.digits + '...'+'—'+'»'+'«'+"–" + '**' + '⁃' + '=)'+'•'+'-'])
df["tokens"] = df["tokens"].apply(lambda row: [token for token in row if not re.match(r'^-+$', token)])

Удаление английских слов

df['tokens'] = df['tokens'].apply( lambda row: [ token for token in row if token == "it" or not re.search(r"[a-zA-Z]+", token)])

Лемматизация

!pip install pymorphy3 pymorphy3-dicts-ru pymorphy2

import pymorphy2
from tqdm.notebook import tqdm

# ну... бывает
def pymorphy2_311_hotfix():
    from inspect import getfullargspec
    from pymorphy2.units.base import BaseAnalyzerUnit

    def _get_param_names_311(klass):
        if klass.__init__ is object.__init__:
            return []
        args = getfullargspec(klass.__init__).args
        return sorted(args[1:])

    setattr(BaseAnalyzerUnit, '_get_param_names', _get_param_names_311)

pymorphy2_311_hotfix()
analyzer = pymorphy2.MorphAnalyzer()

df["tokens"] = tqdm(df.tokens.apply(lambda row: [analyzer.parse(token)[0].normal_form for token in row if token]))

Удаление стоп слов

import nltk
from nltk.corpus import stopwords

nltk.download('stopwords')
nltk.download('punkt')

additional_stops=["это", "который", "наш", "мочь", "год","такой", "мы", "свой", "один", "другой", "человек", "всё", "все", "весь", "очень", "каждый", "день", "её", "ваш", "ваше", "день", "самый", "ещё","также", "нужно","например", "вещь", "хороший", 'новый', "спасибо", 'твой', 'любой','что-то','че','какой-то','какой-то', 'привет', 'час', 'месяц','неделя','сегодня']
om_stops=['сообщество', 'ом', 'антон','назаров']
stops = list(string.ascii_lowercase) + list('абвгдеёжзийклмнопрстуфхцчшщъыьэюя') + stopwords.words("russian") + om_stops + additional_stops

df["tokens"] = df.tokens.apply(lambda row: [token for token in row if token not in stops])

Для дальнейшего использования

all_words = []
for doc in df.tokens.tolist():
  all_words.extend(doc)

import gensim.corpora as corpora
from gensim.models import LdaMulticore, CoherenceModel
from tqdm import tqdm
import warnings
import matplotlib.pyplot as plt

id2word = corpora.Dictionary(df.tokens.tolist())
texts = df.tokens.tolist()
corpus = [id2word.doc2bow(text) for text in texts]

warnings.filterwarnings("ignore")

def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    coherence_values = []
    model_list = []
    for num_topics in tqdm(range(start, limit, step)):
        model=LdaMulticore(corpus=corpus,id2word=dictionary, num_topics=num_topics)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())
    return model_list, coherence_values

model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=texts, start=2, limit=20, step=1)

limit=20; start=2; step=1;
x = range(start, limit, step)

plt.figure(facecolor='black')
ax = plt.gca()
ax.set_facecolor('black')
plt.plot(x, coherence_values, color='#DEB040')
plt.xlabel("Количество тем", color='white')
plt.ylabel("Согласованность", color='white')
plt.tick_params(labelcolor='white')
plt.grid(color='white', linestyle='--', linewidth=0.5)
plt.title("График согласованности", color='white')
plt.show()

!pip install matplotlib pyLDAvis==2.1.2

from gensim.models import LdaMulticore, CoherenceModel
import pyLDAvis.gensim
import pickle
import pyLDAvis
import os

n_topics= 10
lda_model = LdaMulticore(corpus=corpus, id2word=id2word, num_topics=n_topics)

pyLDAvis.enable_notebook()
LDAvis_prepared = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
LDAvis_prepared

from transformers import pipeline

sentiment_pipeline = pipeline("sentiment-analysis", model="blanchefort/rubert-base-cased-sentiment", tokenizer="blanchefort/rubert-base-cased-sentiment")
tokens = df.tokens.tolist()
messages = [" ".join(inner_list) for inner_list in tokens]
sentiment = [item['label'] for item in sentiment_pipeline(messages)]

vis = pd.DataFrame(pd.Series(sentiment).value_counts()).reset_index()
vis.columns = ['sentiment', 'counts']
vis=vis.sort_values(by="counts")

vis_res=vis.drop([0]) # убираем нейтральные сообщения

colors = ["#DEB040", "yellow"]
plt.figure(facecolor='black')
ax = plt.gca()
ax.set_facecolor('black')
plt.tick_params(labelcolor='white')
plt.grid(color='white', linestyle='--', linewidth=0.5)
plt.title("Анализ тональности (кол-во сообщений)", color='white')

plt.barh(y=vis_res.sentiment, width=vis_res.counts, color=colors)
plt.show()

Для второго графика

from matplotlib.ticker import FuncFormatter

positive_counts = []
negative_counts = []
step = 100
 
for i in range(0, len(sent), step):
    inner_list = sent[i:i + step]
    positive_count = inner_list.count('POSITIVE')/len(inner_list)*100
    negative_count = inner_list.count('NEGATIVE')/len(inner_list)*100
    
    positive_counts.append(positive_count)
    negative_counts.append(negative_count)

indices = np.arange(len(sent) / step)

plt.figure(facecolor='black')
ax = plt.gca()
ax.set_facecolor('black')
plt.tick_params(labelcolor='white')
plt.title("Анализ тональности (подсчет по 100 сообщений)", color='white')
ax.set_xticklabels([])
ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: f'{int(x)}%'))
plt.plot(indices, positive_counts, label='Positive', color='#DEB040')
plt.plot(indices, negative_counts, label='Negative', color='yellow')
plt.xticks(indices)
plt.legend(facecolor='black', labelcolor='white')
plt.show()

!pip install wordcloud

from collections import Counter
from wordcloud import WordCloud
from PIL import Image
import requests
from io import BytesIO
import numpy as np
import matplotlib.pyplot as plt

words = dict(Counter(all_words))

url='/content/wolf.png'
if os.path.exists(url):
    image_pil = Image.open(url)
    cloud_mask = np.array(image_pil)

wc = WordCloud(background_color="black", max_words=200, mask=cloud_mask, colormap="Wistia_r")
wc.generate_from_frequencies(words)

plt.figure(figsize=(10, 10))
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.show()

Облако слов по частям речи

def get_words(functors_pos):
  morph=pymorphy2.MorphAnalyzer()
  words=[]
  for word in all_words:
    if morph.parse(word)[0].tag.POS in functors_pos:
      words.append(word)
  words = dict(Counter(words))
  return words

noun = get_words({'NOUN'})
adjf = get_words({'ADJF', 'ADJS'})
verb = get_words({'VERB', 'INFN'})

url = "https://www.pinclipart.com/picdir/middle/560-5604427_money-bag-clip-art-png-download.png"
response = requests.get(url)
cloud_mask = np.array(Image.open(BytesIO(response.content)))

wc = WordCloud(background_color="black", max_words=200, mask=cloud_mask, colormap="Wistia_r")
wc.generate_from_frequencies(words)

wc.generate_from_frequencies(verb)
fig, axs = plt.subplots(nrows= 1 , ncols= 3, figsize=(20, 20))
axs[0].imshow(wc, interpolation="bilinear")
axs[0].axis("off")
wc.generate_from_frequencies(adjf)
axs[1].imshow(wc, interpolation="bilinear")
axs[1].axis("off")
wc.generate_from_frequencies(noun)
axs[2].imshow(wc, interpolation="bilinear")
axs[2].axis("off")

!pip  install  networkx">=2.5"

from collections import Counter
import networkx as nx
import sys
import matplotlib.pyplot as plt
import matplotlib

pairs = []
for doc in df.tokens.tolist():
  if doc:
    b = list((nltk.bigrams(doc)))
    if b:
      pairs.extend(b)

pairs = [tuple(sorted(pair)) for pair in pairs]
word_pairs = dict(Counter(pairs))
word_pairs = [(pair[0], pair[1], val) for pair, val in word_pairs.items() if val > 5]

G = nx.Graph()
edges = word_pairs

G.add_weighted_edges_from(edges)

# Удаляем узлы с степенью менее 2
remove = [node for node, degree in dict(G.degree()).items() if degree < 2]
G.remove_nodes_from(remove)

# Удаляем рёбра, которые соединяют удаленные узлы
remove_edge = [pair for pair in G.edges() if pair[0] in remove and pair[1] in remove]
G.remove_edges_from(remove_edge)

# Удаляем петли
remove = [pair for pair in G.edges() if pair[0] == pair[1]]
G.remove_edges_from(remove)

node_sizes = [deg*50 for node, deg in dict(G.degree()).items()]

# Строим график
plt.figure(figsize=(40,40))
pos = nx.layout.spring_layout(G)
edges, weights = zip(*nx.get_edge_attributes(G,'weight').items())
for key in pos:
    x, y = pos[key]
    plt.text(x,y,key, ha="center", va="center", fontsize=6)
nx.draw(G, pos, node_color='#DEB040', edgelist=list(G.edges()), edge_color=range(len(G.edges())), width=1.0, with_labels=False, edge_cmap=plt.cm.Blues, node_size=node_sizes)
plt.show()

Граф с узлами цвета тем

def get_color(node):
    color_map = {
        9: '#526DC0',
        8: '#FFFACD',
        7: '#FFB6C1',
        6: '#98FF98',
        5: '#FFDAB9',
        4: '#E6E6FA',
        3: '#FFE5B4',
        2: '#F5F5DC',
        1: '#D3D3D3',
        0: '#ADD8E6'
    }

    for topic_index, color in color_map.items():
        if node in topics[topic_index][1]:
            return color

    return '#FFFFF0'

topics=lda_model.print_topics(num_words=100)
color_map = [get_color(node) for node in G]

plt.figure(figsize=(30,30))
edges, weights = zip(*nx.get_edge_attributes(G,'weight').items())
nx.draw(G, pos, edgelist=list(G.edges()), edge_color=range(len(G.edges())), width=1.0, with_labels=False, edge_cmap=plt.cm.Blues, node_size=node_sizes, node_color=color_map)
for key in pos:
    x, y = pos[key]
    plt.text(x,y,key, ha="center", va="center", fontsize=6)
plt.show()

Я не упоминала ОМ всуе