Рекурентні нейронні мережі
У цьому занятті буде розглянуто рекурентні нейронні мережі та їхні відмінності від лінійних моделей. Детально проаналізуємо архітектури LSTM і GRU та покажемо, як за їх допомогою розв’язувати практичні задачі роботи з послідовними даними.
Code and results
Published version after Merge. Results were saved by the author; code does not run on this page.
HTML, SVG, віджети та інтерактивні результати не переносяться. Збережено безпечний текст або зображення.
Python · 1
import torch
from torch import nn
import pandas as pd
import kagglehub
from tqdm import tqdm
import os
from string import punctuation
import nltk
from nltk.corpus import stopwords
from nltk import FreqDist
from bs4 import BeautifulSoup
device = "cuda" if torch.cuda.is_available() else "cpu"Python · 2
path = kagglehub.dataset_download("lakshmi25npathi/imdb-dataset-of-50k-movie-reviews")Results saved by the author in Colab
Using Colab cache for faster access to the 'imdb-dataset-of-50k-movie-reviews' dataset.
Python · 3
df = pd.read_csv(os.path.join(path, os.listdir(path)[0]), sep=",")
df.head(3)Results saved by the author in Colab
review sentiment 0 One of the other reviewers has mentioned that ... positive 1 A wonderful little production. <br /><br />The... positive 2 I thought this was a wonderful way to spend ti... positive
Python · 4
df.isna().sum()Results saved by the author in Colab
review 0 sentiment 0 dtype: int64
Python · 5
df["sentiment"].value_counts()Results saved by the author in Colab
sentiment positive 25000 negative 25000 Name: count, dtype: int64
Python · 6
puncts = set(punctuation)
nltk.download("stopwords")
sw = set(stopwords.words("english"))
def preprocess_text(sample):
text = BeautifulSoup(sample, "html.parser").get_text(separator=" ")
text = text.lower()
text = "".join([c for c in text if c not in puncts])
text = text.split()
text = [w for w in text if w not in sw]
text = " ".join(text)
return textResults saved by the author in Colab
[nltk_data] Downloading package stopwords to /root/nltk_data... [nltk_data] Unzipping corpora/stopwords.zip.
Python · 7
tqdm.pandas()
df["review"] = df["review"].progress_apply(preprocess_text)Results saved by the author in Colab
100%|██████████| 50000/50000 [00:13<00:00, 3600.40it/s]
Python · 8
corpus = " ".join(df["review"])
tokens_dictionary = FreqDist(corpus.split()).most_common(10000)
vocab = {word[0]: idx + 2 for idx, word in enumerate(tokens_dictionary)}
vocab["<pad>"] = 0
vocab["<unk>"] = 1Python · 9
class DataWrapper(torch.utils.data.Dataset):
def __init__(self, reviews, labels, vocab, max_len=256):
super().__init__()
self.reviews = reviews
self.labels = labels
self.vocab = vocab
self.max_len = max_len
def __len__(self):
return len(self.labels)
def __getitem__(self, index):
review, label = self.reviews[index].split(), self.labels[index]
tokens = [self.vocab.get(token, 1) for token in review] # 1 - <unk>
tokens = torch.tensor(tokens, dtype=torch.long)[:self.max_len]
if label == "positive":
label = 1
else:
label = 0
label = torch.LongTensor([label])
return tokens, labelPython · 10
dataset = DataWrapper(df["review"], df["sentiment"], vocab)
train_dataset, valid_dataset = torch.utils.data.random_split(
dataset, [0.9, 0.1]
)Python · 11
def collate_fn(batch):
tokens, labels = zip(*batch)
B = len(tokens)
L = [t.size(0) for t in tokens]
M = max(L)
tokens_padded = torch.zeros(B, M, dtype=torch.long)
for i, t in enumerate(tokens):
tokens_padded[i, :t.size(0)] = t
labels = torch.cat(labels, dim=0)
return tokens_padded, labelsPython · 12
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=64,
shuffle=True,
collate_fn=collate_fn
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset,
batch_size=16,
shuffle=False,
collate_fn=collate_fn
)Python · 13
src, tgt = next(iter(train_loader))
print(src.shape, tgt.shape)Results saved by the author in Colab
torch.Size([64, 256]) torch.Size([64])
Python · 14
class ManualLSTMCell(nn.Module):
"""Одна комірка LSTM із явно реалізованими gates."""
def __init__(self, input_size, hidden_size):
super().__init__()
self.hidden_size = hidden_size
# Для підвищення ефективності чотири gates обчислюються разом.
self.x2gates = nn.Linear(input_size, 4 * hidden_size)
self.h2gates = nn.Linear(hidden_size, 4 * hidden_size)
def _step_from_gates(self, input_gates, h_prev, c_prev):
# input_gates вже містить x_t @ W_x + b_x.
gates = input_gates + self.h2gates(h_prev)
# Порядок gates: вхід, забування, комірка-кандидат, вихід.
input_gate, forget_gate, candidate, output_gate = gates.chunk(4, dim=-1)
input_gate = torch.sigmoid(input_gate)
forget_gate = torch.sigmoid(forget_gate)
candidate = torch.tanh(candidate)
output_gate = torch.sigmoid(output_gate)
# Оновити довгострокову пам'ять (cell state) та короткострокову пам'ять (hidden state).
c_t = forget_gate * c_prev + input_gate * candidate
h_t = output_gate * torch.tanh(c_t)
return h_t, c_t
def forward(self, x_t, h_prev, c_prev):
"""Обробка одного часового кроку: x_t має розмір (batch_size, input_size)."""
input_gates = self.x2gates(x_t)
return self._step_from_gates(input_gates, h_prev, c_prev)
def forward_sequence(self, x, reverse=False):
"""Обробити повну послідовність: x має форму (batch, seq_len, input_size)."""
batch_size, seq_len, _ = x.shape
if seq_len == 0:
raise ValueError("The input sequence must contain at least one time step.")
h = x.new_zeros(batch_size, self.hidden_size)
c = x.new_zeros(batch_size, self.hidden_size)
# Лише проекцію «hidden-to-gate» потрібно обчислювати рекурсивно.
all_input_gates = self.x2gates(x)
outputs = [None] * seq_len
time_steps = range(seq_len - 1, -1, -1) if reverse else range(seq_len)
for t in time_steps:
h, c = self._step_from_gates(all_input_gates[:, t, :], h, c)
outputs[t] = h
return torch.stack(outputs, dim=1)
class ManualLSTMLayer(nn.Module):
"""Один повторюваний шар; за вибором працює в обох напрямках."""
def __init__(self, input_size, hidden_size, bidirectional=True):
super().__init__()
self.bidirectional = bidirectional
self.forward_cell = ManualLSTMCell(input_size, hidden_size)
if bidirectional:
self.backward_cell = ManualLSTMCell(input_size, hidden_size)
def forward(self, x):
forward_output = self.forward_cell.forward_sequence(x)
if not self.bidirectional:
return forward_output
backward_output = self.backward_cell.forward_sequence(x, reverse=True)
# (batch, seq_len, 2 * hidden_size)
return torch.cat((forward_output, backward_output), dim=-1)
class ManualLSTM(nn.Module):
"""Багаторівнева мережа LSTM, побудована з екземплярів ManualLSTMCell."""
def __init__(
self,
input_size,
hidden_size,
num_layers=1,
bidirectional=False,
dropout=0.0,
):
super().__init__()
if num_layers < 1:
raise ValueError("num_layers must be at least 1.")
if not 0.0 <= dropout < 1.0:
raise ValueError("dropout must be in the interval [0, 1).")
self.layers = nn.ModuleList()
layer_input_size = input_size
layer_output_size = hidden_size * (2 if bidirectional else 1)
for _ in range(num_layers):
self.layers.append(
ManualLSTMLayer(
input_size=layer_input_size,
hidden_size=hidden_size,
bidirectional=bidirectional,
)
)
layer_input_size = layer_output_size
# Dropout застосовується між рекурентними шарами, але не після останнього шару.
self.inter_layer_dropout = nn.Dropout(dropout)
def forward(self, x):
for layer_index, layer in enumerate(self.layers):
x = layer(x)
if layer_index < len(self.layers) - 1:
x = self.inter_layer_dropout(x)
return x
class Net(nn.Module):
def __init__(self, vocab_size, d_model):
super().__init__()
self.embedding = nn.Embedding(vocab_size, d_model, padding_idx=0)
self.lstm = ManualLSTM(
input_size=d_model,
hidden_size=d_model,
bidirectional=True,
dropout=0.3,
num_layers=3,
)
self.output_fc = nn.Linear(d_model * 2, 1)
def forward(self, x):
# (batch_size, seq_len) -> (batch_size, seq_len, d_model)
x = self.embedding(x)
# (batch_size, seq_len, d_model * 2)
lstm_out = self.lstm(x)
# Mean pooling over the sequence dimension.
lstm_out = torch.mean(lstm_out, dim=1)
return self.output_fc(lstm_out)Python · 15
model = Net(
vocab_size = len(vocab),
d_model = 128
).to(device)Python · 16
sum([param.numel() for param in model.parameters()])Results saved by the author in Colab
2335233
Python · 17
criterion = nn.BCEWithLogitsLoss()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)Python · 18
def train(loader):
model.train()
total_loss = 0
total_accuracy = 0
for batch in tqdm(loader):
optimizer.zero_grad()
src, tgt = batch[0].to(device), batch[1].to(device)
outputs = model(src).squeeze(1) # 64, 1 -> 64
loss = criterion(outputs, tgt.float())
total_loss += loss.item()
loss.backward()
optimizer.step()
probs = torch.sigmoid(outputs)
predicted_labels = (probs >= 0.5).int()
accuracy = (predicted_labels == tgt).float().mean() * 100
total_accuracy += accuracy.item()
return total_loss / len(loader), total_accuracy / len(loader)
def evaluate(loader):
model.eval()
total_loss = 0
total_accuracy = 0
with torch.no_grad():
for batch in tqdm(loader):
src, tgt = batch[0].to(device), batch[1].to(device)
outputs = model(src).squeeze(1)
loss = criterion(outputs, tgt.float())
total_loss += loss.item()
probs = torch.sigmoid(outputs)
predicted_labels = (probs >= 0.5).int()
accuracy = (predicted_labels == tgt).float().mean() * 100
total_accuracy += accuracy.item()
return total_loss / len(loader), total_accuracy / len(loader)Python · 19
epochs = 5
for epoch in range(epochs):
loss, acc = train(train_loader)
val_loss, val_acc = evaluate(valid_loader)
print(f"epoch: {epoch + 1} | loss: {loss:.3f} | accuracy: {acc:.3f} | val_loss: {val_loss:.3f} | val_accuracy: {val_acc:.3f}")Results saved by the author in Colab
100%|██████████| 704/704 [16:18<00:00, 1.39s/it] 100%|██████████| 313/313 [01:26<00:00, 3.61it/s]
epoch: 1 | loss: 0.552 | accuracy: 69.871 | val_loss: 0.431 | val_accuracy: 80.431
100%|██████████| 704/704 [16:11<00:00, 1.38s/it] 100%|██████████| 313/313 [01:24<00:00, 3.71it/s]
epoch: 2 | loss: 0.398 | accuracy: 82.400 | val_loss: 0.364 | val_accuracy: 83.826
100%|██████████| 704/704 [16:06<00:00, 1.37s/it] 100%|██████████| 313/313 [01:23<00:00, 3.73it/s]
epoch: 3 | loss: 0.340 | accuracy: 85.556 | val_loss: 0.333 | val_accuracy: 86.182
100%|██████████| 704/704 [16:06<00:00, 1.37s/it] 100%|██████████| 313/313 [01:24<00:00, 3.71it/s]
epoch: 4 | loss: 0.300 | accuracy: 87.518 | val_loss: 0.309 | val_accuracy: 87.001
100%|██████████| 704/704 [16:04<00:00, 1.37s/it] 100%|██████████| 313/313 [01:22<00:00, 3.79it/s]
epoch: 5 | loss: 0.274 | accuracy: 88.792 | val_loss: 0.310 | val_accuracy: 87.520
Python · 20
model.eval()
def inference(
input_review,
vocab=vocab,
model=model,
device=device
):
input_review = preprocess_text(input_review).split()
tokens = [vocab.get(token, 1) for token in input_review][:256]
tokens = torch.tensor(tokens, dtype=torch.long).to(device)
tokens = tokens.unsqueeze(0)
with torch.no_grad():
output = model(tokens)
prob = torch.sigmoid(output).item()
sentiment = 1 if prob >= 0.5 else 0
return {
"sentiment": "positive" if sentiment == 1 else "negative"
}Python · 21
inference(df["review"].iloc[-33])Results saved by the author in Colab
{'sentiment': 'positive'}Python · 22
df["sentiment"].iloc[-33]Results saved by the author in Colab
'positive'
Python · 23
inference(df["review"].iloc[-333])Results saved by the author in Colab
{'sentiment': 'negative'}Python · 24
df["sentiment"].iloc[-333]Results saved by the author in Colab
'negative'
Community reviews
Approved. Good work.
