penkovsky_sa/llm-arch-research
1
0
Fork 0
mirror of https://github.com/pese-git/llm-arch-research.git synced 2026-01-23 13:00:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Рефакторинг: единообразие оформления кода (пробелы, кавычки, пустые строки), без изменения логики по всему проекту.

Browse Source
This commit is contained in:
Sergey Penkovsky
2025-10-06 22:57:19 +03:00
parent 332cad6159
commit 712278e33c
49 changed files with 2324 additions and 2004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
experiments/hf_integration/generate_with_hf_tools.py
View File

@@ -14,12 +14,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from hf_proxy import HFAdapter, HFTokenizerAdapter, create_hf_pipeline
from shared.configs import (
TEST_PROMPTS, GENERATION_CONFIG, PATHS
)
from shared.data import (
print_experiment_info, ensure_directories, ExperimentLogger
)
from shared.configs import TEST_PROMPTS, GENERATION_CONFIG, PATHS
from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
def load_hf_model_and_tokenizer() -> tuple:
@@ -41,9 +37,7 @@ def load_hf_model_and_tokenizer() -> tuple:
)
if not os.path.exists(tokenizer_path):
raise FileNotFoundError(
f"Токенизатор не найден: {tokenizer_path}"
)
raise FileNotFoundError(f"Токенизатор не найден: {tokenizer_path}")
# Загружаем адаптированный токенизатор
print("🔧 Загрузка адаптированного токенизатора...")
@@ -52,8 +46,9 @@ def load_hf_model_and_tokenizer() -> tuple:
# Загружаем конфигурацию модели
import json
config_path = os.path.join(model_path, "config.json")
with open(config_path, 'r', encoding='utf-8') as f:
with open(config_path, "r", encoding="utf-8") as f:
model_config = json.load(f)
# Загружаем модель через HFAdapter с правильной конфигурацией
@@ -62,6 +57,7 @@ def load_hf_model_and_tokenizer() -> tuple:
# Создаем конфигурацию из сохраненного config.json
from hf_proxy import HFAdapterConfig
hf_config = HFAdapterConfig(
vocab_size=model_config["vocab_size"],
hidden_size=model_config["hidden_size"],
@@ -69,7 +65,9 @@ def load_hf_model_and_tokenizer() -> tuple:
num_attention_heads=model_config["num_attention_heads"],
max_position_embeddings=model_config["max_position_embeddings"],
hidden_dropout_prob=model_config.get("hidden_dropout_prob", 0.1),
attention_probs_dropout_prob=model_config.get("attention_probs_dropout_prob", 0.1),
attention_probs_dropout_prob=model_config.get(
"attention_probs_dropout_prob", 0.1
),
)
hf_model = HFAdapter.from_pretrained(model_bin_path, hf_config=hf_config)
@@ -97,7 +95,7 @@ def test_hf_pipeline(hf_model, hf_tokenizer):
device="cpu",
max_length=50,
do_sample=True,
temperature=0.7
temperature=0.7,
)
print("✅ HuggingFace pipeline создан")
@@ -132,8 +130,10 @@ def generate_with_hf_model(hf_model, hf_tokenizer, prompt: str, config: dict) ->
str: Сгенерированный текст
"""
print(f"🔤 Промпт: '{prompt}'")
print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}")
print(
f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}"
)
# Кодируем через адаптированный токенизатор
inputs = hf_tokenizer(prompt, return_tensors="pt")
@@ -144,12 +144,12 @@ def generate_with_hf_model(hf_model, hf_tokenizer, prompt: str, config: dict) ->
# Генерируем через адаптированную модель
with torch.no_grad():
generated_ids = hf_model.generate(
input_ids=inputs['input_ids'],
input_ids=inputs["input_ids"],
max_new_tokens=config["max_new_tokens"],
do_sample=config["do_sample"],
temperature=config["temperature"],
top_k=config["top_k"],
top_p=config["top_p"]
top_p=config["top_p"],
)
# Декодируем через адаптированный токенизатор
@@ -174,23 +174,29 @@ def test_different_hf_strategies(hf_model, hf_tokenizer, prompt: str):
{"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
{"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
{"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
{"name": "❄️ Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
{
"name": "❄️ Детерминированная (temp=0.3)",
"do_sample": True,
"temperature": 0.3,
},
]
for strategy in strategies:
print(f"\n{strategy['name']}:")
try:
config = GENERATION_CONFIG.copy()
config.update({
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20
})
config.update(
{
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20,
}
)
generated = generate_with_hf_model(hf_model, hf_tokenizer, prompt, config)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f" 📤 Промпт: '{prompt}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
print(f" 📄 Полный текст: '{generated}'")
@@ -215,7 +221,7 @@ def analyze_hf_tokenization(hf_tokenizer, texts: list):
# Токенизация через адаптер
inputs = hf_tokenizer(text, return_tensors="pt")
tokens = inputs['input_ids'].tolist()[0]
tokens = inputs["input_ids"].tolist()[0]
token_strings = hf_tokenizer.tokenize(text)
print(f" Токены (ID): {tokens}")
@@ -247,7 +253,7 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
try:
user_input = input("\n🔤 Введите промпт: ").strip()
if user_input.lower() in ['exit', 'quit', 'выход']:
if user_input.lower() in ["exit", "quit", "выход"]:
break
if not user_input:
@@ -258,7 +264,7 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
temperature = float(input("🌡️ Температура [0.7]: ") or "0.7")
do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
do_sample = do_sample_input != 'n'
do_sample = do_sample_input != "n"
except:
max_tokens = 50
temperature = 0.7
@@ -266,15 +272,19 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
print("⚠️ Использую параметры по умолчанию")
config = GENERATION_CONFIG.copy()
config.update({
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample
})
config.update(
{
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample,
}
)
generated = generate_with_hf_model(hf_model, hf_tokenizer, user_input, config)
generated = generate_with_hf_model(
hf_model, hf_tokenizer, user_input, config
)
generated_part = generated[len(user_input):]
generated_part = generated[len(user_input) :]
print(f"\n🎯 Результат:")
print(f" 📤 Промпт: '{user_input}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
@@ -295,7 +305,7 @@ def main():
"model": "GPT через HFAdapter",
"tokenizer": "BPE через HFTokenizerAdapter",
"инструменты": "HuggingFace pipeline & генерация",
"стратегия": "интеграция с HF экосистемой"
"стратегия": "интеграция с HF экосистемой",
}
print_experiment_info(experiment_name, experiment_config)
@@ -310,7 +320,7 @@ def main():
analysis_texts = [
"Искусственный интеллект",
"Нейронные сети",
"Машинное обучение"
"Машинное обучение",
]
analyze_hf_tokenization(hf_tokenizer, analysis_texts)
@@ -326,10 +336,12 @@ def main():
print("-" * 40)
try:
generated = generate_with_hf_model(hf_model, hf_tokenizer, prompt, GENERATION_CONFIG)
generated = generate_with_hf_model(
hf_model, hf_tokenizer, prompt, GENERATION_CONFIG
)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f"📤 Промпт: '{prompt}'")
print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -365,6 +377,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

72
experiments/hf_integration/simple_hf_training.py
View File

@@ -19,8 +19,12 @@ from llm.tokenizers import BPETokenizer
from hf_proxy import HFAdapter, HFTokenizerAdapter
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TRAINING_CONFIG, PATHS, TEST_PROMPTS
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TRAINING_CONFIG,
PATHS,
TEST_PROMPTS,
)
@@ -45,18 +49,15 @@ def create_dataset(hf_tokenizer, texts, max_length=128):
max_length=max_length,
truncation=True,
padding=False,
return_tensors="pt"
return_tensors="pt",
)
input_ids = inputs['input_ids'][0]
input_ids = inputs["input_ids"][0]
# Создаем метки для языкового моделирования
labels = input_ids.clone()
dataset.append({
'input_ids': input_ids,
'labels': labels
})
dataset.append({"input_ids": input_ids, "labels": labels})
return dataset
@@ -84,10 +85,7 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
print(f"📊 Данные: {len(train_dataset)} train, {len(val_dataset)} validation")
# Оптимизатор
optimizer = torch.optim.AdamW(
hf_model.parameters(),
lr=config["learning_rate"]
)
optimizer = torch.optim.AdamW(hf_model.parameters(), lr=config["learning_rate"])
# Функция потерь
loss_fn = nn.CrossEntropyLoss()
@@ -105,8 +103,8 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
for i, batch in enumerate(train_dataset):
optimizer.zero_grad()
input_ids = batch['input_ids'].unsqueeze(0) # [1, seq_len]
labels = batch['labels'].unsqueeze(0) # [1, seq_len]
input_ids = batch["input_ids"].unsqueeze(0) # [1, seq_len]
labels = batch["labels"].unsqueeze(0) # [1, seq_len]
# Forward pass
outputs = hf_model(input_ids=input_ids, labels=labels)
@@ -130,8 +128,8 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
epoch_val_loss = 0
with torch.no_grad():
for batch in val_dataset:
input_ids = batch['input_ids'].unsqueeze(0)
labels = batch['labels'].unsqueeze(0)
input_ids = batch["input_ids"].unsqueeze(0)
labels = batch["labels"].unsqueeze(0)
outputs = hf_model(input_ids=input_ids, labels=labels)
epoch_val_loss += outputs.loss.item()
@@ -143,10 +141,10 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
hf_model.train()
return {
'train_losses': train_losses,
'val_losses': val_losses,
'final_train_loss': train_losses[-1],
'final_val_loss': val_losses[-1]
"train_losses": train_losses,
"val_losses": val_losses,
"final_train_loss": train_losses[-1],
"final_val_loss": val_losses[-1],
}
@@ -170,10 +168,10 @@ def test_generation_after_training(hf_model, hf_tokenizer, test_prompts):
with torch.no_grad():
generated = hf_model.generate(
input_ids=inputs['input_ids'],
input_ids=inputs["input_ids"],
max_new_tokens=20,
do_sample=True,
temperature=0.8
temperature=0.8,
)
generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
@@ -192,7 +190,9 @@ def main():
try:
# === Подготовка данных ===
print("🔧 Подготовка данных...")
train_texts = TRAIN_TEXTS[:10] # Используем меньше данных для быстрого тестирования
train_texts = TRAIN_TEXTS[
:10
] # Используем меньше данных для быстрого тестирования
val_texts = TRAIN_TEXTS[10:12]
print(f"📊 Данные: {len(train_texts)} train, {len(val_texts)} validation")
@@ -203,7 +203,7 @@ def main():
llm_tokenizer.train(
texts=train_texts,
vocab_size=BPE_CONFIG["vocab_size"],
special_tokens=BPE_CONFIG["special_tokens"]
special_tokens=BPE_CONFIG["special_tokens"],
)
hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -227,7 +227,7 @@ def main():
training_config = {
"learning_rate": TRAINING_CONFIG["learning_rate"],
"num_epochs": 2, # Меньше эпох для быстрого тестирования
"batch_size": TRAINING_CONFIG["batch_size"]
"batch_size": TRAINING_CONFIG["batch_size"],
}
results = manual_training_loop(
@@ -255,20 +255,23 @@ def main():
# Сохраняем модель
HFAdapter.save_pretrained(
hf_model,
"checkpoints/hf_simple_trained",
tokenizer=hf_tokenizer
hf_model, "checkpoints/hf_simple_trained", tokenizer=hf_tokenizer
)
print("✅ Модель сохранена")
# Сохраняем результаты
results_path = "checkpoints/simple_training_results.json"
with open(results_path, 'w', encoding='utf-8') as f:
json.dump({
'training_config': training_config,
'model_config': model_config,
'results': results
}, f, indent=2, ensure_ascii=False)
with open(results_path, "w", encoding="utf-8") as f:
json.dump(
{
"training_config": training_config,
"model_config": model_config,
"results": results,
},
f,
indent=2,
ensure_ascii=False,
)
print(f"✅ Результаты сохранены в {results_path}")
print(f"\n🎉 Упрощенное обучение завершено успешно!")
@@ -278,6 +281,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

22
experiments/hf_integration/test_hf_proxy.py
View File

@@ -16,8 +16,11 @@ from llm.tokenizers import BPETokenizer
from hf_proxy import HFAdapter, HFTokenizerAdapter
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TEST_PROMPTS, GENERATION_CONFIG
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TEST_PROMPTS,
GENERATION_CONFIG,
)
@@ -31,7 +34,7 @@ def test_basic_hf_integration():
llm_tokenizer.train(
texts=TRAIN_TEXTS,
vocab_size=BPE_CONFIG["vocab_size"],
special_tokens=BPE_CONFIG["special_tokens"]
special_tokens=BPE_CONFIG["special_tokens"],
)
hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -62,7 +65,7 @@ def test_basic_hf_integration():
print(f" HF адаптер: {hf_inputs['input_ids'].shape}")
# Декодирование
decoded = hf_tokenizer.decode(hf_inputs['input_ids'][0])
decoded = hf_tokenizer.decode(hf_inputs["input_ids"][0])
print(f" Декодированный: '{decoded}'")
# === Тестирование forward pass ===
@@ -87,10 +90,10 @@ def test_basic_hf_integration():
with torch.no_grad():
generated = hf_model.generate(
input_ids=inputs['input_ids'],
input_ids=inputs["input_ids"],
max_new_tokens=10,
do_sample=True,
temperature=0.8
temperature=0.8,
)
generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
@@ -123,7 +126,9 @@ def test_basic_hf_integration():
test_input = hf_tokenizer("Тест", return_tensors="pt")
with torch.no_grad():
loaded_outputs = loaded_model(**test_input)
print(f" ✅ Загруженная модель работает (logits: {loaded_outputs.logits.shape})")
print(
f" ✅ Загруженная модель работает (logits: {loaded_outputs.logits.shape})"
)
except Exception as e:
print(f" ❌ Ошибка сохранения/загрузки: {e}")
@@ -140,7 +145,7 @@ def test_hf_tokenizer_methods():
llm_tokenizer.train(
texts=TRAIN_TEXTS[:5],
vocab_size=500,
special_tokens=BPE_CONFIG["special_tokens"]
special_tokens=BPE_CONFIG["special_tokens"],
)
hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -199,6 +204,7 @@ def main():
except Exception as e:
print(f"\n❌ Ошибка в тестировании: {e}")
import traceback
traceback.print_exc()

42
experiments/hf_integration/train_with_hf_trainer.py
View File

@@ -17,12 +17,18 @@ from llm.tokenizers import BPETokenizer
from hf_proxy import HFAdapter, HFTokenizerAdapter
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TRAINING_CONFIG, PATHS, TEST_PROMPTS
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TRAINING_CONFIG,
PATHS,
TEST_PROMPTS,
)
from shared.data import (
load_training_data, ensure_directories,
print_experiment_info, ExperimentLogger
load_training_data,
ensure_directories,
print_experiment_info,
ExperimentLogger,
)
@@ -50,7 +56,7 @@ def setup_hf_training():
llm_tokenizer.train(
texts=TRAIN_TEXTS,
vocab_size=BPE_CONFIG["vocab_size"],
special_tokens=BPE_CONFIG["special_tokens"]
special_tokens=BPE_CONFIG["special_tokens"],
)
llm_tokenizer.save(PATHS["bpe_tokenizer"])
print(f"✅ Токенизатор обучен и сохранен")
@@ -117,7 +123,7 @@ def main():
"tokenizer": "BPE через HFTokenizerAdapter",
"trainer": "HuggingFace Trainer",
"vocab_size": BPE_CONFIG["vocab_size"],
"training_epochs": TRAINING_CONFIG["num_epochs"]
"training_epochs": TRAINING_CONFIG["num_epochs"],
}
print_experiment_info(experiment_name, experiment_config)
@@ -126,7 +132,14 @@ def main():
try:
# Настраиваем обучение
hf_model, hf_tokenizer, llm_tokenizer, model_config, train_texts, val_texts = setup_hf_training()
(
hf_model,
hf_tokenizer,
llm_tokenizer,
model_config,
train_texts,
val_texts,
) = setup_hf_training()
# Тестируем интеграцию
test_hf_integration(hf_model, hf_tokenizer, llm_tokenizer)
@@ -173,7 +186,7 @@ def main():
from transformers import (
Trainer,
TrainingArguments,
DataCollatorForLanguageModeling
DataCollatorForLanguageModeling,
)
# Data collator для языкового моделирования
@@ -261,13 +274,15 @@ def main():
with torch.no_grad():
generated = hf_model.generate(
input_ids=inputs['input_ids'],
input_ids=inputs["input_ids"],
max_new_tokens=20,
do_sample=True,
temperature=0.8
temperature=0.8,
)
generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
generated_text = hf_tokenizer.decode(
generated[0], skip_special_tokens=True
)
print(f"🎯 Результат: '{generated_text}'")
except Exception as e:
@@ -278,8 +293,8 @@ def main():
"experiment": experiment_name,
"model_config": model_config,
"training_config": TRAINING_CONFIG,
"final_loss": train_result.metrics.get('train_loss', 'N/A'),
"eval_loss": train_result.metrics.get('eval_loss', 'N/A')
"final_loss": train_result.metrics.get("train_loss", "N/A"),
"eval_loss": train_result.metrics.get("eval_loss", "N/A"),
}
logger.save_logs("checkpoints/hf_integration_training_logs.json")
@@ -291,6 +306,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

73
experiments/llm_only/generate_gpt2_bpe.py
View File

@@ -15,12 +15,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from llm.models.gpt import GPT2
from llm.tokenizers import BPETokenizer
from shared.configs import (
BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
)
from shared.data import (
print_experiment_info, ensure_directories, ExperimentLogger
)
from shared.configs import BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
def load_model_and_tokenizer() -> tuple:
@@ -38,13 +34,12 @@ def load_model_and_tokenizer() -> tuple:
)
if not os.path.exists(PATHS["bpe_tokenizer"]):
raise FileNotFoundError(
f"Токенизатор не найден: {PATHS['bpe_tokenizer']}"
)
raise FileNotFoundError(f"Токенизатор не найден: {PATHS['bpe_tokenizer']}")
# Загружаем конфигурацию модели
import json
with open(PATHS["gpt_bpe_config"], 'r', encoding='utf-8') as f:
with open(PATHS["gpt_bpe_config"], "r", encoding="utf-8") as f:
model_config = json.load(f)
# Загружаем токенизатор
@@ -55,7 +50,7 @@ def load_model_and_tokenizer() -> tuple:
# Загружаем модель
print("🔧 Загрузка GPT2 модели...")
model = GPT2(model_config)
model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location='cpu'))
model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location="cpu"))
model.eval()
print("✅ Модель загружена")
@@ -63,10 +58,7 @@ def load_model_and_tokenizer() -> tuple:
def generate_text(
model: GPT2,
tokenizer: BPETokenizer,
prompt: str,
config: dict
model: GPT2, tokenizer: BPETokenizer, prompt: str, config: dict
) -> str:
"""
Генерирует текст на основе промпта.
@@ -81,8 +73,10 @@ def generate_text(
str: Сгенерированный текст
"""
print(f"🔤 Промпт: '{prompt}'")
print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}")
print(
f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}"
)
# Кодируем промпт
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
@@ -100,7 +94,7 @@ def generate_text(
do_sample=config["do_sample"],
temperature=config["temperature"],
top_k=config["top_k"],
top_p=config["top_p"]
top_p=config["top_p"],
)
# Декодируем результат
@@ -125,23 +119,29 @@ def test_different_strategies(model: GPT2, tokenizer: BPETokenizer, prompt: str)
{"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
{"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
{"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
{"name": "❄️ Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
{
"name": "❄️ Детерминированная (temp=0.3)",
"do_sample": True,
"temperature": 0.3,
},
]
for strategy in strategies:
print(f"\n{strategy['name']}:")
try:
config = GENERATION_CONFIG.copy()
config.update({
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20
})
config.update(
{
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20,
}
)
generated = generate_text(model, tokenizer, prompt, config)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f" 📤 Промпт: '{prompt}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
print(f" 📄 Полный текст: '{generated}'")
@@ -196,7 +196,7 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
try:
user_input = input("\n🔤 Введите промпт: ").strip()
if user_input.lower() in ['exit', 'quit', 'выход']:
if user_input.lower() in ["exit", "quit", "выход"]:
break
if not user_input:
@@ -207,7 +207,7 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
temperature = float(input("🌡️ Температура [0.7]: ") or "0.7")
do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
do_sample = do_sample_input != 'n'
do_sample = do_sample_input != "n"
except:
max_tokens = 50
temperature = 0.7
@@ -215,15 +215,17 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
print("⚠️ Использую параметры по умолчанию")
config = GENERATION_CONFIG.copy()
config.update({
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample
})
config.update(
{
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample,
}
)
generated = generate_text(model, tokenizer, user_input, config)
generated_part = generated[len(user_input):]
generated_part = generated[len(user_input) :]
print(f"\n🎯 Результат:")
print(f" 📤 Промпт: '{user_input}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
@@ -244,7 +246,7 @@ def main():
"model": "GPT2 с BPE токенизатором",
"стратегия": "автономная генерация",
"вход": "промпты",
"выход": "сгенерированный текст"
"выход": "сгенерированный текст",
}
print_experiment_info(experiment_name, experiment_config)
@@ -275,7 +277,7 @@ def main():
generated = generate_text(model, tokenizer, prompt, GENERATION_CONFIG)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f"📤 Промпт: '{prompt}'")
print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -306,6 +308,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

73
experiments/llm_only/generate_gpt_bpe.py
View File

@@ -15,12 +15,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from llm.models.gpt import GPT
from llm.tokenizers import BPETokenizer
from shared.configs import (
BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
)
from shared.data import (
print_experiment_info, ensure_directories, ExperimentLogger
)
from shared.configs import BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
def load_model_and_tokenizer() -> tuple:
@@ -38,13 +34,12 @@ def load_model_and_tokenizer() -> tuple:
)
if not os.path.exists(PATHS["bpe_tokenizer"]):
raise FileNotFoundError(
f"Токенизатор не найден: {PATHS['bpe_tokenizer']}"
)
raise FileNotFoundError(f"Токенизатор не найден: {PATHS['bpe_tokenizer']}")
# Загружаем конфигурацию модели
import json
with open(PATHS["gpt_bpe_config"], 'r', encoding='utf-8') as f:
with open(PATHS["gpt_bpe_config"], "r", encoding="utf-8") as f:
model_config = json.load(f)
# Загружаем токенизатор
@@ -55,7 +50,7 @@ def load_model_and_tokenizer() -> tuple:
# Загружаем модель
print("🔧 Загрузка GPT модели...")
model = GPT(model_config)
model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location='cpu'))
model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location="cpu"))
model.eval()
print("✅ Модель загружена")
@@ -63,10 +58,7 @@ def load_model_and_tokenizer() -> tuple:
def generate_text(
model: GPT,
tokenizer: BPETokenizer,
prompt: str,
config: dict
model: GPT, tokenizer: BPETokenizer, prompt: str, config: dict
) -> str:
"""
Генерирует текст на основе промпта.
@@ -81,8 +73,10 @@ def generate_text(
str: Сгенерированный текст
"""
print(f"🔤 Промпт: '{prompt}'")
print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}")
print(
f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
f"temp={config['temperature']}, sample={config['do_sample']}"
)
# Кодируем промпт
input_ids = tokenizer.encode(prompt, add_special_tokens=False)
@@ -100,7 +94,7 @@ def generate_text(
do_sample=config["do_sample"],
temperature=config["temperature"],
top_k=config["top_k"],
top_p=config["top_p"]
top_p=config["top_p"],
)
# Декодируем результат
@@ -125,23 +119,29 @@ def test_different_strategies(model: GPT, tokenizer: BPETokenizer, prompt: str):
{"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
{"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
{"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
{"name": "❄️ Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
{
"name": "❄️ Детерминированная (temp=0.3)",
"do_sample": True,
"temperature": 0.3,
},
]
for strategy in strategies:
print(f"\n{strategy['name']}:")
try:
config = GENERATION_CONFIG.copy()
config.update({
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20
})
config.update(
{
"do_sample": strategy["do_sample"],
"temperature": strategy["temperature"],
"max_new_tokens": 20,
}
)
generated = generate_text(model, tokenizer, prompt, config)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f" 📤 Промпт: '{prompt}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
print(f" 📄 Полный текст: '{generated}'")
@@ -196,7 +196,7 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
try:
user_input = input("\n🔤 Введите промпт: ").strip()
if user_input.lower() in ['exit', 'quit', 'выход']:
if user_input.lower() in ["exit", "quit", "выход"]:
break
if not user_input:
@@ -207,7 +207,7 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
temperature = float(input("🌡️ Температура [0.7]: ") or "0.7")
do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
do_sample = do_sample_input != 'n'
do_sample = do_sample_input != "n"
except:
max_tokens = 50
temperature = 0.7
@@ -215,15 +215,17 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
print("⚠️ Использую параметры по умолчанию")
config = GENERATION_CONFIG.copy()
config.update({
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample
})
config.update(
{
"max_new_tokens": max_tokens,
"temperature": temperature,
"do_sample": do_sample,
}
)
generated = generate_text(model, tokenizer, user_input, config)
generated_part = generated[len(user_input):]
generated_part = generated[len(user_input) :]
print(f"\n🎯 Результат:")
print(f" 📤 Промпт: '{user_input}'")
print(f" 🎯 Сгенерировано: '{generated_part}'")
@@ -244,7 +246,7 @@ def main():
"model": "GPT с BPE токенизатором",
"стратегия": "автономная генерация",
"вход": "промпты",
"выход": "сгенерированный текст"
"выход": "сгенерированный текст",
}
print_experiment_info(experiment_name, experiment_config)
@@ -275,7 +277,7 @@ def main():
generated = generate_text(model, tokenizer, prompt, GENERATION_CONFIG)
# Выделяем сгенерированную часть
generated_part = generated[len(prompt):]
generated_part = generated[len(prompt) :]
print(f"📤 Промпт: '{prompt}'")
print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -306,6 +308,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

34
experiments/llm_only/train_gpt2_bpe.py
View File

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
from llm.training.trainer import Trainer
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TRAINING_CONFIG, PATHS, TEST_PROMPTS
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TRAINING_CONFIG,
PATHS,
TEST_PROMPTS,
)
from shared.data import (
load_training_data, ensure_directories,
print_experiment_info, ExperimentLogger
load_training_data,
ensure_directories,
print_experiment_info,
ExperimentLogger,
)
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
tokenizer.train(
texts=texts,
vocab_size=config["vocab_size"],
special_tokens=config["special_tokens"]
special_tokens=config["special_tokens"],
)
# Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
"vocab_size": BPE_CONFIG["vocab_size"],
"training_epochs": TRAINING_CONFIG["num_epochs"],
"batch_size": TRAINING_CONFIG["batch_size"],
"learning_rate": TRAINING_CONFIG["learning_rate"]
"learning_rate": TRAINING_CONFIG["learning_rate"],
}
print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
# === Подготовка датасета ===
print(f"\n📊 Подготовка датасета...")
train_dataset = TextDataset(
train_texts,
tokenizer,
block_size=model_config["max_position_embeddings"]
train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
)
print(f" Размер train датасета: {len(train_dataset)} примеров")
@@ -152,7 +156,7 @@ def main():
lr=TRAINING_CONFIG["learning_rate"],
batch_size=TRAINING_CONFIG["batch_size"],
num_epochs=TRAINING_CONFIG["num_epochs"],
warmup_steps=TRAINING_CONFIG["warmup_steps"]
warmup_steps=TRAINING_CONFIG["warmup_steps"],
)
# Запускаем обучение
@@ -167,7 +171,8 @@ def main():
# Сохраняем конфигурацию
import json
with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
json.dump(model_config, f, indent=2, ensure_ascii=False)
print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
x=input_tensor,
max_new_tokens=20,
do_sample=True,
temperature=0.8
temperature=0.8,
)
# Декодируем результат
generated_text = tokenizer.decode(generated_ids[0].tolist())
generated_part = generated_text[len(prompt):]
generated_part = generated_text[len(prompt) :]
print(f"🎯 Сгенерировано: '{generated_part}'")
print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
"model_config": model_config,
"training_config": TRAINING_CONFIG,
"tokenizer_vocab_size": tokenizer.get_vocab_size(),
"final_loss": "см. логи обучения" # В реальном эксперименте можно сохранить final loss
"final_loss": "см. логи обучения", # В реальном эксперименте можно сохранить final loss
}
logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

34
experiments/llm_only/train_gpt_bpe.py
View File

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
from llm.training.trainer import Trainer
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TRAINING_CONFIG, PATHS, TEST_PROMPTS
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TRAINING_CONFIG,
PATHS,
TEST_PROMPTS,
)
from shared.data import (
load_training_data, ensure_directories,
print_experiment_info, ExperimentLogger
load_training_data,
ensure_directories,
print_experiment_info,
ExperimentLogger,
)
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
tokenizer.train(
texts=texts,
vocab_size=config["vocab_size"],
special_tokens=config["special_tokens"]
special_tokens=config["special_tokens"],
)
# Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
"vocab_size": BPE_CONFIG["vocab_size"],
"training_epochs": TRAINING_CONFIG["num_epochs"],
"batch_size": TRAINING_CONFIG["batch_size"],
"learning_rate": TRAINING_CONFIG["learning_rate"]
"learning_rate": TRAINING_CONFIG["learning_rate"],
}
print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
# === Подготовка датасета ===
print(f"\n📊 Подготовка датасета...")
train_dataset = TextDataset(
train_texts,
tokenizer,
block_size=model_config["max_position_embeddings"]
train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
)
print(f" Размер train датасета: {len(train_dataset)} примеров")
@@ -152,7 +156,7 @@ def main():
lr=TRAINING_CONFIG["learning_rate"],
batch_size=TRAINING_CONFIG["batch_size"],
num_epochs=TRAINING_CONFIG["num_epochs"],
warmup_steps=TRAINING_CONFIG["warmup_steps"]
warmup_steps=TRAINING_CONFIG["warmup_steps"],
)
# Запускаем обучение
@@ -167,7 +171,8 @@ def main():
# Сохраняем конфигурацию
import json
with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
json.dump(model_config, f, indent=2, ensure_ascii=False)
print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
x=input_tensor,
max_new_tokens=20,
do_sample=True,
temperature=0.8
temperature=0.8,
)
# Декодируем результат
generated_text = tokenizer.decode(generated_ids[0].tolist())
generated_part = generated_text[len(prompt):]
generated_part = generated_text[len(prompt) :]
print(f"🎯 Сгенерировано: '{generated_part}'")
print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
"model_config": model_config,
"training_config": TRAINING_CONFIG,
"tokenizer_vocab_size": tokenizer.get_vocab_size(),
"final_loss": "см. логи обучения" # В реальном эксперименте можно сохранить final loss
"final_loss": "см. логи обучения", # В реальном эксперименте можно сохранить final loss
}
logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

34
experiments/llm_only/train_llama_bpe.py
View File

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
from llm.training.trainer import Trainer
from shared.configs import (
TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG,
TRAINING_CONFIG, PATHS, TEST_PROMPTS
TRAIN_TEXTS,
BASE_GPT_CONFIG,
BPE_CONFIG,
TRAINING_CONFIG,
PATHS,
TEST_PROMPTS,
)
from shared.data import (
load_training_data, ensure_directories,
print_experiment_info, ExperimentLogger
load_training_data,
ensure_directories,
print_experiment_info,
ExperimentLogger,
)
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
tokenizer.train(
texts=texts,
vocab_size=config["vocab_size"],
special_tokens=config["special_tokens"]
special_tokens=config["special_tokens"],
)
# Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
"vocab_size": BPE_CONFIG["vocab_size"],
"training_epochs": TRAINING_CONFIG["num_epochs"],
"batch_size": TRAINING_CONFIG["batch_size"],
"learning_rate": TRAINING_CONFIG["learning_rate"]
"learning_rate": TRAINING_CONFIG["learning_rate"],
}
print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
# === Подготовка датасета ===
print(f"\n📊 Подготовка датасета...")
train_dataset = TextDataset(
train_texts,
tokenizer,
block_size=model_config["max_position_embeddings"]
train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
)
print(f" Размер train датасета: {len(train_dataset)} примеров")
@@ -152,7 +156,7 @@ def main():
lr=TRAINING_CONFIG["learning_rate"],
batch_size=TRAINING_CONFIG["batch_size"],
num_epochs=TRAINING_CONFIG["num_epochs"],
warmup_steps=TRAINING_CONFIG["warmup_steps"]
warmup_steps=TRAINING_CONFIG["warmup_steps"],
)
# Запускаем обучение
@@ -167,7 +171,8 @@ def main():
# Сохраняем конфигурацию
import json
with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
json.dump(model_config, f, indent=2, ensure_ascii=False)
print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
x=input_tensor,
max_new_tokens=20,
do_sample=True,
temperature=0.8
temperature=0.8,
)
# Декодируем результат
generated_text = tokenizer.decode(generated_ids[0].tolist())
generated_part = generated_text[len(prompt):]
generated_part = generated_text[len(prompt) :]
print(f"🎯 Сгенерировано: '{generated_part}'")
print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
"model_config": model_config,
"training_config": TRAINING_CONFIG,
"tokenizer_vocab_size": tokenizer.get_vocab_size(),
"final_loss": "см. логи обучения" # В реальном эксперименте можно сохранить final loss
"final_loss": "см. логи обучения", # В реальном эксперименте можно сохранить final loss
}
logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
except Exception as e:
print(f"❌ Ошибка в эксперименте: {e}")
import traceback
traceback.print_exc()

14
experiments/shared/configs.py
View File

@@ -30,7 +30,7 @@ BASE_GPT_CONFIG = {
"num_heads": 4,
"num_layers": 4,
"max_position_embeddings": 128,
"dropout": 0.1
"dropout": 0.1,
}
# Конфигурация для маленькой модели (быстрое тестирование)
@@ -40,7 +40,7 @@ SMALL_GPT_CONFIG = {
"num_heads": 2,
"num_layers": 2,
"max_position_embeddings": 64,
"dropout": 0.1
"dropout": 0.1,
}
# Конфигурация для большой модели (качественное обучение)
@@ -50,13 +50,13 @@ LARGE_GPT_CONFIG = {
"num_heads": 8,
"num_layers": 6,
"max_position_embeddings": 256,
"dropout": 0.1
"dropout": 0.1,
}
# === Конфигурации токенизатора ===
BPE_CONFIG = {
"vocab_size": 1000,
"special_tokens": ["<pad>", "<unk>", "<bos>", "<eos>"]
"special_tokens": ["<pad>", "<unk>", "<bos>", "<eos>"],
}
# === Конфигурации обучения ===
@@ -65,7 +65,7 @@ TRAINING_CONFIG = {
"batch_size": 2,
"num_epochs": 3,
"warmup_steps": 50,
"gradient_clip": 1.0
"gradient_clip": 1.0,
}
# === Конфигурации генерации ===
@@ -74,7 +74,7 @@ GENERATION_CONFIG = {
"temperature": 0.7,
"do_sample": True,
"top_k": None,
"top_p": None
"top_p": None,
}
# === Пути для сохранения ===
@@ -84,7 +84,7 @@ PATHS = {
"gpt_bpe_config": "checkpoints/gpt-bpe/config.json",
"hf_tokenizer": "checkpoints/hf-bpe-tokenizer",
"hf_model": "checkpoints/hf-trained",
"hf_proxy_model": "checkpoints/hf-trained-proxy"
"hf_proxy_model": "checkpoints/hf-trained-proxy",
}
# === Тестовые промпты ===

30
experiments/shared/data.py
View File

@@ -32,7 +32,7 @@ def ensure_directories():
"checkpoints/hf-bpe-tokenizer",
"checkpoints/hf-trained",
"checkpoints/hf-trained-proxy",
"logs"
"logs",
]
for directory in directories:
@@ -52,15 +52,16 @@ def get_model_paths(experiment_type: str = "llm_only") -> dict:
base_paths = PATHS.copy()
if experiment_type == "hf_integration":
base_paths.update({
"model": base_paths["hf_model"],
"tokenizer": base_paths["hf_tokenizer"]
})
base_paths.update(
{"model": base_paths["hf_model"], "tokenizer": base_paths["hf_tokenizer"]}
)
else: # llm_only
base_paths.update({
"model": base_paths["gpt_bpe_model"],
"tokenizer": base_paths["bpe_tokenizer"]
})
base_paths.update(
{
"model": base_paths["gpt_bpe_model"],
"tokenizer": base_paths["bpe_tokenizer"],
}
)
return base_paths
@@ -92,7 +93,7 @@ def save_experiment_results(results: dict, filepath: str):
"""
import json
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(f"✅ Результаты эксперимента сохранены: {filepath}")
@@ -113,7 +114,7 @@ def load_experiment_results(filepath: str) -> dict:
if not os.path.exists(filepath):
return {}
with open(filepath, 'r', encoding='utf-8') as f:
with open(filepath, "r", encoding="utf-8") as f:
return json.load(f)
@@ -151,12 +152,9 @@ class ExperimentLogger:
"""Сохраняет логи эксперимента."""
import json
logs = {
"experiment_name": self.experiment_name,
"metrics": self.metrics
}
logs = {"experiment_name": self.experiment_name, "metrics": self.metrics}
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(logs, f, ensure_ascii=False, indent=2)
print(f"✅ Логи эксперимента сохранены: {filepath}")

3
hf-proxy/src/hf_proxy/__init__.py
View File

@@ -27,16 +27,13 @@ __all__ = [
# Основные классы адаптера
"HFAdapter",
"HFGPTAdapter",
# Конфигурации
"HFAdapterConfig",
"HFPretrainedConfig",
# Адаптеры токенизаторов
"HFTokenizerAdapter",
"create_hf_tokenizer",
"convert_to_hf_format",
# Утилиты
"HFUtils",
"TokenizerWrapper",

55
hf-proxy/src/hf_proxy/hf_adapter.py
View File

@@ -11,7 +11,7 @@ from transformers import (
GPT2Config,
GenerationConfig,
LogitsProcessorList,
StoppingCriteriaList
StoppingCriteriaList,
)
from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions
@@ -24,6 +24,7 @@ class HFGPTAdapter(PreTrainedModel):
Адаптер для модели GPT из библиотеки llm.
Позволяет использовать кастомные GPT модели с HuggingFace Transformers.
"""
config_class = HFPretrainedConfig
def __init__(self, config: HFPretrainedConfig, llm_model: Optional[GPT] = None):
@@ -46,7 +47,7 @@ class HFGPTAdapter(PreTrainedModel):
self.llm_model = llm_model
# Устанавливаем веса если они есть в конфигурации
if hasattr(config, 'state_dict') and config.state_dict is not None:
if hasattr(config, "state_dict") and config.state_dict is not None:
self.llm_model.load_state_dict(config.state_dict)
def _hf_to_llm_config(self, hf_config: HFPretrainedConfig) -> dict:
@@ -78,7 +79,7 @@ class HFGPTAdapter(PreTrainedModel):
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None,
**kwargs
**kwargs,
) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
"""
Прямой проход модели.
@@ -96,7 +97,9 @@ class HFGPTAdapter(PreTrainedModel):
Returns:
CausalLMOutputWithCrossAttentions или кортеж
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
return_dict = (
return_dict if return_dict is not None else self.config.use_return_dict
)
# Основной forward pass
outputs = self.llm_model(input_ids)
@@ -114,8 +117,7 @@ class HFGPTAdapter(PreTrainedModel):
# Вычисляем cross-entropy loss
loss_fct = nn.CrossEntropyLoss()
loss = loss_fct(
shift_logits.view(-1, shift_logits.size(-1)),
shift_labels.view(-1)
shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
)
if not return_dict:
@@ -134,10 +136,7 @@ class HFGPTAdapter(PreTrainedModel):
)
def prepare_inputs_for_generation(
self,
input_ids: torch.Tensor,
past_key_values: Optional[Tuple] = None,
**kwargs
self, input_ids: torch.Tensor, past_key_values: Optional[Tuple] = None, **kwargs
) -> dict:
"""
Подготавливает входные данные для генерации.
@@ -163,7 +162,7 @@ class HFGPTAdapter(PreTrainedModel):
generation_config: Optional[GenerationConfig] = None,
logits_processor: Optional[LogitsProcessorList] = None,
stopping_criteria: Optional[StoppingCriteriaList] = None,
**kwargs
**kwargs,
) -> torch.Tensor:
"""
Генерация текста с поддержкой HuggingFace интерфейса.
@@ -179,8 +178,8 @@ class HFGPTAdapter(PreTrainedModel):
torch.Tensor: Сгенерированные токены
"""
# Извлекаем обязательные параметры из kwargs или используем значения по умолчанию
max_new_tokens = kwargs.pop('max_new_tokens', 50)
do_sample = kwargs.pop('do_sample', True)
max_new_tokens = kwargs.pop("max_new_tokens", 50)
do_sample = kwargs.pop("do_sample", True)
# Используем встроенную генерацию llm модели
return self.llm_model.generate(
@@ -188,7 +187,7 @@ class HFGPTAdapter(PreTrainedModel):
max_new_tokens=max_new_tokens,
do_sample=do_sample,
attention_mask=attention_mask,
**kwargs
**kwargs,
)
@@ -199,8 +198,7 @@ class HFAdapter:
@staticmethod
def from_llm_model(
llm_model: GPT,
hf_config: Optional[HFAdapterConfig] = None
llm_model: GPT, hf_config: Optional[HFAdapterConfig] = None
) -> HFGPTAdapter:
"""
Создает адаптер из существующей llm модели.
@@ -223,8 +221,7 @@ class HFAdapter:
@staticmethod
def from_pretrained(
model_path: str,
hf_config: Optional[HFAdapterConfig] = None
model_path: str, hf_config: Optional[HFAdapterConfig] = None
) -> HFGPTAdapter:
"""
Загружает модель из чекпоинта и создает адаптер.
@@ -237,14 +234,18 @@ class HFAdapter:
HFGPTAdapter: Адаптированная модель
"""
# Загружаем состояние модели
state_dict = torch.load(model_path, map_location='cpu')
state_dict = torch.load(model_path, map_location="cpu")
# Определяем конфигурацию из состояния модели или используем переданную
if hf_config is None:
# Пытаемся определить конфигурацию из состояния модели
# Это упрощенный подход - в реальности нужно сохранять конфигурацию отдельно
vocab_size = state_dict.get('_token_embeddings._embedding.weight', torch.zeros(50257, 768)).shape[0]
embed_dim = state_dict.get('_token_embeddings._embedding.weight', torch.zeros(50257, 768)).shape[1]
vocab_size = state_dict.get(
"_token_embeddings._embedding.weight", torch.zeros(50257, 768)
).shape[0]
embed_dim = state_dict.get(
"_token_embeddings._embedding.weight", torch.zeros(50257, 768)
).shape[1]
hf_config = HFAdapterConfig(
vocab_size=vocab_size,
@@ -270,11 +271,7 @@ class HFAdapter:
return HFGPTAdapter(pretrained_config, llm_model)
@staticmethod
def save_pretrained(
model: HFGPTAdapter,
save_directory: str,
**kwargs
):
def save_pretrained(model: HFGPTAdapter, save_directory: str, **kwargs):
"""
Сохраняет адаптированную модель в формате HuggingFace.
@@ -291,7 +288,7 @@ class HFAdapter:
# Сохраняем конфигурацию
config_path = os.path.join(save_directory, "config.json")
with open(config_path, 'w', encoding='utf-8') as f:
with open(config_path, "w", encoding="utf-8") as f:
json.dump(model.config.to_dict(), f, indent=2, ensure_ascii=False)
# Сохраняем веса модели
@@ -299,5 +296,5 @@ class HFAdapter:
torch.save(model.llm_model.state_dict(), model_path)
# Сохраняем токенизатор если передан
if hasattr(kwargs, 'tokenizer') and kwargs['tokenizer'] is not None:
kwargs['tokenizer'].save_pretrained(save_directory)
if hasattr(kwargs, "tokenizer") and kwargs["tokenizer"] is not None:
kwargs["tokenizer"].save_pretrained(save_directory)

14
hf-proxy/src/hf_proxy/hf_config.py
View File

@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
from typing import Dict, Any, Optional
from transformers import PretrainedConfig
@dataclass
class HFAdapterConfig:
"""
@@ -28,6 +29,7 @@ class HFAdapterConfig:
eos_token_id: ID токена конца строки
bos_token_id: ID токена начала строки
"""
model_type: str = "gpt"
vocab_size: int = 50257
hidden_size: int = 768
@@ -52,8 +54,9 @@ class HFAdapterConfig:
def to_dict(self) -> Dict[str, Any]:
"""Преобразует конфигурацию в словарь."""
return {
k: v for k, v in self.__dict__.items()
if not k.startswith('_') and not callable(v)
k: v
for k, v in self.__dict__.items()
if not k.startswith("_") and not callable(v)
}
@classmethod
@@ -74,7 +77,7 @@ class HFAdapterConfig:
"num_heads": "num_attention_heads",
"max_position_embeddings": "max_position_embeddings",
"dropout": "hidden_dropout_prob",
"vocab_size": "vocab_size"
"vocab_size": "vocab_size",
}
hf_config_dict = {}
@@ -94,6 +97,7 @@ class HFPretrainedConfig(PretrainedConfig):
Конфигурация для предобученных моделей HuggingFace.
Наследуется от PretrainedConfig для полной совместимости.
"""
model_type = "gpt"
def __init__(
@@ -112,13 +116,13 @@ class HFPretrainedConfig(PretrainedConfig):
pad_token_id=50256,
eos_token_id=50256,
bos_token_id=50256,
**kwargs
**kwargs,
):
super().__init__(
pad_token_id=pad_token_id,
eos_token_id=eos_token_id,
bos_token_id=bos_token_id,
**kwargs
**kwargs,
)
self.vocab_size = vocab_size

127
hf-proxy/src/hf_proxy/hf_tokenizer.py
View File

@@ -27,16 +27,16 @@ class HFTokenizerAdapter:
self.vocab_size = llm_tokenizer.get_vocab_size()
# Устанавливаем специальные токены
self.pad_token = getattr(llm_tokenizer, 'pad_token', '<pad>')
self.unk_token = getattr(llm_tokenizer, 'unk_token', '<unk>')
self.bos_token = getattr(llm_tokenizer, 'bos_token', '<bos>')
self.eos_token = getattr(llm_tokenizer, 'eos_token', '<eos>')
self.pad_token = getattr(llm_tokenizer, "pad_token", "<pad>")
self.unk_token = getattr(llm_tokenizer, "unk_token", "<unk>")
self.bos_token = getattr(llm_tokenizer, "bos_token", "<bos>")
self.eos_token = getattr(llm_tokenizer, "eos_token", "<eos>")
# Сохраняем ID специальных токенов
self.pad_token_id = getattr(llm_tokenizer, 'pad_token_id', 0)
self.unk_token_id = getattr(llm_tokenizer, 'unk_token_id', 1)
self.bos_token_id = getattr(llm_tokenizer, 'bos_token_id', 2)
self.eos_token_id = getattr(llm_tokenizer, 'eos_token_id', 3)
self.pad_token_id = getattr(llm_tokenizer, "pad_token_id", 0)
self.unk_token_id = getattr(llm_tokenizer, "unk_token_id", 1)
self.bos_token_id = getattr(llm_tokenizer, "bos_token_id", 2)
self.eos_token_id = getattr(llm_tokenizer, "eos_token_id", 3)
def __call__(self, text: str, **kwargs):
"""
@@ -49,30 +49,27 @@ class HFTokenizerAdapter:
Returns:
dict: Словарь с токенами
"""
return_tensors = kwargs.get('return_tensors', None)
padding = kwargs.get('padding', False)
truncation = kwargs.get('truncation', False)
max_length = kwargs.get('max_length', None)
add_special_tokens = kwargs.get('add_special_tokens', True)
return_tensors = kwargs.get("return_tensors", None)
padding = kwargs.get("padding", False)
truncation = kwargs.get("truncation", False)
max_length = kwargs.get("max_length", None)
add_special_tokens = kwargs.get("add_special_tokens", True)
# Кодируем текст
#input_ids = self.llm_tokenizer.encode(
# input_ids = self.llm_tokenizer.encode(
# text,
# add_special_tokens=add_special_tokens
#)
# )
if isinstance(text, str):
input_ids = self.llm_tokenizer.encode(
text,
add_special_tokens=add_special_tokens
text, add_special_tokens=add_special_tokens
)
input_ids = [input_ids] # <-- оборачиваем в batch
else:
# Список строк, батч-режим!
input_ids = [
self.llm_tokenizer.encode(
t,
add_special_tokens=add_special_tokens
) for t in text
self.llm_tokenizer.encode(t, add_special_tokens=add_special_tokens)
for t in text
]
# Применяем truncation
@@ -86,6 +83,7 @@ class HFTokenizerAdapter:
# Конвертируем в тензоры если нужно
if return_tensors == "pt":
import torch
input_ids = torch.tensor([input_ids])
return {"input_ids": input_ids}
@@ -99,7 +97,7 @@ class HFTokenizerAdapter:
truncation: bool = False,
max_length: Optional[int] = None,
return_tensors: Optional[str] = None,
**kwargs
**kwargs,
) -> Union[List[int], List[List[int]]]:
"""
Кодирует текст в последовательность токенов.
@@ -118,16 +116,12 @@ class HFTokenizerAdapter:
"""
# Кодируем основной текст
token_ids = self.llm_tokenizer.encode(
text,
add_special_tokens=add_special_tokens
text, add_special_tokens=add_special_tokens
)
# Обрабатываем text_pair если есть
if text_pair is not None:
pair_ids = self.llm_tokenizer.encode(
text_pair,
add_special_tokens=False
)
pair_ids = self.llm_tokenizer.encode(text_pair, add_special_tokens=False)
token_ids.extend(pair_ids)
# Применяем truncation
@@ -141,9 +135,11 @@ class HFTokenizerAdapter:
# Конвертируем в тензоры если нужно
if return_tensors == "pt":
import torch
return torch.tensor([token_ids])
elif return_tensors == "np":
import numpy as np
return np.array([token_ids])
return token_ids
@@ -152,7 +148,7 @@ class HFTokenizerAdapter:
self,
token_ids: Union[int, List[int], List[List[int]]],
skip_special_tokens: bool = True,
**kwargs
**kwargs,
) -> str:
"""
Декодирует последовательность токенов в текст.
@@ -167,13 +163,22 @@ class HFTokenizerAdapter:
# Обрабатываем разные форматы входных данных
if isinstance(token_ids, int):
token_ids = [token_ids]
elif isinstance(token_ids, list) and len(token_ids) > 0 and isinstance(token_ids[0], list):
elif (
isinstance(token_ids, list)
and len(token_ids) > 0
and isinstance(token_ids[0], list)
):
# Список списков - берем первый элемент
token_ids = token_ids[0]
# Фильтруем специальные токены если нужно
if skip_special_tokens:
special_ids = {self.pad_token_id, self.unk_token_id, self.bos_token_id, self.eos_token_id}
special_ids = {
self.pad_token_id,
self.unk_token_id,
self.bos_token_id,
self.eos_token_id,
}
token_ids = [tid for tid in token_ids if tid not in special_ids]
return self.llm_tokenizer.decode(token_ids)
@@ -224,8 +229,12 @@ class HFTokenizerAdapter:
# Обрабатываем разные типы данных
if isinstance(input_ids, int):
seq_len = 1
elif hasattr(input_ids, 'shape'):
seq_len = input_ids.shape[-1] if len(input_ids.shape) > 1 else len(input_ids)
elif hasattr(input_ids, "shape"):
seq_len = (
input_ids.shape[-1]
if len(input_ids.shape) > 1
else len(input_ids)
)
else:
seq_len = len(input_ids)
max_len = max(max_len, seq_len)
@@ -240,8 +249,12 @@ class HFTokenizerAdapter:
# Получаем текущую длину
if isinstance(input_ids, int):
current_len = 1
elif hasattr(input_ids, 'shape'):
current_len = input_ids.shape[-1] if len(input_ids.shape) > 1 else len(input_ids)
elif hasattr(input_ids, "shape"):
current_len = (
input_ids.shape[-1]
if len(input_ids.shape) > 1
else len(input_ids)
)
else:
current_len = len(input_ids)
@@ -251,20 +264,27 @@ class HFTokenizerAdapter:
# Обрабатываем разные типы данных
if isinstance(input_ids, int):
item["input_ids"] = [input_ids] + [self.pad_token_id] * padding_length
elif hasattr(input_ids, 'shape'):
item["input_ids"] = [input_ids] + [
self.pad_token_id
] * padding_length
elif hasattr(input_ids, "shape"):
import torch
padding_tensor = torch.full((padding_length,), self.pad_token_id, dtype=input_ids.dtype)
padding_tensor = torch.full(
(padding_length,), self.pad_token_id, dtype=input_ids.dtype
)
item["input_ids"] = torch.cat([input_ids, padding_tensor])
else:
item["input_ids"] = input_ids + [self.pad_token_id] * padding_length
item["input_ids"] = (
input_ids + [self.pad_token_id] * padding_length
)
# Добавляем attention_mask если требуется
if "attention_mask" in item:
mask = item["attention_mask"]
if isinstance(mask, int):
item["attention_mask"] = [mask] + [0] * padding_length
elif hasattr(mask, 'shape'):
elif hasattr(mask, "shape"):
padding_mask = torch.zeros(padding_length, dtype=mask.dtype)
item["attention_mask"] = torch.cat([mask, padding_mask])
else:
@@ -272,16 +292,21 @@ class HFTokenizerAdapter:
elif return_attention_mask:
if isinstance(input_ids, int):
item["attention_mask"] = [1] + [0] * padding_length
elif hasattr(input_ids, 'shape'):
elif hasattr(input_ids, "shape"):
attention_mask = torch.ones(current_len, dtype=torch.long)
padding_mask = torch.zeros(padding_length, dtype=torch.long)
item["attention_mask"] = torch.cat([attention_mask, padding_mask])
item["attention_mask"] = torch.cat(
[attention_mask, padding_mask]
)
else:
item["attention_mask"] = [1] * current_len + [0] * padding_length
item["attention_mask"] = [1] * current_len + [
0
] * padding_length
# Конвертируем в тензоры если требуется
if return_tensors == "pt":
import torch
for key in list(encoded_inputs[0].keys()):
if isinstance(encoded_inputs[0][key], list):
for i in range(len(encoded_inputs)):
@@ -326,12 +351,12 @@ class HFTokenizerAdapter:
}
config_path = os.path.join(save_directory, "tokenizer_config.json")
with open(config_path, 'w', encoding='utf-8') as f:
with open(config_path, "w", encoding="utf-8") as f:
json.dump(tokenizer_config, f, ensure_ascii=False, indent=2)
# Сохраняем словарь
vocab_path = os.path.join(save_directory, "vocab.json")
with open(vocab_path, 'w', encoding='utf-8') as f:
with open(vocab_path, "w", encoding="utf-8") as f:
json.dump(self._vocab, f, ensure_ascii=False, indent=2)
print(f"✅ Токенизатор сохранен в {save_directory}")
@@ -353,7 +378,9 @@ class HFTokenizerAdapter:
# Проверяем, является ли путь директорией с файлами токенизатора
if os.path.isdir(pretrained_model_name_or_path):
# Загружаем из директории
config_path = os.path.join(pretrained_model_name_or_path, "tokenizer_config.json")
config_path = os.path.join(
pretrained_model_name_or_path, "tokenizer_config.json"
)
vocab_path = os.path.join(pretrained_model_name_or_path, "vocab.json")
if not os.path.exists(config_path) or not os.path.exists(vocab_path):
@@ -362,7 +389,7 @@ class HFTokenizerAdapter:
)
# Загружаем конфигурацию
with open(config_path, 'r', encoding='utf-8') as f:
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
# Определяем тип токенизатора llm
@@ -373,7 +400,7 @@ class HFTokenizerAdapter:
llm_tokenizer = BPETokenizer()
# Загружаем словарь
with open(vocab_path, 'r', encoding='utf-8') as f:
with open(vocab_path, "r", encoding="utf-8") as f:
vocab = json.load(f)
llm_tokenizer.vocab = vocab
@@ -393,7 +420,9 @@ class HFTokenizerAdapter:
return cls(llm_tokenizer, **kwargs)
else:
raise ValueError(f"Неподдерживаемый тип токенизатора: {llm_tokenizer_type}")
raise ValueError(
f"Неподдерживаемый тип токенизатора: {llm_tokenizer_type}"
)
else:
# Пытаемся загрузить как файл llm токенизатора

59
hf-proxy/src/hf_proxy/hf_utils.py
View File

@@ -31,9 +31,7 @@ class HFUtils:
@staticmethod
def convert_to_hf_format(
llm_model,
tokenizer = None,
model_name: str = "custom-gpt"
llm_model, tokenizer=None, model_name: str = "custom-gpt"
) -> tuple:
"""
Конвертирует llm модель в формат HuggingFace.
@@ -52,13 +50,17 @@ class HFUtils:
# Если токенизатор не передан, создаем стандартный
if tokenizer is None:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")
# Устанавливаем специальные токены
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
elif hasattr(tokenizer, '__class__') and 'BPETokenizer' in str(tokenizer.__class__):
elif hasattr(tokenizer, "__class__") and "BPETokenizer" in str(
tokenizer.__class__
):
# Если передан наш кастомный токенизатор, создаем адаптер
from .hf_tokenizer import create_hf_tokenizer
tokenizer = create_hf_tokenizer(tokenizer)
return hf_model, tokenizer
@@ -70,7 +72,7 @@ class HFUtils:
repo_name: str,
organization: Optional[str] = None,
private: bool = False,
**kwargs
**kwargs,
):
"""
Загружает модель в HuggingFace Hub.
@@ -116,7 +118,7 @@ class HFUtils:
api.upload_folder(
folder_path=tmp_dir,
repo_id=repo_id,
commit_message="Initial commit with custom GPT model"
commit_message="Initial commit with custom GPT model",
)
print(f"✅ Модель успешно загружена в HuggingFace Hub: {repo_id}")
@@ -128,10 +130,7 @@ class HFUtils:
)
@staticmethod
def load_from_hub(
repo_id: str,
**kwargs
) -> tuple:
def load_from_hub(repo_id: str, **kwargs) -> tuple:
"""
Загружает модель из HuggingFace Hub.
@@ -162,17 +161,14 @@ class HFUtils:
# Загружаем модель через адаптер
model = HFAdapter.from_pretrained(
f"{repo_id}/pytorch_model.bin",
HFAdapterConfig.from_llm_config(llm_config)
f"{repo_id}/pytorch_model.bin", HFAdapterConfig.from_llm_config(llm_config)
)
return model, tokenizer
@staticmethod
def compare_with_hf_model(
llm_model,
hf_model_name: str = "gpt2",
test_input: str = "Hello world"
llm_model, hf_model_name: str = "gpt2", test_input: str = "Hello world"
) -> Dict[str, Any]:
"""
Сравнивает llm модель с эталонной моделью из HuggingFace.
@@ -197,7 +193,7 @@ class HFUtils:
# Получаем логиты от обеих моделей
with torch.no_grad():
hf_logits = hf_model(**inputs).logits
llm_logits = llm_model(inputs['input_ids'])
llm_logits = llm_model(inputs["input_ids"])
# Сравниваем результаты
hf_probs = torch.softmax(hf_logits[0, -1], dim=-1)
@@ -205,15 +201,11 @@ class HFUtils:
# Вычисляем метрики
kl_divergence = torch.nn.functional.kl_div(
torch.log(llm_probs + 1e-8),
hf_probs,
reduction='batchmean'
torch.log(llm_probs + 1e-8), hf_probs, reduction="batchmean"
)
cosine_similarity = torch.nn.functional.cosine_similarity(
hf_logits.flatten(),
llm_logits.flatten(),
dim=0
hf_logits.flatten(), llm_logits.flatten(), dim=0
)
return {
@@ -244,11 +236,7 @@ class TokenizerWrapper:
Dict: Токенизированные данные
"""
return self.tokenizer(
texts,
padding=True,
truncation=True,
return_tensors="pt",
**kwargs
texts, padding=True, truncation=True, return_tensors="pt", **kwargs
)
def decode_batch(self, token_ids: torch.Tensor, **kwargs) -> List[str]:
@@ -268,9 +256,7 @@ class TokenizerWrapper:
texts = []
for i in range(token_ids.size(0)):
text = self.tokenizer.decode(
token_ids[i],
skip_special_tokens=True,
**kwargs
token_ids[i], skip_special_tokens=True, **kwargs
)
texts.append(text)
@@ -290,12 +276,7 @@ class TokenizerWrapper:
}
def create_hf_pipeline(
llm_model,
tokenizer=None,
device: str = "auto",
**kwargs
):
def create_hf_pipeline(llm_model, tokenizer=None, device: str = "auto", **kwargs):
"""
Создает HuggingFace pipeline из llm модели.
@@ -315,11 +296,7 @@ def create_hf_pipeline(
# Создаем pipeline
pipe = pipeline(
"text-generation",
model=hf_model,
tokenizer=tokenizer,
device=device,
**kwargs
"text-generation", model=hf_model, tokenizer=tokenizer, device=device, **kwargs
)
return pipe

6
llm/src/llm/core/base_model.py
View File

@@ -19,6 +19,7 @@ from abc import ABC, abstractmethod
from typing import Optional, Tuple
import torch
class BaseModel(nn.Module, ABC):
"""
Абстрактный класс — стандарт для всех архитектур LLM.
@@ -32,6 +33,7 @@ class BaseModel(nn.Module, ABC):
Attributes:
config (dict): Конфиг модели
"""
def __init__(self, config: dict):
"""
Инициализация модели.
@@ -43,7 +45,9 @@ class BaseModel(nn.Module, ABC):
self.config = config
@abstractmethod
def forward(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
def forward(
self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""
Прямой проход — получение логитов для входных токенов.

2
llm/src/llm/core/cached_decoder.py
View File

@@ -6,6 +6,7 @@ from .feed_forward import FeedForward
from .multi_head_attention import MultiHeadAttention
from .rope import RoPE
class CachedDecoder(nn.Module):
"""
Универсальный декодерный блок для современных LLM (GPT, LLaMA, др.), поддерживает кэширование key-value для эффективной генерации.
@@ -36,6 +37,7 @@ class CachedDecoder(nn.Module):
... num_heads=4, emb_size=256, head_size=64, max_seq_len=128)
>>> out, cache = decoder(x, use_cache=True)
"""
def __init__(
self,
feed_forward_layer: nn.Module,

9
llm/src/llm/core/decoder.py
View File

@@ -3,6 +3,7 @@ import torch
from .feed_forward import FeedForward
from .multi_head_attention import MultiHeadAttention
class Decoder(nn.Module):
"""
Базовый автогерессивный блок-декодер трансформера (без кэша KV).
@@ -24,12 +25,14 @@ class Decoder(nn.Module):
>>> out = decoder(x)
>>> print(out.shape) # torch.Size([1, 10, 512])
"""
def __init__(self,
def __init__(
self,
num_heads: int,
emb_size: int,
head_size: int,
max_seq_len: int,
dropout: float = 0.1
dropout: float = 0.1,
):
"""
Инициализация декодера.
@@ -47,7 +50,7 @@ class Decoder(nn.Module):
emb_size=emb_size,
head_size=head_size,
max_seq_len=max_seq_len,
dropout=dropout
dropout=dropout,
)
self._ff = FeedForward(emb_size=emb_size, dropout=dropout)
self._norm1 = nn.LayerNorm(emb_size)

1
llm/src/llm/core/feed_forward.py
View File

@@ -45,6 +45,7 @@ class FeedForward(nn.Module):
>>> output = ff(x)
>>> print(output.shape) # torch.Size([32, 10, 512])
"""
def __init__(self, emb_size: int, dropout: float = 0.1, activation: str = "relu"):
"""
Инициализация слоя Feed Forward Network.

10
llm/src/llm/core/gelu.py
View File

@@ -1,6 +1,7 @@
import torch
from torch import nn
class GELU(nn.Module):
"""
Гауссовская Эрф-активация (GELU, Gaussian Error Linear Unit).
@@ -17,11 +18,14 @@ class GELU(nn.Module):
>>> y = gelu(torch.tensor([-1.0, 0.0, 1.0]))
>>> print(y)
"""
def __init__(self):
super().__init__()
self.sqrt_2_over_pi = torch.sqrt(torch.tensor(2.0) / math.pi)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return 0.5 * x * (1 + torch.tanh(
self.sqrt_2_over_pi * (x + 0.044715 * torch.pow(x, 3))
))
return (
0.5
* x
* (1 + torch.tanh(self.sqrt_2_over_pi * (x + 0.044715 * torch.pow(x, 3))))
)

22
llm/src/llm/core/head_attention.py
View File

@@ -4,6 +4,7 @@ import torch.nn.functional as F
from math import sqrt
from .rope import RoPE
class HeadAttention(nn.Module):
"""
Одноголовый механизм внимания (scaled dot-product attention) — фундаментальный строительный блок всех современных Transformer.
@@ -35,7 +36,10 @@ class HeadAttention(nn.Module):
>>> output, _ = attention(x)
>>> print(output.shape) # torch.Size([1, 10, 32])
"""
def __init__(self, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None):
def __init__(
self, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None
):
super().__init__()
self._emb_size = emb_size
self._head_size = head_size
@@ -49,9 +53,13 @@ class HeadAttention(nn.Module):
# Создание causal маски
mask = torch.tril(torch.ones(max_seq_len, max_seq_len))
self.register_buffer('_tril_mask', mask.bool() if hasattr(torch, 'bool') else mask.byte())
self.register_buffer(
"_tril_mask", mask.bool() if hasattr(torch, "bool") else mask.byte()
)
def forward(self, x: torch.Tensor, use_cache: bool = True, cache: tuple = None) -> tuple:
def forward(
self, x: torch.Tensor, use_cache: bool = True, cache: tuple = None
) -> tuple:
"""
Прямой проход через слой внимания.
@@ -73,7 +81,9 @@ class HeadAttention(nn.Module):
"""
seq_len = x.shape[1]
if seq_len > self._max_seq_len:
raise ValueError(f"Длина последовательности {seq_len} превышает максимум {self._max_seq_len}")
raise ValueError(
f"Длина последовательности {seq_len} превышает максимум {self._max_seq_len}"
)
k = self._k(x) # [B, T, hs]
q = self._q(x) # [B, T, hs]
@@ -92,7 +102,9 @@ class HeadAttention(nn.Module):
scores = q @ k.transpose(-2, -1) / sqrt(self._head_size)
if cache is None:
scores = scores.masked_fill(~self._tril_mask[:seq_len, :seq_len], float('-inf'))
scores = scores.masked_fill(
~self._tril_mask[:seq_len, :seq_len], float("-inf")
)
weights = F.softmax(scores, dim=-1)
x_out = weights @ v # [B, T, hs]

39
llm/src/llm/core/multi_head_attention.py
View File

@@ -3,6 +3,7 @@ import torch
from .head_attention import HeadAttention
from .rope import RoPE
class MultiHeadAttention(nn.Module):
"""
Мультиголовый (многоголовый) механизм внимания — ключевой компонент любого Transformer.
@@ -32,7 +33,16 @@ class MultiHeadAttention(nn.Module):
>>> out, cache = mha(x)
>>> print(out.shape)
"""
def __init__(self, num_heads: int, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None, dropout: float = 0.1):
def __init__(
self,
num_heads: int,
emb_size: int,
head_size: int,
max_seq_len: int,
rope: RoPE = None,
dropout: float = 0.1,
):
"""
Инициализация многоголового внимания.
@@ -49,18 +59,27 @@ class MultiHeadAttention(nn.Module):
- max_seq_len зависит от задачи (512 для BERT, 2048 для GPT-3)
"""
super().__init__()
self._heads = nn.ModuleList([
HeadAttention(
emb_size=emb_size,
head_size=head_size,
max_seq_len=max_seq_len,
rope=rope,
) for _ in range(num_heads)
])
self._heads = nn.ModuleList(
[
HeadAttention(
emb_size=emb_size,
head_size=head_size,
max_seq_len=max_seq_len,
rope=rope,
)
for _ in range(num_heads)
]
)
self._layer = nn.Linear(head_size * num_heads, emb_size)
self._dropout = nn.Dropout(dropout)
def forward(self, x: torch.Tensor, mask: torch.Tensor = None, use_cache: bool = True, cache: list = None):
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor = None,
use_cache: bool = True,
cache: list = None,
):
"""
Прямой проход (forward):
Для каждого токена оценивает "важность" остальных токенов сразу через несколько attention-блоков.

10
llm/src/llm/core/positional_embeddings.py
View File

@@ -1,6 +1,7 @@
import torch
from torch import nn, Tensor
class PositionalEmbeddings(nn.Module):
"""
Обучаемые позиционные эмбеддинги (learnable positional embeddings).
@@ -36,8 +37,7 @@ class PositionalEmbeddings(nn.Module):
self.max_seq_len = max_seq_len
self.emb_size = emb_size
self.embedding = nn.Embedding(
num_embeddings=max_seq_len,
embedding_dim=emb_size
num_embeddings=max_seq_len, embedding_dim=emb_size
)
def forward(self, seq_len: int, start_pos: int = 0) -> Tensor:
@@ -62,5 +62,9 @@ class PositionalEmbeddings(nn.Module):
if start_pos == 0:
positions = torch.arange(seq_len, device=self.embedding.weight.device)
else:
positions = torch.arange(start=start_pos, end=start_pos + seq_len, device=self.embedding.weight.device)
positions = torch.arange(
start=start_pos,
end=start_pos + seq_len,
device=self.embedding.weight.device,
)
return self.embedding(positions)

2
llm/src/llm/core/rms_norm.py
View File

@@ -80,4 +80,4 @@ class RMSNorm(nn.Module):
def extra_repr(self) -> str:
"""Строковое представление для отладки."""
return f'dim={self._w.shape[0]}, eps={self._eps}'
return f"dim={self._w.shape[0]}, eps={self._eps}"

6
llm/src/llm/core/rope.py
View File

@@ -65,8 +65,8 @@ class RoPE(nn.Module):
freq_matrix = positions.unsqueeze(1) * freqs.unsqueeze(0)
# Предвычисление матриц косинусов и синусов
self.register_buffer('cos_matrix', torch.cos(freq_matrix))
self.register_buffer('sin_matrix', torch.sin(freq_matrix))
self.register_buffer("cos_matrix", torch.cos(freq_matrix))
self.register_buffer("sin_matrix", torch.sin(freq_matrix))
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
@@ -91,7 +91,7 @@ class RoPE(nn.Module):
# Разделяем на четные и нечетные компоненты
x_even = x[:, :, 0::2] # [batch_size, seq_len, head_size//2]
x_odd = x[:, :, 1::2] # [batch_size, seq_len, head_size//2]
x_odd = x[:, :, 1::2] # [batch_size, seq_len, head_size//2]
# Применяем поворот: q' = q * cos(mθ) + rotate(q) * sin(mθ)
x_rotated_even = x_even * cos - x_odd * sin

2
llm/src/llm/core/silu.py
View File

@@ -1,6 +1,7 @@
import torch
from torch import nn
class SiLU(nn.Module):
"""
SiLU (Swish) — современная активационная функция для нейросетей.
@@ -15,5 +16,6 @@ class SiLU(nn.Module):
>>> x = torch.tensor([-1.0, 0.0, 1.0])
>>> print(act(x))
"""
def forward(self, x: torch.Tensor):
return torch.sigmoid(x) * x

12
llm/src/llm/core/swi_glu.py
View File

@@ -83,19 +83,19 @@ class SwiGLU(nn.Module):
4. apply dropout
"""
# Gate ветвь: линейное преобразование + активация
gate_out = self._gate(x) # [batch, seq, 4*emb]
activation_out = self._activation(gate_out) # [batch, seq, 4*emb]
gate_out = self._gate(x) # [batch, seq, 4*emb]
activation_out = self._activation(gate_out) # [batch, seq, 4*emb]
# Up ветвь: линейное преобразование
up_out = self._up(x) # [batch, seq, 4*emb]
up_out = self._up(x) # [batch, seq, 4*emb]
# Element-wise multiplication (gating mechanism)
out = up_out * activation_out # поэлементное умножение!
out = up_out * activation_out # поэлементное умножение!
# Final projection and dropout
out = self._down(out) # [batch, seq, emb]
out = self._down(out) # [batch, seq, emb]
return self._dropout(out)
def extra_repr(self) -> str:
"""Строковое представление для отладки."""
return f'emb_size={self._gate.in_features}, dropout={self._dropout.p}'
return f"emb_size={self._gate.in_features}, dropout={self._dropout.p}"

10
llm/src/llm/core/token_embeddings.py
View File

@@ -2,6 +2,7 @@ import torch
from torch import nn
from torch import Tensor
class TokenEmbeddings(nn.Module):
"""
Токеновые эмбеддинги — обучаемые векторные представления для каждого токена словаря.
@@ -29,11 +30,11 @@ class TokenEmbeddings(nn.Module):
>>> vecs = emb(tokens)
>>> vecs.shape # torch.Size([1, 3, 256])
"""
def __init__(self, vocab_size: int, emb_size: int):
super().__init__()
self._embedding = nn.Embedding(
num_embeddings=vocab_size,
embedding_dim=emb_size
num_embeddings=vocab_size, embedding_dim=emb_size
)
def forward(self, x: Tensor) -> Tensor:
@@ -55,10 +56,7 @@ if __name__ == "__main__":
embedding = TokenEmbeddings(vocab_size=100, emb_size=128)
# Создаем тензор с индексами в пределах vocab_size (0-99)
tensor = torch.tensor([
[11, 45, 76, 34],
[34, 67, 45, 54]
])
tensor = torch.tensor([[11, 45, 76, 34], [34, 67, 45, 54]])
# Проверяем индексы
if (tensor >= 100).any():

96
llm/src/llm/models/gpt/gpt.py
View File

@@ -54,28 +54,32 @@ class GPT(BaseModel):
_norm: Финальный слой нормализации
_linear: Выходной линейный слой
"""
def __init__(self, config):
super().__init__(config)
# Инициализация слоев
self._max_seq_len = config["max_position_embeddings"]
self._token_embeddings = TokenEmbeddings(
vocab_size=config["vocab_size"],
emb_size=config["embed_dim"]
vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
)
self._position_embeddings = PositionalEmbeddings(
max_seq_len=config["max_position_embeddings"],
emb_size=config["embed_dim"]
max_seq_len=config["max_position_embeddings"], emb_size=config["embed_dim"]
)
self._dropout = nn.Dropout(config["dropout"])
# head_size = emb_size // num_heads
self._decoders = nn.ModuleList([Decoder(
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
max_seq_len=config["max_position_embeddings"],
dropout=config["dropout"]
) for _ in range(config["num_layers"])])
self._decoders = nn.ModuleList(
[
Decoder(
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
max_seq_len=config["max_position_embeddings"],
dropout=config["dropout"],
)
for _ in range(config["num_layers"])
]
)
self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
@property
@@ -94,14 +98,18 @@ class GPT(BaseModel):
"""
# Проверка длины последовательности
if x.size(1) > self._max_seq_len:
raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}")
raise ValueError(
f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}"
)
# Эмбеддинги токенов и позиций
tok_out = self._token_embeddings(x) # [batch, seq_len, emb_size]
pos_out = self._position_embeddings(x.size(1)) # [seq_len, emb_size]
# Комбинирование
out = self._dropout(tok_out + pos_out.unsqueeze(0)) # [batch, seq_len, emb_size]
out = self._dropout(
tok_out + pos_out.unsqueeze(0)
) # [batch, seq_len, emb_size]
# Стек декодеров
for decoder in self._decoders:
@@ -109,22 +117,21 @@ class GPT(BaseModel):
return self._linear(out) # [batch, seq_len, vocab_size]
# def forward(self, input_ids, attention_mask=None):
# B, T = input_ids.size()
# pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
#
# x = self.token_emb(input_ids) + self.pos_emb(pos)
#
# for block in self.blocks:
# x = block(x, attention_mask)
#
# x = self.ln_f(x)
# logits = self.head(x)
# return logits
# def forward(self, input_ids, attention_mask=None):
# B, T = input_ids.size()
# pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
#
# x = self.token_emb(input_ids) + self.pos_emb(pos)
#
# for block in self.blocks:
# x = block(x, attention_mask)
#
# x = self.ln_f(x)
# logits = self.head(x)
# return logits
def generate(self,
def generate(
self,
x: torch.Tensor,
max_new_tokens: int,
do_sample: bool,
@@ -132,7 +139,7 @@ class GPT(BaseModel):
top_k: int = None,
top_p: float = None,
attention_mask: torch.Tensor = None, # Добавляем для совместимости с HF
**kwargs # Игнорируем остальные параметры
**kwargs, # Игнорируем остальные параметры
) -> torch.Tensor:
"""Авторегрессивная генерация текста.
@@ -228,7 +235,7 @@ class GPT(BaseModel):
"""
for _ in range(max_new_tokens):
# 1. Обрезаем вход, если последовательность слишком длинная
x_cond = x[:, -self._max_seq_len:]
x_cond = x[:, -self._max_seq_len :]
# 2. Передаем последовательность в метод forward класса GPT и полуаем логиты.
logits = self.forward(x_cond)
@@ -250,9 +257,14 @@ class GPT(BaseModel):
vocab_size = logits_scaled.size(-1)
# создаём маску: True, если токен НЕ в topk_indices
mask = torch.ones_like(logits_scaled, dtype=torch.bool if hasattr(torch, 'bool') else torch.uint8)
mask.scatter_(1, topk_indices, False if hasattr(torch, 'bool') else 0) # False там, где top-k индексы
masked_logits[mask] = float('-inf')
mask = torch.ones_like(
logits_scaled,
dtype=torch.bool if hasattr(torch, "bool") else torch.uint8,
)
mask.scatter_(
1, topk_indices, False if hasattr(torch, "bool") else 0
) # False там, где top-k индексы
masked_logits[mask] = float("-inf")
logits_scaled = masked_logits
@@ -260,36 +272,42 @@ class GPT(BaseModel):
# 1. Применим softmax, чтобы получить вероятности:
probs = F.softmax(logits_scaled, dim=-1) # [B, vocab_size]
# 2. Отсортируем токены по убыванию вероятностей:
sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
sorted_probs, sorted_indices = torch.sort(
probs, descending=True, dim=-1
)
# 3. Посчитаем кумулятивную сумму вероятностей:
cum_probs = torch.cumsum(sorted_probs, dim=-1) # [B, vocab_size]
# 4. Определим маску: оставить токены, пока сумма < top_p
sorted_mask = (cum_probs <= top_p) # [B, vocab_size]
sorted_mask = cum_probs <= top_p # [B, vocab_size]
# Гарантируем, что хотя бы первый токен останется
sorted_mask[:, 0] = True
# 5. Преобразуем маску обратно в оригинальный порядок:
# Создаём полную маску из False
mask = torch.zeros_like(probs, dtype=torch.bool if hasattr(torch, 'bool') else torch.uint8)
mask = torch.zeros_like(
probs, dtype=torch.bool if hasattr(torch, "bool") else torch.uint8
)
# Устанавливаем True в местах нужных токенов
mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
# 6. Зануляем логиты токенов вне топ-p:
logits_scaled[~mask] = float('-inf')
logits_scaled[~mask] = float("-inf")
# 4. Применяем Softmax
probs = F.softmax(logits_scaled, dim=-1) # [batch_size, vocab_size]
if do_sample == True:
# 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
next_token = torch.multinomial(probs, num_samples=1) # [batch_size, 1]
else:
# 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
next_token = torch.argmax(probs, dim=-1, keepdim=True) # [batch_size, 1]
next_token = torch.argmax(
probs, dim=-1, keepdim=True
) # [batch_size, 1]
# 6. Добавляем его к последовательности
x = torch.cat([x, next_token], dim=1) # [batch_size, seq_len+1]
return x
# def generate(self, input_ids, max_length=50):
# for _ in range(max_length):
# logits = self.forward(input_ids)

72
llm/src/llm/models/gpt/gpt2.py
View File

@@ -27,6 +27,7 @@ from llm.core.positional_embeddings import PositionalEmbeddings
from llm.core.cached_decoder import CachedDecoder
from llm.core.feed_forward import FeedForward
class GPT2(BaseModel):
"""
GPT2 — автогерессивная языковая модель, архитектура Transformer, предложенная OpenAI.
@@ -44,37 +45,43 @@ class GPT2(BaseModel):
>>> logits = model(input_ids)
>>> out = model.generate(input_ids, max_length=20)
"""
def __init__(self, config):
super().__init__(config)
# Инициализация слоев
self._max_seq_len = config["max_position_embeddings"]
self._token_embeddings = TokenEmbeddings(
vocab_size=config["vocab_size"],
emb_size=config["embed_dim"]
vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
)
self._position_embeddings = PositionalEmbeddings(
max_seq_len=config["max_position_embeddings"],
emb_size=config["embed_dim"]
max_seq_len=config["max_position_embeddings"], emb_size=config["embed_dim"]
)
self._dropout = nn.Dropout(config["dropout"])
# head_size = emb_size // num_heads
self._decoders = nn.ModuleList([CachedDecoder(
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
feed_forward_layer=FeedForward(
emb_size=config["embed_dim"],
dropout=config["dropout"],
activation="gelu"
),
max_seq_len=config["max_position_embeddings"],
dropout=config["dropout"]
) for _ in range(config["num_layers"])])
self._decoders = nn.ModuleList(
[
CachedDecoder(
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
feed_forward_layer=FeedForward(
emb_size=config["embed_dim"],
dropout=config["dropout"],
activation="gelu",
),
max_seq_len=config["max_position_embeddings"],
dropout=config["dropout"],
)
for _ in range(config["num_layers"])
]
)
self._norm = nn.LayerNorm(config["embed_dim"])
self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
def forward(self, x: torch.Tensor, use_cache: bool = True, cache: list = None) -> tuple:
def forward(
self, x: torch.Tensor, use_cache: bool = True, cache: list = None
) -> tuple:
"""
Прямой проход GPT2:
- Все слои работают как autoregressive transformer (masked self-attention).
@@ -91,8 +98,9 @@ class GPT2(BaseModel):
"""
# Проверка длины последовательности (только при отсутствии кэша)
if cache is None and x.size(1) > self._max_seq_len:
raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}")
raise ValueError(
f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}"
)
# Вычисление start_pos из кэша (если кэш передан)
if cache is not None:
@@ -111,10 +119,14 @@ class GPT2(BaseModel):
# Эмбеддинги токенов и позиций
tok_out = self._token_embeddings(x) # [batch, seq_len, emb_size]
pos_out = self._position_embeddings(seq_len, start_pos=start_pos) # [seq_len, emb_size]
pos_out = self._position_embeddings(
seq_len, start_pos=start_pos
) # [seq_len, emb_size]
# Комбинирование
out = self._dropout(tok_out + pos_out.unsqueeze(0)) # [batch, seq_len, emb_size]
out = self._dropout(
tok_out + pos_out.unsqueeze(0)
) # [batch, seq_len, emb_size]
# Стек декодеров с передачей кэша
new_cache = []
@@ -138,14 +150,15 @@ class GPT2(BaseModel):
else:
return (logits, None)
def generate(self,
def generate(
self,
x: torch.Tensor,
max_new_tokens: int,
do_sample: bool,
temperature: float = 1.0,
top_k: int = None,
top_p: float = None,
use_cache: bool = True
use_cache: bool = True,
) -> torch.Tensor:
"""
Генерация текста с использованием autoregressive трансформера (GPT2).
@@ -200,7 +213,7 @@ class GPT2(BaseModel):
# создаём маску: 1, если токен НЕ в topk_indices
mask = torch.ones_like(logits_scaled, dtype=torch.uint8)
mask.scatter_(1, topk_indices, 0) # 0 там, где top-k индексы
masked_logits[mask.byte()] = float('-inf')
masked_logits[mask.byte()] = float("-inf")
logits_scaled = masked_logits
@@ -208,7 +221,9 @@ class GPT2(BaseModel):
# 1. Применим softmax, чтобы получить вероятности:
probs = F.softmax(logits_scaled, dim=-1) # [B, vocab_size]
# 2. Отсортируем токены по убыванию вероятностей:
sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
sorted_probs, sorted_indices = torch.sort(
probs, descending=True, dim=-1
)
# 3. Посчитаем кумулятивную сумму вероятностей:
cum_probs = torch.cumsum(sorted_probs, dim=-1) # [B, vocab_size]
# 4. Определим маску: оставить токены, пока сумма < top_p
@@ -221,18 +236,19 @@ class GPT2(BaseModel):
# Устанавливаем 1 в местах нужных токенов
mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
# 6. Зануляем логиты токенов вне топ-p:
logits_scaled[~mask] = float('-inf')
logits_scaled[~mask] = float("-inf")
# 4. Применяем Softmax
probs = F.softmax(logits_scaled, dim=-1) # [batch_size, vocab_size]
if do_sample == True:
# 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
next_token = torch.multinomial(probs, num_samples=1) # [batch_size, 1]
else:
# 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
next_token = torch.argmax(probs, dim=-1, keepdim=True) # [batch_size, 1]
next_token = torch.argmax(
probs, dim=-1, keepdim=True
) # [batch_size, 1]
# 6. Добавляем его к последовательности
x = torch.cat([x, next_token], dim=1) # [batch_size, seq_len+1]

111
llm/src/llm/models/llama/llama.py
View File

@@ -10,7 +10,6 @@ from llm.core.rope import RoPE
from llm.core.cached_decoder import CachedDecoder
class Llama(BaseModel):
"""
LLaMA (Large Language Model Meta AI) — высокоэффективная масштабируемая языковая модель, разработанная Meta AI Research.
@@ -29,38 +28,45 @@ class Llama(BaseModel):
>>> logits, cache = model(input_ids, use_cache=True)
>>> out = model.generate(input_ids, max_new_tokens=20)
"""
def __init__(self,config):
def __init__(self, config):
super().__init__(config)
# Инициализация слоев
self._max_seq_len = config["max_position_embeddings"]
self._token_embeddings = TokenEmbeddings(
vocab_size=config["vocab_size"],
emb_size=config["embed_dim"]
vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
)
self._position_embeddings = RoPE(
head_size=config["embed_dim"] // config["num_heads"],
max_seq_len=config["max_position_embeddings"]
max_seq_len=config["max_position_embeddings"],
)
self._dropout = nn.Dropout(config["dropout"])
self._decoders = nn.ModuleList([CachedDecoder(
norm_layer=RMSNorm,
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
feed_forward_layer=SwiGLU(
emb_size=config["embed_dim"],
dropout=config["dropout"],
),
max_seq_len=config["max_position_embeddings"],
rope=self._position_embeddings,
dropout=config["dropout"],
) for _ in range(config["num_layers"])])
self._decoders = nn.ModuleList(
[
CachedDecoder(
norm_layer=RMSNorm,
num_heads=config["num_heads"],
emb_size=config["embed_dim"],
head_size=config["embed_dim"] // config["num_heads"],
feed_forward_layer=SwiGLU(
emb_size=config["embed_dim"],
dropout=config["dropout"],
),
max_seq_len=config["max_position_embeddings"],
rope=self._position_embeddings,
dropout=config["dropout"],
)
for _ in range(config["num_layers"])
]
)
self._norm = RMSNorm(config["embed_dim"])
self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
def forward(self, x: torch.Tensor, use_cache: bool = True, cache: list = None) -> tuple:
def forward(
self, x: torch.Tensor, use_cache: bool = True, cache: list = None
) -> tuple:
"""
Прямой проход через LLaMA (inference/train): авторегрессионное предсказание токенов.
@@ -76,11 +82,12 @@ class Llama(BaseModel):
"""
# Проверка длины последовательности (только при отсутствии кэша)
if cache is None and x.size(1) > self._max_seq_len:
raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}")
raise ValueError(
f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}"
)
# Вычисление start_pos из кэша (если кэш передан)
#if cache is not None:
# if cache is not None:
# # При кэше обрабатываем только один токен (последний)
# seq_len = 1
# # Вычисляем start_pos из самого нижнего уровня кэша
@@ -89,14 +96,14 @@ class Llama(BaseModel):
# start_pos = key_cache.size(1) # cache_len
# else:
# start_pos = 0
#else:
# else:
# # Без кэша работаем как раньше
# start_pos = 0
# seq_len = x.size(1)
# Эмбеддинги токенов и позиций
tok_out = self._token_embeddings(x) # [batch, seq_len, emb_size]
#pos_out = self._position_embeddings(x) # [batch, seq_len, emb_size]
# pos_out = self._position_embeddings(x) # [batch, seq_len, emb_size]
# Комбинирование
out = self._dropout(tok_out) # [batch, seq_len, emb_size]
@@ -123,35 +130,36 @@ class Llama(BaseModel):
else:
return (logits, None)
def generate(self,
def generate(
self,
x: torch.Tensor,
max_new_tokens: int,
do_sample: bool,
temperature: float = 1.0,
top_k: int = None,
top_p: float = None,
use_cache: bool = True
use_cache: bool = True,
) -> torch.Tensor:
"""
Генерация текста c помощью LLaMA (autoregressive Transformer).
Поддерживается:
- greedy и вероятностное сэмплирование (top-k, top-p, temperature)
- кэш attention для ускорения генерации длинных последовательностей
Генерация текста c помощью LLaMA (autoregressive Transformer).
Поддерживается:
- greedy и вероятностное сэмплирование (top-k, top-p, temperature)
- кэш attention для ускорения генерации длинных последовательностей
Args:
x (Tensor[int]): начальная последовательность [batch, seq_len]
max_new_tokens (int): сколько новых токенов сгенерировать
do_sample (bool): использовать стохастику (True) или жадный выбор (False)
temperature (float): масштаб для softmax (важно для sampling)
top_k (int|None): ограничение на количество кандидатов (top-k sampling)
top_p (float|None): nucleus sampling
use_cache (bool): ускоряет autoregressive при длинной генерации
Returns:
output (Tensor[int]): [batch, seq_len + max_new_tokens]
Пример:
>>> prompt = tokenizer.encode('Meta AI', return_tensors="pt")
>>> generated = model.generate(prompt, max_new_tokens=30, do_sample=True)
>>> print(tokenizer.decode(generated[0]))
Args:
x (Tensor[int]): начальная последовательность [batch, seq_len]
max_new_tokens (int): сколько новых токенов сгенерировать
do_sample (bool): использовать стохастику (True) или жадный выбор (False)
temperature (float): масштаб для softmax (важно для sampling)
top_k (int|None): ограничение на количество кандидатов (top-k sampling)
top_p (float|None): nucleus sampling
use_cache (bool): ускоряет autoregressive при длинной генерации
Returns:
output (Tensor[int]): [batch, seq_len + max_new_tokens]
Пример:
>>> prompt = tokenizer.encode('Meta AI', return_tensors="pt")
>>> generated = model.generate(prompt, max_new_tokens=30, do_sample=True)
>>> print(tokenizer.decode(generated[0]))
"""
cache = None
@@ -188,7 +196,7 @@ class Llama(BaseModel):
# создаём маску: 1, если токен НЕ в topk_indices
mask = torch.ones_like(logits_scaled, dtype=torch.uint8)
mask.scatter_(1, topk_indices, 0) # 0 там, где top-k индексы
masked_logits[mask.byte()] = float('-inf')
masked_logits[mask.byte()] = float("-inf")
logits_scaled = masked_logits
@@ -196,7 +204,9 @@ class Llama(BaseModel):
# 1. Применим softmax, чтобы получить вероятности:
probs = F.softmax(logits_scaled, dim=-1) # [B, vocab_size]
# 2. Отсортируем токены по убыванию вероятностей:
sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
sorted_probs, sorted_indices = torch.sort(
probs, descending=True, dim=-1
)
# 3. Посчитаем кумулятивную сумму вероятностей:
cum_probs = torch.cumsum(sorted_probs, dim=-1) # [B, vocab_size]
# 4. Определим маску: оставить токены, пока сумма < top_p
@@ -209,25 +219,24 @@ class Llama(BaseModel):
# Устанавливаем 1 в местах нужных токенов
mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
# 6. Зануляем логиты токенов вне топ-p:
logits_scaled[~mask] = float('-inf')
logits_scaled[~mask] = float("-inf")
# 4. Применяем Softmax
probs = F.softmax(logits_scaled, dim=-1) # [batch_size, vocab_size]
if do_sample == True:
# 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
next_token = torch.multinomial(probs, num_samples=1) # [batch_size, 1]
else:
# 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
next_token = torch.argmax(probs, dim=-1, keepdim=True) # [batch_size, 1]
next_token = torch.argmax(
probs, dim=-1, keepdim=True
) # [batch_size, 1]
# 6. Добавляем его к последовательности
x = torch.cat([x, next_token], dim=1) # [batch_size, seq_len+1]
return x
@property
def max_seq_len(self) -> int:
return self._max_seq_len

34
llm/src/llm/tokenizers/base_tokenizer.py
View File

@@ -82,7 +82,9 @@ class BaseTokenizer(ABC):
List[str]: Список токенов
"""
token_ids = self.encode(text, **kwargs)
return [self.inverse_vocab.get(token_id, self.unk_token) for token_id in token_ids]
return [
self.inverse_vocab.get(token_id, self.unk_token) for token_id in token_ids
]
def get_vocab(self) -> Dict[str, int]:
"""Возвращает словарь токенизатора."""
@@ -120,16 +122,16 @@ class BaseTokenizer(ABC):
filepath: Путь для сохранения
"""
config = {
'vocab': self.vocab,
'vocab_size': self.vocab_size,
'pad_token': self.pad_token,
'unk_token': self.unk_token,
'bos_token': self.bos_token,
'eos_token': self.eos_token,
'tokenizer_type': self.__class__.__name__
"vocab": self.vocab,
"vocab_size": self.vocab_size,
"pad_token": self.pad_token,
"unk_token": self.unk_token,
"bos_token": self.bos_token,
"eos_token": self.eos_token,
"tokenizer_type": self.__class__.__name__,
}
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(config, f, ensure_ascii=False, indent=2)
@classmethod
@@ -143,17 +145,17 @@ class BaseTokenizer(ABC):
Returns:
BaseTokenizer: Загруженный токенизатор
"""
with open(filepath, 'r', encoding='utf-8') as f:
with open(filepath, "r", encoding="utf-8") as f:
config = json.load(f)
# Создаем экземпляр токенизатора
tokenizer = cls()
tokenizer.vocab = config['vocab']
tokenizer.vocab_size = config['vocab_size']
tokenizer.pad_token = config['pad_token']
tokenizer.unk_token = config['unk_token']
tokenizer.bos_token = config['bos_token']
tokenizer.eos_token = config['eos_token']
tokenizer.vocab = config["vocab"]
tokenizer.vocab_size = config["vocab_size"]
tokenizer.pad_token = config["pad_token"]
tokenizer.unk_token = config["unk_token"]
tokenizer.bos_token = config["bos_token"]
tokenizer.eos_token = config["eos_token"]
# Создаем обратный словарь
tokenizer.inverse_vocab = {v: k for k, v in tokenizer.vocab.items()}

104
llm/src/llm/tokenizers/bpe_tokenizer copy.py
View File

@@ -44,7 +44,10 @@ class BPETokenizer(BaseTokenizer):
self._initialize_vocab()
# Добавляем специальные токены если указаны
special_tokens = kwargs.get('special_tokens', [self.pad_token, self.unk_token, self.bos_token, self.eos_token])
special_tokens = kwargs.get(
"special_tokens",
[self.pad_token, self.unk_token, self.bos_token, self.eos_token],
)
self.add_special_tokens(special_tokens)
# Предобработка текстов
@@ -54,7 +57,7 @@ class BPETokenizer(BaseTokenizer):
vocab = self._get_initial_vocab(words)
# Выполняем BPE мерджи
self._perform_merges(vocab, vocab_size, kwargs.get('min_frequency', 2))
self._perform_merges(vocab, vocab_size, kwargs.get("min_frequency", 2))
# Строим финальный словарь
self._build_final_vocab()
@@ -99,11 +102,13 @@ class BPETokenizer(BaseTokenizer):
for word_list in words:
for word in word_list:
# Разбиваем слово на символы и добавляем специальный символ конца слова
chars = list(word) + ['</w>']
vocab.update([''.join(chars[i:i+1]) for i in range(len(chars))])
chars = list(word) + ["</w>"]
vocab.update(["".join(chars[i : i + 1]) for i in range(len(chars))])
return vocab
def _perform_merges(self, vocab: Dict[str, int], target_vocab_size: int, min_frequency: int):
def _perform_merges(
self, vocab: Dict[str, int], target_vocab_size: int, min_frequency: int
):
"""
Выполняет BPE мерджи до достижения целевого размера словаря.
@@ -146,7 +151,9 @@ class BPETokenizer(BaseTokenizer):
pairs[symbols[i], symbols[i + 1]] += freq
return pairs
def _merge_vocab(self, vocab: Dict[str, int], pair: Tuple[str, str]) -> Dict[str, int]:
def _merge_vocab(
self, vocab: Dict[str, int], pair: Tuple[str, str]
) -> Dict[str, int]:
"""
Объединяет пару символов в словаре.
@@ -158,7 +165,9 @@ class BPETokenizer(BaseTokenizer):
Dict[str, int]: Обновленный словарь
"""
new_vocab = {}
bigram = re.compile(r'(?<!\\S)' + re.escape(pair[0]) + r' ' + re.escape(pair[1]) + r'(?!\\S)')
bigram = re.compile(
r"(?<!\\S)" + re.escape(pair[0]) + r" " + re.escape(pair[1]) + r"(?!\\S)"
)
replacement = pair[0] + pair[1]
for word in vocab:
@@ -173,7 +182,9 @@ class BPETokenizer(BaseTokenizer):
all_tokens = set()
# Добавляем специальные токены
all_tokens.update([self.pad_token, self.unk_token, self.bos_token, self.eos_token])
all_tokens.update(
[self.pad_token, self.unk_token, self.bos_token, self.eos_token]
)
# Добавляем токены из мерджей
for pair in self.merges:
@@ -204,7 +215,7 @@ class BPETokenizer(BaseTokenizer):
Returns:
List[int]: Список идентификаторов токенов
"""
add_special_tokens = kwargs.get('add_special_tokens', False)
add_special_tokens = kwargs.get("add_special_tokens", False)
# Токенизация текста
tokens = self.compiled_pattern.findall(text)
@@ -243,8 +254,8 @@ class BPETokenizer(BaseTokenizer):
List[str]: Список BPE токенов
"""
# Простая реализация - в реальной реализации нужно применять обученные мерджи
word = token + '</w>'
tokens = [word[i:i+1] for i in range(len(word))]
word = token + "</w>"
tokens = [word[i : i + 1] for i in range(len(word))]
# Применяем мерджи (упрощенная версия)
# В полной реализации нужно применять все обученные мерджи
@@ -271,7 +282,7 @@ class BPETokenizer(BaseTokenizer):
Returns:
str: Декодированный текст
"""
skip_special_tokens = kwargs.get('skip_special_tokens', True)
skip_special_tokens = kwargs.get("skip_special_tokens", True)
# Конвертируем ID в токены
token_strings = []
@@ -279,16 +290,21 @@ class BPETokenizer(BaseTokenizer):
token = self.inverse_vocab.get(token_id, self.unk_token)
# Пропускаем специальные токены если нужно
if skip_special_tokens and token in [self.pad_token, self.unk_token, self.bos_token, self.eos_token]:
if skip_special_tokens and token in [
self.pad_token,
self.unk_token,
self.bos_token,
self.eos_token,
]:
continue
token_strings.append(token)
# Объединяем токены в текст
text = ''.join(token_strings)
text = "".join(token_strings)
# Убираем маркер конца слова
text = text.replace('</w>', ' ')
text = text.replace("</w>", " ")
return text.strip()
@@ -302,18 +318,18 @@ class BPETokenizer(BaseTokenizer):
import json
config = {
'vocab': self.vocab,
'merges': {f"{k[0]} {k[1]}": v for k, v in self.merges.items()},
'vocab_size': self.vocab_size,
'pad_token': self.pad_token,
'unk_token': self.unk_token,
'bos_token': self.bos_token,
'eos_token': self.eos_token,
'pattern': self.pattern,
'tokenizer_type': self.__class__.__name__
"vocab": self.vocab,
"merges": {f"{k[0]} {k[1]}": v for k, v in self.merges.items()},
"vocab_size": self.vocab_size,
"pad_token": self.pad_token,
"unk_token": self.unk_token,
"bos_token": self.bos_token,
"eos_token": self.eos_token,
"pattern": self.pattern,
"tokenizer_type": self.__class__.__name__,
}
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(config, f, ensure_ascii=False, indent=2)
@classmethod
@@ -329,21 +345,21 @@ class BPETokenizer(BaseTokenizer):
"""
import json
with open(filepath, 'r', encoding='utf-8') as f:
with open(filepath, "r", encoding="utf-8") as f:
config = json.load(f)
tokenizer = cls()
tokenizer.vocab = config['vocab']
tokenizer.vocab_size = config['vocab_size']
tokenizer.pad_token = config['pad_token']
tokenizer.unk_token = config['unk_token']
tokenizer.bos_token = config['bos_token']
tokenizer.eos_token = config['eos_token']
tokenizer.pattern = config.get('pattern', tokenizer.pattern)
tokenizer.vocab = config["vocab"]
tokenizer.vocab_size = config["vocab_size"]
tokenizer.pad_token = config["pad_token"]
tokenizer.unk_token = config["unk_token"]
tokenizer.bos_token = config["bos_token"]
tokenizer.eos_token = config["eos_token"]
tokenizer.pattern = config.get("pattern", tokenizer.pattern)
tokenizer.compiled_pattern = re.compile(tokenizer.pattern, re.UNICODE)
# Восстанавливаем мерджи
merges = config.get('merges', {})
merges = config.get("merges", {})
tokenizer.merges = {}
for k, v in merges.items():
parts = k.split()
@@ -374,7 +390,12 @@ class SimpleBPETokenizer(BPETokenizer):
self._initialize_vocab()
# Добавляем базовые токены
special_tokens = [self.pad_token, self.unk_token, self.bos_token, self.eos_token]
special_tokens = [
self.pad_token,
self.unk_token,
self.bos_token,
self.eos_token,
]
self.add_special_tokens(special_tokens)
# Простая реализация - собираем все символы
@@ -398,7 +419,7 @@ class SimpleBPETokenizer(BPETokenizer):
def encode(self, text: str, **kwargs) -> List[int]:
"""Упрощенное кодирование - разбиваем на символы."""
add_special_tokens = kwargs.get('add_special_tokens', False)
add_special_tokens = kwargs.get("add_special_tokens", False)
token_ids = []
for char in text:
@@ -416,13 +437,18 @@ class SimpleBPETokenizer(BPETokenizer):
def decode(self, tokens: List[int], **kwargs) -> str:
"""Упрощенное декодирование."""
skip_special_tokens = kwargs.get('skip_special_tokens', True)
skip_special_tokens = kwargs.get("skip_special_tokens", True)
chars = []
for token_id in tokens:
char = self.inverse_vocab.get(token_id, self.unk_token)
if skip_special_tokens and char in [self.pad_token, self.unk_token, self.bos_token, self.eos_token]:
if skip_special_tokens and char in [
self.pad_token,
self.unk_token,
self.bos_token,
self.eos_token,
]:
continue
chars.append(char)
return ''.join(chars)
return "".join(chars)

82
llm/src/llm/tokenizers/bpe_tokenizer.py
View File

@@ -61,7 +61,10 @@ class BPETokenizer(BaseTokenizer):
break # нет пар — выходим
# Находим самую частую пару (в случае равенства — та, что встретилась первой)
most_frequent_pair = max(pair_freq.items(), key=lambda x: (x[1], -self._pair_first_index(sequence, x[0])))[0]
most_frequent_pair = max(
pair_freq.items(),
key=lambda x: (x[1], -self._pair_first_index(sequence, x[0])),
)[0]
# Создаем новый токен
new_token = most_frequent_pair[0] + most_frequent_pair[1]
@@ -71,7 +74,10 @@ class BPETokenizer(BaseTokenizer):
new_sequence = []
while i < len(sequence):
if i < len(sequence) - 1 and (sequence[i], sequence[i + 1]) == most_frequent_pair:
if (
i < len(sequence) - 1
and (sequence[i], sequence[i + 1]) == most_frequent_pair
):
new_sequence.append(new_token)
i += 2 # пропускаем два символа — заменённую пару
else:
@@ -86,7 +92,10 @@ class BPETokenizer(BaseTokenizer):
self.vocab_size = len(self.vocab)
# Добавляем специальные токены если указаны
special_tokens = kwargs.get('special_tokens', [self.pad_token, self.unk_token, self.bos_token, self.eos_token])
special_tokens = kwargs.get(
"special_tokens",
[self.pad_token, self.unk_token, self.bos_token, self.eos_token],
)
self.add_special_tokens(special_tokens)
def _pair_first_index(self, sequence, pair):
@@ -94,7 +103,7 @@ class BPETokenizer(BaseTokenizer):
for i in range(len(sequence) - 1):
if (sequence[i], sequence[i + 1]) == pair:
return i
return float('inf') # если пара не найдена (в теории не должно случиться)
return float("inf") # если пара не найдена (в теории не должно случиться)
def encode(self, text: str, **kwargs) -> List[int]:
"""
@@ -108,7 +117,7 @@ class BPETokenizer(BaseTokenizer):
Returns:
List[int]: Список идентификаторов токенов
"""
add_special_tokens = kwargs.get('add_special_tokens', False)
add_special_tokens = kwargs.get("add_special_tokens", False)
# 1. Разбиваем текст на токены-символы
sequence = list(text)
@@ -119,7 +128,9 @@ class BPETokenizer(BaseTokenizer):
while i < len(text):
# 3.1 Найти все токены в словаре, начинающиеся с text[i]
start_char = text[i]
result = [token for token in self.vocab_list if token.startswith(start_char)]
result = [
token for token in self.vocab_list if token.startswith(start_char)
]
# 3.2 Выбрать самый длинный подходящий токен
find_token = self._find_max_matching_token(text[i:], result)
if find_token is None:
@@ -174,19 +185,27 @@ class BPETokenizer(BaseTokenizer):
Returns:
str: Декодированный текст
"""
skip_special_tokens = kwargs.get('skip_special_tokens', True)
skip_special_tokens = kwargs.get("skip_special_tokens", True)
# Фильтруем специальные токены если нужно
if skip_special_tokens:
tokens = [tid for tid in tokens if tid not in [
self.pad_token_id, self.unk_token_id, self.bos_token_id, self.eos_token_id
]]
tokens = [
tid
for tid in tokens
if tid
not in [
self.pad_token_id,
self.unk_token_id,
self.bos_token_id,
self.eos_token_id,
]
]
# Конвертируем ID в токены
token_strings = self._ids_to_tokens(tokens)
# Объединяем токены в текст
return ''.join(token_strings)
return "".join(token_strings)
def _ids_to_tokens(self, ids: List[int]) -> List[str]:
"""Конвертирует список Ids в их tokens"""
@@ -211,18 +230,18 @@ class BPETokenizer(BaseTokenizer):
merges_serializable = {f"{k[0]},{k[1]}": v for k, v in self.merges.items()}
config = {
'vocab': self.vocab,
'vocab_size': self.vocab_size,
'pad_token': self.pad_token,
'unk_token': self.unk_token,
'bos_token': self.bos_token,
'eos_token': self.eos_token,
'tokenizer_type': self.__class__.__name__,
'merges': merges_serializable,
'vocab_list': self.vocab_list
"vocab": self.vocab,
"vocab_size": self.vocab_size,
"pad_token": self.pad_token,
"unk_token": self.unk_token,
"bos_token": self.bos_token,
"eos_token": self.eos_token,
"tokenizer_type": self.__class__.__name__,
"merges": merges_serializable,
"vocab_list": self.vocab_list,
}
with open(filepath, 'w', encoding='utf-8') as f:
with open(filepath, "w", encoding="utf-8") as f:
json.dump(config, f, ensure_ascii=False, indent=2)
@classmethod
@@ -238,23 +257,23 @@ class BPETokenizer(BaseTokenizer):
"""
import json
with open(filepath, 'r', encoding='utf-8') as f:
with open(filepath, "r", encoding="utf-8") as f:
config = json.load(f)
# Создаем экземпляр токенизатора
tokenizer = cls()
tokenizer.vocab = config['vocab']
tokenizer.vocab_size = config['vocab_size']
tokenizer.pad_token = config['pad_token']
tokenizer.unk_token = config['unk_token']
tokenizer.bos_token = config['bos_token']
tokenizer.eos_token = config['eos_token']
tokenizer.vocab_list = config['vocab_list']
tokenizer.vocab = config["vocab"]
tokenizer.vocab_size = config["vocab_size"]
tokenizer.pad_token = config["pad_token"]
tokenizer.unk_token = config["unk_token"]
tokenizer.bos_token = config["bos_token"]
tokenizer.eos_token = config["eos_token"]
tokenizer.vocab_list = config["vocab_list"]
# Восстанавливаем кортежи из строк
tokenizer.merges = {}
for k, v in config['merges'].items():
parts = k.split(',')
for k, v in config["merges"].items():
parts = k.split(",")
if len(parts) == 2:
tokenizer.merges[(parts[0], parts[1])] = v
@@ -275,4 +294,5 @@ class SimpleBPETokenizer(BPETokenizer):
Упрощенная версия BPE токенизатора для демонстрации.
Наследует вашу реализацию, но может быть упрощена при необходимости.
"""
pass

39
llm/src/llm/training/dataset.py
View File

@@ -31,7 +31,7 @@ class TextDataset(Dataset):
input_ids = input_ids[:block_size]
else:
# Дополняем pad_token_id
pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
pad_token_id = getattr(tokenizer, "pad_token_id", 0)
input_ids = input_ids + [pad_token_id] * (block_size - len(input_ids))
self.examples.append(input_ids)
@@ -57,7 +57,7 @@ class StreamingTextDataset(Dataset):
self.block_size = block_size
# Получаем pad_token_id из токенизатора
self.pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
self.pad_token_id = getattr(tokenizer, "pad_token_id", 0)
def __len__(self):
return len(self.texts)
@@ -70,9 +70,11 @@ class StreamingTextDataset(Dataset):
# Обрезаем или дополняем до нужной длины
if len(input_ids) > self.block_size:
input_ids = input_ids[:self.block_size]
input_ids = input_ids[: self.block_size]
else:
input_ids = input_ids + [self.pad_token_id] * (self.block_size - len(input_ids))
input_ids = input_ids + [self.pad_token_id] * (
self.block_size - len(input_ids)
)
input_ids = torch.tensor(input_ids, dtype=torch.long)
labels = input_ids.clone()
@@ -85,8 +87,14 @@ class TextDatasetWithSpecialTokens(TextDataset):
Расширенная версия TextDataset с поддержкой специальных токенов.
"""
def __init__(self, texts: List[str], tokenizer: Any, block_size: int = 128,
add_bos: bool = False, add_eos: bool = False):
def __init__(
self,
texts: List[str],
tokenizer: Any,
block_size: int = 128,
add_bos: bool = False,
add_eos: bool = False,
):
"""
Args:
texts: Список текстов
@@ -104,10 +112,7 @@ class TextDatasetWithSpecialTokens(TextDataset):
for text in texts:
# Кодируем с специальными токенами
input_ids = tokenizer.encode(
text,
add_special_tokens=True,
add_bos_token=add_bos,
add_eos_token=eos
text, add_special_tokens=True, add_bos_token=add_bos, add_eos_token=eos
)
# Учитываем специальные токены при обрезке/дополнении
@@ -121,13 +126,21 @@ class TextDatasetWithSpecialTokens(TextDataset):
input_ids = input_ids[:effective_block_size]
# Добавляем специальные токены если нужно
if add_bos and hasattr(tokenizer, 'bos_token_id') and tokenizer.bos_token_id is not None:
if (
add_bos
and hasattr(tokenizer, "bos_token_id")
and tokenizer.bos_token_id is not None
):
input_ids = [tokenizer.bos_token_id] + input_ids
if add_eos and hasattr(tokenizer, 'eos_token_id') and tokenizer.eos_token_id is not None:
if (
add_eos
and hasattr(tokenizer, "eos_token_id")
and tokenizer.eos_token_id is not None
):
input_ids = input_ids + [tokenizer.eos_token_id]
# Дополняем до полной длины
pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
pad_token_id = getattr(tokenizer, "pad_token_id", 0)
if len(input_ids) < block_size:
input_ids = input_ids + [pad_token_id] * (block_size - len(input_ids))

1
llm/src/llm/training/optimizer.py
View File

@@ -1,5 +1,6 @@
import torch.optim as optim
def get_optimizer(model, lr=3e-4, weight_decay=0.01, optimizer_type="adamw"):
"""
Возвращает оптимизатор для обучения модели.

7
llm/src/llm/training/scheduler.py
View File

@@ -1,5 +1,6 @@
from torch.optim.lr_scheduler import LambdaLR
def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps):
"""
Линейный планировщик обучения с warmup.
@@ -8,6 +9,10 @@ def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_st
def lr_lambda(current_step):
if current_step < num_warmup_steps:
return float(current_step) / float(max(1, num_warmup_steps))
return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)))
return max(
0.0,
float(num_training_steps - current_step)
/ float(max(1, num_training_steps - num_warmup_steps)),
)
return LambdaLR(optimizer, lr_lambda)

30
llm/src/llm/training/trainer.py
View File

@@ -5,15 +5,29 @@ from tqdm import tqdm
from llm.training.optimizer import get_optimizer
from llm.training.scheduler import get_linear_schedule_with_warmup
class Trainer:
"""
Универсальный класс обучения LLM (GPT, LLaMA, Mistral и т.д.)
"""
def __init__(self, model, train_dataset, val_dataset=None, lr=3e-4, batch_size=8, num_epochs=3, warmup_steps=100):
def __init__(
self,
model,
train_dataset,
val_dataset=None,
lr=3e-4,
batch_size=8,
num_epochs=3,
warmup_steps=100,
):
self.model = model
self.train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
self.val_loader = DataLoader(val_dataset, batch_size=batch_size) if val_dataset else None
self.train_loader = DataLoader(
train_dataset, batch_size=batch_size, shuffle=True
)
self.val_loader = (
DataLoader(val_dataset, batch_size=batch_size) if val_dataset else None
)
self.optimizer = get_optimizer(model, lr=lr)
self.scheduler = None
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -34,19 +48,23 @@ class Trainer:
loss = F.cross_entropy(
shift_logits.view(-1, shift_logits.size(-1)),
shift_labels.view(-1),
ignore_index=-100 # Игнорируем padding tokens
ignore_index=-100, # Игнорируем padding tokens
)
return loss
def train(self):
total_steps = len(self.train_loader) * self.num_epochs
self.scheduler = get_linear_schedule_with_warmup(self.optimizer, self.warmup_steps, total_steps)
self.scheduler = get_linear_schedule_with_warmup(
self.optimizer, self.warmup_steps, total_steps
)
for epoch in range(self.num_epochs):
self.model.train()
total_loss = 0
progress_bar = tqdm(self.train_loader, desc=f"Epoch {epoch+1}/{self.num_epochs}")
progress_bar = tqdm(
self.train_loader, desc=f"Epoch {epoch+1}/{self.num_epochs}"
)
for batch in progress_bar:
self.optimizer.zero_grad()

4
llm/tests/conftest.py
View File

@@ -58,7 +58,7 @@ def gpt_config(vocab_size, embed_dim, num_heads, num_layers):
"num_heads": num_heads,
"num_layers": num_layers,
"max_position_embeddings": 1024,
"dropout": 0.1
"dropout": 0.1,
}
@@ -68,12 +68,14 @@ def random_inputs(batch_size, seq_len, vocab_size):
input_ids = torch.randint(0, vocab_size, (batch_size, seq_len))
return input_ids
@pytest.fixture
def random_float_inputs(batch_size, seq_len, embed_dim):
"""Generate random floating point input tensors for testing feed forward."""
inputs = torch.randn(batch_size, seq_len, embed_dim)
return inputs
@pytest.fixture
def random_embeddings(batch_size, seq_len, embed_dim):
"""Generate random embedding tensors for testing attention modules."""

97
llm/tests/core/test_decoder.py
View File

@@ -14,20 +14,30 @@ class TestDecoder:
"""Test that Decoder can be initialized."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
assert decoder is not None
# Check internal components
assert hasattr(decoder, '_heads')
assert hasattr(decoder, '_ff')
assert hasattr(decoder, '_norm1')
assert hasattr(decoder, '_norm2')
assert hasattr(decoder, "_heads")
assert hasattr(decoder, "_ff")
assert hasattr(decoder, "_norm1")
assert hasattr(decoder, "_norm2")
def test_forward_pass(self, embed_dim, num_heads, random_embeddings):
"""Test forward pass of Decoder."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
# Forward pass
output = decoder(random_embeddings)
@@ -40,7 +50,12 @@ class TestDecoder:
"""Test forward pass with causal mask."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
batch_size, seq_len = random_embeddings.shape[:2]
# Create causal mask
@@ -56,7 +71,12 @@ class TestDecoder:
"""Test that residual connections are properly applied."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
output = decoder(random_embeddings)
@@ -72,7 +92,12 @@ class TestDecoder:
"""Test that layer normalization is applied."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
output = decoder(random_embeddings)
@@ -89,7 +114,12 @@ class TestDecoder:
"""Test that gradients flow through Decoder."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
# Forward pass
output = decoder(random_embeddings)
@@ -109,7 +139,12 @@ class TestDecoder:
"""Test that Decoder works on correct device."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len).to(device)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
).to(device)
inputs = random_embeddings.to(device)
# Forward pass
@@ -122,7 +157,7 @@ class TestDecoder:
def test_different_configurations(self):
"""Test Decoder with different configurations."""
test_cases = [
(64, 2), # embed_dim=64, num_heads=2
(64, 2), # embed_dim=64, num_heads=2
(128, 4), # embed_dim=128, num_heads=4
(256, 8), # embed_dim=256, num_heads=8
]
@@ -130,7 +165,12 @@ class TestDecoder:
for embed_dim, num_heads in test_cases:
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
batch_size, seq_len = 2, 16
inputs = torch.randn(batch_size, seq_len, embed_dim)
@@ -143,7 +183,12 @@ class TestDecoder:
"""Test Decoder with different input shapes."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
inputs = torch.randn(batch_size, seq_len, embed_dim)
output = decoder(inputs)
@@ -154,7 +199,13 @@ class TestDecoder:
"""Test that Decoder behaves differently in train vs eval mode."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len, dropout=0.5)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
dropout=0.5,
)
# Training mode
decoder.train()
@@ -171,18 +222,20 @@ class TestDecoder:
"""Test that parameters are properly initialized."""
head_size = embed_dim // num_heads
max_seq_len = 1024
decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
decoder = Decoder(
num_heads=num_heads,
emb_size=embed_dim,
head_size=head_size,
max_seq_len=max_seq_len,
)
# Check that various components have non-zero parameters
assert not torch.allclose(
decoder._heads._layer.weight,
torch.zeros_like(decoder._heads._layer.weight)
decoder._heads._layer.weight, torch.zeros_like(decoder._heads._layer.weight)
)
assert not torch.allclose(
decoder._ff._layer1.weight,
torch.zeros_like(decoder._ff._layer1.weight)
decoder._ff._layer1.weight, torch.zeros_like(decoder._ff._layer1.weight)
)
assert not torch.allclose(
decoder._norm1.weight,
torch.zeros_like(decoder._norm1.weight)
decoder._norm1.weight, torch.zeros_like(decoder._norm1.weight)
)

34
llm/tests/core/test_feed_forward.py
View File

@@ -17,10 +17,10 @@ class TestFeedForward:
assert ff is not None
# Check internal layers
assert hasattr(ff, '_layer1')
assert hasattr(ff, '_layer2')
assert hasattr(ff, '_activation')
assert hasattr(ff, '_dropout')
assert hasattr(ff, "_layer1")
assert hasattr(ff, "_layer2")
assert hasattr(ff, "_activation")
assert hasattr(ff, "_dropout")
# Check layer dimensions
expected_hidden_dim = embed_dim * 4 # Default expansion factor
@@ -101,10 +101,12 @@ class TestFeedForward:
# Check that gradients are computed for learnable parameters
assert ff._layer1.weight.grad is not None
assert ff._layer2.weight.grad is not None
assert not torch.allclose(ff._layer1.weight.grad,
torch.zeros_like(ff._layer1.weight.grad))
assert not torch.allclose(ff._layer2.weight.grad,
torch.zeros_like(ff._layer2.weight.grad))
assert not torch.allclose(
ff._layer1.weight.grad, torch.zeros_like(ff._layer1.weight.grad)
)
assert not torch.allclose(
ff._layer2.weight.grad, torch.zeros_like(ff._layer2.weight.grad)
)
def test_device_consistency(self, embed_dim, random_float_inputs, device):
"""Test that FeedForward works on correct device."""
@@ -167,11 +169,19 @@ class TestFeedForward:
ff = FeedForward(embed_dim)
# Check that weights are not all zeros
assert not torch.allclose(ff._layer1.weight, torch.zeros_like(ff._layer1.weight))
assert not torch.allclose(ff._layer2.weight, torch.zeros_like(ff._layer2.weight))
assert not torch.allclose(
ff._layer1.weight, torch.zeros_like(ff._layer1.weight)
)
assert not torch.allclose(
ff._layer2.weight, torch.zeros_like(ff._layer2.weight)
)
# Check that biases are not all zeros (they should be initialized with some values)
if ff._layer1.bias is not None:
assert not torch.allclose(ff._layer1.bias, torch.zeros_like(ff._layer1.bias))
assert not torch.allclose(
ff._layer1.bias, torch.zeros_like(ff._layer1.bias)
)
if ff._layer2.bias is not None:
assert not torch.allclose(ff._layer2.bias, torch.zeros_like(ff._layer2.bias))
assert not torch.allclose(
ff._layer2.bias, torch.zeros_like(ff._layer2.bias)
)

52
llm/tests/core/test_multi_head_attention.py
View File

@@ -13,7 +13,9 @@ class TestMultiHeadAttention:
def test_initialization(self, embed_dim, num_heads):
"""Test that MultiHeadAttention can be initialized."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
assert attention is not None
# Check internal attributes
@@ -24,7 +26,9 @@ class TestMultiHeadAttention:
def test_forward_pass(self, embed_dim, num_heads, random_embeddings):
"""Test forward pass of MultiHeadAttention."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
# Forward pass
output, _ = attention(random_embeddings)
@@ -36,7 +40,9 @@ class TestMultiHeadAttention:
def test_forward_with_mask(self, embed_dim, num_heads, random_embeddings):
"""Test forward pass with attention mask."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
# Create a simple mask
seq_len = random_embeddings.shape[1]
@@ -51,7 +57,9 @@ class TestMultiHeadAttention:
def test_causal_mask(self, embed_dim, num_heads, random_embeddings):
"""Test that causal mask prevents attending to future positions."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
# Create causal mask
seq_len = random_embeddings.shape[1]
@@ -63,10 +71,14 @@ class TestMultiHeadAttention:
# Check output shape
assert output.shape == random_embeddings.shape
def test_attention_weights_normalization(self, embed_dim, num_heads, random_embeddings):
def test_attention_weights_normalization(
self, embed_dim, num_heads, random_embeddings
):
"""Test that attention weights are properly normalized."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
# Forward pass
output, _ = attention(random_embeddings)
@@ -77,7 +89,9 @@ class TestMultiHeadAttention:
def test_gradient_flow(self, embed_dim, num_heads, random_embeddings):
"""Test that gradients flow through MultiHeadAttention."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
# Forward pass
output, _ = attention(random_embeddings)
@@ -94,7 +108,9 @@ class TestMultiHeadAttention:
def test_device_consistency(self, embed_dim, num_heads, random_embeddings, device):
"""Test that MultiHeadAttention works on correct device."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024).to(device)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
).to(device)
inputs = random_embeddings.to(device)
# Forward pass
@@ -107,15 +123,17 @@ class TestMultiHeadAttention:
def test_different_embed_dim_and_heads(self):
"""Test MultiHeadAttention with different embed_dim and num_heads combinations."""
test_cases = [
(64, 2), # embed_dim=64, num_heads=2
(64, 2), # embed_dim=64, num_heads=2
(128, 4), # embed_dim=128, num_heads=4
(256, 8), # embed_dim=256, num_heads=8
(512, 16), # embed_dim=512, num_heads=16
(512, 16), # embed_dim=512, num_heads=16
]
for embed_dim, num_heads in test_cases:
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
batch_size, seq_len = 2, 16
inputs = torch.randn(batch_size, seq_len, embed_dim)
@@ -126,7 +144,9 @@ class TestMultiHeadAttention:
def test_attention_output_range(self, embed_dim, num_heads, random_embeddings):
"""Test that attention output is in reasonable range."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
output, _ = attention(random_embeddings)
@@ -137,7 +157,9 @@ class TestMultiHeadAttention:
def test_different_input_shapes(self, embed_dim, num_heads, batch_size, seq_len):
"""Test MultiHeadAttention with different input shapes."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024
)
inputs = torch.randn(batch_size, seq_len, embed_dim)
output, _ = attention(inputs)
@@ -147,7 +169,9 @@ class TestMultiHeadAttention:
def test_parameter_sharing(self, embed_dim, num_heads):
"""Test that parameters are properly shared across the sequence."""
head_size = embed_dim // num_heads
attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024, dropout=0.0) # No dropout for deterministic test
attention = MultiHeadAttention(
num_heads, embed_dim, head_size, max_seq_len=1024, dropout=0.0
) # No dropout for deterministic test
# Create two identical sequences
seq_len = 10

27
llm/tests/core/test_positional_embeddings.py
View File

@@ -18,7 +18,7 @@ class TestPositionalEmbeddings:
assert embeddings is not None
# Check that positional embeddings are created
assert hasattr(embeddings, 'embedding')
assert hasattr(embeddings, "embedding")
assert embeddings.embedding.weight.shape == (max_seq_len, embed_dim)
def test_forward_pass(self, embed_dim):
@@ -52,7 +52,7 @@ class TestPositionalEmbeddings:
def test_different_sequence_lengths(self, embed_dim):
"""Test PositionalEmbeddings with different sequence lengths."""
test_cases = [
(10, 5), # seq_len < max_seq_len
(10, 5), # seq_len < max_seq_len
(10, 10), # seq_len == max_seq_len
]
@@ -80,8 +80,10 @@ class TestPositionalEmbeddings:
# Positional embeddings should have gradients (they're learnable)
assert embeddings.embedding.weight.grad is not None
assert not torch.allclose(embeddings.embedding.weight.grad,
torch.zeros_like(embeddings.embedding.weight.grad))
assert not torch.allclose(
embeddings.embedding.weight.grad,
torch.zeros_like(embeddings.embedding.weight.grad),
)
def test_device_consistency(self, embed_dim, device):
"""Test that PositionalEmbeddings works on correct device."""
@@ -103,7 +105,9 @@ class TestPositionalEmbeddings:
embeddings2 = PositionalEmbeddings(max_seq_len, embed_dim)
# Different instances should have different embeddings (random initialization)
assert not torch.allclose(embeddings1.embedding.weight, embeddings2.embedding.weight)
assert not torch.allclose(
embeddings1.embedding.weight, embeddings2.embedding.weight
)
# But same instance should produce same output for same input
seq_len = 50
@@ -122,11 +126,14 @@ class TestPositionalEmbeddings:
assert not torch.allclose(pe[0], pe[1], rtol=1e-4)
assert not torch.allclose(pe[10], pe[20], rtol=1e-4)
@pytest.mark.parametrize("max_seq_len,seq_len,embed_dim", [
(64, 10, 64),
(128, 50, 128),
(256, 100, 256),
])
@pytest.mark.parametrize(
"max_seq_len,seq_len,embed_dim",
[
(64, 10, 64),
(128, 50, 128),
(256, 100, 256),
],
)
def test_different_configurations(self, max_seq_len, seq_len, embed_dim):
"""Test PositionalEmbeddings with different configurations."""
embeddings = PositionalEmbeddings(max_seq_len, embed_dim)

24
llm/tests/core/test_token_embeddings.py
View File

@@ -16,7 +16,7 @@ class TestTokenEmbeddings:
assert embeddings is not None
# Check embedding layer
assert hasattr(embeddings, '_embedding')
assert hasattr(embeddings, "_embedding")
assert embeddings._embedding.weight.shape == (vocab_size, embed_dim)
def test_forward_pass(self, vocab_size, embed_dim, random_inputs):
@@ -27,7 +27,11 @@ class TestTokenEmbeddings:
output = embeddings(random_inputs)
# Check output shape
assert output.shape == (random_inputs.shape[0], random_inputs.shape[1], embed_dim)
assert output.shape == (
random_inputs.shape[0],
random_inputs.shape[1],
embed_dim,
)
assert isinstance(output, torch.Tensor)
def test_embedding_weights(self, vocab_size, embed_dim):
@@ -42,11 +46,7 @@ class TestTokenEmbeddings:
def test_different_vocab_sizes(self):
"""Test TokenEmbeddings with different vocabulary sizes."""
test_cases = [
(100, 128),
(1000, 256),
(50000, 512)
]
test_cases = [(100, 128), (1000, 256), (50000, 512)]
for vocab_size, embed_dim in test_cases:
embeddings = TokenEmbeddings(vocab_size, embed_dim)
@@ -65,8 +65,10 @@ class TestTokenEmbeddings:
# Check that gradients are computed
assert embeddings._embedding.weight.grad is not None
assert not torch.allclose(embeddings._embedding.weight.grad,
torch.zeros_like(embeddings._embedding.weight.grad))
assert not torch.allclose(
embeddings._embedding.weight.grad,
torch.zeros_like(embeddings._embedding.weight.grad),
)
def test_device_consistency(self, vocab_size, embed_dim, random_inputs, device):
"""Test that TokenEmbeddings works on correct device."""
@@ -85,7 +87,9 @@ class TestTokenEmbeddings:
embeddings = TokenEmbeddings(vocab_size, embed_dim)
# Test lookup for specific tokens
test_tokens = torch.tensor([[0, 1, 2], [vocab_size - 1, vocab_size - 2, vocab_size - 3]])
test_tokens = torch.tensor(
[[0, 1, 2], [vocab_size - 1, vocab_size - 2, vocab_size - 3]]
)
output = embeddings(test_tokens)

82
llm/tests/models/test_gpt.py
View File

@@ -16,14 +16,14 @@ class TestGPT:
assert model is not None
# Check that model has required components
assert hasattr(model, '_token_embeddings')
assert hasattr(model, '_position_embeddings')
assert hasattr(model, '_decoders')
assert hasattr(model, '_linear')
assert hasattr(model, '_dropout')
assert hasattr(model, "_token_embeddings")
assert hasattr(model, "_position_embeddings")
assert hasattr(model, "_decoders")
assert hasattr(model, "_linear")
assert hasattr(model, "_dropout")
# Check number of decoder layers
assert len(model._decoders) == gpt_config['num_layers']
assert len(model._decoders) == gpt_config["num_layers"]
def test_forward_pass(self, gpt_config, random_inputs):
"""Test forward pass of GPT."""
@@ -34,11 +34,13 @@ class TestGPT:
# Check output shape
batch_size, seq_len = random_inputs.shape
vocab_size = gpt_config['vocab_size']
vocab_size = gpt_config["vocab_size"]
assert logits.shape == (batch_size, seq_len, vocab_size)
assert isinstance(logits, torch.Tensor)
def test_forward_with_attention_mask(self, gpt_config, random_inputs, attention_mask):
def test_forward_with_attention_mask(
self, gpt_config, random_inputs, attention_mask
):
"""Test forward pass with attention mask."""
model = GPT(gpt_config)
@@ -47,7 +49,7 @@ class TestGPT:
# Check output shape
batch_size, seq_len = random_inputs.shape
vocab_size = gpt_config['vocab_size']
vocab_size = gpt_config["vocab_size"]
assert logits.shape == (batch_size, seq_len, vocab_size)
def test_generate_text(self, gpt_config):
@@ -58,14 +60,16 @@ class TestGPT:
# Create initial input
batch_size = 2
initial_seq_len = 5
input_ids = torch.randint(0, gpt_config['vocab_size'], (batch_size, initial_seq_len))
input_ids = torch.randint(
0, gpt_config["vocab_size"], (batch_size, initial_seq_len)
)
# Generate text
with torch.no_grad():
generated = model.generate(
x=input_ids,
max_new_tokens=10,
do_sample=False # Use greedy for deterministic testing
do_sample=False, # Use greedy for deterministic testing
)
# Check output shape
@@ -81,15 +85,12 @@ class TestGPT:
model.eval()
# Create initial input
input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
# Generate with temperature
with torch.no_grad():
generated = model.generate(
x=input_ids,
max_new_tokens=5,
do_sample=True,
temperature=0.8
x=input_ids, max_new_tokens=5, do_sample=True, temperature=0.8
)
assert generated.shape == (1, 8) # 3 initial + 5 new tokens
@@ -100,15 +101,12 @@ class TestGPT:
model.eval()
# Create initial input
input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
# Generate with top-k
with torch.no_grad():
generated = model.generate(
x=input_ids,
max_new_tokens=5,
do_sample=True,
top_k=10
x=input_ids, max_new_tokens=5, do_sample=True, top_k=10
)
assert generated.shape == (1, 8)
@@ -119,15 +117,12 @@ class TestGPT:
model.eval()
# Create initial input
input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
# Generate with top-p
with torch.no_grad():
generated = model.generate(
x=input_ids,
max_new_tokens=5,
do_sample=True,
top_p=0.9
x=input_ids, max_new_tokens=5, do_sample=True, top_p=0.9
)
assert generated.shape == (1, 8)
@@ -140,10 +135,9 @@ class TestGPT:
logits = model(random_inputs)
# Create a dummy loss and backward pass
targets = torch.randint(0, gpt_config['vocab_size'], random_inputs.shape)
targets = torch.randint(0, gpt_config["vocab_size"], random_inputs.shape)
loss = torch.nn.functional.cross_entropy(
logits.view(-1, logits.size(-1)),
targets.view(-1)
logits.view(-1, logits.size(-1)), targets.view(-1)
)
loss.backward()
@@ -174,7 +168,7 @@ class TestGPT:
"num_heads": 2,
"num_layers": 2,
"max_position_embeddings": 256,
"dropout": 0.1
"dropout": 0.1,
},
{
"vocab_size": 5000,
@@ -182,7 +176,7 @@ class TestGPT:
"num_heads": 4,
"num_layers": 4,
"max_position_embeddings": 512,
"dropout": 0.1
"dropout": 0.1,
},
{
"vocab_size": 10000,
@@ -190,18 +184,18 @@ class TestGPT:
"num_heads": 8,
"num_layers": 6,
"max_position_embeddings": 1024,
"dropout": 0.1
}
"dropout": 0.1,
},
]
for config in test_configs:
model = GPT(config)
batch_size, seq_len = 2, 16
inputs = torch.randint(0, config['vocab_size'], (batch_size, seq_len))
inputs = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
logits = model(inputs)
expected_shape = (batch_size, seq_len, config['vocab_size'])
expected_shape = (batch_size, seq_len, config["vocab_size"])
assert logits.shape == expected_shape
@pytest.mark.parametrize("batch_size,seq_len", [(1, 8), (2, 16), (4, 32)])
@@ -209,10 +203,10 @@ class TestGPT:
"""Test GPT with different input shapes."""
model = GPT(gpt_config)
inputs = torch.randint(0, gpt_config['vocab_size'], (batch_size, seq_len))
inputs = torch.randint(0, gpt_config["vocab_size"], (batch_size, seq_len))
logits = model(inputs)
expected_shape = (batch_size, seq_len, gpt_config['vocab_size'])
expected_shape = (batch_size, seq_len, gpt_config["vocab_size"])
assert logits.shape == expected_shape
def test_training_vs_evaluation(self, gpt_config, random_inputs):
@@ -237,10 +231,10 @@ class TestGPT:
total_params = sum(p.numel() for p in model.parameters())
# For a small GPT model, parameters should be in reasonable range
vocab_size = gpt_config['vocab_size']
embed_dim = gpt_config['embed_dim']
num_layers = gpt_config['num_layers']
num_heads = gpt_config['num_heads']
vocab_size = gpt_config["vocab_size"]
embed_dim = gpt_config["embed_dim"]
num_layers = gpt_config["num_layers"]
num_heads = gpt_config["num_heads"]
# Rough estimate: token_embeddings + output_layer + (attention + ff) * layers
expected_min = vocab_size * embed_dim * 2 # embeddings and output
@@ -264,11 +258,7 @@ class TestGPT:
# We can't directly test attention masks in the public API,
# but we can verify the generation works correctly
generated = model.generate(
x=input_ids,
max_new_tokens=3,
do_sample=False
)
generated = model.generate(x=input_ids, max_new_tokens=3, do_sample=False)
# Generated sequence should be longer than input
assert generated.shape[1] == input_ids.shape[1] + 3

30
llm/tests/test_basic.py
View File

@@ -18,7 +18,7 @@ def test_gpt_model_creation():
"num_heads": 4,
"num_layers": 2,
"max_position_embeddings": 256,
"dropout": 0.1
"dropout": 0.1,
}
model = GPT(config)
@@ -41,16 +41,10 @@ def test_bpe_tokenizer_basic():
tokenizer = BPETokenizer()
# Train on simple texts
texts = [
"hello world",
"test tokenization",
"simple example"
]
texts = ["hello world", "test tokenization", "simple example"]
tokenizer.train(
texts=texts,
vocab_size=50,
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
texts=texts, vocab_size=50, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
)
# Test encoding/decoding
@@ -132,7 +126,7 @@ def test_gpt_generation():
"num_heads": 4,
"num_layers": 2,
"max_position_embeddings": 256,
"dropout": 0.1
"dropout": 0.1,
}
model = GPT(config)
@@ -142,11 +136,7 @@ def test_gpt_generation():
input_ids = torch.randint(0, config["vocab_size"], (1, 5))
with torch.no_grad():
generated = model.generate(
x=input_ids,
max_new_tokens=3,
do_sample=False
)
generated = model.generate(x=input_ids, max_new_tokens=3, do_sample=False)
assert generated.shape == (1, 8) # 5 initial + 3 new tokens
print("✅ GPT generation test passed")
@@ -161,9 +151,7 @@ def test_bpe_tokenizer_save_load():
# Train on simple texts
texts = ["hello world", "test save load"]
tokenizer.train(
texts=texts,
vocab_size=30,
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
texts=texts, vocab_size=30, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
)
with tempfile.TemporaryDirectory() as temp_dir:
@@ -211,9 +199,7 @@ def test_gpt_with_tokenizer():
tokenizer = BPETokenizer()
texts = ["hello world", "test integration"]
tokenizer.train(
texts=texts,
vocab_size=50,
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
texts=texts, vocab_size=50, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
)
vocab_size = tokenizer.get_vocab_size()
@@ -225,7 +211,7 @@ def test_gpt_with_tokenizer():
"num_heads": 4,
"num_layers": 2,
"max_position_embeddings": 256,
"dropout": 0.1
"dropout": 0.1,
}
model = GPT(config)

10
llm/tests/tokenizers/test_bpe_tokenizer.py
View File

@@ -19,7 +19,7 @@ class TestBPETokenizer:
"Нейронные сети",
"Машинное обучение",
"Глубокое обучение",
"Трансформеры"
"Трансформеры",
]
@pytest.fixture
@@ -29,7 +29,7 @@ class TestBPETokenizer:
tokenizer.train(
texts=sample_texts,
vocab_size=100,
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"],
)
return tokenizer
@@ -44,7 +44,7 @@ class TestBPETokenizer:
tokenizer.train(
texts=sample_texts,
vocab_size=50,
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"],
)
assert tokenizer.get_vocab_size() > 0
@@ -117,7 +117,9 @@ class TestBPETokenizer:
loaded_vocab = loaded_tokenizer.get_vocab()
assert original_vocab == loaded_vocab
assert trained_tokenizer.get_vocab_size() == loaded_tokenizer.get_vocab_size()
assert (
trained_tokenizer.get_vocab_size() == loaded_tokenizer.get_vocab_size()
)
# Test encoding consistency
text = sample_texts[0]