Рефакторинг: единообразие оформления кода (пробелы, кавычки, пустые строки), без изменения логики по всему проекту.

experiments/hf_integration/generate_with_hf_tools.py

@@ -14,12 +14,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from hf_proxy import HFAdapter, HFTokenizerAdapter, create_hf_pipeline
 
-from shared.configs import (
+from shared.configs import TEST_PROMPTS, GENERATION_CONFIG, PATHS
-    TEST_PROMPTS, GENERATION_CONFIG, PATHS
+from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
-)
-from shared.data import (
-    print_experiment_info, ensure_directories, ExperimentLogger
-)
 
 
 def load_hf_model_and_tokenizer() -> tuple:
@@ -41,9 +37,7 @@ def load_hf_model_and_tokenizer() -> tuple:
         )
 
     if not os.path.exists(tokenizer_path):
-        raise FileNotFoundError(
+        raise FileNotFoundError(f"Токенизатор не найден: {tokenizer_path}")
-            f"Токенизатор не найден: {tokenizer_path}"
-        )
 
     # Загружаем адаптированный токенизатор
     print("🔧 Загрузка адаптированного токенизатора...")
@@ -52,8 +46,9 @@ def load_hf_model_and_tokenizer() -> tuple:
 
     # Загружаем конфигурацию модели
     import json
+
     config_path = os.path.join(model_path, "config.json")
-    with open(config_path, 'r', encoding='utf-8') as f:
+    with open(config_path, "r", encoding="utf-8") as f:
         model_config = json.load(f)
 
     # Загружаем модель через HFAdapter с правильной конфигурацией
@@ -62,6 +57,7 @@ def load_hf_model_and_tokenizer() -> tuple:
 
     # Создаем конфигурацию из сохраненного config.json
     from hf_proxy import HFAdapterConfig
+
     hf_config = HFAdapterConfig(
         vocab_size=model_config["vocab_size"],
         hidden_size=model_config["hidden_size"],
@@ -69,7 +65,9 @@ def load_hf_model_and_tokenizer() -> tuple:
         num_attention_heads=model_config["num_attention_heads"],
         max_position_embeddings=model_config["max_position_embeddings"],
         hidden_dropout_prob=model_config.get("hidden_dropout_prob", 0.1),
-        attention_probs_dropout_prob=model_config.get("attention_probs_dropout_prob", 0.1),
+        attention_probs_dropout_prob=model_config.get(
+            "attention_probs_dropout_prob", 0.1
+        ),
     )
 
     hf_model = HFAdapter.from_pretrained(model_bin_path, hf_config=hf_config)
@@ -97,7 +95,7 @@ def test_hf_pipeline(hf_model, hf_tokenizer):
             device="cpu",
             max_length=50,
             do_sample=True,
-            temperature=0.7
+            temperature=0.7,
         )
 
         print("✅ HuggingFace pipeline создан")
@@ -132,8 +130,10 @@ def generate_with_hf_model(hf_model, hf_tokenizer, prompt: str, config: dict) ->
         str: Сгенерированный текст
     """
     print(f"🔤 Промпт: '{prompt}'")
-    print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+    print(
-          f"temp={config['temperature']}, sample={config['do_sample']}")
+        f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+        f"temp={config['temperature']}, sample={config['do_sample']}"
+    )
 
     # Кодируем через адаптированный токенизатор
     inputs = hf_tokenizer(prompt, return_tensors="pt")
@@ -144,12 +144,12 @@ def generate_with_hf_model(hf_model, hf_tokenizer, prompt: str, config: dict) ->
     # Генерируем через адаптированную модель
     with torch.no_grad():
         generated_ids = hf_model.generate(
-            input_ids=inputs['input_ids'],
+            input_ids=inputs["input_ids"],
             max_new_tokens=config["max_new_tokens"],
             do_sample=config["do_sample"],
             temperature=config["temperature"],
             top_k=config["top_k"],
-            top_p=config["top_p"]
+            top_p=config["top_p"],
         )
 
     # Декодируем через адаптированный токенизатор
@@ -174,23 +174,29 @@ def test_different_hf_strategies(hf_model, hf_tokenizer, prompt: str):
         {"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
         {"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
         {"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
-        {"name": "❄️  Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
+        {
+            "name": "❄️  Детерминированная (temp=0.3)",
+            "do_sample": True,
+            "temperature": 0.3,
+        },
     ]
 
     for strategy in strategies:
         print(f"\n{strategy['name']}:")
         try:
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "do_sample": strategy["do_sample"],
+                {
-                "temperature": strategy["temperature"],
+                    "do_sample": strategy["do_sample"],
-                "max_new_tokens": 20
+                    "temperature": strategy["temperature"],
-            })
+                    "max_new_tokens": 20,
+                }
+            )
 
             generated = generate_with_hf_model(hf_model, hf_tokenizer, prompt, config)
 
             # Выделяем сгенерированную часть
-            generated_part = generated[len(prompt):]
+            generated_part = generated[len(prompt) :]
             print(f"   📤 Промпт: '{prompt}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
             print(f"   📄 Полный текст: '{generated}'")
@@ -215,7 +221,7 @@ def analyze_hf_tokenization(hf_tokenizer, texts: list):
 
         # Токенизация через адаптер
         inputs = hf_tokenizer(text, return_tensors="pt")
-        tokens = inputs['input_ids'].tolist()[0]
+        tokens = inputs["input_ids"].tolist()[0]
         token_strings = hf_tokenizer.tokenize(text)
 
         print(f"   Токены (ID): {tokens}")
@@ -247,7 +253,7 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
         try:
             user_input = input("\n🔤 Введите промпт: ").strip()
 
-            if user_input.lower() in ['exit', 'quit', 'выход']:
+            if user_input.lower() in ["exit", "quit", "выход"]:
                 break
 
             if not user_input:
@@ -258,7 +264,7 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
                 max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
                 temperature = float(input("🌡️  Температура [0.7]: ") or "0.7")
                 do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
-                do_sample = do_sample_input != 'n'
+                do_sample = do_sample_input != "n"
             except:
                 max_tokens = 50
                 temperature = 0.7
@@ -266,15 +272,19 @@ def interactive_hf_generation(hf_model, hf_tokenizer):
                 print("⚠️  Использую параметры по умолчанию")
 
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "max_new_tokens": max_tokens,
+                {
-                "temperature": temperature,
+                    "max_new_tokens": max_tokens,
-                "do_sample": do_sample
+                    "temperature": temperature,
-            })
+                    "do_sample": do_sample,
+                }
+            )
 
-            generated = generate_with_hf_model(hf_model, hf_tokenizer, user_input, config)
+            generated = generate_with_hf_model(
+                hf_model, hf_tokenizer, user_input, config
+            )
 
-            generated_part = generated[len(user_input):]
+            generated_part = generated[len(user_input) :]
             print(f"\n🎯 Результат:")
             print(f"   📤 Промпт: '{user_input}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
@@ -295,7 +305,7 @@ def main():
         "model": "GPT через HFAdapter",
         "tokenizer": "BPE через HFTokenizerAdapter",
         "инструменты": "HuggingFace pipeline & генерация",
-        "стратегия": "интеграция с HF экосистемой"
+        "стратегия": "интеграция с HF экосистемой",
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -310,7 +320,7 @@ def main():
         analysis_texts = [
             "Искусственный интеллект",
             "Нейронные сети",
-            "Машинное обучение"
+            "Машинное обучение",
         ]
         analyze_hf_tokenization(hf_tokenizer, analysis_texts)
 
@@ -326,10 +336,12 @@ def main():
             print("-" * 40)
 
             try:
-                generated = generate_with_hf_model(hf_model, hf_tokenizer, prompt, GENERATION_CONFIG)
+                generated = generate_with_hf_model(
+                    hf_model, hf_tokenizer, prompt, GENERATION_CONFIG
+                )
 
                 # Выделяем сгенерированную часть
-                generated_part = generated[len(prompt):]
+                generated_part = generated[len(prompt) :]
 
                 print(f"📤 Промпт: '{prompt}'")
                 print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -365,6 +377,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/hf_integration/simple_hf_training.py

@@ -19,8 +19,12 @@ from llm.tokenizers import BPETokenizer
 from hf_proxy import HFAdapter, HFTokenizerAdapter
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TRAINING_CONFIG, PATHS, TEST_PROMPTS
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TRAINING_CONFIG,
+    PATHS,
+    TEST_PROMPTS,
 )
 
 
@@ -45,18 +49,15 @@ def create_dataset(hf_tokenizer, texts, max_length=128):
             max_length=max_length,
             truncation=True,
             padding=False,
-            return_tensors="pt"
+            return_tensors="pt",
         )
 
-        input_ids = inputs['input_ids'][0]
+        input_ids = inputs["input_ids"][0]
 
         # Создаем метки для языкового моделирования
         labels = input_ids.clone()
 
-        dataset.append({
+        dataset.append({"input_ids": input_ids, "labels": labels})
-            'input_ids': input_ids,
-            'labels': labels
-        })
 
     return dataset
 
@@ -84,10 +85,7 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
     print(f"📊 Данные: {len(train_dataset)} train, {len(val_dataset)} validation")
 
     # Оптимизатор
-    optimizer = torch.optim.AdamW(
+    optimizer = torch.optim.AdamW(hf_model.parameters(), lr=config["learning_rate"])
-        hf_model.parameters(), 
-        lr=config["learning_rate"]
-    )
 
     # Функция потерь
     loss_fn = nn.CrossEntropyLoss()
@@ -105,8 +103,8 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
         for i, batch in enumerate(train_dataset):
             optimizer.zero_grad()
 
-            input_ids = batch['input_ids'].unsqueeze(0)  # [1, seq_len]
+            input_ids = batch["input_ids"].unsqueeze(0)  # [1, seq_len]
-            labels = batch['labels'].unsqueeze(0)        # [1, seq_len]
+            labels = batch["labels"].unsqueeze(0)  # [1, seq_len]
 
             # Forward pass
             outputs = hf_model(input_ids=input_ids, labels=labels)
@@ -130,8 +128,8 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
         epoch_val_loss = 0
         with torch.no_grad():
             for batch in val_dataset:
-                input_ids = batch['input_ids'].unsqueeze(0)
+                input_ids = batch["input_ids"].unsqueeze(0)
-                labels = batch['labels'].unsqueeze(0)
+                labels = batch["labels"].unsqueeze(0)
 
                 outputs = hf_model(input_ids=input_ids, labels=labels)
                 epoch_val_loss += outputs.loss.item()
@@ -143,10 +141,10 @@ def manual_training_loop(hf_model, hf_tokenizer, train_texts, val_texts, config)
         hf_model.train()
 
     return {
-        'train_losses': train_losses,
+        "train_losses": train_losses,
-        'val_losses': val_losses,
+        "val_losses": val_losses,
-        'final_train_loss': train_losses[-1],
+        "final_train_loss": train_losses[-1],
-        'final_val_loss': val_losses[-1]
+        "final_val_loss": val_losses[-1],
     }
 
 
@@ -170,10 +168,10 @@ def test_generation_after_training(hf_model, hf_tokenizer, test_prompts):
 
             with torch.no_grad():
                 generated = hf_model.generate(
-                    input_ids=inputs['input_ids'],
+                    input_ids=inputs["input_ids"],
                     max_new_tokens=20,
                     do_sample=True,
-                    temperature=0.8
+                    temperature=0.8,
                 )
 
             generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
@@ -192,7 +190,9 @@ def main():
     try:
         # === Подготовка данных ===
         print("🔧 Подготовка данных...")
-        train_texts = TRAIN_TEXTS[:10]  # Используем меньше данных для быстрого тестирования
+        train_texts = TRAIN_TEXTS[
+            :10
+        ]  # Используем меньше данных для быстрого тестирования
         val_texts = TRAIN_TEXTS[10:12]
 
         print(f"📊 Данные: {len(train_texts)} train, {len(val_texts)} validation")
@@ -203,7 +203,7 @@ def main():
         llm_tokenizer.train(
             texts=train_texts,
             vocab_size=BPE_CONFIG["vocab_size"],
-            special_tokens=BPE_CONFIG["special_tokens"]
+            special_tokens=BPE_CONFIG["special_tokens"],
         )
 
         hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -227,7 +227,7 @@ def main():
         training_config = {
             "learning_rate": TRAINING_CONFIG["learning_rate"],
             "num_epochs": 2,  # Меньше эпох для быстрого тестирования
-            "batch_size": TRAINING_CONFIG["batch_size"]
+            "batch_size": TRAINING_CONFIG["batch_size"],
         }
 
         results = manual_training_loop(
@@ -255,20 +255,23 @@ def main():
 
         # Сохраняем модель
         HFAdapter.save_pretrained(
-            hf_model, 
+            hf_model, "checkpoints/hf_simple_trained", tokenizer=hf_tokenizer
-            "checkpoints/hf_simple_trained", 
-            tokenizer=hf_tokenizer
         )
         print("✅ Модель сохранена")
 
         # Сохраняем результаты
         results_path = "checkpoints/simple_training_results.json"
-        with open(results_path, 'w', encoding='utf-8') as f:
+        with open(results_path, "w", encoding="utf-8") as f:
-            json.dump({
+            json.dump(
-                'training_config': training_config,
+                {
-                'model_config': model_config,
+                    "training_config": training_config,
-                'results': results
+                    "model_config": model_config,
-            }, f, indent=2, ensure_ascii=False)
+                    "results": results,
+                },
+                f,
+                indent=2,
+                ensure_ascii=False,
+            )
         print(f"✅ Результаты сохранены в {results_path}")
 
         print(f"\n🎉 Упрощенное обучение завершено успешно!")
@@ -278,6 +281,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/hf_integration/test_hf_proxy.py

@@ -16,8 +16,11 @@ from llm.tokenizers import BPETokenizer
 from hf_proxy import HFAdapter, HFTokenizerAdapter
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TEST_PROMPTS, GENERATION_CONFIG
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TEST_PROMPTS,
+    GENERATION_CONFIG,
 )
 
 
@@ -31,7 +34,7 @@ def test_basic_hf_integration():
     llm_tokenizer.train(
         texts=TRAIN_TEXTS,
         vocab_size=BPE_CONFIG["vocab_size"],
-        special_tokens=BPE_CONFIG["special_tokens"]
+        special_tokens=BPE_CONFIG["special_tokens"],
     )
 
     hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -62,7 +65,7 @@ def test_basic_hf_integration():
         print(f"      HF адаптер: {hf_inputs['input_ids'].shape}")
 
         # Декодирование
-        decoded = hf_tokenizer.decode(hf_inputs['input_ids'][0])
+        decoded = hf_tokenizer.decode(hf_inputs["input_ids"][0])
         print(f"      Декодированный: '{decoded}'")
 
     # === Тестирование forward pass ===
@@ -87,10 +90,10 @@ def test_basic_hf_integration():
 
             with torch.no_grad():
                 generated = hf_model.generate(
-                    input_ids=inputs['input_ids'],
+                    input_ids=inputs["input_ids"],
                     max_new_tokens=10,
                     do_sample=True,
-                    temperature=0.8
+                    temperature=0.8,
                 )
 
             generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
@@ -123,7 +126,9 @@ def test_basic_hf_integration():
         test_input = hf_tokenizer("Тест", return_tensors="pt")
         with torch.no_grad():
             loaded_outputs = loaded_model(**test_input)
-        print(f"   ✅ Загруженная модель работает (logits: {loaded_outputs.logits.shape})")
+        print(
+            f"   ✅ Загруженная модель работает (logits: {loaded_outputs.logits.shape})"
+        )
 
     except Exception as e:
         print(f"   ❌ Ошибка сохранения/загрузки: {e}")
@@ -140,7 +145,7 @@ def test_hf_tokenizer_methods():
     llm_tokenizer.train(
         texts=TRAIN_TEXTS[:5],
         vocab_size=500,
-        special_tokens=BPE_CONFIG["special_tokens"]
+        special_tokens=BPE_CONFIG["special_tokens"],
     )
 
     hf_tokenizer = HFTokenizerAdapter(llm_tokenizer)
@@ -199,6 +204,7 @@ def main():
     except Exception as e:
         print(f"\n❌ Ошибка в тестировании: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/hf_integration/train_with_hf_trainer.py

@@ -17,12 +17,18 @@ from llm.tokenizers import BPETokenizer
 from hf_proxy import HFAdapter, HFTokenizerAdapter
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TRAINING_CONFIG, PATHS, TEST_PROMPTS
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TRAINING_CONFIG,
+    PATHS,
+    TEST_PROMPTS,
 )
 from shared.data import (
-    load_training_data, ensure_directories, 
+    load_training_data,
-    print_experiment_info, ExperimentLogger
+    ensure_directories,
+    print_experiment_info,
+    ExperimentLogger,
 )
 
 
@@ -50,7 +56,7 @@ def setup_hf_training():
         llm_tokenizer.train(
             texts=TRAIN_TEXTS,
             vocab_size=BPE_CONFIG["vocab_size"],
-            special_tokens=BPE_CONFIG["special_tokens"]
+            special_tokens=BPE_CONFIG["special_tokens"],
         )
         llm_tokenizer.save(PATHS["bpe_tokenizer"])
         print(f"✅ Токенизатор обучен и сохранен")
@@ -117,7 +123,7 @@ def main():
         "tokenizer": "BPE через HFTokenizerAdapter",
         "trainer": "HuggingFace Trainer",
         "vocab_size": BPE_CONFIG["vocab_size"],
-        "training_epochs": TRAINING_CONFIG["num_epochs"]
+        "training_epochs": TRAINING_CONFIG["num_epochs"],
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -126,7 +132,14 @@ def main():
 
     try:
         # Настраиваем обучение
-        hf_model, hf_tokenizer, llm_tokenizer, model_config, train_texts, val_texts = setup_hf_training()
+        (
+            hf_model,
+            hf_tokenizer,
+            llm_tokenizer,
+            model_config,
+            train_texts,
+            val_texts,
+        ) = setup_hf_training()
 
         # Тестируем интеграцию
         test_hf_integration(hf_model, hf_tokenizer, llm_tokenizer)
@@ -173,7 +186,7 @@ def main():
         from transformers import (
             Trainer,
             TrainingArguments,
-            DataCollatorForLanguageModeling
+            DataCollatorForLanguageModeling,
         )
 
         # Data collator для языкового моделирования
@@ -261,13 +274,15 @@ def main():
 
                 with torch.no_grad():
                     generated = hf_model.generate(
-                        input_ids=inputs['input_ids'],
+                        input_ids=inputs["input_ids"],
                         max_new_tokens=20,
                         do_sample=True,
-                        temperature=0.8
+                        temperature=0.8,
                     )
 
-                generated_text = hf_tokenizer.decode(generated[0], skip_special_tokens=True)
+                generated_text = hf_tokenizer.decode(
+                    generated[0], skip_special_tokens=True
+                )
                 print(f"🎯 Результат: '{generated_text}'")
 
             except Exception as e:
@@ -278,8 +293,8 @@ def main():
             "experiment": experiment_name,
             "model_config": model_config,
             "training_config": TRAINING_CONFIG,
-            "final_loss": train_result.metrics.get('train_loss', 'N/A'),
+            "final_loss": train_result.metrics.get("train_loss", "N/A"),
-            "eval_loss": train_result.metrics.get('eval_loss', 'N/A')
+            "eval_loss": train_result.metrics.get("eval_loss", "N/A"),
         }
 
         logger.save_logs("checkpoints/hf_integration_training_logs.json")
@@ -291,6 +306,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/llm_only/generate_gpt2_bpe.py

@@ -15,12 +15,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from llm.models.gpt import GPT2
 from llm.tokenizers import BPETokenizer
 
-from shared.configs import (
+from shared.configs import BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
-    BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
+from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
-)
-from shared.data import (
-    print_experiment_info, ensure_directories, ExperimentLogger
-)
 
 
 def load_model_and_tokenizer() -> tuple:
@@ -38,13 +34,12 @@ def load_model_and_tokenizer() -> tuple:
         )
 
     if not os.path.exists(PATHS["bpe_tokenizer"]):
-        raise FileNotFoundError(
+        raise FileNotFoundError(f"Токенизатор не найден: {PATHS['bpe_tokenizer']}")
-            f"Токенизатор не найден: {PATHS['bpe_tokenizer']}"
-        )
 
     # Загружаем конфигурацию модели
     import json
-    with open(PATHS["gpt_bpe_config"], 'r', encoding='utf-8') as f:
+
+    with open(PATHS["gpt_bpe_config"], "r", encoding="utf-8") as f:
         model_config = json.load(f)
 
     # Загружаем токенизатор
@@ -55,7 +50,7 @@ def load_model_and_tokenizer() -> tuple:
     # Загружаем модель
     print("🔧 Загрузка GPT2 модели...")
     model = GPT2(model_config)
-    model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location='cpu'))
+    model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location="cpu"))
     model.eval()
     print("✅ Модель загружена")
 
@@ -63,10 +58,7 @@ def load_model_and_tokenizer() -> tuple:
 
 
 def generate_text(
-    model: GPT2, 
+    model: GPT2, tokenizer: BPETokenizer, prompt: str, config: dict
-    tokenizer: BPETokenizer, 
-    prompt: str, 
-    config: dict
 ) -> str:
     """
     Генерирует текст на основе промпта.
@@ -81,8 +73,10 @@ def generate_text(
         str: Сгенерированный текст
     """
     print(f"🔤 Промпт: '{prompt}'")
-    print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+    print(
-          f"temp={config['temperature']}, sample={config['do_sample']}")
+        f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+        f"temp={config['temperature']}, sample={config['do_sample']}"
+    )
 
     # Кодируем промпт
     input_ids = tokenizer.encode(prompt, add_special_tokens=False)
@@ -100,7 +94,7 @@ def generate_text(
             do_sample=config["do_sample"],
             temperature=config["temperature"],
             top_k=config["top_k"],
-            top_p=config["top_p"]
+            top_p=config["top_p"],
         )
 
     # Декодируем результат
@@ -125,23 +119,29 @@ def test_different_strategies(model: GPT2, tokenizer: BPETokenizer, prompt: str)
         {"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
         {"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
         {"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
-        {"name": "❄️  Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
+        {
+            "name": "❄️  Детерминированная (temp=0.3)",
+            "do_sample": True,
+            "temperature": 0.3,
+        },
     ]
 
     for strategy in strategies:
         print(f"\n{strategy['name']}:")
         try:
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "do_sample": strategy["do_sample"],
+                {
-                "temperature": strategy["temperature"],
+                    "do_sample": strategy["do_sample"],
-                "max_new_tokens": 20
+                    "temperature": strategy["temperature"],
-            })
+                    "max_new_tokens": 20,
+                }
+            )
 
             generated = generate_text(model, tokenizer, prompt, config)
 
             # Выделяем сгенерированную часть
-            generated_part = generated[len(prompt):]
+            generated_part = generated[len(prompt) :]
             print(f"   📤 Промпт: '{prompt}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
             print(f"   📄 Полный текст: '{generated}'")
@@ -196,7 +196,7 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
         try:
             user_input = input("\n🔤 Введите промпт: ").strip()
 
-            if user_input.lower() in ['exit', 'quit', 'выход']:
+            if user_input.lower() in ["exit", "quit", "выход"]:
                 break
 
             if not user_input:
@@ -207,7 +207,7 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
                 max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
                 temperature = float(input("🌡️  Температура [0.7]: ") or "0.7")
                 do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
-                do_sample = do_sample_input != 'n'
+                do_sample = do_sample_input != "n"
             except:
                 max_tokens = 50
                 temperature = 0.7
@@ -215,15 +215,17 @@ def interactive_generation(model: GPT2, tokenizer: BPETokenizer):
                 print("⚠️  Использую параметры по умолчанию")
 
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "max_new_tokens": max_tokens,
+                {
-                "temperature": temperature,
+                    "max_new_tokens": max_tokens,
-                "do_sample": do_sample
+                    "temperature": temperature,
-            })
+                    "do_sample": do_sample,
+                }
+            )
 
             generated = generate_text(model, tokenizer, user_input, config)
 
-            generated_part = generated[len(user_input):]
+            generated_part = generated[len(user_input) :]
             print(f"\n🎯 Результат:")
             print(f"   📤 Промпт: '{user_input}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
@@ -244,7 +246,7 @@ def main():
         "model": "GPT2 с BPE токенизатором",
         "стратегия": "автономная генерация",
         "вход": "промпты",
-        "выход": "сгенерированный текст"
+        "выход": "сгенерированный текст",
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -275,7 +277,7 @@ def main():
                 generated = generate_text(model, tokenizer, prompt, GENERATION_CONFIG)
 
                 # Выделяем сгенерированную часть
-                generated_part = generated[len(prompt):]
+                generated_part = generated[len(prompt) :]
 
                 print(f"📤 Промпт: '{prompt}'")
                 print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -306,6 +308,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/llm_only/generate_gpt_bpe.py

@@ -15,12 +15,8 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from llm.models.gpt import GPT
 from llm.tokenizers import BPETokenizer
 
-from shared.configs import (
+from shared.configs import BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
-    BASE_GPT_CONFIG, TEST_PROMPTS, GENERATION_CONFIG, PATHS
+from shared.data import print_experiment_info, ensure_directories, ExperimentLogger
-)
-from shared.data import (
-    print_experiment_info, ensure_directories, ExperimentLogger
-)
 
 
 def load_model_and_tokenizer() -> tuple:
@@ -38,13 +34,12 @@ def load_model_and_tokenizer() -> tuple:
         )
 
     if not os.path.exists(PATHS["bpe_tokenizer"]):
-        raise FileNotFoundError(
+        raise FileNotFoundError(f"Токенизатор не найден: {PATHS['bpe_tokenizer']}")
-            f"Токенизатор не найден: {PATHS['bpe_tokenizer']}"
-        )
 
     # Загружаем конфигурацию модели
     import json
-    with open(PATHS["gpt_bpe_config"], 'r', encoding='utf-8') as f:
+
+    with open(PATHS["gpt_bpe_config"], "r", encoding="utf-8") as f:
         model_config = json.load(f)
 
     # Загружаем токенизатор
@@ -55,7 +50,7 @@ def load_model_and_tokenizer() -> tuple:
     # Загружаем модель
     print("🔧 Загрузка GPT модели...")
     model = GPT(model_config)
-    model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location='cpu'))
+    model.load_state_dict(torch.load(PATHS["gpt_bpe_model"], map_location="cpu"))
     model.eval()
     print("✅ Модель загружена")
 
@@ -63,10 +58,7 @@ def load_model_and_tokenizer() -> tuple:
 
 
 def generate_text(
-    model: GPT, 
+    model: GPT, tokenizer: BPETokenizer, prompt: str, config: dict
-    tokenizer: BPETokenizer, 
-    prompt: str, 
-    config: dict
 ) -> str:
     """
     Генерирует текст на основе промпта.
@@ -81,8 +73,10 @@ def generate_text(
         str: Сгенерированный текст
     """
     print(f"🔤 Промпт: '{prompt}'")
-    print(f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+    print(
-          f"temp={config['temperature']}, sample={config['do_sample']}")
+        f"📊 Параметры: max_tokens={config['max_new_tokens']}, "
+        f"temp={config['temperature']}, sample={config['do_sample']}"
+    )
 
     # Кодируем промпт
     input_ids = tokenizer.encode(prompt, add_special_tokens=False)
@@ -100,7 +94,7 @@ def generate_text(
             do_sample=config["do_sample"],
             temperature=config["temperature"],
             top_k=config["top_k"],
-            top_p=config["top_p"]
+            top_p=config["top_p"],
         )
 
     # Декодируем результат
@@ -125,23 +119,29 @@ def test_different_strategies(model: GPT, tokenizer: BPETokenizer, prompt: str):
         {"name": "🎯 Жадный поиск", "do_sample": False, "temperature": 1.0},
         {"name": "🎲 Вероятностная (temp=0.7)", "do_sample": True, "temperature": 0.7},
         {"name": "🔥 Случайная (temp=1.2)", "do_sample": True, "temperature": 1.2},
-        {"name": "❄️  Детерминированная (temp=0.3)", "do_sample": True, "temperature": 0.3},
+        {
+            "name": "❄️  Детерминированная (temp=0.3)",
+            "do_sample": True,
+            "temperature": 0.3,
+        },
     ]
 
     for strategy in strategies:
         print(f"\n{strategy['name']}:")
         try:
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "do_sample": strategy["do_sample"],
+                {
-                "temperature": strategy["temperature"],
+                    "do_sample": strategy["do_sample"],
-                "max_new_tokens": 20
+                    "temperature": strategy["temperature"],
-            })
+                    "max_new_tokens": 20,
+                }
+            )
 
             generated = generate_text(model, tokenizer, prompt, config)
 
             # Выделяем сгенерированную часть
-            generated_part = generated[len(prompt):]
+            generated_part = generated[len(prompt) :]
             print(f"   📤 Промпт: '{prompt}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
             print(f"   📄 Полный текст: '{generated}'")
@@ -196,7 +196,7 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
         try:
             user_input = input("\n🔤 Введите промпт: ").strip()
 
-            if user_input.lower() in ['exit', 'quit', 'выход']:
+            if user_input.lower() in ["exit", "quit", "выход"]:
                 break
 
             if not user_input:
@@ -207,7 +207,7 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
                 max_tokens = int(input("📏 Макс. токенов [50]: ") or "50")
                 temperature = float(input("🌡️  Температура [0.7]: ") or "0.7")
                 do_sample_input = input("🎲 Сэмплирование (y/n) [y]: ").lower()
-                do_sample = do_sample_input != 'n'
+                do_sample = do_sample_input != "n"
             except:
                 max_tokens = 50
                 temperature = 0.7
@@ -215,15 +215,17 @@ def interactive_generation(model: GPT, tokenizer: BPETokenizer):
                 print("⚠️  Использую параметры по умолчанию")
 
             config = GENERATION_CONFIG.copy()
-            config.update({
+            config.update(
-                "max_new_tokens": max_tokens,
+                {
-                "temperature": temperature,
+                    "max_new_tokens": max_tokens,
-                "do_sample": do_sample
+                    "temperature": temperature,
-            })
+                    "do_sample": do_sample,
+                }
+            )
 
             generated = generate_text(model, tokenizer, user_input, config)
 
-            generated_part = generated[len(user_input):]
+            generated_part = generated[len(user_input) :]
             print(f"\n🎯 Результат:")
             print(f"   📤 Промпт: '{user_input}'")
             print(f"   🎯 Сгенерировано: '{generated_part}'")
@@ -244,7 +246,7 @@ def main():
         "model": "GPT с BPE токенизатором",
         "стратегия": "автономная генерация",
         "вход": "промпты",
-        "выход": "сгенерированный текст"
+        "выход": "сгенерированный текст",
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -275,7 +277,7 @@ def main():
                 generated = generate_text(model, tokenizer, prompt, GENERATION_CONFIG)
 
                 # Выделяем сгенерированную часть
-                generated_part = generated[len(prompt):]
+                generated_part = generated[len(prompt) :]
 
                 print(f"📤 Промпт: '{prompt}'")
                 print(f"🎯 Сгенерировано: '{generated_part}'")
@@ -306,6 +308,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/llm_only/train_gpt2_bpe.py

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
 from llm.training.trainer import Trainer
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TRAINING_CONFIG, PATHS, TEST_PROMPTS
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TRAINING_CONFIG,
+    PATHS,
+    TEST_PROMPTS,
 )
 from shared.data import (
-    load_training_data, ensure_directories, 
+    load_training_data,
-    print_experiment_info, ExperimentLogger
+    ensure_directories,
+    print_experiment_info,
+    ExperimentLogger,
 )
 
 
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
     tokenizer.train(
         texts=texts,
         vocab_size=config["vocab_size"],
-        special_tokens=config["special_tokens"]
+        special_tokens=config["special_tokens"],
     )
 
     # Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
         "vocab_size": BPE_CONFIG["vocab_size"],
         "training_epochs": TRAINING_CONFIG["num_epochs"],
         "batch_size": TRAINING_CONFIG["batch_size"],
-        "learning_rate": TRAINING_CONFIG["learning_rate"]
+        "learning_rate": TRAINING_CONFIG["learning_rate"],
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
         # === Подготовка датасета ===
         print(f"\n📊 Подготовка датасета...")
         train_dataset = TextDataset(
-            train_texts, 
+            train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
-            tokenizer, 
-            block_size=model_config["max_position_embeddings"]
         )
         print(f"   Размер train датасета: {len(train_dataset)} примеров")
 
@@ -152,7 +156,7 @@ def main():
             lr=TRAINING_CONFIG["learning_rate"],
             batch_size=TRAINING_CONFIG["batch_size"],
             num_epochs=TRAINING_CONFIG["num_epochs"],
-            warmup_steps=TRAINING_CONFIG["warmup_steps"]
+            warmup_steps=TRAINING_CONFIG["warmup_steps"],
         )
 
         # Запускаем обучение
@@ -167,7 +171,8 @@ def main():
 
         # Сохраняем конфигурацию
         import json
-        with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
+
+        with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
             json.dump(model_config, f, indent=2, ensure_ascii=False)
 
         print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
                         x=input_tensor,
                         max_new_tokens=20,
                         do_sample=True,
-                        temperature=0.8
+                        temperature=0.8,
                     )
 
                 # Декодируем результат
                 generated_text = tokenizer.decode(generated_ids[0].tolist())
-                generated_part = generated_text[len(prompt):]
+                generated_part = generated_text[len(prompt) :]
 
                 print(f"🎯 Сгенерировано: '{generated_part}'")
                 print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
             "model_config": model_config,
             "training_config": TRAINING_CONFIG,
             "tokenizer_vocab_size": tokenizer.get_vocab_size(),
-            "final_loss": "см. логи обучения"  # В реальном эксперименте можно сохранить final loss
+            "final_loss": "см. логи обучения",  # В реальном эксперименте можно сохранить final loss
         }
 
         logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/llm_only/train_gpt_bpe.py

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
 from llm.training.trainer import Trainer
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TRAINING_CONFIG, PATHS, TEST_PROMPTS
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TRAINING_CONFIG,
+    PATHS,
+    TEST_PROMPTS,
 )
 from shared.data import (
-    load_training_data, ensure_directories, 
+    load_training_data,
-    print_experiment_info, ExperimentLogger
+    ensure_directories,
+    print_experiment_info,
+    ExperimentLogger,
 )
 
 
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
     tokenizer.train(
         texts=texts,
         vocab_size=config["vocab_size"],
-        special_tokens=config["special_tokens"]
+        special_tokens=config["special_tokens"],
     )
 
     # Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
         "vocab_size": BPE_CONFIG["vocab_size"],
         "training_epochs": TRAINING_CONFIG["num_epochs"],
         "batch_size": TRAINING_CONFIG["batch_size"],
-        "learning_rate": TRAINING_CONFIG["learning_rate"]
+        "learning_rate": TRAINING_CONFIG["learning_rate"],
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
         # === Подготовка датасета ===
         print(f"\n📊 Подготовка датасета...")
         train_dataset = TextDataset(
-            train_texts, 
+            train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
-            tokenizer, 
-            block_size=model_config["max_position_embeddings"]
         )
         print(f"   Размер train датасета: {len(train_dataset)} примеров")
 
@@ -152,7 +156,7 @@ def main():
             lr=TRAINING_CONFIG["learning_rate"],
             batch_size=TRAINING_CONFIG["batch_size"],
             num_epochs=TRAINING_CONFIG["num_epochs"],
-            warmup_steps=TRAINING_CONFIG["warmup_steps"]
+            warmup_steps=TRAINING_CONFIG["warmup_steps"],
         )
 
         # Запускаем обучение
@@ -167,7 +171,8 @@ def main():
 
         # Сохраняем конфигурацию
         import json
-        with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
+
+        with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
             json.dump(model_config, f, indent=2, ensure_ascii=False)
 
         print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
                         x=input_tensor,
                         max_new_tokens=20,
                         do_sample=True,
-                        temperature=0.8
+                        temperature=0.8,
                     )
 
                 # Декодируем результат
                 generated_text = tokenizer.decode(generated_ids[0].tolist())
-                generated_part = generated_text[len(prompt):]
+                generated_part = generated_text[len(prompt) :]
 
                 print(f"🎯 Сгенерировано: '{generated_part}'")
                 print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
             "model_config": model_config,
             "training_config": TRAINING_CONFIG,
             "tokenizer_vocab_size": tokenizer.get_vocab_size(),
-            "final_loss": "см. логи обучения"  # В реальном эксперименте можно сохранить final loss
+            "final_loss": "см. логи обучения",  # В реальном эксперименте можно сохранить final loss
         }
 
         logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/llm_only/train_llama_bpe.py

@@ -18,12 +18,18 @@ from llm.training.dataset import TextDataset
 from llm.training.trainer import Trainer
 
 from shared.configs import (
-    TRAIN_TEXTS, BASE_GPT_CONFIG, BPE_CONFIG, 
+    TRAIN_TEXTS,
-    TRAINING_CONFIG, PATHS, TEST_PROMPTS
+    BASE_GPT_CONFIG,
+    BPE_CONFIG,
+    TRAINING_CONFIG,
+    PATHS,
+    TEST_PROMPTS,
 )
 from shared.data import (
-    load_training_data, ensure_directories, 
+    load_training_data,
-    print_experiment_info, ExperimentLogger
+    ensure_directories,
+    print_experiment_info,
+    ExperimentLogger,
 )
 
 
@@ -44,7 +50,7 @@ def train_bpe_tokenizer(texts: list, config: dict) -> BPETokenizer:
     tokenizer.train(
         texts=texts,
         vocab_size=config["vocab_size"],
-        special_tokens=config["special_tokens"]
+        special_tokens=config["special_tokens"],
     )
 
     # Сохраняем токенизатор
@@ -99,7 +105,7 @@ def main():
         "vocab_size": BPE_CONFIG["vocab_size"],
         "training_epochs": TRAINING_CONFIG["num_epochs"],
         "batch_size": TRAINING_CONFIG["batch_size"],
-        "learning_rate": TRAINING_CONFIG["learning_rate"]
+        "learning_rate": TRAINING_CONFIG["learning_rate"],
     }
 
     print_experiment_info(experiment_name, experiment_config)
@@ -137,9 +143,7 @@ def main():
         # === Подготовка датасета ===
         print(f"\n📊 Подготовка датасета...")
         train_dataset = TextDataset(
-            train_texts, 
+            train_texts, tokenizer, block_size=model_config["max_position_embeddings"]
-            tokenizer, 
-            block_size=model_config["max_position_embeddings"]
         )
         print(f"   Размер train датасета: {len(train_dataset)} примеров")
 
@@ -152,7 +156,7 @@ def main():
             lr=TRAINING_CONFIG["learning_rate"],
             batch_size=TRAINING_CONFIG["batch_size"],
             num_epochs=TRAINING_CONFIG["num_epochs"],
-            warmup_steps=TRAINING_CONFIG["warmup_steps"]
+            warmup_steps=TRAINING_CONFIG["warmup_steps"],
         )
 
         # Запускаем обучение
@@ -167,7 +171,8 @@ def main():
 
         # Сохраняем конфигурацию
         import json
-        with open(PATHS["gpt_bpe_config"], 'w', encoding='utf-8') as f:
+
+        with open(PATHS["gpt_bpe_config"], "w", encoding="utf-8") as f:
             json.dump(model_config, f, indent=2, ensure_ascii=False)
 
         print(f"✅ Модель сохранена:")
@@ -193,12 +198,12 @@ def main():
                         x=input_tensor,
                         max_new_tokens=20,
                         do_sample=True,
-                        temperature=0.8
+                        temperature=0.8,
                     )
 
                 # Декодируем результат
                 generated_text = tokenizer.decode(generated_ids[0].tolist())
-                generated_part = generated_text[len(prompt):]
+                generated_part = generated_text[len(prompt) :]
 
                 print(f"🎯 Сгенерировано: '{generated_part}'")
                 print(f"📄 Полный текст: '{generated_text}'")
@@ -212,7 +217,7 @@ def main():
             "model_config": model_config,
             "training_config": TRAINING_CONFIG,
             "tokenizer_vocab_size": tokenizer.get_vocab_size(),
-            "final_loss": "см. логи обучения"  # В реальном эксперименте можно сохранить final loss
+            "final_loss": "см. логи обучения",  # В реальном эксперименте можно сохранить final loss
         }
 
         logger.save_logs("checkpoints/llm_only_training_logs.json")
@@ -224,6 +229,7 @@ def main():
     except Exception as e:
         print(f"❌ Ошибка в эксперименте: {e}")
         import traceback
+
         traceback.print_exc()
 
 

experiments/shared/configs.py

@@ -30,7 +30,7 @@ BASE_GPT_CONFIG = {
     "num_heads": 4,
     "num_layers": 4,
     "max_position_embeddings": 128,
-    "dropout": 0.1
+    "dropout": 0.1,
 }
 
 # Конфигурация для маленькой модели (быстрое тестирование)
@@ -40,7 +40,7 @@ SMALL_GPT_CONFIG = {
     "num_heads": 2,
     "num_layers": 2,
     "max_position_embeddings": 64,
-    "dropout": 0.1
+    "dropout": 0.1,
 }
 
 # Конфигурация для большой модели (качественное обучение)
@@ -50,13 +50,13 @@ LARGE_GPT_CONFIG = {
     "num_heads": 8,
     "num_layers": 6,
     "max_position_embeddings": 256,
-    "dropout": 0.1
+    "dropout": 0.1,
 }
 
 # === Конфигурации токенизатора ===
 BPE_CONFIG = {
     "vocab_size": 1000,
-    "special_tokens": ["<pad>", "<unk>", "<bos>", "<eos>"]
+    "special_tokens": ["<pad>", "<unk>", "<bos>", "<eos>"],
 }
 
 # === Конфигурации обучения ===
@@ -65,7 +65,7 @@ TRAINING_CONFIG = {
     "batch_size": 2,
     "num_epochs": 3,
     "warmup_steps": 50,
-    "gradient_clip": 1.0
+    "gradient_clip": 1.0,
 }
 
 # === Конфигурации генерации ===
@@ -74,7 +74,7 @@ GENERATION_CONFIG = {
     "temperature": 0.7,
     "do_sample": True,
     "top_k": None,
-    "top_p": None
+    "top_p": None,
 }
 
 # === Пути для сохранения ===
@@ -84,7 +84,7 @@ PATHS = {
     "gpt_bpe_config": "checkpoints/gpt-bpe/config.json",
     "hf_tokenizer": "checkpoints/hf-bpe-tokenizer",
     "hf_model": "checkpoints/hf-trained",
-    "hf_proxy_model": "checkpoints/hf-trained-proxy"
+    "hf_proxy_model": "checkpoints/hf-trained-proxy",
 }
 
 # === Тестовые промпты ===

experiments/shared/data.py

@@ -32,7 +32,7 @@ def ensure_directories():
         "checkpoints/hf-bpe-tokenizer",
         "checkpoints/hf-trained",
         "checkpoints/hf-trained-proxy",
-        "logs"
+        "logs",
     ]
 
     for directory in directories:
@@ -52,15 +52,16 @@ def get_model_paths(experiment_type: str = "llm_only") -> dict:
     base_paths = PATHS.copy()
 
     if experiment_type == "hf_integration":
-        base_paths.update({
+        base_paths.update(
-            "model": base_paths["hf_model"],
+            {"model": base_paths["hf_model"], "tokenizer": base_paths["hf_tokenizer"]}
-            "tokenizer": base_paths["hf_tokenizer"]
+        )
-        })
     else:  # llm_only
-        base_paths.update({
+        base_paths.update(
-            "model": base_paths["gpt_bpe_model"],
+            {
-            "tokenizer": base_paths["bpe_tokenizer"]
+                "model": base_paths["gpt_bpe_model"],
-        })
+                "tokenizer": base_paths["bpe_tokenizer"],
+            }
+        )
 
     return base_paths
 
@@ -92,7 +93,7 @@ def save_experiment_results(results: dict, filepath: str):
     """
     import json
 
-    with open(filepath, 'w', encoding='utf-8') as f:
+    with open(filepath, "w", encoding="utf-8") as f:
         json.dump(results, f, ensure_ascii=False, indent=2)
 
     print(f"✅ Результаты эксперимента сохранены: {filepath}")
@@ -113,7 +114,7 @@ def load_experiment_results(filepath: str) -> dict:
     if not os.path.exists(filepath):
         return {}
 
-    with open(filepath, 'r', encoding='utf-8') as f:
+    with open(filepath, "r", encoding="utf-8") as f:
         return json.load(f)
 
 
@@ -151,12 +152,9 @@ class ExperimentLogger:
         """Сохраняет логи эксперимента."""
         import json
 
-        logs = {
+        logs = {"experiment_name": self.experiment_name, "metrics": self.metrics}
-            "experiment_name": self.experiment_name,
-            "metrics": self.metrics
-        }
 
-        with open(filepath, 'w', encoding='utf-8') as f:
+        with open(filepath, "w", encoding="utf-8") as f:
             json.dump(logs, f, ensure_ascii=False, indent=2)
 
         print(f"✅ Логи эксперимента сохранены: {filepath}")

hf-proxy/src/hf_proxy/__init__.py

@@ -27,16 +27,13 @@ __all__ = [
     # Основные классы адаптера
     "HFAdapter",
     "HFGPTAdapter",
-    
     # Конфигурации
     "HFAdapterConfig",
     "HFPretrainedConfig",
-    
     # Адаптеры токенизаторов
     "HFTokenizerAdapter",
     "create_hf_tokenizer",
     "convert_to_hf_format",
-    
     # Утилиты
     "HFUtils",
     "TokenizerWrapper",

hf-proxy/src/hf_proxy/hf_adapter.py

@@ -11,7 +11,7 @@ from transformers import (
     GPT2Config,
     GenerationConfig,
     LogitsProcessorList,
-    StoppingCriteriaList
+    StoppingCriteriaList,
 )
 from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions
 
@@ -24,6 +24,7 @@ class HFGPTAdapter(PreTrainedModel):
     Адаптер для модели GPT из библиотеки llm.
     Позволяет использовать кастомные GPT модели с HuggingFace Transformers.
     """
+
     config_class = HFPretrainedConfig
 
     def __init__(self, config: HFPretrainedConfig, llm_model: Optional[GPT] = None):
@@ -46,7 +47,7 @@ class HFGPTAdapter(PreTrainedModel):
             self.llm_model = llm_model
 
         # Устанавливаем веса если они есть в конфигурации
-        if hasattr(config, 'state_dict') and config.state_dict is not None:
+        if hasattr(config, "state_dict") and config.state_dict is not None:
             self.llm_model.load_state_dict(config.state_dict)
 
     def _hf_to_llm_config(self, hf_config: HFPretrainedConfig) -> dict:
@@ -78,7 +79,7 @@ class HFGPTAdapter(PreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
-        **kwargs
+        **kwargs,
     ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
         """
         Прямой проход модели.
@@ -96,7 +97,9 @@ class HFGPTAdapter(PreTrainedModel):
         Returns:
             CausalLMOutputWithCrossAttentions или кортеж
         """
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
 
         # Основной forward pass
         outputs = self.llm_model(input_ids)
@@ -114,8 +117,7 @@ class HFGPTAdapter(PreTrainedModel):
             # Вычисляем cross-entropy loss
             loss_fct = nn.CrossEntropyLoss()
             loss = loss_fct(
-                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
-                shift_labels.view(-1)
             )
 
         if not return_dict:
@@ -134,10 +136,7 @@ class HFGPTAdapter(PreTrainedModel):
         )
 
     def prepare_inputs_for_generation(
-        self, 
+        self, input_ids: torch.Tensor, past_key_values: Optional[Tuple] = None, **kwargs
-        input_ids: torch.Tensor, 
-        past_key_values: Optional[Tuple] = None,
-        **kwargs
     ) -> dict:
         """
         Подготавливает входные данные для генерации.
@@ -163,7 +162,7 @@ class HFGPTAdapter(PreTrainedModel):
         generation_config: Optional[GenerationConfig] = None,
         logits_processor: Optional[LogitsProcessorList] = None,
         stopping_criteria: Optional[StoppingCriteriaList] = None,
-        **kwargs
+        **kwargs,
     ) -> torch.Tensor:
         """
         Генерация текста с поддержкой HuggingFace интерфейса.
@@ -179,8 +178,8 @@ class HFGPTAdapter(PreTrainedModel):
             torch.Tensor: Сгенерированные токены
         """
         # Извлекаем обязательные параметры из kwargs или используем значения по умолчанию
-        max_new_tokens = kwargs.pop('max_new_tokens', 50)
+        max_new_tokens = kwargs.pop("max_new_tokens", 50)
-        do_sample = kwargs.pop('do_sample', True)
+        do_sample = kwargs.pop("do_sample", True)
 
         # Используем встроенную генерацию llm модели
         return self.llm_model.generate(
@@ -188,7 +187,7 @@ class HFGPTAdapter(PreTrainedModel):
             max_new_tokens=max_new_tokens,
             do_sample=do_sample,
             attention_mask=attention_mask,
-            **kwargs
+            **kwargs,
         )
 
 
@@ -199,8 +198,7 @@ class HFAdapter:
 
     @staticmethod
     def from_llm_model(
-        llm_model: GPT, 
+        llm_model: GPT, hf_config: Optional[HFAdapterConfig] = None
-        hf_config: Optional[HFAdapterConfig] = None
     ) -> HFGPTAdapter:
         """
         Создает адаптер из существующей llm модели.
@@ -223,8 +221,7 @@ class HFAdapter:
 
     @staticmethod
     def from_pretrained(
-        model_path: str,
+        model_path: str, hf_config: Optional[HFAdapterConfig] = None
-        hf_config: Optional[HFAdapterConfig] = None
     ) -> HFGPTAdapter:
         """
         Загружает модель из чекпоинта и создает адаптер.
@@ -237,14 +234,18 @@ class HFAdapter:
             HFGPTAdapter: Адаптированная модель
         """
         # Загружаем состояние модели
-        state_dict = torch.load(model_path, map_location='cpu')
+        state_dict = torch.load(model_path, map_location="cpu")
 
         # Определяем конфигурацию из состояния модели или используем переданную
         if hf_config is None:
             # Пытаемся определить конфигурацию из состояния модели
             # Это упрощенный подход - в реальности нужно сохранять конфигурацию отдельно
-            vocab_size = state_dict.get('_token_embeddings._embedding.weight', torch.zeros(50257, 768)).shape[0]
+            vocab_size = state_dict.get(
-            embed_dim = state_dict.get('_token_embeddings._embedding.weight', torch.zeros(50257, 768)).shape[1]
+                "_token_embeddings._embedding.weight", torch.zeros(50257, 768)
+            ).shape[0]
+            embed_dim = state_dict.get(
+                "_token_embeddings._embedding.weight", torch.zeros(50257, 768)
+            ).shape[1]
 
             hf_config = HFAdapterConfig(
                 vocab_size=vocab_size,
@@ -270,11 +271,7 @@ class HFAdapter:
         return HFGPTAdapter(pretrained_config, llm_model)
 
     @staticmethod
-    def save_pretrained(
+    def save_pretrained(model: HFGPTAdapter, save_directory: str, **kwargs):
-        model: HFGPTAdapter,
-        save_directory: str,
-        **kwargs
-    ):
         """
         Сохраняет адаптированную модель в формате HuggingFace.
 
@@ -291,7 +288,7 @@ class HFAdapter:
 
         # Сохраняем конфигурацию
         config_path = os.path.join(save_directory, "config.json")
-        with open(config_path, 'w', encoding='utf-8') as f:
+        with open(config_path, "w", encoding="utf-8") as f:
             json.dump(model.config.to_dict(), f, indent=2, ensure_ascii=False)
 
         # Сохраняем веса модели
@@ -299,5 +296,5 @@ class HFAdapter:
         torch.save(model.llm_model.state_dict(), model_path)
 
         # Сохраняем токенизатор если передан
-        if hasattr(kwargs, 'tokenizer') and kwargs['tokenizer'] is not None:
+        if hasattr(kwargs, "tokenizer") and kwargs["tokenizer"] is not None:
-            kwargs['tokenizer'].save_pretrained(save_directory)
+            kwargs["tokenizer"].save_pretrained(save_directory)

hf-proxy/src/hf_proxy/hf_config.py

@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
 from typing import Dict, Any, Optional
 from transformers import PretrainedConfig
 
+
 @dataclass
 class HFAdapterConfig:
     """
@@ -28,6 +29,7 @@ class HFAdapterConfig:
         eos_token_id: ID токена конца строки
         bos_token_id: ID токена начала строки
     """
+
     model_type: str = "gpt"
     vocab_size: int = 50257
     hidden_size: int = 768
@@ -52,8 +54,9 @@ class HFAdapterConfig:
     def to_dict(self) -> Dict[str, Any]:
         """Преобразует конфигурацию в словарь."""
         return {
-            k: v for k, v in self.__dict__.items() 
+            k: v
-            if not k.startswith('_') and not callable(v)
+            for k, v in self.__dict__.items()
+            if not k.startswith("_") and not callable(v)
         }
 
     @classmethod
@@ -74,7 +77,7 @@ class HFAdapterConfig:
             "num_heads": "num_attention_heads",
             "max_position_embeddings": "max_position_embeddings",
             "dropout": "hidden_dropout_prob",
-            "vocab_size": "vocab_size"
+            "vocab_size": "vocab_size",
         }
 
         hf_config_dict = {}
@@ -94,6 +97,7 @@ class HFPretrainedConfig(PretrainedConfig):
     Конфигурация для предобученных моделей HuggingFace.
     Наследуется от PretrainedConfig для полной совместимости.
     """
+
     model_type = "gpt"
 
     def __init__(
@@ -112,13 +116,13 @@ class HFPretrainedConfig(PretrainedConfig):
         pad_token_id=50256,
         eos_token_id=50256,
         bos_token_id=50256,
-        **kwargs
+        **kwargs,
     ):
         super().__init__(
             pad_token_id=pad_token_id,
             eos_token_id=eos_token_id,
             bos_token_id=bos_token_id,
-            **kwargs
+            **kwargs,
         )
 
         self.vocab_size = vocab_size

hf-proxy/src/hf_proxy/hf_tokenizer.py

@@ -27,16 +27,16 @@ class HFTokenizerAdapter:
         self.vocab_size = llm_tokenizer.get_vocab_size()
 
         # Устанавливаем специальные токены
-        self.pad_token = getattr(llm_tokenizer, 'pad_token', '<pad>')
+        self.pad_token = getattr(llm_tokenizer, "pad_token", "<pad>")
-        self.unk_token = getattr(llm_tokenizer, 'unk_token', '<unk>') 
+        self.unk_token = getattr(llm_tokenizer, "unk_token", "<unk>")
-        self.bos_token = getattr(llm_tokenizer, 'bos_token', '<bos>')
+        self.bos_token = getattr(llm_tokenizer, "bos_token", "<bos>")
-        self.eos_token = getattr(llm_tokenizer, 'eos_token', '<eos>')
+        self.eos_token = getattr(llm_tokenizer, "eos_token", "<eos>")
 
         # Сохраняем ID специальных токенов
-        self.pad_token_id = getattr(llm_tokenizer, 'pad_token_id', 0)
+        self.pad_token_id = getattr(llm_tokenizer, "pad_token_id", 0)
-        self.unk_token_id = getattr(llm_tokenizer, 'unk_token_id', 1)
+        self.unk_token_id = getattr(llm_tokenizer, "unk_token_id", 1)
-        self.bos_token_id = getattr(llm_tokenizer, 'bos_token_id', 2)
+        self.bos_token_id = getattr(llm_tokenizer, "bos_token_id", 2)
-        self.eos_token_id = getattr(llm_tokenizer, 'eos_token_id', 3)
+        self.eos_token_id = getattr(llm_tokenizer, "eos_token_id", 3)
 
     def __call__(self, text: str, **kwargs):
         """
@@ -49,30 +49,27 @@ class HFTokenizerAdapter:
         Returns:
             dict: Словарь с токенами
         """
-        return_tensors = kwargs.get('return_tensors', None)
+        return_tensors = kwargs.get("return_tensors", None)
-        padding = kwargs.get('padding', False)
+        padding = kwargs.get("padding", False)
-        truncation = kwargs.get('truncation', False)
+        truncation = kwargs.get("truncation", False)
-        max_length = kwargs.get('max_length', None)
+        max_length = kwargs.get("max_length", None)
-        add_special_tokens = kwargs.get('add_special_tokens', True)
+        add_special_tokens = kwargs.get("add_special_tokens", True)
 
         # Кодируем текст
-        #input_ids = self.llm_tokenizer.encode(
+        # input_ids = self.llm_tokenizer.encode(
         #    text,
         #    add_special_tokens=add_special_tokens
-        #)
+        # )
         if isinstance(text, str):
             input_ids = self.llm_tokenizer.encode(
-                text, 
+                text, add_special_tokens=add_special_tokens
-                add_special_tokens=add_special_tokens
             )
             input_ids = [input_ids]  # <-- оборачиваем в batch
         else:
             # Список строк, батч-режим!
             input_ids = [
-                self.llm_tokenizer.encode(
+                self.llm_tokenizer.encode(t, add_special_tokens=add_special_tokens)
-                    t,
+                for t in text
-                    add_special_tokens=add_special_tokens
-                ) for t in text
             ]
 
         # Применяем truncation
@@ -86,6 +83,7 @@ class HFTokenizerAdapter:
         # Конвертируем в тензоры если нужно
         if return_tensors == "pt":
             import torch
+
             input_ids = torch.tensor([input_ids])
 
         return {"input_ids": input_ids}
@@ -99,7 +97,7 @@ class HFTokenizerAdapter:
         truncation: bool = False,
         max_length: Optional[int] = None,
         return_tensors: Optional[str] = None,
-        **kwargs
+        **kwargs,
     ) -> Union[List[int], List[List[int]]]:
         """
         Кодирует текст в последовательность токенов.
@@ -118,16 +116,12 @@ class HFTokenizerAdapter:
         """
         # Кодируем основной текст
         token_ids = self.llm_tokenizer.encode(
-            text, 
+            text, add_special_tokens=add_special_tokens
-            add_special_tokens=add_special_tokens
         )
 
         # Обрабатываем text_pair если есть
         if text_pair is not None:
-            pair_ids = self.llm_tokenizer.encode(
+            pair_ids = self.llm_tokenizer.encode(text_pair, add_special_tokens=False)
-                text_pair,
-                add_special_tokens=False
-            )
             token_ids.extend(pair_ids)
 
         # Применяем truncation
@@ -141,9 +135,11 @@ class HFTokenizerAdapter:
         # Конвертируем в тензоры если нужно
         if return_tensors == "pt":
             import torch
+
             return torch.tensor([token_ids])
         elif return_tensors == "np":
             import numpy as np
+
             return np.array([token_ids])
 
         return token_ids
@@ -152,7 +148,7 @@ class HFTokenizerAdapter:
         self,
         token_ids: Union[int, List[int], List[List[int]]],
         skip_special_tokens: bool = True,
-        **kwargs
+        **kwargs,
     ) -> str:
         """
         Декодирует последовательность токенов в текст.
@@ -167,13 +163,22 @@ class HFTokenizerAdapter:
         # Обрабатываем разные форматы входных данных
         if isinstance(token_ids, int):
             token_ids = [token_ids]
-        elif isinstance(token_ids, list) and len(token_ids) > 0 and isinstance(token_ids[0], list):
+        elif (
+            isinstance(token_ids, list)
+            and len(token_ids) > 0
+            and isinstance(token_ids[0], list)
+        ):
             # Список списков - берем первый элемент
             token_ids = token_ids[0]
 
         # Фильтруем специальные токены если нужно
         if skip_special_tokens:
-            special_ids = {self.pad_token_id, self.unk_token_id, self.bos_token_id, self.eos_token_id}
+            special_ids = {
+                self.pad_token_id,
+                self.unk_token_id,
+                self.bos_token_id,
+                self.eos_token_id,
+            }
             token_ids = [tid for tid in token_ids if tid not in special_ids]
 
         return self.llm_tokenizer.decode(token_ids)
@@ -224,8 +229,12 @@ class HFTokenizerAdapter:
                 # Обрабатываем разные типы данных
                 if isinstance(input_ids, int):
                     seq_len = 1
-                elif hasattr(input_ids, 'shape'):
+                elif hasattr(input_ids, "shape"):
-                    seq_len = input_ids.shape[-1] if len(input_ids.shape) > 1 else len(input_ids)
+                    seq_len = (
+                        input_ids.shape[-1]
+                        if len(input_ids.shape) > 1
+                        else len(input_ids)
+                    )
                 else:
                     seq_len = len(input_ids)
                 max_len = max(max_len, seq_len)
@@ -240,8 +249,12 @@ class HFTokenizerAdapter:
                 # Получаем текущую длину
                 if isinstance(input_ids, int):
                     current_len = 1
-                elif hasattr(input_ids, 'shape'):
+                elif hasattr(input_ids, "shape"):
-                    current_len = input_ids.shape[-1] if len(input_ids.shape) > 1 else len(input_ids)
+                    current_len = (
+                        input_ids.shape[-1]
+                        if len(input_ids.shape) > 1
+                        else len(input_ids)
+                    )
                 else:
                     current_len = len(input_ids)
 
@@ -251,20 +264,27 @@ class HFTokenizerAdapter:
 
                     # Обрабатываем разные типы данных
                     if isinstance(input_ids, int):
-                        item["input_ids"] = [input_ids] + [self.pad_token_id] * padding_length
+                        item["input_ids"] = [input_ids] + [
-                    elif hasattr(input_ids, 'shape'):
+                            self.pad_token_id
+                        ] * padding_length
+                    elif hasattr(input_ids, "shape"):
                         import torch
-                        padding_tensor = torch.full((padding_length,), self.pad_token_id, dtype=input_ids.dtype)
+
+                        padding_tensor = torch.full(
+                            (padding_length,), self.pad_token_id, dtype=input_ids.dtype
+                        )
                         item["input_ids"] = torch.cat([input_ids, padding_tensor])
                     else:
-                        item["input_ids"] = input_ids + [self.pad_token_id] * padding_length
+                        item["input_ids"] = (
+                            input_ids + [self.pad_token_id] * padding_length
+                        )
 
                     # Добавляем attention_mask если требуется
                     if "attention_mask" in item:
                         mask = item["attention_mask"]
                         if isinstance(mask, int):
                             item["attention_mask"] = [mask] + [0] * padding_length
-                        elif hasattr(mask, 'shape'):
+                        elif hasattr(mask, "shape"):
                             padding_mask = torch.zeros(padding_length, dtype=mask.dtype)
                             item["attention_mask"] = torch.cat([mask, padding_mask])
                         else:
@@ -272,16 +292,21 @@ class HFTokenizerAdapter:
                     elif return_attention_mask:
                         if isinstance(input_ids, int):
                             item["attention_mask"] = [1] + [0] * padding_length
-                        elif hasattr(input_ids, 'shape'):
+                        elif hasattr(input_ids, "shape"):
                             attention_mask = torch.ones(current_len, dtype=torch.long)
                             padding_mask = torch.zeros(padding_length, dtype=torch.long)
-                            item["attention_mask"] = torch.cat([attention_mask, padding_mask])
+                            item["attention_mask"] = torch.cat(
+                                [attention_mask, padding_mask]
+                            )
                         else:
-                            item["attention_mask"] = [1] * current_len + [0] * padding_length
+                            item["attention_mask"] = [1] * current_len + [
+                                0
+                            ] * padding_length
 
         # Конвертируем в тензоры если требуется
         if return_tensors == "pt":
             import torch
+
             for key in list(encoded_inputs[0].keys()):
                 if isinstance(encoded_inputs[0][key], list):
                     for i in range(len(encoded_inputs)):
@@ -326,12 +351,12 @@ class HFTokenizerAdapter:
         }
 
         config_path = os.path.join(save_directory, "tokenizer_config.json")
-        with open(config_path, 'w', encoding='utf-8') as f:
+        with open(config_path, "w", encoding="utf-8") as f:
             json.dump(tokenizer_config, f, ensure_ascii=False, indent=2)
 
         # Сохраняем словарь
         vocab_path = os.path.join(save_directory, "vocab.json")
-        with open(vocab_path, 'w', encoding='utf-8') as f:
+        with open(vocab_path, "w", encoding="utf-8") as f:
             json.dump(self._vocab, f, ensure_ascii=False, indent=2)
 
         print(f"✅ Токенизатор сохранен в {save_directory}")
@@ -353,7 +378,9 @@ class HFTokenizerAdapter:
         # Проверяем, является ли путь директорией с файлами токенизатора
         if os.path.isdir(pretrained_model_name_or_path):
             # Загружаем из директории
-            config_path = os.path.join(pretrained_model_name_or_path, "tokenizer_config.json")
+            config_path = os.path.join(
+                pretrained_model_name_or_path, "tokenizer_config.json"
+            )
             vocab_path = os.path.join(pretrained_model_name_or_path, "vocab.json")
 
             if not os.path.exists(config_path) or not os.path.exists(vocab_path):
@@ -362,7 +389,7 @@ class HFTokenizerAdapter:
                 )
 
             # Загружаем конфигурацию
-            with open(config_path, 'r', encoding='utf-8') as f:
+            with open(config_path, "r", encoding="utf-8") as f:
                 config = json.load(f)
 
             # Определяем тип токенизатора llm
@@ -373,7 +400,7 @@ class HFTokenizerAdapter:
                 llm_tokenizer = BPETokenizer()
 
                 # Загружаем словарь
-                with open(vocab_path, 'r', encoding='utf-8') as f:
+                with open(vocab_path, "r", encoding="utf-8") as f:
                     vocab = json.load(f)
 
                 llm_tokenizer.vocab = vocab
@@ -393,7 +420,9 @@ class HFTokenizerAdapter:
 
                 return cls(llm_tokenizer, **kwargs)
             else:
-                raise ValueError(f"Неподдерживаемый тип токенизатора: {llm_tokenizer_type}")
+                raise ValueError(
+                    f"Неподдерживаемый тип токенизатора: {llm_tokenizer_type}"
+                )
 
         else:
             # Пытаемся загрузить как файл llm токенизатора

hf-proxy/src/hf_proxy/hf_utils.py

@@ -31,9 +31,7 @@ class HFUtils:
 
     @staticmethod
     def convert_to_hf_format(
-        llm_model,
+        llm_model, tokenizer=None, model_name: str = "custom-gpt"
-        tokenizer = None,
-        model_name: str = "custom-gpt"
     ) -> tuple:
         """
         Конвертирует llm модель в формат HuggingFace.
@@ -52,13 +50,17 @@ class HFUtils:
         # Если токенизатор не передан, создаем стандартный
         if tokenizer is None:
             from transformers import AutoTokenizer
+
             tokenizer = AutoTokenizer.from_pretrained("gpt2")
             # Устанавливаем специальные токены
             if tokenizer.pad_token is None:
                 tokenizer.pad_token = tokenizer.eos_token
-        elif hasattr(tokenizer, '__class__') and 'BPETokenizer' in str(tokenizer.__class__):
+        elif hasattr(tokenizer, "__class__") and "BPETokenizer" in str(
+            tokenizer.__class__
+        ):
             # Если передан наш кастомный токенизатор, создаем адаптер
             from .hf_tokenizer import create_hf_tokenizer
+
             tokenizer = create_hf_tokenizer(tokenizer)
 
         return hf_model, tokenizer
@@ -70,7 +72,7 @@ class HFUtils:
         repo_name: str,
         organization: Optional[str] = None,
         private: bool = False,
-        **kwargs
+        **kwargs,
     ):
         """
         Загружает модель в HuggingFace Hub.
@@ -116,7 +118,7 @@ class HFUtils:
                 api.upload_folder(
                     folder_path=tmp_dir,
                     repo_id=repo_id,
-                    commit_message="Initial commit with custom GPT model"
+                    commit_message="Initial commit with custom GPT model",
                 )
 
             print(f"✅ Модель успешно загружена в HuggingFace Hub: {repo_id}")
@@ -128,10 +130,7 @@ class HFUtils:
             )
 
     @staticmethod
-    def load_from_hub(
+    def load_from_hub(repo_id: str, **kwargs) -> tuple:
-        repo_id: str,
-        **kwargs
-    ) -> tuple:
         """
         Загружает модель из HuggingFace Hub.
 
@@ -162,17 +161,14 @@ class HFUtils:
 
         # Загружаем модель через адаптер
         model = HFAdapter.from_pretrained(
-            f"{repo_id}/pytorch_model.bin",
+            f"{repo_id}/pytorch_model.bin", HFAdapterConfig.from_llm_config(llm_config)
-            HFAdapterConfig.from_llm_config(llm_config)
         )
 
         return model, tokenizer
 
     @staticmethod
     def compare_with_hf_model(
-        llm_model,
+        llm_model, hf_model_name: str = "gpt2", test_input: str = "Hello world"
-        hf_model_name: str = "gpt2",
-        test_input: str = "Hello world"
     ) -> Dict[str, Any]:
         """
         Сравнивает llm модель с эталонной моделью из HuggingFace.
@@ -197,7 +193,7 @@ class HFUtils:
         # Получаем логиты от обеих моделей
         with torch.no_grad():
             hf_logits = hf_model(**inputs).logits
-            llm_logits = llm_model(inputs['input_ids'])
+            llm_logits = llm_model(inputs["input_ids"])
 
         # Сравниваем результаты
         hf_probs = torch.softmax(hf_logits[0, -1], dim=-1)
@@ -205,15 +201,11 @@ class HFUtils:
 
         # Вычисляем метрики
         kl_divergence = torch.nn.functional.kl_div(
-            torch.log(llm_probs + 1e-8),
+            torch.log(llm_probs + 1e-8), hf_probs, reduction="batchmean"
-            hf_probs,
-            reduction='batchmean'
         )
 
         cosine_similarity = torch.nn.functional.cosine_similarity(
-            hf_logits.flatten(),
+            hf_logits.flatten(), llm_logits.flatten(), dim=0
-            llm_logits.flatten(),
-            dim=0
         )
 
         return {
@@ -244,11 +236,7 @@ class TokenizerWrapper:
             Dict: Токенизированные данные
         """
         return self.tokenizer(
-            texts,
+            texts, padding=True, truncation=True, return_tensors="pt", **kwargs
-            padding=True,
-            truncation=True,
-            return_tensors="pt",
-            **kwargs
         )
 
     def decode_batch(self, token_ids: torch.Tensor, **kwargs) -> List[str]:
@@ -268,9 +256,7 @@ class TokenizerWrapper:
         texts = []
         for i in range(token_ids.size(0)):
             text = self.tokenizer.decode(
-                token_ids[i],
+                token_ids[i], skip_special_tokens=True, **kwargs
-                skip_special_tokens=True,
-                **kwargs
             )
             texts.append(text)
 
@@ -290,12 +276,7 @@ class TokenizerWrapper:
         }
 
 
-def create_hf_pipeline(
+def create_hf_pipeline(llm_model, tokenizer=None, device: str = "auto", **kwargs):
-    llm_model,
-    tokenizer=None,
-    device: str = "auto",
-    **kwargs
-):
     """
     Создает HuggingFace pipeline из llm модели.
 
@@ -315,11 +296,7 @@ def create_hf_pipeline(
 
     # Создаем pipeline
     pipe = pipeline(
-        "text-generation",
+        "text-generation", model=hf_model, tokenizer=tokenizer, device=device, **kwargs
-        model=hf_model,
-        tokenizer=tokenizer,
-        device=device,
-        **kwargs
     )
 
     return pipe

llm/src/llm/core/base_model.py

@@ -19,6 +19,7 @@ from abc import ABC, abstractmethod
 from typing import Optional, Tuple
 import torch
 
+
 class BaseModel(nn.Module, ABC):
     """
     Абстрактный класс — стандарт для всех архитектур LLM.
@@ -32,6 +33,7 @@ class BaseModel(nn.Module, ABC):
     Attributes:
         config (dict): Конфиг модели
     """
+
     def __init__(self, config: dict):
         """
         Инициализация модели.
@@ -43,7 +45,9 @@ class BaseModel(nn.Module, ABC):
         self.config = config
 
     @abstractmethod
-    def forward(self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+    def forward(
+        self, input_ids: torch.Tensor, attention_mask: Optional[torch.Tensor] = None
+    ) -> torch.Tensor:
         """
         Прямой проход — получение логитов для входных токенов.
 

llm/src/llm/core/cached_decoder.py

@@ -6,6 +6,7 @@ from .feed_forward import FeedForward
 from .multi_head_attention import MultiHeadAttention
 from .rope import RoPE
 
+
 class CachedDecoder(nn.Module):
     """
     Универсальный декодерный блок для современных LLM (GPT, LLaMA, др.), поддерживает кэширование key-value для эффективной генерации.
@@ -36,6 +37,7 @@ class CachedDecoder(nn.Module):
         ...   num_heads=4, emb_size=256, head_size=64, max_seq_len=128)
         >>> out, cache = decoder(x, use_cache=True)
     """
+
     def __init__(
         self,
         feed_forward_layer: nn.Module,

llm/src/llm/core/decoder.py

@@ -3,6 +3,7 @@ import torch
 from .feed_forward import FeedForward
 from .multi_head_attention import MultiHeadAttention
 
+
 class Decoder(nn.Module):
     """
     Базовый автогерессивный блок-декодер трансформера (без кэша KV).
@@ -24,12 +25,14 @@ class Decoder(nn.Module):
         >>> out = decoder(x)
         >>> print(out.shape)  # torch.Size([1, 10, 512])
     """
-    def __init__(self, 
+
+    def __init__(
+        self,
         num_heads: int,
         emb_size: int,
         head_size: int,
         max_seq_len: int,
-        dropout: float = 0.1
+        dropout: float = 0.1,
     ):
         """
         Инициализация декодера.
@@ -47,7 +50,7 @@ class Decoder(nn.Module):
             emb_size=emb_size,
             head_size=head_size,
             max_seq_len=max_seq_len,
-            dropout=dropout
+            dropout=dropout,
         )
         self._ff = FeedForward(emb_size=emb_size, dropout=dropout)
         self._norm1 = nn.LayerNorm(emb_size)

llm/src/llm/core/feed_forward.py

@@ -45,6 +45,7 @@ class FeedForward(nn.Module):
         >>> output = ff(x)
         >>> print(output.shape)  # torch.Size([32, 10, 512])
     """
+
     def __init__(self, emb_size: int, dropout: float = 0.1, activation: str = "relu"):
         """
         Инициализация слоя Feed Forward Network.

llm/src/llm/core/gelu.py

@@ -1,6 +1,7 @@
 import torch
 from torch import nn
 
+
 class GELU(nn.Module):
     """
     Гауссовская Эрф-активация (GELU, Gaussian Error Linear Unit).
@@ -17,11 +18,14 @@ class GELU(nn.Module):
         >>> y = gelu(torch.tensor([-1.0, 0.0, 1.0]))
         >>> print(y)
     """
+
     def __init__(self):
         super().__init__()
         self.sqrt_2_over_pi = torch.sqrt(torch.tensor(2.0) / math.pi)
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
-        return 0.5 * x * (1 + torch.tanh(
+        return (
-            self.sqrt_2_over_pi * (x + 0.044715 * torch.pow(x, 3))
+            0.5
-        ))
+            * x
+            * (1 + torch.tanh(self.sqrt_2_over_pi * (x + 0.044715 * torch.pow(x, 3))))
+        )

llm/src/llm/core/head_attention.py

@@ -4,6 +4,7 @@ import torch.nn.functional as F
 from math import sqrt
 from .rope import RoPE
 
+
 class HeadAttention(nn.Module):
     """
     Одноголовый механизм внимания (scaled dot-product attention) — фундаментальный строительный блок всех современных Transformer.
@@ -35,7 +36,10 @@ class HeadAttention(nn.Module):
         >>> output, _ = attention(x)
         >>> print(output.shape)  # torch.Size([1, 10, 32])
     """
-    def __init__(self, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None):
+
+    def __init__(
+        self, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None
+    ):
         super().__init__()
         self._emb_size = emb_size
         self._head_size = head_size
@@ -49,9 +53,13 @@ class HeadAttention(nn.Module):
 
         # Создание causal маски
         mask = torch.tril(torch.ones(max_seq_len, max_seq_len))
-        self.register_buffer('_tril_mask', mask.bool() if hasattr(torch, 'bool') else mask.byte())
+        self.register_buffer(
+            "_tril_mask", mask.bool() if hasattr(torch, "bool") else mask.byte()
+        )
 
-    def forward(self, x: torch.Tensor, use_cache: bool = True, cache: tuple = None) -> tuple:
+    def forward(
+        self, x: torch.Tensor, use_cache: bool = True, cache: tuple = None
+    ) -> tuple:
         """
         Прямой проход через слой внимания.
 
@@ -73,7 +81,9 @@ class HeadAttention(nn.Module):
         """
         seq_len = x.shape[1]
         if seq_len > self._max_seq_len:
-            raise ValueError(f"Длина последовательности {seq_len} превышает максимум {self._max_seq_len}")
+            raise ValueError(
+                f"Длина последовательности {seq_len} превышает максимум {self._max_seq_len}"
+            )
 
         k = self._k(x)  # [B, T, hs]
         q = self._q(x)  # [B, T, hs]
@@ -92,7 +102,9 @@ class HeadAttention(nn.Module):
         scores = q @ k.transpose(-2, -1) / sqrt(self._head_size)
 
         if cache is None:
-            scores = scores.masked_fill(~self._tril_mask[:seq_len, :seq_len], float('-inf'))
+            scores = scores.masked_fill(
+                ~self._tril_mask[:seq_len, :seq_len], float("-inf")
+            )
 
         weights = F.softmax(scores, dim=-1)
         x_out = weights @ v  # [B, T, hs]

llm/src/llm/core/multi_head_attention.py

@@ -3,6 +3,7 @@ import torch
 from .head_attention import HeadAttention
 from .rope import RoPE
 
+
 class MultiHeadAttention(nn.Module):
     """
     Мультиголовый (многоголовый) механизм внимания — ключевой компонент любого Transformer.
@@ -32,7 +33,16 @@ class MultiHeadAttention(nn.Module):
         >>> out, cache = mha(x)
         >>> print(out.shape)
     """
-    def __init__(self, num_heads: int, emb_size: int, head_size: int, max_seq_len: int, rope: RoPE = None, dropout: float = 0.1):
+
+    def __init__(
+        self,
+        num_heads: int,
+        emb_size: int,
+        head_size: int,
+        max_seq_len: int,
+        rope: RoPE = None,
+        dropout: float = 0.1,
+    ):
         """
         Инициализация многоголового внимания.
 
@@ -49,18 +59,27 @@ class MultiHeadAttention(nn.Module):
             - max_seq_len зависит от задачи (512 для BERT, 2048 для GPT-3)
         """
         super().__init__()
-        self._heads = nn.ModuleList([
+        self._heads = nn.ModuleList(
-            HeadAttention(
+            [
-                emb_size=emb_size, 
+                HeadAttention(
-                head_size=head_size, 
+                    emb_size=emb_size,
-                max_seq_len=max_seq_len,
+                    head_size=head_size,
-                rope=rope,
+                    max_seq_len=max_seq_len,
-            ) for _ in range(num_heads)
+                    rope=rope,
-        ])
+                )
+                for _ in range(num_heads)
+            ]
+        )
         self._layer = nn.Linear(head_size * num_heads, emb_size)
         self._dropout = nn.Dropout(dropout)
 
-    def forward(self, x: torch.Tensor, mask: torch.Tensor = None, use_cache: bool = True, cache: list = None):
+    def forward(
+        self,
+        x: torch.Tensor,
+        mask: torch.Tensor = None,
+        use_cache: bool = True,
+        cache: list = None,
+    ):
         """
         Прямой проход (forward):
         Для каждого токена оценивает "важность" остальных токенов сразу через несколько attention-блоков.

llm/src/llm/core/positional_embeddings.py

@@ -1,6 +1,7 @@
 import torch
 from torch import nn, Tensor
 
+
 class PositionalEmbeddings(nn.Module):
     """
     Обучаемые позиционные эмбеддинги (learnable positional embeddings).
@@ -36,8 +37,7 @@ class PositionalEmbeddings(nn.Module):
         self.max_seq_len = max_seq_len
         self.emb_size = emb_size
         self.embedding = nn.Embedding(
-            num_embeddings=max_seq_len,
+            num_embeddings=max_seq_len, embedding_dim=emb_size
-            embedding_dim=emb_size
         )
 
     def forward(self, seq_len: int, start_pos: int = 0) -> Tensor:
@@ -62,5 +62,9 @@ class PositionalEmbeddings(nn.Module):
         if start_pos == 0:
             positions = torch.arange(seq_len, device=self.embedding.weight.device)
         else:
-            positions = torch.arange(start=start_pos, end=start_pos + seq_len, device=self.embedding.weight.device)
+            positions = torch.arange(
+                start=start_pos,
+                end=start_pos + seq_len,
+                device=self.embedding.weight.device,
+            )
         return self.embedding(positions)

llm/src/llm/core/rms_norm.py

@@ -80,4 +80,4 @@ class RMSNorm(nn.Module):
 
     def extra_repr(self) -> str:
         """Строковое представление для отладки."""
-        return f'dim={self._w.shape[0]}, eps={self._eps}'
+        return f"dim={self._w.shape[0]}, eps={self._eps}"

llm/src/llm/core/rope.py

@@ -65,8 +65,8 @@ class RoPE(nn.Module):
         freq_matrix = positions.unsqueeze(1) * freqs.unsqueeze(0)
 
         # Предвычисление матриц косинусов и синусов
-        self.register_buffer('cos_matrix', torch.cos(freq_matrix))
+        self.register_buffer("cos_matrix", torch.cos(freq_matrix))
-        self.register_buffer('sin_matrix', torch.sin(freq_matrix))
+        self.register_buffer("sin_matrix", torch.sin(freq_matrix))
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         """
@@ -91,7 +91,7 @@ class RoPE(nn.Module):
 
         # Разделяем на четные и нечетные компоненты
         x_even = x[:, :, 0::2]  # [batch_size, seq_len, head_size//2]
-        x_odd = x[:, :, 1::2]   # [batch_size, seq_len, head_size//2]
+        x_odd = x[:, :, 1::2]  # [batch_size, seq_len, head_size//2]
 
         # Применяем поворот: q' = q * cos(mθ) + rotate(q) * sin(mθ)
         x_rotated_even = x_even * cos - x_odd * sin

llm/src/llm/core/silu.py

@@ -1,6 +1,7 @@
 import torch
 from torch import nn
 
+
 class SiLU(nn.Module):
     """
     SiLU (Swish) — современная активационная функция для нейросетей.
@@ -15,5 +16,6 @@ class SiLU(nn.Module):
         >>> x = torch.tensor([-1.0, 0.0, 1.0])
         >>> print(act(x))
     """
+
     def forward(self, x: torch.Tensor):
         return torch.sigmoid(x) * x

llm/src/llm/core/swi_glu.py

@@ -83,19 +83,19 @@ class SwiGLU(nn.Module):
         4. apply dropout
         """
         # Gate ветвь: линейное преобразование + активация
-        gate_out = self._gate(x)                          # [batch, seq, 4*emb]
+        gate_out = self._gate(x)  # [batch, seq, 4*emb]
-        activation_out = self._activation(gate_out)       # [batch, seq, 4*emb]
+        activation_out = self._activation(gate_out)  # [batch, seq, 4*emb]
 
         # Up ветвь: линейное преобразование
-        up_out = self._up(x)                              # [batch, seq, 4*emb]
+        up_out = self._up(x)  # [batch, seq, 4*emb]
 
         # Element-wise multiplication (gating mechanism)
-        out = up_out * activation_out                     # поэлементное умножение!
+        out = up_out * activation_out  # поэлементное умножение!
 
         # Final projection and dropout
-        out = self._down(out)                             # [batch, seq, emb]
+        out = self._down(out)  # [batch, seq, emb]
         return self._dropout(out)
 
     def extra_repr(self) -> str:
         """Строковое представление для отладки."""
-        return f'emb_size={self._gate.in_features}, dropout={self._dropout.p}'
+        return f"emb_size={self._gate.in_features}, dropout={self._dropout.p}"

llm/src/llm/core/token_embeddings.py

@@ -2,6 +2,7 @@ import torch
 from torch import nn
 from torch import Tensor
 
+
 class TokenEmbeddings(nn.Module):
     """
     Токеновые эмбеддинги — обучаемые векторные представления для каждого токена словаря.
@@ -29,11 +30,11 @@ class TokenEmbeddings(nn.Module):
         >>> vecs = emb(tokens)
         >>> vecs.shape  # torch.Size([1, 3, 256])
     """
+
     def __init__(self, vocab_size: int, emb_size: int):
         super().__init__()
         self._embedding = nn.Embedding(
-            num_embeddings=vocab_size,
+            num_embeddings=vocab_size, embedding_dim=emb_size
-            embedding_dim=emb_size
         )
 
     def forward(self, x: Tensor) -> Tensor:
@@ -55,10 +56,7 @@ if __name__ == "__main__":
     embedding = TokenEmbeddings(vocab_size=100, emb_size=128)
 
     # Создаем тензор с индексами в пределах vocab_size (0-99)
-    tensor = torch.tensor([
+    tensor = torch.tensor([[11, 45, 76, 34], [34, 67, 45, 54]])
-        [11, 45, 76, 34],
-        [34, 67, 45, 54]
-    ])
 
     # Проверяем индексы
     if (tensor >= 100).any():

llm/src/llm/models/gpt/gpt.py

@@ -54,28 +54,32 @@ class GPT(BaseModel):
         _norm: Финальный слой нормализации
         _linear: Выходной линейный слой
     """
+
     def __init__(self, config):
         super().__init__(config)
 
         # Инициализация слоев
         self._max_seq_len = config["max_position_embeddings"]
         self._token_embeddings = TokenEmbeddings(
-            vocab_size=config["vocab_size"], 
+            vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
-            emb_size=config["embed_dim"]
         )
         self._position_embeddings = PositionalEmbeddings(
-            max_seq_len=config["max_position_embeddings"], 
+            max_seq_len=config["max_position_embeddings"], emb_size=config["embed_dim"]
-            emb_size=config["embed_dim"]
         )
         self._dropout = nn.Dropout(config["dropout"])
         # head_size = emb_size // num_heads
-        self._decoders = nn.ModuleList([Decoder(
+        self._decoders = nn.ModuleList(
-            num_heads=config["num_heads"],
+            [
-            emb_size=config["embed_dim"],
+                Decoder(
-            head_size=config["embed_dim"] // config["num_heads"],
+                    num_heads=config["num_heads"],
-            max_seq_len=config["max_position_embeddings"],
+                    emb_size=config["embed_dim"],
-            dropout=config["dropout"] 
+                    head_size=config["embed_dim"] // config["num_heads"],
-        ) for _ in range(config["num_layers"])])
+                    max_seq_len=config["max_position_embeddings"],
+                    dropout=config["dropout"],
+                )
+                for _ in range(config["num_layers"])
+            ]
+        )
         self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
 
     @property
@@ -94,14 +98,18 @@ class GPT(BaseModel):
         """
         # Проверка длины последовательности
         if x.size(1) > self._max_seq_len:
-            raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}")
+            raise ValueError(
+                f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}"
+            )
 
         # Эмбеддинги токенов и позиций
         tok_out = self._token_embeddings(x)  # [batch, seq_len, emb_size]
         pos_out = self._position_embeddings(x.size(1))  # [seq_len, emb_size]
 
         # Комбинирование
-        out = self._dropout(tok_out + pos_out.unsqueeze(0))  # [batch, seq_len, emb_size]
+        out = self._dropout(
+            tok_out + pos_out.unsqueeze(0)
+        )  # [batch, seq_len, emb_size]
 
         # Стек декодеров
         for decoder in self._decoders:
@@ -109,22 +117,21 @@ class GPT(BaseModel):
 
         return self._linear(out)  # [batch, seq_len, vocab_size]
 
+    #    def forward(self, input_ids, attention_mask=None):
+    #        B, T = input_ids.size()
+    #        pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
+    #
+    #        x = self.token_emb(input_ids) + self.pos_emb(pos)
+    #
+    #        for block in self.blocks:
+    #            x = block(x, attention_mask)
+    #
+    #        x = self.ln_f(x)
+    #        logits = self.head(x)
+    #        return logits
 
-#    def forward(self, input_ids, attention_mask=None):
+    def generate(
-#        B, T = input_ids.size()
+        self,
-#        pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
-#
-#        x = self.token_emb(input_ids) + self.pos_emb(pos)
-#
-#        for block in self.blocks:
-#            x = block(x, attention_mask)
-#
-#        x = self.ln_f(x)
-#        logits = self.head(x)
-#        return logits
-
-
-    def generate(self,
         x: torch.Tensor,
         max_new_tokens: int,
         do_sample: bool,
@@ -132,7 +139,7 @@ class GPT(BaseModel):
         top_k: int = None,
         top_p: float = None,
         attention_mask: torch.Tensor = None,  # Добавляем для совместимости с HF
-        **kwargs  # Игнорируем остальные параметры
+        **kwargs,  # Игнорируем остальные параметры
     ) -> torch.Tensor:
         """Авторегрессивная генерация текста.
 
@@ -228,7 +235,7 @@ class GPT(BaseModel):
         """
         for _ in range(max_new_tokens):
             # 1. Обрезаем вход, если последовательность слишком длинная
-            x_cond = x[:, -self._max_seq_len:]
+            x_cond = x[:, -self._max_seq_len :]
 
             # 2. Передаем последовательность в метод forward класса GPT и полуаем логиты.
             logits = self.forward(x_cond)
@@ -250,9 +257,14 @@ class GPT(BaseModel):
                 vocab_size = logits_scaled.size(-1)
 
                 # создаём маску: True, если токен НЕ в topk_indices
-                mask = torch.ones_like(logits_scaled, dtype=torch.bool if hasattr(torch, 'bool') else torch.uint8)
+                mask = torch.ones_like(
-                mask.scatter_(1, topk_indices, False if hasattr(torch, 'bool') else 0)  # False там, где top-k индексы
+                    logits_scaled,
-                masked_logits[mask] = float('-inf')
+                    dtype=torch.bool if hasattr(torch, "bool") else torch.uint8,
+                )
+                mask.scatter_(
+                    1, topk_indices, False if hasattr(torch, "bool") else 0
+                )  # False там, где top-k индексы
+                masked_logits[mask] = float("-inf")
 
                 logits_scaled = masked_logits
 
@@ -260,36 +272,42 @@ class GPT(BaseModel):
                 # 1. Применим softmax, чтобы получить вероятности:
                 probs = F.softmax(logits_scaled, dim=-1)  # [B, vocab_size]
                 # 2. Отсортируем токены по убыванию вероятностей:
-                sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
+                sorted_probs, sorted_indices = torch.sort(
+                    probs, descending=True, dim=-1
+                )
                 # 3. Посчитаем кумулятивную сумму вероятностей:
                 cum_probs = torch.cumsum(sorted_probs, dim=-1)  # [B, vocab_size]
                 # 4. Определим маску: оставить токены, пока сумма < top_p
-                sorted_mask = (cum_probs <= top_p)  # [B, vocab_size]
+                sorted_mask = cum_probs <= top_p  # [B, vocab_size]
                 # Гарантируем, что хотя бы первый токен останется
                 sorted_mask[:, 0] = True
                 # 5. Преобразуем маску обратно в оригинальный порядок:
                 # Создаём полную маску из False
-                mask = torch.zeros_like(probs, dtype=torch.bool if hasattr(torch, 'bool') else torch.uint8)
+                mask = torch.zeros_like(
+                    probs, dtype=torch.bool if hasattr(torch, "bool") else torch.uint8
+                )
                 # Устанавливаем True в местах нужных токенов
                 mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
                 # 6. Зануляем логиты токенов вне топ-p:
-                logits_scaled[~mask] = float('-inf')
+                logits_scaled[~mask] = float("-inf")
 
             # 4. Применяем Softmax
             probs = F.softmax(logits_scaled, dim=-1)  # [batch_size, vocab_size]
 
-
             if do_sample == True:
                 # 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
                 next_token = torch.multinomial(probs, num_samples=1)  # [batch_size, 1]
             else:
                 # 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
-                next_token = torch.argmax(probs, dim=-1, keepdim=True)  # [batch_size, 1]
+                next_token = torch.argmax(
+                    probs, dim=-1, keepdim=True
+                )  # [batch_size, 1]
 
             # 6. Добавляем его к последовательности
             x = torch.cat([x, next_token], dim=1)  # [batch_size, seq_len+1]
         return x
 
+
 #    def generate(self, input_ids, max_length=50):
 #        for _ in range(max_length):
 #            logits = self.forward(input_ids)

llm/src/llm/models/gpt/gpt2.py

@@ -27,6 +27,7 @@ from llm.core.positional_embeddings import PositionalEmbeddings
 from llm.core.cached_decoder import CachedDecoder
 from llm.core.feed_forward import FeedForward
 
+
 class GPT2(BaseModel):
     """
     GPT2 — автогерессивная языковая модель, архитектура Transformer, предложенная OpenAI.
@@ -44,37 +45,43 @@ class GPT2(BaseModel):
         >>> logits = model(input_ids)
         >>> out = model.generate(input_ids, max_length=20)
     """
+
     def __init__(self, config):
         super().__init__(config)
 
         # Инициализация слоев
         self._max_seq_len = config["max_position_embeddings"]
         self._token_embeddings = TokenEmbeddings(
-            vocab_size=config["vocab_size"], 
+            vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
-            emb_size=config["embed_dim"]
         )
         self._position_embeddings = PositionalEmbeddings(
-            max_seq_len=config["max_position_embeddings"], 
+            max_seq_len=config["max_position_embeddings"], emb_size=config["embed_dim"]
-            emb_size=config["embed_dim"]
         )
         self._dropout = nn.Dropout(config["dropout"])
         # head_size = emb_size // num_heads
-        self._decoders = nn.ModuleList([CachedDecoder(
+        self._decoders = nn.ModuleList(
-            num_heads=config["num_heads"],
+            [
-            emb_size=config["embed_dim"],
+                CachedDecoder(
-            head_size=config["embed_dim"] // config["num_heads"],
+                    num_heads=config["num_heads"],
-            feed_forward_layer=FeedForward(
+                    emb_size=config["embed_dim"],
-                emb_size=config["embed_dim"], 
+                    head_size=config["embed_dim"] // config["num_heads"],
-                dropout=config["dropout"], 
+                    feed_forward_layer=FeedForward(
-                activation="gelu"
+                        emb_size=config["embed_dim"],
-            ),
+                        dropout=config["dropout"],
-            max_seq_len=config["max_position_embeddings"],
+                        activation="gelu",
-            dropout=config["dropout"] 
+                    ),
-        ) for _ in range(config["num_layers"])])
+                    max_seq_len=config["max_position_embeddings"],
+                    dropout=config["dropout"],
+                )
+                for _ in range(config["num_layers"])
+            ]
+        )
         self._norm = nn.LayerNorm(config["embed_dim"])
         self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
 
-    def forward(self, x: torch.Tensor, use_cache: bool = True, cache: list = None) -> tuple:
+    def forward(
+        self, x: torch.Tensor, use_cache: bool = True, cache: list = None
+    ) -> tuple:
         """
         Прямой проход GPT2:
         - Все слои работают как autoregressive transformer (masked self-attention).
@@ -91,8 +98,9 @@ class GPT2(BaseModel):
         """
         # Проверка длины последовательности (только при отсутствии кэша)
         if cache is None and x.size(1) > self._max_seq_len:
-            raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}")
+            raise ValueError(
-        
+                f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}"
+            )
 
         # Вычисление start_pos из кэша (если кэш передан)
         if cache is not None:
@@ -111,10 +119,14 @@ class GPT2(BaseModel):
 
         # Эмбеддинги токенов и позиций
         tok_out = self._token_embeddings(x)  # [batch, seq_len, emb_size]
-        pos_out = self._position_embeddings(seq_len, start_pos=start_pos)  # [seq_len, emb_size]
+        pos_out = self._position_embeddings(
+            seq_len, start_pos=start_pos
+        )  # [seq_len, emb_size]
 
         # Комбинирование
-        out = self._dropout(tok_out + pos_out.unsqueeze(0))  # [batch, seq_len, emb_size]
+        out = self._dropout(
+            tok_out + pos_out.unsqueeze(0)
+        )  # [batch, seq_len, emb_size]
 
         # Стек декодеров с передачей кэша
         new_cache = []
@@ -138,14 +150,15 @@ class GPT2(BaseModel):
         else:
             return (logits, None)
 
-    def generate(self,
+    def generate(
+        self,
         x: torch.Tensor,
         max_new_tokens: int,
         do_sample: bool,
         temperature: float = 1.0,
         top_k: int = None,
         top_p: float = None,
-        use_cache: bool = True
+        use_cache: bool = True,
     ) -> torch.Tensor:
         """
         Генерация текста с использованием autoregressive трансформера (GPT2).
@@ -200,7 +213,7 @@ class GPT2(BaseModel):
                 # создаём маску: 1, если токен НЕ в topk_indices
                 mask = torch.ones_like(logits_scaled, dtype=torch.uint8)
                 mask.scatter_(1, topk_indices, 0)  # 0 там, где top-k индексы
-                masked_logits[mask.byte()] = float('-inf')
+                masked_logits[mask.byte()] = float("-inf")
 
                 logits_scaled = masked_logits
 
@@ -208,7 +221,9 @@ class GPT2(BaseModel):
                 # 1. Применим softmax, чтобы получить вероятности:
                 probs = F.softmax(logits_scaled, dim=-1)  # [B, vocab_size]
                 # 2. Отсортируем токены по убыванию вероятностей:
-                sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
+                sorted_probs, sorted_indices = torch.sort(
+                    probs, descending=True, dim=-1
+                )
                 # 3. Посчитаем кумулятивную сумму вероятностей:
                 cum_probs = torch.cumsum(sorted_probs, dim=-1)  # [B, vocab_size]
                 # 4. Определим маску: оставить токены, пока сумма < top_p
@@ -221,18 +236,19 @@ class GPT2(BaseModel):
                 # Устанавливаем 1 в местах нужных токенов
                 mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
                 # 6. Зануляем логиты токенов вне топ-p:
-                logits_scaled[~mask] = float('-inf')
+                logits_scaled[~mask] = float("-inf")
 
             # 4. Применяем Softmax
             probs = F.softmax(logits_scaled, dim=-1)  # [batch_size, vocab_size]
 
-
             if do_sample == True:
                 # 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
                 next_token = torch.multinomial(probs, num_samples=1)  # [batch_size, 1]
             else:
                 # 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
-                next_token = torch.argmax(probs, dim=-1, keepdim=True)  # [batch_size, 1]
+                next_token = torch.argmax(
+                    probs, dim=-1, keepdim=True
+                )  # [batch_size, 1]
 
             # 6. Добавляем его к последовательности
             x = torch.cat([x, next_token], dim=1)  # [batch_size, seq_len+1]

llm/src/llm/models/llama/llama.py

@@ -10,7 +10,6 @@ from llm.core.rope import RoPE
 from llm.core.cached_decoder import CachedDecoder
 
 
-
 class Llama(BaseModel):
     """
     LLaMA (Large Language Model Meta AI) — высокоэффективная масштабируемая языковая модель, разработанная Meta AI Research.
@@ -29,38 +28,45 @@ class Llama(BaseModel):
         >>> logits, cache = model(input_ids, use_cache=True)
         >>> out = model.generate(input_ids, max_new_tokens=20)
     """
-    def __init__(self,config):
+
+    def __init__(self, config):
         super().__init__(config)
 
         # Инициализация слоев
         self._max_seq_len = config["max_position_embeddings"]
         self._token_embeddings = TokenEmbeddings(
-         vocab_size=config["vocab_size"], 
+            vocab_size=config["vocab_size"], emb_size=config["embed_dim"]
-            emb_size=config["embed_dim"]
         )
         self._position_embeddings = RoPE(
             head_size=config["embed_dim"] // config["num_heads"],
-            max_seq_len=config["max_position_embeddings"]
+            max_seq_len=config["max_position_embeddings"],
         )
 
         self._dropout = nn.Dropout(config["dropout"])
-        self._decoders = nn.ModuleList([CachedDecoder(
+        self._decoders = nn.ModuleList(
-            norm_layer=RMSNorm,
+            [
-            num_heads=config["num_heads"],
+                CachedDecoder(
-            emb_size=config["embed_dim"],
+                    norm_layer=RMSNorm,
-            head_size=config["embed_dim"] // config["num_heads"],
+                    num_heads=config["num_heads"],
-            feed_forward_layer=SwiGLU(
+                    emb_size=config["embed_dim"],
-                emb_size=config["embed_dim"], 
+                    head_size=config["embed_dim"] // config["num_heads"],
-                dropout=config["dropout"], 
+                    feed_forward_layer=SwiGLU(
-            ),
+                        emb_size=config["embed_dim"],
-            max_seq_len=config["max_position_embeddings"],
+                        dropout=config["dropout"],
-            rope=self._position_embeddings,
+                    ),
-            dropout=config["dropout"], 
+                    max_seq_len=config["max_position_embeddings"],
-        ) for _ in range(config["num_layers"])])
+                    rope=self._position_embeddings,
+                    dropout=config["dropout"],
+                )
+                for _ in range(config["num_layers"])
+            ]
+        )
         self._norm = RMSNorm(config["embed_dim"])
         self._linear = nn.Linear(config["embed_dim"], config["vocab_size"])
 
-    def forward(self, x: torch.Tensor, use_cache: bool = True, cache: list = None) -> tuple:
+    def forward(
+        self, x: torch.Tensor, use_cache: bool = True, cache: list = None
+    ) -> tuple:
         """
         Прямой проход через LLaMA (inference/train): авторегрессионное предсказание токенов.
 
@@ -76,11 +82,12 @@ class Llama(BaseModel):
         """
         # Проверка длины последовательности (только при отсутствии кэша)
         if cache is None and x.size(1) > self._max_seq_len:
-            raise ValueError(f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}")
+            raise ValueError(
-        
+                f"Длина последовательности {x.size(1)} превышает максимальную {self.max_seq_len}"
+            )
 
         # Вычисление start_pos из кэша (если кэш передан)
-        #if cache is not None:
+        # if cache is not None:
         #    # При кэше обрабатываем только один токен (последний)
         #    seq_len = 1
         #    # Вычисляем start_pos из самого нижнего уровня кэша
@@ -89,14 +96,14 @@ class Llama(BaseModel):
         #        start_pos = key_cache.size(1)  # cache_len
         #    else:
         #        start_pos = 0
-        #else:
+        # else:
         #    # Без кэша работаем как раньше
         #    start_pos = 0
         #    seq_len = x.size(1)
 
         # Эмбеддинги токенов и позиций
         tok_out = self._token_embeddings(x)  # [batch, seq_len, emb_size]
-       #pos_out = self._position_embeddings(x)  # [batch, seq_len, emb_size]
+        # pos_out = self._position_embeddings(x)  # [batch, seq_len, emb_size]
 
         # Комбинирование
         out = self._dropout(tok_out)  # [batch, seq_len, emb_size]
@@ -123,35 +130,36 @@ class Llama(BaseModel):
         else:
             return (logits, None)
 
-    def generate(self,
+    def generate(
+        self,
         x: torch.Tensor,
         max_new_tokens: int,
         do_sample: bool,
         temperature: float = 1.0,
         top_k: int = None,
         top_p: float = None,
-        use_cache: bool = True
+        use_cache: bool = True,
     ) -> torch.Tensor:
         """
-        Генерация текста c помощью LLaMA (autoregressive Transformer).
+         Генерация текста c помощью LLaMA (autoregressive Transformer).
-       Поддерживается:
+        Поддерживается:
-        - greedy и вероятностное сэмплирование (top-k, top-p, temperature)
+         - greedy и вероятностное сэмплирование (top-k, top-p, temperature)
-        - кэш attention для ускорения генерации длинных последовательностей
+         - кэш attention для ускорения генерации длинных последовательностей
 
-        Args:
+         Args:
-            x (Tensor[int]): начальная последовательность [batch, seq_len]
+             x (Tensor[int]): начальная последовательность [batch, seq_len]
-            max_new_tokens (int): сколько новых токенов сгенерировать
+             max_new_tokens (int): сколько новых токенов сгенерировать
-            do_sample (bool): использовать стохастику (True) или жадный выбор (False)
+             do_sample (bool): использовать стохастику (True) или жадный выбор (False)
-            temperature (float): масштаб для softmax (важно для sampling)
+             temperature (float): масштаб для softmax (важно для sampling)
-            top_k (int|None): ограничение на количество кандидатов (top-k sampling)
+             top_k (int|None): ограничение на количество кандидатов (top-k sampling)
-            top_p (float|None): nucleus sampling
+             top_p (float|None): nucleus sampling
-            use_cache (bool): ускоряет autoregressive при длинной генерации
+             use_cache (bool): ускоряет autoregressive при длинной генерации
-        Returns:
+         Returns:
-            output (Tensor[int]): [batch, seq_len + max_new_tokens]
+             output (Tensor[int]): [batch, seq_len + max_new_tokens]
-        Пример:
+         Пример:
-            >>> prompt = tokenizer.encode('Meta AI', return_tensors="pt")
+             >>> prompt = tokenizer.encode('Meta AI', return_tensors="pt")
-            >>> generated = model.generate(prompt, max_new_tokens=30, do_sample=True)
+             >>> generated = model.generate(prompt, max_new_tokens=30, do_sample=True)
-            >>> print(tokenizer.decode(generated[0]))
+             >>> print(tokenizer.decode(generated[0]))
         """
         cache = None
 
@@ -188,7 +196,7 @@ class Llama(BaseModel):
                 # создаём маску: 1, если токен НЕ в topk_indices
                 mask = torch.ones_like(logits_scaled, dtype=torch.uint8)
                 mask.scatter_(1, topk_indices, 0)  # 0 там, где top-k индексы
-                masked_logits[mask.byte()] = float('-inf')
+                masked_logits[mask.byte()] = float("-inf")
 
                 logits_scaled = masked_logits
 
@@ -196,7 +204,9 @@ class Llama(BaseModel):
                 # 1. Применим softmax, чтобы получить вероятности:
                 probs = F.softmax(logits_scaled, dim=-1)  # [B, vocab_size]
                 # 2. Отсортируем токены по убыванию вероятностей:
-                sorted_probs, sorted_indices = torch.sort(probs, descending=True, dim=-1)
+                sorted_probs, sorted_indices = torch.sort(
+                    probs, descending=True, dim=-1
+                )
                 # 3. Посчитаем кумулятивную сумму вероятностей:
                 cum_probs = torch.cumsum(sorted_probs, dim=-1)  # [B, vocab_size]
                 # 4. Определим маску: оставить токены, пока сумма < top_p
@@ -209,25 +219,24 @@ class Llama(BaseModel):
                 # Устанавливаем 1 в местах нужных токенов
                 mask.scatter_(dim=1, index=sorted_indices, src=sorted_mask)
                 # 6. Зануляем логиты токенов вне топ-p:
-                logits_scaled[~mask] = float('-inf')
+                logits_scaled[~mask] = float("-inf")
 
             # 4. Применяем Softmax
             probs = F.softmax(logits_scaled, dim=-1)  # [batch_size, vocab_size]
 
-
             if do_sample == True:
                 # 5. Если do_sample равен True, то отбираем токен случайно с помощью torch.multinomial
                 next_token = torch.multinomial(probs, num_samples=1)  # [batch_size, 1]
             else:
                 # 5. Если do_sample равен False, то выбираем токен с максимальной вероятностью
-                next_token = torch.argmax(probs, dim=-1, keepdim=True)  # [batch_size, 1]
+                next_token = torch.argmax(
+                    probs, dim=-1, keepdim=True
+                )  # [batch_size, 1]
 
             # 6. Добавляем его к последовательности
             x = torch.cat([x, next_token], dim=1)  # [batch_size, seq_len+1]
         return x
 
-
-
     @property
     def max_seq_len(self) -> int:
         return self._max_seq_len

llm/src/llm/tokenizers/base_tokenizer.py

@@ -82,7 +82,9 @@ class BaseTokenizer(ABC):
             List[str]: Список токенов
         """
         token_ids = self.encode(text, **kwargs)
-        return [self.inverse_vocab.get(token_id, self.unk_token) for token_id in token_ids]
+        return [
+            self.inverse_vocab.get(token_id, self.unk_token) for token_id in token_ids
+        ]
 
     def get_vocab(self) -> Dict[str, int]:
         """Возвращает словарь токенизатора."""
@@ -120,16 +122,16 @@ class BaseTokenizer(ABC):
             filepath: Путь для сохранения
         """
         config = {
-            'vocab': self.vocab,
+            "vocab": self.vocab,
-            'vocab_size': self.vocab_size,
+            "vocab_size": self.vocab_size,
-            'pad_token': self.pad_token,
+            "pad_token": self.pad_token,
-            'unk_token': self.unk_token,
+            "unk_token": self.unk_token,
-            'bos_token': self.bos_token,
+            "bos_token": self.bos_token,
-            'eos_token': self.eos_token,
+            "eos_token": self.eos_token,
-            'tokenizer_type': self.__class__.__name__
+            "tokenizer_type": self.__class__.__name__,
         }
 
-        with open(filepath, 'w', encoding='utf-8') as f:
+        with open(filepath, "w", encoding="utf-8") as f:
             json.dump(config, f, ensure_ascii=False, indent=2)
 
     @classmethod
@@ -143,17 +145,17 @@ class BaseTokenizer(ABC):
         Returns:
             BaseTokenizer: Загруженный токенизатор
         """
-        with open(filepath, 'r', encoding='utf-8') as f:
+        with open(filepath, "r", encoding="utf-8") as f:
             config = json.load(f)
 
         # Создаем экземпляр токенизатора
         tokenizer = cls()
-        tokenizer.vocab = config['vocab']
+        tokenizer.vocab = config["vocab"]
-        tokenizer.vocab_size = config['vocab_size']
+        tokenizer.vocab_size = config["vocab_size"]
-        tokenizer.pad_token = config['pad_token']
+        tokenizer.pad_token = config["pad_token"]
-        tokenizer.unk_token = config['unk_token']
+        tokenizer.unk_token = config["unk_token"]
-        tokenizer.bos_token = config['bos_token']
+        tokenizer.bos_token = config["bos_token"]
-        tokenizer.eos_token = config['eos_token']
+        tokenizer.eos_token = config["eos_token"]
 
         # Создаем обратный словарь
         tokenizer.inverse_vocab = {v: k for k, v in tokenizer.vocab.items()}

llm/src/llm/tokenizers/bpe_tokenizer copy.py

@@ -44,7 +44,10 @@ class BPETokenizer(BaseTokenizer):
         self._initialize_vocab()
 
         # Добавляем специальные токены если указаны
-        special_tokens = kwargs.get('special_tokens', [self.pad_token, self.unk_token, self.bos_token, self.eos_token])
+        special_tokens = kwargs.get(
+            "special_tokens",
+            [self.pad_token, self.unk_token, self.bos_token, self.eos_token],
+        )
         self.add_special_tokens(special_tokens)
 
         # Предобработка текстов
@@ -54,7 +57,7 @@ class BPETokenizer(BaseTokenizer):
         vocab = self._get_initial_vocab(words)
 
         # Выполняем BPE мерджи
-        self._perform_merges(vocab, vocab_size, kwargs.get('min_frequency', 2))
+        self._perform_merges(vocab, vocab_size, kwargs.get("min_frequency", 2))
 
         # Строим финальный словарь
         self._build_final_vocab()
@@ -99,11 +102,13 @@ class BPETokenizer(BaseTokenizer):
         for word_list in words:
             for word in word_list:
                 # Разбиваем слово на символы и добавляем специальный символ конца слова
-                chars = list(word) + ['</w>']
+                chars = list(word) + ["</w>"]
-                vocab.update([''.join(chars[i:i+1]) for i in range(len(chars))])
+                vocab.update(["".join(chars[i : i + 1]) for i in range(len(chars))])
         return vocab
 
-    def _perform_merges(self, vocab: Dict[str, int], target_vocab_size: int, min_frequency: int):
+    def _perform_merges(
+        self, vocab: Dict[str, int], target_vocab_size: int, min_frequency: int
+    ):
         """
         Выполняет BPE мерджи до достижения целевого размера словаря.
 
@@ -146,7 +151,9 @@ class BPETokenizer(BaseTokenizer):
                 pairs[symbols[i], symbols[i + 1]] += freq
         return pairs
 
-    def _merge_vocab(self, vocab: Dict[str, int], pair: Tuple[str, str]) -> Dict[str, int]:
+    def _merge_vocab(
+        self, vocab: Dict[str, int], pair: Tuple[str, str]
+    ) -> Dict[str, int]:
         """
         Объединяет пару символов в словаре.
 
@@ -158,7 +165,9 @@ class BPETokenizer(BaseTokenizer):
             Dict[str, int]: Обновленный словарь
         """
         new_vocab = {}
-        bigram = re.compile(r'(?<!\\S)' + re.escape(pair[0]) + r' ' + re.escape(pair[1]) + r'(?!\\S)')
+        bigram = re.compile(
+            r"(?<!\\S)" + re.escape(pair[0]) + r" " + re.escape(pair[1]) + r"(?!\\S)"
+        )
         replacement = pair[0] + pair[1]
 
         for word in vocab:
@@ -173,7 +182,9 @@ class BPETokenizer(BaseTokenizer):
         all_tokens = set()
 
         # Добавляем специальные токены
-        all_tokens.update([self.pad_token, self.unk_token, self.bos_token, self.eos_token])
+        all_tokens.update(
+            [self.pad_token, self.unk_token, self.bos_token, self.eos_token]
+        )
 
         # Добавляем токены из мерджей
         for pair in self.merges:
@@ -204,7 +215,7 @@ class BPETokenizer(BaseTokenizer):
         Returns:
             List[int]: Список идентификаторов токенов
         """
-        add_special_tokens = kwargs.get('add_special_tokens', False)
+        add_special_tokens = kwargs.get("add_special_tokens", False)
 
         # Токенизация текста
         tokens = self.compiled_pattern.findall(text)
@@ -243,8 +254,8 @@ class BPETokenizer(BaseTokenizer):
             List[str]: Список BPE токенов
         """
         # Простая реализация - в реальной реализации нужно применять обученные мерджи
-        word = token + '</w>'
+        word = token + "</w>"
-        tokens = [word[i:i+1] for i in range(len(word))]
+        tokens = [word[i : i + 1] for i in range(len(word))]
 
         # Применяем мерджи (упрощенная версия)
         # В полной реализации нужно применять все обученные мерджи
@@ -271,7 +282,7 @@ class BPETokenizer(BaseTokenizer):
         Returns:
             str: Декодированный текст
         """
-        skip_special_tokens = kwargs.get('skip_special_tokens', True)
+        skip_special_tokens = kwargs.get("skip_special_tokens", True)
 
         # Конвертируем ID в токены
         token_strings = []
@@ -279,16 +290,21 @@ class BPETokenizer(BaseTokenizer):
             token = self.inverse_vocab.get(token_id, self.unk_token)
 
             # Пропускаем специальные токены если нужно
-            if skip_special_tokens and token in [self.pad_token, self.unk_token, self.bos_token, self.eos_token]:
+            if skip_special_tokens and token in [
+                self.pad_token,
+                self.unk_token,
+                self.bos_token,
+                self.eos_token,
+            ]:
                 continue
 
             token_strings.append(token)
 
         # Объединяем токены в текст
-        text = ''.join(token_strings)
+        text = "".join(token_strings)
 
         # Убираем маркер конца слова
-        text = text.replace('</w>', ' ')
+        text = text.replace("</w>", " ")
 
         return text.strip()
 
@@ -302,18 +318,18 @@ class BPETokenizer(BaseTokenizer):
         import json
 
         config = {
-            'vocab': self.vocab,
+            "vocab": self.vocab,
-            'merges': {f"{k[0]} {k[1]}": v for k, v in self.merges.items()},
+            "merges": {f"{k[0]} {k[1]}": v for k, v in self.merges.items()},
-            'vocab_size': self.vocab_size,
+            "vocab_size": self.vocab_size,
-            'pad_token': self.pad_token,
+            "pad_token": self.pad_token,
-            'unk_token': self.unk_token,
+            "unk_token": self.unk_token,
-            'bos_token': self.bos_token,
+            "bos_token": self.bos_token,
-            'eos_token': self.eos_token,
+            "eos_token": self.eos_token,
-            'pattern': self.pattern,
+            "pattern": self.pattern,
-            'tokenizer_type': self.__class__.__name__
+            "tokenizer_type": self.__class__.__name__,
         }
 
-        with open(filepath, 'w', encoding='utf-8') as f:
+        with open(filepath, "w", encoding="utf-8") as f:
             json.dump(config, f, ensure_ascii=False, indent=2)
 
     @classmethod
@@ -329,21 +345,21 @@ class BPETokenizer(BaseTokenizer):
         """
         import json
 
-        with open(filepath, 'r', encoding='utf-8') as f:
+        with open(filepath, "r", encoding="utf-8") as f:
             config = json.load(f)
 
         tokenizer = cls()
-        tokenizer.vocab = config['vocab']
+        tokenizer.vocab = config["vocab"]
-        tokenizer.vocab_size = config['vocab_size']
+        tokenizer.vocab_size = config["vocab_size"]
-        tokenizer.pad_token = config['pad_token']
+        tokenizer.pad_token = config["pad_token"]
-        tokenizer.unk_token = config['unk_token']
+        tokenizer.unk_token = config["unk_token"]
-        tokenizer.bos_token = config['bos_token']
+        tokenizer.bos_token = config["bos_token"]
-        tokenizer.eos_token = config['eos_token']
+        tokenizer.eos_token = config["eos_token"]
-        tokenizer.pattern = config.get('pattern', tokenizer.pattern)
+        tokenizer.pattern = config.get("pattern", tokenizer.pattern)
         tokenizer.compiled_pattern = re.compile(tokenizer.pattern, re.UNICODE)
 
         # Восстанавливаем мерджи
-        merges = config.get('merges', {})
+        merges = config.get("merges", {})
         tokenizer.merges = {}
         for k, v in merges.items():
             parts = k.split()
@@ -374,7 +390,12 @@ class SimpleBPETokenizer(BPETokenizer):
         self._initialize_vocab()
 
         # Добавляем базовые токены
-        special_tokens = [self.pad_token, self.unk_token, self.bos_token, self.eos_token]
+        special_tokens = [
+            self.pad_token,
+            self.unk_token,
+            self.bos_token,
+            self.eos_token,
+        ]
         self.add_special_tokens(special_tokens)
 
         # Простая реализация - собираем все символы
@@ -398,7 +419,7 @@ class SimpleBPETokenizer(BPETokenizer):
 
     def encode(self, text: str, **kwargs) -> List[int]:
         """Упрощенное кодирование - разбиваем на символы."""
-        add_special_tokens = kwargs.get('add_special_tokens', False)
+        add_special_tokens = kwargs.get("add_special_tokens", False)
 
         token_ids = []
         for char in text:
@@ -416,13 +437,18 @@ class SimpleBPETokenizer(BPETokenizer):
 
     def decode(self, tokens: List[int], **kwargs) -> str:
         """Упрощенное декодирование."""
-        skip_special_tokens = kwargs.get('skip_special_tokens', True)
+        skip_special_tokens = kwargs.get("skip_special_tokens", True)
 
         chars = []
         for token_id in tokens:
             char = self.inverse_vocab.get(token_id, self.unk_token)
-            if skip_special_tokens and char in [self.pad_token, self.unk_token, self.bos_token, self.eos_token]:
+            if skip_special_tokens and char in [
+                self.pad_token,
+                self.unk_token,
+                self.bos_token,
+                self.eos_token,
+            ]:
                 continue
             chars.append(char)
 
-        return ''.join(chars)
+        return "".join(chars)

llm/src/llm/tokenizers/bpe_tokenizer.py

@@ -61,7 +61,10 @@ class BPETokenizer(BaseTokenizer):
                 break  # нет пар — выходим
 
             # Находим самую частую пару (в случае равенства — та, что встретилась первой)
-            most_frequent_pair = max(pair_freq.items(), key=lambda x: (x[1], -self._pair_first_index(sequence, x[0])))[0]
+            most_frequent_pair = max(
+                pair_freq.items(),
+                key=lambda x: (x[1], -self._pair_first_index(sequence, x[0])),
+            )[0]
 
             # Создаем новый токен
             new_token = most_frequent_pair[0] + most_frequent_pair[1]
@@ -71,7 +74,10 @@ class BPETokenizer(BaseTokenizer):
             new_sequence = []
 
             while i < len(sequence):
-                if i < len(sequence) - 1 and (sequence[i], sequence[i + 1]) == most_frequent_pair:
+                if (
+                    i < len(sequence) - 1
+                    and (sequence[i], sequence[i + 1]) == most_frequent_pair
+                ):
                     new_sequence.append(new_token)
                     i += 2  # пропускаем два символа — заменённую пару
                 else:
@@ -86,7 +92,10 @@ class BPETokenizer(BaseTokenizer):
         self.vocab_size = len(self.vocab)
 
         # Добавляем специальные токены если указаны
-        special_tokens = kwargs.get('special_tokens', [self.pad_token, self.unk_token, self.bos_token, self.eos_token])
+        special_tokens = kwargs.get(
+            "special_tokens",
+            [self.pad_token, self.unk_token, self.bos_token, self.eos_token],
+        )
         self.add_special_tokens(special_tokens)
 
     def _pair_first_index(self, sequence, pair):
@@ -94,7 +103,7 @@ class BPETokenizer(BaseTokenizer):
         for i in range(len(sequence) - 1):
             if (sequence[i], sequence[i + 1]) == pair:
                 return i
-        return float('inf')  # если пара не найдена (в теории не должно случиться)
+        return float("inf")  # если пара не найдена (в теории не должно случиться)
 
     def encode(self, text: str, **kwargs) -> List[int]:
         """
@@ -108,7 +117,7 @@ class BPETokenizer(BaseTokenizer):
         Returns:
             List[int]: Список идентификаторов токенов
         """
-        add_special_tokens = kwargs.get('add_special_tokens', False)
+        add_special_tokens = kwargs.get("add_special_tokens", False)
 
         # 1. Разбиваем текст на токены-символы
         sequence = list(text)
@@ -119,7 +128,9 @@ class BPETokenizer(BaseTokenizer):
         while i < len(text):
             # 3.1 Найти все токены в словаре, начинающиеся с text[i]
             start_char = text[i]
-            result = [token for token in self.vocab_list if token.startswith(start_char)]
+            result = [
+                token for token in self.vocab_list if token.startswith(start_char)
+            ]
             # 3.2 Выбрать самый длинный подходящий токен
             find_token = self._find_max_matching_token(text[i:], result)
             if find_token is None:
@@ -174,19 +185,27 @@ class BPETokenizer(BaseTokenizer):
         Returns:
             str: Декодированный текст
         """
-        skip_special_tokens = kwargs.get('skip_special_tokens', True)
+        skip_special_tokens = kwargs.get("skip_special_tokens", True)
 
         # Фильтруем специальные токены если нужно
         if skip_special_tokens:
-            tokens = [tid for tid in tokens if tid not in [
+            tokens = [
-                self.pad_token_id, self.unk_token_id, self.bos_token_id, self.eos_token_id
+                tid
-            ]]
+                for tid in tokens
+                if tid
+                not in [
+                    self.pad_token_id,
+                    self.unk_token_id,
+                    self.bos_token_id,
+                    self.eos_token_id,
+                ]
+            ]
 
         # Конвертируем ID в токены
         token_strings = self._ids_to_tokens(tokens)
 
         # Объединяем токены в текст
-        return ''.join(token_strings)
+        return "".join(token_strings)
 
     def _ids_to_tokens(self, ids: List[int]) -> List[str]:
         """Конвертирует список Ids в их tokens"""
@@ -211,18 +230,18 @@ class BPETokenizer(BaseTokenizer):
         merges_serializable = {f"{k[0]},{k[1]}": v for k, v in self.merges.items()}
 
         config = {
-            'vocab': self.vocab,
+            "vocab": self.vocab,
-            'vocab_size': self.vocab_size,
+            "vocab_size": self.vocab_size,
-            'pad_token': self.pad_token,
+            "pad_token": self.pad_token,
-            'unk_token': self.unk_token,
+            "unk_token": self.unk_token,
-            'bos_token': self.bos_token,
+            "bos_token": self.bos_token,
-            'eos_token': self.eos_token,
+            "eos_token": self.eos_token,
-            'tokenizer_type': self.__class__.__name__,
+            "tokenizer_type": self.__class__.__name__,
-            'merges': merges_serializable,
+            "merges": merges_serializable,
-            'vocab_list': self.vocab_list
+            "vocab_list": self.vocab_list,
         }
 
-        with open(filepath, 'w', encoding='utf-8') as f:
+        with open(filepath, "w", encoding="utf-8") as f:
             json.dump(config, f, ensure_ascii=False, indent=2)
 
     @classmethod
@@ -238,23 +257,23 @@ class BPETokenizer(BaseTokenizer):
         """
         import json
 
-        with open(filepath, 'r', encoding='utf-8') as f:
+        with open(filepath, "r", encoding="utf-8") as f:
             config = json.load(f)
 
         # Создаем экземпляр токенизатора
         tokenizer = cls()
-        tokenizer.vocab = config['vocab']
+        tokenizer.vocab = config["vocab"]
-        tokenizer.vocab_size = config['vocab_size']
+        tokenizer.vocab_size = config["vocab_size"]
-        tokenizer.pad_token = config['pad_token']
+        tokenizer.pad_token = config["pad_token"]
-        tokenizer.unk_token = config['unk_token']
+        tokenizer.unk_token = config["unk_token"]
-        tokenizer.bos_token = config['bos_token']
+        tokenizer.bos_token = config["bos_token"]
-        tokenizer.eos_token = config['eos_token']
+        tokenizer.eos_token = config["eos_token"]
-        tokenizer.vocab_list = config['vocab_list']
+        tokenizer.vocab_list = config["vocab_list"]
 
         # Восстанавливаем кортежи из строк
         tokenizer.merges = {}
-        for k, v in config['merges'].items():
+        for k, v in config["merges"].items():
-            parts = k.split(',')
+            parts = k.split(",")
             if len(parts) == 2:
                 tokenizer.merges[(parts[0], parts[1])] = v
 
@@ -275,4 +294,5 @@ class SimpleBPETokenizer(BPETokenizer):
     Упрощенная версия BPE токенизатора для демонстрации.
     Наследует вашу реализацию, но может быть упрощена при необходимости.
     """
+
     pass

llm/src/llm/training/dataset.py

@@ -31,7 +31,7 @@ class TextDataset(Dataset):
                 input_ids = input_ids[:block_size]
             else:
                 # Дополняем pad_token_id
-                pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
+                pad_token_id = getattr(tokenizer, "pad_token_id", 0)
                 input_ids = input_ids + [pad_token_id] * (block_size - len(input_ids))
 
             self.examples.append(input_ids)
@@ -57,7 +57,7 @@ class StreamingTextDataset(Dataset):
         self.block_size = block_size
 
         # Получаем pad_token_id из токенизатора
-        self.pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
+        self.pad_token_id = getattr(tokenizer, "pad_token_id", 0)
 
     def __len__(self):
         return len(self.texts)
@@ -70,9 +70,11 @@ class StreamingTextDataset(Dataset):
 
         # Обрезаем или дополняем до нужной длины
         if len(input_ids) > self.block_size:
-            input_ids = input_ids[:self.block_size]
+            input_ids = input_ids[: self.block_size]
         else:
-            input_ids = input_ids + [self.pad_token_id] * (self.block_size - len(input_ids))
+            input_ids = input_ids + [self.pad_token_id] * (
+                self.block_size - len(input_ids)
+            )
 
         input_ids = torch.tensor(input_ids, dtype=torch.long)
         labels = input_ids.clone()
@@ -85,8 +87,14 @@ class TextDatasetWithSpecialTokens(TextDataset):
     Расширенная версия TextDataset с поддержкой специальных токенов.
     """
 
-    def __init__(self, texts: List[str], tokenizer: Any, block_size: int = 128, 
+    def __init__(
-                 add_bos: bool = False, add_eos: bool = False):
+        self,
+        texts: List[str],
+        tokenizer: Any,
+        block_size: int = 128,
+        add_bos: bool = False,
+        add_eos: bool = False,
+    ):
         """
         Args:
             texts: Список текстов
@@ -104,10 +112,7 @@ class TextDatasetWithSpecialTokens(TextDataset):
         for text in texts:
             # Кодируем с специальными токенами
             input_ids = tokenizer.encode(
-                text, 
+                text, add_special_tokens=True, add_bos_token=add_bos, add_eos_token=eos
-                add_special_tokens=True,
-                add_bos_token=add_bos,
-                add_eos_token=eos
             )
 
             # Учитываем специальные токены при обрезке/дополнении
@@ -121,13 +126,21 @@ class TextDatasetWithSpecialTokens(TextDataset):
                 input_ids = input_ids[:effective_block_size]
 
             # Добавляем специальные токены если нужно
-            if add_bos and hasattr(tokenizer, 'bos_token_id') and tokenizer.bos_token_id is not None:
+            if (
+                add_bos
+                and hasattr(tokenizer, "bos_token_id")
+                and tokenizer.bos_token_id is not None
+            ):
                 input_ids = [tokenizer.bos_token_id] + input_ids
-            if add_eos and hasattr(tokenizer, 'eos_token_id') and tokenizer.eos_token_id is not None:
+            if (
+                add_eos
+                and hasattr(tokenizer, "eos_token_id")
+                and tokenizer.eos_token_id is not None
+            ):
                 input_ids = input_ids + [tokenizer.eos_token_id]
 
             # Дополняем до полной длины
-            pad_token_id = getattr(tokenizer, 'pad_token_id', 0)
+            pad_token_id = getattr(tokenizer, "pad_token_id", 0)
             if len(input_ids) < block_size:
                 input_ids = input_ids + [pad_token_id] * (block_size - len(input_ids))
 

llm/src/llm/training/optimizer.py

@@ -1,5 +1,6 @@
 import torch.optim as optim
 
+
 def get_optimizer(model, lr=3e-4, weight_decay=0.01, optimizer_type="adamw"):
     """
     Возвращает оптимизатор для обучения модели.

llm/src/llm/training/scheduler.py

@@ -1,5 +1,6 @@
 from torch.optim.lr_scheduler import LambdaLR
 
+
 def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps):
     """
     Линейный планировщик обучения с warmup.
@@ -8,6 +9,10 @@ def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_st
     def lr_lambda(current_step):
         if current_step < num_warmup_steps:
             return float(current_step) / float(max(1, num_warmup_steps))
-        return max(0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)))
+        return max(
+            0.0,
+            float(num_training_steps - current_step)
+            / float(max(1, num_training_steps - num_warmup_steps)),
+        )
 
     return LambdaLR(optimizer, lr_lambda)

llm/src/llm/training/trainer.py

@@ -5,15 +5,29 @@ from tqdm import tqdm
 from llm.training.optimizer import get_optimizer
 from llm.training.scheduler import get_linear_schedule_with_warmup
 
+
 class Trainer:
     """
     Универсальный класс обучения LLM (GPT, LLaMA, Mistral и т.д.)
     """
 
-    def __init__(self, model, train_dataset, val_dataset=None, lr=3e-4, batch_size=8, num_epochs=3, warmup_steps=100):
+    def __init__(
+        self,
+        model,
+        train_dataset,
+        val_dataset=None,
+        lr=3e-4,
+        batch_size=8,
+        num_epochs=3,
+        warmup_steps=100,
+    ):
         self.model = model
-        self.train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
+        self.train_loader = DataLoader(
-        self.val_loader = DataLoader(val_dataset, batch_size=batch_size) if val_dataset else None
+            train_dataset, batch_size=batch_size, shuffle=True
+        )
+        self.val_loader = (
+            DataLoader(val_dataset, batch_size=batch_size) if val_dataset else None
+        )
         self.optimizer = get_optimizer(model, lr=lr)
         self.scheduler = None
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
@@ -34,19 +48,23 @@ class Trainer:
         loss = F.cross_entropy(
             shift_logits.view(-1, shift_logits.size(-1)),
             shift_labels.view(-1),
-            ignore_index=-100  # Игнорируем padding tokens
+            ignore_index=-100,  # Игнорируем padding tokens
         )
         return loss
 
     def train(self):
         total_steps = len(self.train_loader) * self.num_epochs
-        self.scheduler = get_linear_schedule_with_warmup(self.optimizer, self.warmup_steps, total_steps)
+        self.scheduler = get_linear_schedule_with_warmup(
+            self.optimizer, self.warmup_steps, total_steps
+        )
 
         for epoch in range(self.num_epochs):
             self.model.train()
             total_loss = 0
 
-            progress_bar = tqdm(self.train_loader, desc=f"Epoch {epoch+1}/{self.num_epochs}")
+            progress_bar = tqdm(
+                self.train_loader, desc=f"Epoch {epoch+1}/{self.num_epochs}"
+            )
             for batch in progress_bar:
                 self.optimizer.zero_grad()
 

llm/tests/conftest.py

@@ -58,7 +58,7 @@ def gpt_config(vocab_size, embed_dim, num_heads, num_layers):
         "num_heads": num_heads,
         "num_layers": num_layers,
         "max_position_embeddings": 1024,
-        "dropout": 0.1
+        "dropout": 0.1,
     }
 
 
@@ -68,12 +68,14 @@ def random_inputs(batch_size, seq_len, vocab_size):
     input_ids = torch.randint(0, vocab_size, (batch_size, seq_len))
     return input_ids
 
+
 @pytest.fixture
 def random_float_inputs(batch_size, seq_len, embed_dim):
     """Generate random floating point input tensors for testing feed forward."""
     inputs = torch.randn(batch_size, seq_len, embed_dim)
     return inputs
 
+
 @pytest.fixture
 def random_embeddings(batch_size, seq_len, embed_dim):
     """Generate random embedding tensors for testing attention modules."""

llm/tests/core/test_decoder.py

@@ -14,20 +14,30 @@ class TestDecoder:
         """Test that Decoder can be initialized."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
         assert decoder is not None
 
         # Check internal components
-        assert hasattr(decoder, '_heads')
+        assert hasattr(decoder, "_heads")
-        assert hasattr(decoder, '_ff')
+        assert hasattr(decoder, "_ff")
-        assert hasattr(decoder, '_norm1')
+        assert hasattr(decoder, "_norm1")
-        assert hasattr(decoder, '_norm2')
+        assert hasattr(decoder, "_norm2")
 
     def test_forward_pass(self, embed_dim, num_heads, random_embeddings):
         """Test forward pass of Decoder."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         # Forward pass
         output = decoder(random_embeddings)
@@ -40,7 +50,12 @@ class TestDecoder:
         """Test forward pass with causal mask."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         batch_size, seq_len = random_embeddings.shape[:2]
         # Create causal mask
@@ -56,7 +71,12 @@ class TestDecoder:
         """Test that residual connections are properly applied."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         output = decoder(random_embeddings)
 
@@ -72,7 +92,12 @@ class TestDecoder:
         """Test that layer normalization is applied."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         output = decoder(random_embeddings)
 
@@ -89,7 +114,12 @@ class TestDecoder:
         """Test that gradients flow through Decoder."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         # Forward pass
         output = decoder(random_embeddings)
@@ -109,7 +139,12 @@ class TestDecoder:
         """Test that Decoder works on correct device."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len).to(device)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        ).to(device)
         inputs = random_embeddings.to(device)
 
         # Forward pass
@@ -122,7 +157,7 @@ class TestDecoder:
     def test_different_configurations(self):
         """Test Decoder with different configurations."""
         test_cases = [
-            (64, 2),   # embed_dim=64, num_heads=2
+            (64, 2),  # embed_dim=64, num_heads=2
             (128, 4),  # embed_dim=128, num_heads=4
             (256, 8),  # embed_dim=256, num_heads=8
         ]
@@ -130,7 +165,12 @@ class TestDecoder:
         for embed_dim, num_heads in test_cases:
             head_size = embed_dim // num_heads
             max_seq_len = 1024
-            decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+            decoder = Decoder(
+                num_heads=num_heads,
+                emb_size=embed_dim,
+                head_size=head_size,
+                max_seq_len=max_seq_len,
+            )
             batch_size, seq_len = 2, 16
             inputs = torch.randn(batch_size, seq_len, embed_dim)
 
@@ -143,7 +183,12 @@ class TestDecoder:
         """Test Decoder with different input shapes."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         inputs = torch.randn(batch_size, seq_len, embed_dim)
         output = decoder(inputs)
@@ -154,7 +199,13 @@ class TestDecoder:
         """Test that Decoder behaves differently in train vs eval mode."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len, dropout=0.5)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+            dropout=0.5,
+        )
 
         # Training mode
         decoder.train()
@@ -171,18 +222,20 @@ class TestDecoder:
         """Test that parameters are properly initialized."""
         head_size = embed_dim // num_heads
         max_seq_len = 1024
-        decoder = Decoder(num_heads=num_heads, emb_size=embed_dim, head_size=head_size, max_seq_len=max_seq_len)
+        decoder = Decoder(
+            num_heads=num_heads,
+            emb_size=embed_dim,
+            head_size=head_size,
+            max_seq_len=max_seq_len,
+        )
 
         # Check that various components have non-zero parameters
         assert not torch.allclose(
-            decoder._heads._layer.weight, 
+            decoder._heads._layer.weight, torch.zeros_like(decoder._heads._layer.weight)
-            torch.zeros_like(decoder._heads._layer.weight)
         )
         assert not torch.allclose(
-            decoder._ff._layer1.weight,
+            decoder._ff._layer1.weight, torch.zeros_like(decoder._ff._layer1.weight)
-            torch.zeros_like(decoder._ff._layer1.weight)
         )
         assert not torch.allclose(
-            decoder._norm1.weight,
+            decoder._norm1.weight, torch.zeros_like(decoder._norm1.weight)
-            torch.zeros_like(decoder._norm1.weight)
         )

llm/tests/core/test_feed_forward.py

@@ -17,10 +17,10 @@ class TestFeedForward:
         assert ff is not None
 
         # Check internal layers
-        assert hasattr(ff, '_layer1')
+        assert hasattr(ff, "_layer1")
-        assert hasattr(ff, '_layer2')
+        assert hasattr(ff, "_layer2")
-        assert hasattr(ff, '_activation')
+        assert hasattr(ff, "_activation")
-        assert hasattr(ff, '_dropout')
+        assert hasattr(ff, "_dropout")
 
         # Check layer dimensions
         expected_hidden_dim = embed_dim * 4  # Default expansion factor
@@ -101,10 +101,12 @@ class TestFeedForward:
         # Check that gradients are computed for learnable parameters
         assert ff._layer1.weight.grad is not None
         assert ff._layer2.weight.grad is not None
-        assert not torch.allclose(ff._layer1.weight.grad, 
+        assert not torch.allclose(
-                                torch.zeros_like(ff._layer1.weight.grad))
+            ff._layer1.weight.grad, torch.zeros_like(ff._layer1.weight.grad)
-        assert not torch.allclose(ff._layer2.weight.grad, 
+        )
-                                torch.zeros_like(ff._layer2.weight.grad))
+        assert not torch.allclose(
+            ff._layer2.weight.grad, torch.zeros_like(ff._layer2.weight.grad)
+        )
 
     def test_device_consistency(self, embed_dim, random_float_inputs, device):
         """Test that FeedForward works on correct device."""
@@ -167,11 +169,19 @@ class TestFeedForward:
         ff = FeedForward(embed_dim)
 
         # Check that weights are not all zeros
-        assert not torch.allclose(ff._layer1.weight, torch.zeros_like(ff._layer1.weight))
+        assert not torch.allclose(
-        assert not torch.allclose(ff._layer2.weight, torch.zeros_like(ff._layer2.weight))
+            ff._layer1.weight, torch.zeros_like(ff._layer1.weight)
+        )
+        assert not torch.allclose(
+            ff._layer2.weight, torch.zeros_like(ff._layer2.weight)
+        )
 
         # Check that biases are not all zeros (they should be initialized with some values)
         if ff._layer1.bias is not None:
-            assert not torch.allclose(ff._layer1.bias, torch.zeros_like(ff._layer1.bias))
+            assert not torch.allclose(
+                ff._layer1.bias, torch.zeros_like(ff._layer1.bias)
+            )
         if ff._layer2.bias is not None:
-            assert not torch.allclose(ff._layer2.bias, torch.zeros_like(ff._layer2.bias))
+            assert not torch.allclose(
+                ff._layer2.bias, torch.zeros_like(ff._layer2.bias)
+            )

llm/tests/core/test_multi_head_attention.py

@@ -13,7 +13,9 @@ class TestMultiHeadAttention:
     def test_initialization(self, embed_dim, num_heads):
         """Test that MultiHeadAttention can be initialized."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
         assert attention is not None
 
         # Check internal attributes
@@ -24,7 +26,9 @@ class TestMultiHeadAttention:
     def test_forward_pass(self, embed_dim, num_heads, random_embeddings):
         """Test forward pass of MultiHeadAttention."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         # Forward pass
         output, _ = attention(random_embeddings)
@@ -36,7 +40,9 @@ class TestMultiHeadAttention:
     def test_forward_with_mask(self, embed_dim, num_heads, random_embeddings):
         """Test forward pass with attention mask."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         # Create a simple mask
         seq_len = random_embeddings.shape[1]
@@ -51,7 +57,9 @@ class TestMultiHeadAttention:
     def test_causal_mask(self, embed_dim, num_heads, random_embeddings):
         """Test that causal mask prevents attending to future positions."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         # Create causal mask
         seq_len = random_embeddings.shape[1]
@@ -63,10 +71,14 @@ class TestMultiHeadAttention:
         # Check output shape
         assert output.shape == random_embeddings.shape
 
-    def test_attention_weights_normalization(self, embed_dim, num_heads, random_embeddings):
+    def test_attention_weights_normalization(
+        self, embed_dim, num_heads, random_embeddings
+    ):
         """Test that attention weights are properly normalized."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         # Forward pass
         output, _ = attention(random_embeddings)
@@ -77,7 +89,9 @@ class TestMultiHeadAttention:
     def test_gradient_flow(self, embed_dim, num_heads, random_embeddings):
         """Test that gradients flow through MultiHeadAttention."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         # Forward pass
         output, _ = attention(random_embeddings)
@@ -94,7 +108,9 @@ class TestMultiHeadAttention:
     def test_device_consistency(self, embed_dim, num_heads, random_embeddings, device):
         """Test that MultiHeadAttention works on correct device."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024).to(device)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        ).to(device)
         inputs = random_embeddings.to(device)
 
         # Forward pass
@@ -107,15 +123,17 @@ class TestMultiHeadAttention:
     def test_different_embed_dim_and_heads(self):
         """Test MultiHeadAttention with different embed_dim and num_heads combinations."""
         test_cases = [
-            (64, 2),   # embed_dim=64, num_heads=2
+            (64, 2),  # embed_dim=64, num_heads=2
             (128, 4),  # embed_dim=128, num_heads=4
             (256, 8),  # embed_dim=256, num_heads=8
-            (512, 16), # embed_dim=512, num_heads=16
+            (512, 16),  # embed_dim=512, num_heads=16
         ]
 
         for embed_dim, num_heads in test_cases:
             head_size = embed_dim // num_heads
-            attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+            attention = MultiHeadAttention(
+                num_heads, embed_dim, head_size, max_seq_len=1024
+            )
             batch_size, seq_len = 2, 16
             inputs = torch.randn(batch_size, seq_len, embed_dim)
 
@@ -126,7 +144,9 @@ class TestMultiHeadAttention:
     def test_attention_output_range(self, embed_dim, num_heads, random_embeddings):
         """Test that attention output is in reasonable range."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         output, _ = attention(random_embeddings)
 
@@ -137,7 +157,9 @@ class TestMultiHeadAttention:
     def test_different_input_shapes(self, embed_dim, num_heads, batch_size, seq_len):
         """Test MultiHeadAttention with different input shapes."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024)
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024
+        )
 
         inputs = torch.randn(batch_size, seq_len, embed_dim)
         output, _ = attention(inputs)
@@ -147,7 +169,9 @@ class TestMultiHeadAttention:
     def test_parameter_sharing(self, embed_dim, num_heads):
         """Test that parameters are properly shared across the sequence."""
         head_size = embed_dim // num_heads
-        attention = MultiHeadAttention(num_heads, embed_dim, head_size, max_seq_len=1024, dropout=0.0)  # No dropout for deterministic test
+        attention = MultiHeadAttention(
+            num_heads, embed_dim, head_size, max_seq_len=1024, dropout=0.0
+        )  # No dropout for deterministic test
 
         # Create two identical sequences
         seq_len = 10

llm/tests/core/test_positional_embeddings.py

@@ -18,7 +18,7 @@ class TestPositionalEmbeddings:
         assert embeddings is not None
 
         # Check that positional embeddings are created
-        assert hasattr(embeddings, 'embedding')
+        assert hasattr(embeddings, "embedding")
         assert embeddings.embedding.weight.shape == (max_seq_len, embed_dim)
 
     def test_forward_pass(self, embed_dim):
@@ -52,7 +52,7 @@ class TestPositionalEmbeddings:
     def test_different_sequence_lengths(self, embed_dim):
         """Test PositionalEmbeddings with different sequence lengths."""
         test_cases = [
-            (10, 5),   # seq_len < max_seq_len
+            (10, 5),  # seq_len < max_seq_len
             (10, 10),  # seq_len == max_seq_len
         ]
 
@@ -80,8 +80,10 @@ class TestPositionalEmbeddings:
 
         # Positional embeddings should have gradients (they're learnable)
         assert embeddings.embedding.weight.grad is not None
-        assert not torch.allclose(embeddings.embedding.weight.grad, 
+        assert not torch.allclose(
-                                torch.zeros_like(embeddings.embedding.weight.grad))
+            embeddings.embedding.weight.grad,
+            torch.zeros_like(embeddings.embedding.weight.grad),
+        )
 
     def test_device_consistency(self, embed_dim, device):
         """Test that PositionalEmbeddings works on correct device."""
@@ -103,7 +105,9 @@ class TestPositionalEmbeddings:
         embeddings2 = PositionalEmbeddings(max_seq_len, embed_dim)
 
         # Different instances should have different embeddings (random initialization)
-        assert not torch.allclose(embeddings1.embedding.weight, embeddings2.embedding.weight)
+        assert not torch.allclose(
+            embeddings1.embedding.weight, embeddings2.embedding.weight
+        )
 
         # But same instance should produce same output for same input
         seq_len = 50
@@ -122,11 +126,14 @@ class TestPositionalEmbeddings:
         assert not torch.allclose(pe[0], pe[1], rtol=1e-4)
         assert not torch.allclose(pe[10], pe[20], rtol=1e-4)
 
-    @pytest.mark.parametrize("max_seq_len,seq_len,embed_dim", [
+    @pytest.mark.parametrize(
-        (64, 10, 64),
+        "max_seq_len,seq_len,embed_dim",
-        (128, 50, 128), 
+        [
-        (256, 100, 256),
+            (64, 10, 64),
-    ])
+            (128, 50, 128),
+            (256, 100, 256),
+        ],
+    )
     def test_different_configurations(self, max_seq_len, seq_len, embed_dim):
         """Test PositionalEmbeddings with different configurations."""
         embeddings = PositionalEmbeddings(max_seq_len, embed_dim)

llm/tests/core/test_token_embeddings.py

@@ -16,7 +16,7 @@ class TestTokenEmbeddings:
         assert embeddings is not None
 
         # Check embedding layer
-        assert hasattr(embeddings, '_embedding')
+        assert hasattr(embeddings, "_embedding")
         assert embeddings._embedding.weight.shape == (vocab_size, embed_dim)
 
     def test_forward_pass(self, vocab_size, embed_dim, random_inputs):
@@ -27,7 +27,11 @@ class TestTokenEmbeddings:
         output = embeddings(random_inputs)
 
         # Check output shape
-        assert output.shape == (random_inputs.shape[0], random_inputs.shape[1], embed_dim)
+        assert output.shape == (
+            random_inputs.shape[0],
+            random_inputs.shape[1],
+            embed_dim,
+        )
         assert isinstance(output, torch.Tensor)
 
     def test_embedding_weights(self, vocab_size, embed_dim):
@@ -42,11 +46,7 @@ class TestTokenEmbeddings:
 
     def test_different_vocab_sizes(self):
         """Test TokenEmbeddings with different vocabulary sizes."""
-        test_cases = [
+        test_cases = [(100, 128), (1000, 256), (50000, 512)]
-            (100, 128),
-            (1000, 256),
-            (50000, 512)
-        ]
 
         for vocab_size, embed_dim in test_cases:
             embeddings = TokenEmbeddings(vocab_size, embed_dim)
@@ -65,8 +65,10 @@ class TestTokenEmbeddings:
 
         # Check that gradients are computed
         assert embeddings._embedding.weight.grad is not None
-        assert not torch.allclose(embeddings._embedding.weight.grad, 
+        assert not torch.allclose(
-                                torch.zeros_like(embeddings._embedding.weight.grad))
+            embeddings._embedding.weight.grad,
+            torch.zeros_like(embeddings._embedding.weight.grad),
+        )
 
     def test_device_consistency(self, vocab_size, embed_dim, random_inputs, device):
         """Test that TokenEmbeddings works on correct device."""
@@ -85,7 +87,9 @@ class TestTokenEmbeddings:
         embeddings = TokenEmbeddings(vocab_size, embed_dim)
 
         # Test lookup for specific tokens
-        test_tokens = torch.tensor([[0, 1, 2], [vocab_size - 1, vocab_size - 2, vocab_size - 3]])
+        test_tokens = torch.tensor(
+            [[0, 1, 2], [vocab_size - 1, vocab_size - 2, vocab_size - 3]]
+        )
 
         output = embeddings(test_tokens)
 

llm/tests/models/test_gpt.py

@@ -16,14 +16,14 @@ class TestGPT:
         assert model is not None
 
         # Check that model has required components
-        assert hasattr(model, '_token_embeddings')
+        assert hasattr(model, "_token_embeddings")
-        assert hasattr(model, '_position_embeddings')
+        assert hasattr(model, "_position_embeddings")
-        assert hasattr(model, '_decoders')
+        assert hasattr(model, "_decoders")
-        assert hasattr(model, '_linear')
+        assert hasattr(model, "_linear")
-        assert hasattr(model, '_dropout')
+        assert hasattr(model, "_dropout")
 
         # Check number of decoder layers
-        assert len(model._decoders) == gpt_config['num_layers']
+        assert len(model._decoders) == gpt_config["num_layers"]
 
     def test_forward_pass(self, gpt_config, random_inputs):
         """Test forward pass of GPT."""
@@ -34,11 +34,13 @@ class TestGPT:
 
         # Check output shape
         batch_size, seq_len = random_inputs.shape
-        vocab_size = gpt_config['vocab_size']
+        vocab_size = gpt_config["vocab_size"]
         assert logits.shape == (batch_size, seq_len, vocab_size)
         assert isinstance(logits, torch.Tensor)
 
-    def test_forward_with_attention_mask(self, gpt_config, random_inputs, attention_mask):
+    def test_forward_with_attention_mask(
+        self, gpt_config, random_inputs, attention_mask
+    ):
         """Test forward pass with attention mask."""
         model = GPT(gpt_config)
 
@@ -47,7 +49,7 @@ class TestGPT:
 
         # Check output shape
         batch_size, seq_len = random_inputs.shape
-        vocab_size = gpt_config['vocab_size']
+        vocab_size = gpt_config["vocab_size"]
         assert logits.shape == (batch_size, seq_len, vocab_size)
 
     def test_generate_text(self, gpt_config):
@@ -58,14 +60,16 @@ class TestGPT:
         # Create initial input
         batch_size = 2
         initial_seq_len = 5
-        input_ids = torch.randint(0, gpt_config['vocab_size'], (batch_size, initial_seq_len))
+        input_ids = torch.randint(
+            0, gpt_config["vocab_size"], (batch_size, initial_seq_len)
+        )
 
         # Generate text
         with torch.no_grad():
             generated = model.generate(
                 x=input_ids,
                 max_new_tokens=10,
-                do_sample=False  # Use greedy for deterministic testing
+                do_sample=False,  # Use greedy for deterministic testing
             )
 
         # Check output shape
@@ -81,15 +85,12 @@ class TestGPT:
         model.eval()
 
         # Create initial input
-        input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
+        input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
 
         # Generate with temperature
         with torch.no_grad():
             generated = model.generate(
-                x=input_ids,
+                x=input_ids, max_new_tokens=5, do_sample=True, temperature=0.8
-                max_new_tokens=5,
-                do_sample=True,
-                temperature=0.8
             )
 
         assert generated.shape == (1, 8)  # 3 initial + 5 new tokens
@@ -100,15 +101,12 @@ class TestGPT:
         model.eval()
 
         # Create initial input
-        input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
+        input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
 
         # Generate with top-k
         with torch.no_grad():
             generated = model.generate(
-                x=input_ids,
+                x=input_ids, max_new_tokens=5, do_sample=True, top_k=10
-                max_new_tokens=5,
-                do_sample=True,
-                top_k=10
             )
 
         assert generated.shape == (1, 8)
@@ -119,15 +117,12 @@ class TestGPT:
         model.eval()
 
         # Create initial input
-        input_ids = torch.randint(0, gpt_config['vocab_size'], (1, 3))
+        input_ids = torch.randint(0, gpt_config["vocab_size"], (1, 3))
 
         # Generate with top-p
         with torch.no_grad():
             generated = model.generate(
-                x=input_ids,
+                x=input_ids, max_new_tokens=5, do_sample=True, top_p=0.9
-                max_new_tokens=5,
-                do_sample=True,
-                top_p=0.9
             )
 
         assert generated.shape == (1, 8)
@@ -140,10 +135,9 @@ class TestGPT:
         logits = model(random_inputs)
 
         # Create a dummy loss and backward pass
-        targets = torch.randint(0, gpt_config['vocab_size'], random_inputs.shape)
+        targets = torch.randint(0, gpt_config["vocab_size"], random_inputs.shape)
         loss = torch.nn.functional.cross_entropy(
-            logits.view(-1, logits.size(-1)), 
+            logits.view(-1, logits.size(-1)), targets.view(-1)
-            targets.view(-1)
         )
         loss.backward()
 
@@ -174,7 +168,7 @@ class TestGPT:
                 "num_heads": 2,
                 "num_layers": 2,
                 "max_position_embeddings": 256,
-                "dropout": 0.1
+                "dropout": 0.1,
             },
             {
                 "vocab_size": 5000,
@@ -182,7 +176,7 @@ class TestGPT:
                 "num_heads": 4,
                 "num_layers": 4,
                 "max_position_embeddings": 512,
-                "dropout": 0.1
+                "dropout": 0.1,
             },
             {
                 "vocab_size": 10000,
@@ -190,18 +184,18 @@ class TestGPT:
                 "num_heads": 8,
                 "num_layers": 6,
                 "max_position_embeddings": 1024,
-                "dropout": 0.1
+                "dropout": 0.1,
-            }
+            },
         ]
 
         for config in test_configs:
             model = GPT(config)
             batch_size, seq_len = 2, 16
-            inputs = torch.randint(0, config['vocab_size'], (batch_size, seq_len))
+            inputs = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
 
             logits = model(inputs)
 
-            expected_shape = (batch_size, seq_len, config['vocab_size'])
+            expected_shape = (batch_size, seq_len, config["vocab_size"])
             assert logits.shape == expected_shape
 
     @pytest.mark.parametrize("batch_size,seq_len", [(1, 8), (2, 16), (4, 32)])
@@ -209,10 +203,10 @@ class TestGPT:
         """Test GPT with different input shapes."""
         model = GPT(gpt_config)
 
-        inputs = torch.randint(0, gpt_config['vocab_size'], (batch_size, seq_len))
+        inputs = torch.randint(0, gpt_config["vocab_size"], (batch_size, seq_len))
         logits = model(inputs)
 
-        expected_shape = (batch_size, seq_len, gpt_config['vocab_size'])
+        expected_shape = (batch_size, seq_len, gpt_config["vocab_size"])
         assert logits.shape == expected_shape
 
     def test_training_vs_evaluation(self, gpt_config, random_inputs):
@@ -237,10 +231,10 @@ class TestGPT:
         total_params = sum(p.numel() for p in model.parameters())
 
         # For a small GPT model, parameters should be in reasonable range
-        vocab_size = gpt_config['vocab_size']
+        vocab_size = gpt_config["vocab_size"]
-        embed_dim = gpt_config['embed_dim']
+        embed_dim = gpt_config["embed_dim"]
-        num_layers = gpt_config['num_layers']
+        num_layers = gpt_config["num_layers"]
-        num_heads = gpt_config['num_heads']
+        num_heads = gpt_config["num_heads"]
 
         # Rough estimate: token_embeddings + output_layer + (attention + ff) * layers
         expected_min = vocab_size * embed_dim * 2  # embeddings and output
@@ -264,11 +258,7 @@ class TestGPT:
             # We can't directly test attention masks in the public API,
             # but we can verify the generation works correctly
 
-            generated = model.generate(
+            generated = model.generate(x=input_ids, max_new_tokens=3, do_sample=False)
-                x=input_ids,
-                max_new_tokens=3,
-                do_sample=False
-            )
 
             # Generated sequence should be longer than input
             assert generated.shape[1] == input_ids.shape[1] + 3

llm/tests/test_basic.py

@@ -18,7 +18,7 @@ def test_gpt_model_creation():
         "num_heads": 4,
         "num_layers": 2,
         "max_position_embeddings": 256,
-        "dropout": 0.1
+        "dropout": 0.1,
     }
 
     model = GPT(config)
@@ -41,16 +41,10 @@ def test_bpe_tokenizer_basic():
     tokenizer = BPETokenizer()
 
     # Train on simple texts
-    texts = [
+    texts = ["hello world", "test tokenization", "simple example"]
-        "hello world",
-        "test tokenization",
-        "simple example"
-    ]
 
     tokenizer.train(
-        texts=texts,
+        texts=texts, vocab_size=50, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
-        vocab_size=50,
-        special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
     )
 
     # Test encoding/decoding
@@ -132,7 +126,7 @@ def test_gpt_generation():
         "num_heads": 4,
         "num_layers": 2,
         "max_position_embeddings": 256,
-        "dropout": 0.1
+        "dropout": 0.1,
     }
 
     model = GPT(config)
@@ -142,11 +136,7 @@ def test_gpt_generation():
     input_ids = torch.randint(0, config["vocab_size"], (1, 5))
 
     with torch.no_grad():
-        generated = model.generate(
+        generated = model.generate(x=input_ids, max_new_tokens=3, do_sample=False)
-            x=input_ids,
-            max_new_tokens=3,
-            do_sample=False
-        )
 
     assert generated.shape == (1, 8)  # 5 initial + 3 new tokens
     print("✅ GPT generation test passed")
@@ -161,9 +151,7 @@ def test_bpe_tokenizer_save_load():
     # Train on simple texts
     texts = ["hello world", "test save load"]
     tokenizer.train(
-        texts=texts,
+        texts=texts, vocab_size=30, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
-        vocab_size=30,
-        special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
     )
 
     with tempfile.TemporaryDirectory() as temp_dir:
@@ -211,9 +199,7 @@ def test_gpt_with_tokenizer():
     tokenizer = BPETokenizer()
     texts = ["hello world", "test integration"]
     tokenizer.train(
-        texts=texts,
+        texts=texts, vocab_size=50, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
-        vocab_size=50,
-        special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
     )
 
     vocab_size = tokenizer.get_vocab_size()
@@ -225,7 +211,7 @@ def test_gpt_with_tokenizer():
         "num_heads": 4,
         "num_layers": 2,
         "max_position_embeddings": 256,
-        "dropout": 0.1
+        "dropout": 0.1,
     }
 
     model = GPT(config)

llm/tests/tokenizers/test_bpe_tokenizer.py

@@ -19,7 +19,7 @@ class TestBPETokenizer:
             "Нейронные сети",
             "Машинное обучение",
             "Глубокое обучение",
-            "Трансформеры"
+            "Трансформеры",
         ]
 
     @pytest.fixture
@@ -29,7 +29,7 @@ class TestBPETokenizer:
         tokenizer.train(
             texts=sample_texts,
             vocab_size=100,
-            special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
+            special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"],
         )
         return tokenizer
 
@@ -44,7 +44,7 @@ class TestBPETokenizer:
         tokenizer.train(
             texts=sample_texts,
             vocab_size=50,
-            special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"]
+            special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"],
         )
 
         assert tokenizer.get_vocab_size() > 0
@@ -117,7 +117,9 @@ class TestBPETokenizer:
             loaded_vocab = loaded_tokenizer.get_vocab()
 
             assert original_vocab == loaded_vocab
-            assert trained_tokenizer.get_vocab_size() == loaded_tokenizer.get_vocab_size()
+            assert (
+                trained_tokenizer.get_vocab_size() == loaded_tokenizer.get_vocab_size()
+            )
 
             # Test encoding consistency
             text = sample_texts[0]