mirror of
https://github.com/pese-git/simple-llm.git
synced 2026-01-24 05:26:02 +00:00
feat: implement bpe algorithm
This commit is contained in:
13
tests/conftest.py
Normal file
13
tests/conftest.py
Normal file
@@ -0,0 +1,13 @@
|
||||
import pytest
|
||||
from simple_llm.tokenizer.simple_bpe import SimpleBPE
|
||||
from simple_llm.tokenizer.optimize_bpe import OptimizeBPE
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def large_text():
|
||||
"""Генерирует большой текст для тестирования"""
|
||||
return " ".join(["мама мыла раму"] * 1000)
|
||||
|
||||
@pytest.fixture(params=[SimpleBPE, OptimizeBPE])
|
||||
def bpe_class(request):
|
||||
"""Возвращает классы BPE для тестирования"""
|
||||
return request.param
|
||||
Reference in New Issue
Block a user