mirror of
https://github.com/pese-git/llm-arch-research.git
synced 2026-01-23 21:10:54 +00:00
Рефакторинг: единообразие оформления кода (пробелы, кавычки, пустые строки), без изменения логики по всему проекту.
This commit is contained in:
@@ -8,15 +8,15 @@ from llm.tokenizers import BaseTokenizer
|
||||
|
||||
class ConcreteTokenizer(BaseTokenizer):
|
||||
"""Concrete implementation for testing BaseTokenizer."""
|
||||
|
||||
|
||||
def train(self, texts: list, vocab_size: int = 1000, **kwargs):
|
||||
"""Dummy implementation for testing."""
|
||||
pass
|
||||
|
||||
|
||||
def encode(self, text: str, **kwargs) -> list:
|
||||
"""Dummy implementation for testing."""
|
||||
return [1, 2, 3]
|
||||
|
||||
|
||||
def decode(self, tokens: list, **kwargs) -> str:
|
||||
"""Dummy implementation for testing."""
|
||||
return "decoded text"
|
||||
@@ -24,33 +24,33 @@ class ConcreteTokenizer(BaseTokenizer):
|
||||
|
||||
class TestBaseTokenizer:
|
||||
"""Test cases for BaseTokenizer."""
|
||||
|
||||
|
||||
def test_initialization(self):
|
||||
"""Test that BaseTokenizer can be initialized through concrete class."""
|
||||
tokenizer = ConcreteTokenizer()
|
||||
assert tokenizer is not None
|
||||
assert tokenizer.vocab == {}
|
||||
assert tokenizer.vocab_size == 0
|
||||
|
||||
|
||||
def test_encode_implemented(self):
|
||||
"""Test that encode method works in concrete implementation."""
|
||||
tokenizer = ConcreteTokenizer()
|
||||
result = tokenizer.encode("test text")
|
||||
assert result == [1, 2, 3]
|
||||
|
||||
|
||||
def test_decode_implemented(self):
|
||||
"""Test that decode method works in concrete implementation."""
|
||||
tokenizer = ConcreteTokenizer()
|
||||
result = tokenizer.decode([1, 2, 3])
|
||||
assert result == "decoded text"
|
||||
|
||||
|
||||
def test_get_vocab_size(self):
|
||||
"""Test that get_vocab_size method works."""
|
||||
tokenizer = ConcreteTokenizer()
|
||||
tokenizer.vocab = {"a": 0, "b": 1, "c": 2}
|
||||
tokenizer.vocab_size = 3
|
||||
assert tokenizer.get_vocab_size() == 3
|
||||
|
||||
|
||||
def test_get_vocab(self):
|
||||
"""Test that get_vocab method works."""
|
||||
tokenizer = ConcreteTokenizer()
|
||||
|
||||
Reference in New Issue
Block a user