feat: implement bpe algorithm

pyproject.toml

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "simple-llm"
+version = "0.1.0"
+description = "Simple BPE tokenizer implementation for educational purposes"
+readme = "README.md"
+authors = [
+    { name = "Sergey Penkovsky", email = "sergey.penkovsky@gmail.com" },
+]
+license = { text = "MIT" }
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Education",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+requires-python = ">=3.8"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/pese-git/simple-llm"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["simple_llm*"]
+exclude = ["tests*", "example*"]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0",
+    "black>=23.0",
+]