mirror of
https://github.com/pese-git/llm-arch-research.git
synced 2026-01-23 21:10:54 +00:00
cfb4b6dfb1
- Add core Gemma model (architecture, attention, GeGLU, RoPE, RMSNorm, etc) - Add configs for training and generation: gemma_train.json, gemma_generate.json - Add Gemma notebook for exploratory analysis and demonstration - Add __init__.py for Gemma submodule - Update run_llm_experiment.py to support Gemma experiment configs test(gemma): add comprehensive unit tests for Gemma - Test forward pass (with/without cache) - Test autoregressive generation (greedy, top-k, top-p) - Test shape correctness and max sequence length errors - Test multi-layer stack and token embeddings docs: add documentation notebook for Gemma usage and analysis Closes: #issue (if applicable)
19 lines
499 B
JSON
19 lines
499 B
JSON
{
|
|
"bpe_tokenizer": "checkpoints/bpe_tokenizer.json",
|
|
"test_prompts": [
|
|
"Open weights",
|
|
"The Llama model is",
|
|
"Efficient transformers"
|
|
],
|
|
"model_config_path": "checkpoints/gemma-bpe/config.json",
|
|
"model_weights": "checkpoints/gemma-bpe/model.pt",
|
|
"generation": {
|
|
"max_new_tokens": 40,
|
|
"temperature": 0.8,
|
|
"do_sample": true,
|
|
"top_k": null,
|
|
"top_p": null
|
|
},
|
|
"log_path": "checkpoints/gemma_only_generation_logs.json"
|
|
}
|
|
|