llm-arch-research/experiments/llm_only/configs/gemma_train.json

{
    "bpe_tokenizer": "checkpoints/bpe_tokenizer.json",
    "bpe_vocab_size": 1000,
    "bpe_special_tokens": ["<pad>", "<unk>", "<bos>", "<eos>"],
    "test_prompts": ["Open source AI", "What is Llama?"],
    "model_config": {
      "vocab_size": null,
      "embed_dim": 256,
      "num_q_heads": 4,
      "num_kv_heads": 2,
      "head_size": 64,
      "num_layers": 4,
      "max_position_embeddings": 512,
      "num_experts": 8,
      "top_k_experts": 2,
      "window_size": 16,
      "dropout": 0.1
    },
    "model_weights": "checkpoints/gemma-bpe/model.pt",
    "model_config_path": "checkpoints/gemma-bpe/config.json",
    "training": {
      "learning_rate": 0.0003,
      "batch_size": 2,
      "num_epochs": 3,
      "warmup_steps": 50
    },
    "log_path": "checkpoints/gemma_only_training_logs.json"
  }