| { | |
| "model": { | |
| "model_name_or_path": "google/gemma-2-2b-it", | |
| "lora_r": 8, | |
| "lora_alpha": 8.0, | |
| "lora_dropout": 0.0, | |
| "target_modules": [ | |
| "down_proj" | |
| ], | |
| "ctx_encoder_layer_idx": null, | |
| "latent_size": 512, | |
| "n_latent_queries": 208, | |
| "num_blocks": 8, | |
| "num_self_attn_per_block": 0, | |
| "shared_weights": false, | |
| "per_rank_gen": true, | |
| "per_layer_processing": true, | |
| "num_pre_head_layers": 1, | |
| "use_bias": true, | |
| "l1_reg_coef": 1.5, | |
| "n_layers": 26, | |
| "ctx_hidden_size": 2304, | |
| "d_in": { | |
| "down_proj": 9216 | |
| }, | |
| "d_out": { | |
| "down_proj": 2304 | |
| } | |
| } | |
| } |