Migrate policy to PolicyProcessorPipeline system (#1)

- Migrate policy to PolicyProcessorPipeline system (413c21db4730b1a293c038748b3291c966dfc1b6)

README.md

@@ -0,0 +1,62 @@
+---
+datasets: unknown
+library_name: lerobot
+license: apache-2.0
+model_name: xvla
+pipeline_tag: robotics
+tags:
+- xvla
+- lerobot
+- robotics
+---
+
+# Model Card for xvla
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+_Model type not recognized — please update this template._
+
+
+This policy has been trained and pushed to the Hub using [LeRobot](https://github.com/huggingface/lerobot).
+See the full documentation at [LeRobot Docs](https://huggingface.co/docs/lerobot/index).
+
+---
+
+## How to Get Started with the Model
+
+For a complete walkthrough, see the [training guide](https://huggingface.co/docs/lerobot/il_robots#train-a-policy).
+Below is the short version on how to train and run inference/eval:
+
+### Train from scratch
+
+```bash
+lerobot-train \
+  --dataset.repo_id=${HF_USER}/<dataset> \
+  --policy.type=act \
+  --output_dir=outputs/train/<desired_policy_repo_id> \
+  --job_name=lerobot_training \
+  --policy.device=cuda \
+  --policy.repo_id=${HF_USER}/<desired_policy_repo_id>
+  --wandb.enable=true
+```
+
+_Writes checkpoints to `outputs/train/<desired_policy_repo_id>/checkpoints/`._
+
+### Evaluate the policy/run inference
+
+```bash
+lerobot-record \
+  --robot.type=so100_follower \
+  --dataset.repo_id=<hf_user>/eval_<dataset> \
+  --policy.path=<hf_user>/<desired_policy_repo_id> \
+  --episodes=10
+```
+
+Prefix the dataset repo with **eval\_** and supply `--policy.path` pointing to a local or hub checkpoint.
+
+---
+
+## Model Details
+
+- **License:** apache-2.0

config.json

@@ -0,0 +1,213 @@
+{
+    "type": "xvla",
+    "n_obs_steps": 1,
+    "input_features": {
+        "observation.images.image": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.images.image2": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                256,
+                256
+            ]
+        },
+        "observation.state": {
+            "type": "STATE",
+            "shape": [
+                8
+            ]
+        },
+        "observation.images.empty_camera_0": {
+            "type": "VISUAL",
+            "shape": [
+                3,
+                224,
+                224
+            ]
+        }
+    },
+    "output_features": {
+        "action": {
+            "type": "ACTION",
+            "shape": [
+                20
+            ]
+        }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "push_to_hub": true,
+    "repo_id": null,
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "chunk_size": 30,
+    "n_action_steps": 30,
+    "normalization_mapping": {
+        "STATE": "IDENTITY",
+        "ACTION": "IDENTITY",
+        "VISUAL": "MEAN_STD"
+    },
+    "florence_config": {
+        "model_type": "florence2",
+        "bos_token_id": 0,
+        "eos_token_id": 2,
+        "ignore_index": -100,
+        "pad_token_id": 1,
+        "projection_dim": 1024,
+        "text_config": {
+            "vocab_size": 51289,
+            "activation_dropout": 0.1,
+            "activation_function": "gelu",
+            "add_bias_logits": false,
+            "add_final_layer_norm": false,
+            "attention_dropout": 0.1,
+            "bos_token_id": 0,
+            "classif_dropout": 0.1,
+            "classifier_dropout": 0.0,
+            "d_model": 1024,
+            "decoder_attention_heads": 16,
+            "decoder_ffn_dim": 4096,
+            "decoder_layerdrop": 0.0,
+            "decoder_layers": 12,
+            "decoder_start_token_id": 2,
+            "dropout": 0.1,
+            "early_stopping": true,
+            "encoder_attention_heads": 16,
+            "encoder_ffn_dim": 4096,
+            "encoder_layerdrop": 0.0,
+            "encoder_layers": 12,
+            "eos_token_id": 2,
+            "forced_eos_token_id": 2,
+            "forced_bos_token_id": 0,
+            "gradient_checkpointing": false,
+            "init_std": 0.02,
+            "is_encoder_decoder": true,
+            "label2id": {
+                "LABEL_0": 0,
+                "LABEL_1": 1,
+                "LABEL_2": 2
+            },
+            "max_position_embeddings": 4096,
+            "no_repeat_ngram_size": 3,
+            "normalize_before": false,
+            "num_hidden_layers": 12,
+            "pad_token_id": 1,
+            "scale_embedding": false,
+            "num_beams": 3
+        },
+        "vision_config": {
+            "model_type": "davit",
+            "drop_path_rate": 0.1,
+            "patch_size": [
+                7,
+                3,
+                3,
+                3
+            ],
+            "patch_stride": [
+                4,
+                2,
+                2,
+                2
+            ],
+            "patch_padding": [
+                3,
+                1,
+                1,
+                1
+            ],
+            "patch_prenorm": [
+                false,
+                true,
+                true,
+                true
+            ],
+            "enable_checkpoint": false,
+            "dim_embed": [
+                256,
+                512,
+                1024,
+                2048
+            ],
+            "num_heads": [
+                8,
+                16,
+                32,
+                64
+            ],
+            "num_groups": [
+                8,
+                16,
+                32,
+                64
+            ],
+            "depths": [
+                1,
+                1,
+                9,
+                1
+            ],
+            "window_size": 12,
+            "projection_dim": 1024,
+            "visual_temporal_embedding": {
+                "type": "COSINE",
+                "max_temporal_embeddings": 100
+            },
+            "image_pos_embed": {
+                "type": "learned_abs_2d",
+                "max_pos_embeddings": 50
+            },
+            "image_feature_source": [
+                "spatial_avg_pool",
+                "temporal_avg_pool"
+            ]
+        },
+        "vocab_size": 51289,
+        "torch_dtype": "float32",
+        "is_encoder_decoder": true
+    },
+    "tokenizer_name": "facebook/bart-large",
+    "tokenizer_max_length": 1024,
+    "tokenizer_padding_side": "right",
+    "pad_language_to": "max_length",
+    "hidden_size": 1024,
+    "depth": 24,
+    "num_heads": 16,
+    "mlp_ratio": 4.0,
+    "num_domains": 30,
+    "len_soft_prompts": 32,
+    "dim_time": 32,
+    "max_len_seq": 512,
+    "use_hetero_proj": false,
+    "action_mode": "ee6d",
+    "num_denoising_steps": 10,
+    "use_proprio": true,
+    "max_state_dim": 20,
+    "domain_feature_key": null,
+    "resize_imgs_with_padding": [
+        224,
+        224
+    ],
+    "num_image_views": 3,
+    "empty_cameras": 1,
+    "optimizer_lr": 0.0001,
+    "optimizer_betas": [
+        0.9,
+        0.95
+    ],
+    "optimizer_eps": 1e-08,
+    "optimizer_weight_decay": 0.0001,
+    "optimizer_grad_clip_norm": 10.0,
+    "scheduler_warmup_steps": 1000,
+    "scheduler_decay_steps": 30000,
+    "scheduler_decay_lr": 2.5e-06
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f05bc0fab1c9523d7f5d6b41a651313641ca2227a88822249845da8e20e036c9
+size 3519073692

policy_postprocessor.json

@@ -0,0 +1,31 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "unnormalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              20
+            ]
+          }
+        },
+        "norm_map": {
+          "STATE": "IDENTITY",
+          "ACTION": "IDENTITY",
+          "VISUAL": "MEAN_STD"
+        }
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_preprocessor.json

@@ -0,0 +1,82 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "tokenizer_processor",
+      "config": {
+        "max_length": 1024,
+        "task_key": "task",
+        "padding_side": "right",
+        "padding": "max_length",
+        "truncation": true,
+        "tokenizer_name": "facebook/bart-large"
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    },
+    {
+      "registry_name": "normalizer_processor",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.images.image": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.images.image2": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              256,
+              256
+            ]
+          },
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              8
+            ]
+          },
+          "observation.images.empty_camera_0": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              224,
+              224
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              20
+            ]
+          }
+        },
+        "norm_map": {
+          "STATE": "IDENTITY",
+          "ACTION": "IDENTITY",
+          "VISUAL": "MEAN_STD"
+        }
+      }
+    }
+  ]
+}