@article{ma2023vip,
  title = "{VIP}: Towards Universal Visual Reward and Representation
                   via {Value-Implicit} {Pre-Training}",
  author = "Ma, Yecheng Jason and Sodhani, Shagun and Jayaraman, Dinesh
                   and Bastani, Osbert and Kumar, Vikash and Zhang, Amy",
  journal = {ICLR},
  year = {2023},
  month = {Apr},
  pub_type = {conference},
  award = {Spotlight Presentation, 5% accept rate},
  abstract = {Reward and representation learning are two long-standing challenges for learning an expanding set of robot manipulation skills from sensory observations. Given the inherent cost and scarcity of in-domain, task-specific robot data, learning from large, diverse, offline human videos has emerged as a promising path towards acquiring a generally useful visual representation for control; however, how these human videos can be used for general-purpose reward learning remains an open question. We introduce Value-Implicit Pre-training (VIP), a self-supervised pre-trained visual representation capable of generating dense and smooth reward functions for unseen robotic tasks. VIP casts representation learning from human videos as an offline goal-conditioned reinforcement learning problem and derives a self-supervised dual goal-conditioned value-function objective that does not depend on actions, enabling pre-training on unlabeled human videos. Theoretically, VIP can be understood as a novel implicit time contrastive objective that generates a temporally smooth embedding, enabling the value function to be implicitly defined via the embedding distance, which can then be used to construct the reward for any goal-image specified downstream task. Trained on large-scale Ego4D human videos and without any fine-tuning on in-domain, task-specific data, VIP's frozen representation can provide dense visual reward for an extensive set of simulated and real-robot tasks, enabling diverse reward-based visual control methods and significantly outperforming all prior pre-trained representations. Notably, VIP can enable simple, few-shot offline RL on a suite of real-world robot tasks with as few as 20 trajectories.},
  url_pdf = {/publication/ma-2023-vip/ma-2023-vip.pdf},
  url_project = {https://sites.google.com/view/vip-rl},
  url_code = {https://github.com/facebookresearch/vip},
  url = {https://arxiv.org/abs/2210.00030},
  url_arxiv = {https://arxiv.org/abs/2210.00030},
}