@article{qian2024soft,
  title = {Recasting Generic Pretrained Vision Transformers As Object-Centric Scene Encoders For Manipulation Policies },
  author = {Jianing Qian and Anastasios Panagopoulos and Dinesh Jayaraman},
  abstract = {Generic re-usable pre-trained image representation encoders have become a standard component of methods for many computer vision tasks. As visual representations for robots however, their utility has been limited, leading to a recent wave of efforts to pre-train robotics-specific image encoders that are better suited to robotic tasks than their generic counterparts. We propose SOFT, a wrapper around pre-trained vision transformer PVT models that bridges this gap without any further training. Rather than construct representations out of only the final layer activations, SOFT individuates and locates object-like entities from PVT attentions, and describes them with PVT activations, producing an object-centric representation. Across standard choices of generic pre-trained vision transformers PVT, we demonstrate in each case that policies trained on SOFT(PVT) far outstrip standard PVT representations for manipulation tasks in simulated and real settings, approaching the state-of-the-art robotics-aware representations.},
  url = {https://sites.google.com/view/robot-soft/},
  journal = {ICRA},
  year = {2024},
  month = {May},
  pub_type = {conference},
  url_pdf = {/publication/qian-2024-soft/qian-2024-soft.pdf},
  url_project = {https://sites.google.com/view/robot-soft/},
  url_arxiv = {https://arxiv.org/abs/2405.15916},
}