Finetuning siamese rpn on your dataset using author's weight and make it work better on your dataset. You can find author's weight here https://github.com/foolwood/DaSiamRPN
@inproceedings{Zhu_2018_ECCV, title={Distractor-aware Siamese Networks for Visual Object Tracking}, author={Zhu, Zheng and Wang, Qiang and Bo, Li and Wu, Wei and Yan, Junjie and Hu, Weiming}, booktitle={European Conference on Computer Vision}, year={2018} }
@InProceedings{Li_2018_CVPR, title = {High Performance Visual Tracking With Siamese Region Proposal Network}, author = {Li, Bo and Yan, Junjie and Wu, Wei and Zhu, Zheng and Hu, Xiaolin}, booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, year = {2018} }