Draft of the Benchmarking AI chapter - still pending some changes.

_quarto.yml

@@ -21,7 +21,7 @@ book:
     search: true
     pinned: true
     collapse: true
-
+    
   sidebar:
     search: true
 
@@ -121,6 +121,7 @@ format:
     citations-hover: false
     fig-width: 8
     fig-height: 6
+    number-depth: 3
 
 editor:
   render-on-save: true

references.bib

@@ -1,3 +1,137 @@
+@inproceedings{lin2014microsoft,
+  title={Microsoft coco: Common objects in context},
+  author={Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\'a}r, Piotr and Zitnick, C Lawrence},
+  booktitle={Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13},
+  pages={740--755},
+  year={2014},
+  organization={Springer}
+}
+
+@article{banbury2020benchmarking,
+  title={Benchmarking tinyml systems: Challenges and direction},
+  author={Banbury, Colby R and Reddi, Vijay Janapa and Lam, Max and Fu, William and Fazel, Amin and Holleman, Jeremy and Huang, Xinyuan and Hurtado, Robert and Kanter, David and Lokhmotov, Anton and others},
+  journal={arXiv preprint arXiv:2003.04821},
+  year={2020}
+}
+
+@inproceedings{hendrycks2021natural,
+  title={Natural adversarial examples},
+  author={Hendrycks, Dan and Zhao, Kevin and Basart, Steven and Steinhardt, Jacob and Song, Dawn},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  pages={15262--15271},
+  year={2021}
+}
+
+@inproceedings{xie2020adversarial,
+  title={Adversarial examples improve image recognition},
+  author={Xie, Cihang and Tan, Mingxing and Gong, Boqing and Wang, Jiang and Yuille, Alan L and Le, Quoc V},
+  booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},
+  pages={819--828},
+  year={2020}
+}
+
+@inproceedings{koh2021wilds,
+  title={Wilds: A benchmark of in-the-wild distribution shifts},
+  author={Koh, Pang Wei and Sagawa, Shiori and Marklund, Henrik and Xie, Sang Michael and Zhang, Marvin and Balsubramani, Akshay and Hu, Weihua and Yasunaga, Michihiro and Phillips, Richard Lanas and Gao, Irena and others},
+  booktitle={International Conference on Machine Learning},
+  pages={5637--5664},
+  year={2021},
+  organization={PMLR}
+}
+
+@inproceedings{antol2015vqa,
+  title={Vqa: Visual question answering},
+  author={Antol, Stanislaw and Agrawal, Aishwarya and Lu, Jiasen and Mitchell, Margaret and Batra, Dhruv and Zitnick, C Lawrence and Parikh, Devi},
+  booktitle={Proceedings of the IEEE international conference on computer vision},
+  pages={2425--2433},
+  year={2015}
+}
+
+@inproceedings{chu2021discovering,
+  title={Discovering multi-hardware mobile models via architecture search},
+  author={Chu, Grace and Arikan, Okan and Bender, Gabriel and Wang, Weijun and Brighton, Achille and Kindermans, Pieter-Jan and Liu, Hanxiao and Akin, Berkin and Gupta, Suyog and Howard, Andrew},
+  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
+  pages={3022--3031},
+  year={2021}
+}
+
+@article{david2021tensorflow,
+  title={Tensorflow lite micro: Embedded machine learning for tinyml systems},
+  author={David, Robert and Duke, Jared and Jain, Advait and Janapa Reddi, Vijay and Jeffries, Nat and Li, Jian and Kreeger, Nick and Nappier, Ian and Natraj, Meghna and Wang, Tiezhen and others},
+  journal={Proceedings of Machine Learning and Systems},
+  volume={3},
+  pages={800--811},
+  year={2021}
+}
+
+@article{warden2018speech,
+  title={Speech commands: A dataset for limited-vocabulary speech recognition},
+  author={Warden, Pete},
+  journal={arXiv preprint arXiv:1804.03209},
+  year={2018}
+}
+
+@inproceedings{adolf2016fathom,
+  title={Fathom: Reference workloads for modern deep learning methods},
+  author={Adolf, Robert and Rama, Saketh and Reagen, Brandon and Wei, Gu-Yeon and Brooks, David},
+  booktitle={2016 IEEE International Symposium on Workload Characterization (IISWC)},
+  pages={1--10},
+  year={2016},
+  organization={IEEE}
+}
+
+@article{coleman2017dawnbench,
+  title={Dawnbench: An end-to-end deep learning benchmark and competition},
+  author={Coleman, Cody and Narayanan, Deepak and Kang, Daniel and Zhao, Tian and Zhang, Jian and Nardi, Luigi and Bailis, Peter and Olukotun, Kunle and R{\'e}, Chris and Zaharia, Matei},
+  journal={Training},
+  volume={100},
+  number={101},
+  pages={102},
+  year={2017}
+}
+
+@article{mattson2020mlperf,
+  title={Mlperf training benchmark},
+  author={Mattson, Peter and Cheng, Christine and Diamos, Gregory and Coleman, Cody and Micikevicius, Paulius and Patterson, David and Tang, Hanlin and Wei, Gu-Yeon and Bailis, Peter and Bittorf, Victor and others},
+  journal={Proceedings of Machine Learning and Systems},
+  volume={2},
+  pages={336--349},
+  year={2020}
+}
+
+@article{brown2020language,
+  title={Language models are few-shot learners},
+  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
+  journal={Advances in neural information processing systems},
+  volume={33},
+  pages={1877--1901},
+  year={2020}
+}
+
+@article{devlin2018bert,
+  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
+  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
+  journal={arXiv preprint arXiv:1810.04805},
+  year={2018}
+}
+
+@inproceedings{ignatov2018ai,
+  title={Ai benchmark: Running deep neural networks on android smartphones},
+  author={Ignatov, Andrey and Timofte, Radu and Chou, William and Wang, Ke and Wu, Max and Hartley, Tim and Van Gool, Luc},
+  booktitle={Proceedings of the European Conference on Computer Vision (ECCV) Workshops},
+  pages={0--0},
+  year={2018}
+}
+
+@inproceedings{reddi2020mlperf,
+  title={Mlperf inference benchmark},
+  author={Reddi, Vijay Janapa and Cheng, Christine and Kanter, David and Mattson, Peter and Schmuelling, Guenther and Wu, Carole-Jean and Anderson, Brian and Breughe, Maximilien and Charlebois, Mark and Chou, William and others},
+  booktitle={2020 ACM/IEEE 47th Annual International Symposium on Computer Architecture (ISCA)},
+  pages={446--459},
+  year={2020},
+  organization={IEEE}
+}
+
 @misc{Thefutur92:online,
 author = {ARM.com},
 title = {The future is being built on Arm: Market diversification continues to drive strong royalty and licensing growth as ecosystem reaches quarter of a trillion chips milestone – Arm®},
@@ -514,3 +648,62 @@ @misc{Verma_2022
 @misc{Rao_2021, url={https://www.wevolver.com/article/tinyml-unlocks-new-possibilities-for-sustainable-development-technologies}, journal={www.wevolver.com}, author={Rao, Ravi}, year={2021}, month={Dec}} 
 
 
+@article{kiela2021dynabench,
+  title={Dynabench: Rethinking benchmarking in NLP},
+  author={Kiela, Douwe and Bartolo, Max and Nie, Yixin and Kaushik, Divyansh and Geiger, Atticus and Wu, Zhengxuan and Vidgen, Bertie and Prasad, Grusha and Singh, Amanpreet and Ringshia, Pratik and others},
+  journal={arXiv preprint arXiv:2104.14337},
+  year={2021}
+}
+
+@article{beyer2020we,
+  title={Are we done with imagenet?},
+  author={Beyer, Lucas and H{\'e}naff, Olivier J and Kolesnikov, Alexander and Zhai, Xiaohua and Oord, A{\"a}ron van den},
+  journal={arXiv preprint arXiv:2006.07159},
+  year={2020}
+}
+@article{gaviria2022dollar,
+  title={The Dollar Street Dataset: Images Representing the Geographic and Socioeconomic Diversity of the World},
+  author={Gaviria Rojas, William and Diamos, Sudnya and Kini, Keertan and Kanter, David and Janapa Reddi, Vijay and Coleman, Cody},
+  journal={Advances in Neural Information Processing Systems},
+  volume={35},
+  pages={12979--12990},
+  year={2022}
+}
+@article{xu2023demystifying,
+  title={Demystifying CLIP Data},
+  author={Xu, Hu and Xie, Saining and Tan, Xiaoqing Ellen and Huang, Po-Yao and Howes, Russell and Sharma, Vasu and Li, Shang-Wen and Ghosh, Gargi and Zettlemoyer, Luke and Feichtenhofer, Christoph},
+  journal={arXiv preprint arXiv:2309.16671},
+  year={2023}
+}
+@inproceedings{coleman2022similarity,
+  title={Similarity search for efficient active learning and search of rare concepts},
+  author={Coleman, Cody and Chou, Edward and Katz-Samuels, Julian and Culatana, Sean and Bailis, Peter and Berg, Alexander C and Nowak, Robert and Sumbaly, Roshan and Zaharia, Matei and Yalniz, I Zeki},
+  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
+  volume={36},
+  number={6},
+  pages={6402--6410},
+  year={2022}
+}
+@inproceedings{ribeiro2016should,
+  title={" Why should i trust you?" Explaining the predictions of any classifier},
+  author={Ribeiro, Marco Tulio and Singh, Sameer and Guestrin, Carlos},
+  booktitle={Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining},
+  pages={1135--1144},
+  year={2016}
+}
+@article{lundberg2017unified,
+  title={A unified approach to interpreting model predictions},
+  author={Lundberg, Scott M and Lee, Su-In},
+  journal={Advances in neural information processing systems},
+  volume={30},
+  year={2017}
+}
+@inproceedings{coleman2022similarity,
+  title={Similarity search for efficient active learning and search of rare concepts},
+  author={Coleman, Cody and Chou, Edward and Katz-Samuels, Julian and Culatana, Sean and Bailis, Peter and Berg, Alexander C and Nowak, Robert and Sumbaly, Roshan and Zaharia, Matei and Yalniz, I Zeki},
+  booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},
+  volume={36},
+  number={6},
+  pages={6402--6410},
+  year={2022}
+}