feat: download all resources (models, dataset) on-demand

flutter/android/android.mk

@@ -90,7 +90,7 @@ flutter_android_apk_test_main_path=${FLUTTER_ANDROID_APK_FOLDER}/${FLUTTER_ANDRO
 .PHONY: flutter/android/test-apk/main
 flutter/android/test-apk/main:
 	mkdir -p $$(dirname ${flutter_android_apk_test_main_path})
-	flutter_android_apk_test_perf_arg=$$(printf enable-perf-test=${PERF_TEST} | base64) && \
+	flutter_android_apk_test_perf_arg=$$(printf PERF_TEST=${PERF_TEST} | base64) && \
 		cd flutter/android && \
 		./gradlew app:assembleDebug \
 		-Ptarget=integration_test/first_test.dart \

flutter/flutter.mk

@@ -159,7 +159,7 @@ flutter_test_device_arg=--device-id "${FLUTTER_TEST_DEVICE}"
 else
 flutter_test_device_arg=
 endif
-flutter_perf_test_arg=--dart-define=enable-perf-test=${PERF_TEST}
+flutter_perf_test_arg=--dart-define=PERF_TEST=${PERF_TEST}
 .PHONY: flutter/test/integration
 flutter/test/integration:
 	cd flutter && ${_start_args} \

flutter/integration_test/first_test.dart

@@ -14,11 +14,6 @@ import 'expected_accuracy.dart';
 import 'expected_throughput.dart';
 import 'utils.dart';
 
-const enablePerfTest = bool.fromEnvironment(
-  'enable-perf-test',
-  defaultValue: false,
-);
-
 void main() {
   final binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized();
   binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
@@ -30,7 +25,7 @@ void main() {
     StoreConstants.testMinDuration: 1,
     StoreConstants.testMinQueryCount: 4,
   };
-  if (enablePerfTest) {
+  if (DartDefine.perfTestEnabled) {
     prefs[StoreConstants.testMinDuration] = 15;
     prefs[StoreConstants.testMinQueryCount] = 64;
     prefs[StoreConstants.testCooldownDuration] = 2;
@@ -72,7 +67,7 @@ void checkTasks(ExtendedResult extendedResult) {
     expect(benchmarkResult.performanceRun!.throughput, isNotNull);
 
     checkAccuracy(benchmarkResult);
-    if (enablePerfTest) {
+    if (DartDefine.perfTestEnabled) {
       checkThroughput(benchmarkResult, extendedResult.environmentInfo);
     }
   }

flutter/integration_test/utils.dart

@@ -68,6 +68,10 @@ Future<void> runBenchmarks(WidgetTester tester) async {
   const downloadTimeout = 20 * 60; // 20 minutes
   const runBenchmarkTimeout = 30 * 60; // 30 minutes
 
+  final state = tester.state(find.byType(MaterialApp));
+  final benchmarkState = state.context.read<BenchmarkState>();
+  await benchmarkState.loadResources(downloadMissing: true);
+
   var goButtonIsPresented =
       await waitFor(tester, downloadTimeout, const Key(WidgetKeys.goButton));
 

flutter/lib/app_constants.dart

@@ -7,6 +7,9 @@ class DartDefine {
       bool.fromEnvironment('FIREBASE_CRASHLYTICS_ENABLED', defaultValue: false);
   static const isFastMode =
       bool.fromEnvironment('FAST_MODE', defaultValue: false);
+
+  static const perfTestEnabled =
+      bool.fromEnvironment('PERF_TEST', defaultValue: false);
 }
 
 class WidgetKeys {

flutter/lib/benchmark/benchmark.dart

@@ -154,13 +154,11 @@ class BenchmarkStore {
 
   List<Resource> listResources({
     required List<BenchmarkRunMode> modes,
-    bool skipInactive = false,
+    required List<Benchmark> benchmarks,
   }) {
     final result = <Resource>[];
 
     for (final b in benchmarks) {
-      if (skipInactive && !b.isActive) continue;
-
       for (var mode in modes) {
         final dataset = mode.chooseDataset(b.taskConfig);
         final data = Resource(

flutter/lib/benchmark/state.dart

@@ -22,7 +22,6 @@ import 'package:mlperfbench/state/task_runner.dart';
 import 'package:mlperfbench/store.dart';
 
 enum BenchmarkStateEnum {
-  downloading,
   waiting,
   running,
   aborting,
@@ -121,7 +120,7 @@ class BenchmarkState extends ChangeNotifier {
   // ignore: avoid_void_async
   void deferredLoadResources() async {
     try {
-      await loadResources();
+      await loadResources(downloadMissing: false);
     } catch (e, trace) {
       print("can't load resources: $e");
       print(trace);
@@ -132,22 +131,24 @@ class BenchmarkState extends ChangeNotifier {
     }
   }
 
-  Future<void> loadResources() async {
+  Future<void> loadResources({required bool downloadMissing}) async {
     final newAppVersion =
         '${BuildInfoHelper.info.version}+${BuildInfoHelper.info.buildNumber}';
     var needToPurgeCache = _store.previousAppVersion != newAppVersion;
     _store.previousAppVersion = newAppVersion;
 
     await Wakelock.enable();
-    print('start loading resources');
+    print('Start loading resources with downloadMissing=$downloadMissing');
+    final resources = _benchmarkStore.listResources(
+      modes: [taskRunner.perfMode, taskRunner.accuracyMode],
+      benchmarks: benchmarks,
+    );
     await resourceManager.handleResources(
-      _benchmarkStore.listResources(
-        modes: [taskRunner.perfMode, taskRunner.accuracyMode],
-        skipInactive: false,
-      ),
+      resources,
       needToPurgeCache,
+      downloadMissing,
     );
-    print('finished loading resources');
+    print('Finished loading resources with downloadMissing=$downloadMissing');
     error = null;
     stackTrace = null;
     taskConfigFailedToLoad = false;
@@ -216,7 +217,6 @@ class BenchmarkState extends ChangeNotifier {
   }
 
   BenchmarkStateEnum get state {
-    if (!resourceManager.done) return BenchmarkStateEnum.downloading;
     switch (_doneRunning) {
       case null:
         return BenchmarkStateEnum.waiting;

flutter/lib/l10n/app_en.arb

@@ -7,6 +7,7 @@
   "menuHome": "MLPerf Mobile",
   "menuHistory": "History",
   "menuSettings": "Settings",
+  "menuResources": "Resources",
   "menuAbout": "About",
   "menuProfile": "Profile",
   "menuSignIn": "Sign In",
@@ -78,7 +79,8 @@
   "settingsTaskConfigError": "Path to config is invalid:",
   "settingsTaskDataFolderTitle": "Data folder",
   "settingsClearCache": "Clear cache",
-  "settingsClearCacheConfirm": "All loaded resources will be deleted and downloaded again. Continue?",
+  "settingsClearCacheConfirm": "All downloaded resources will be deleted. Continue?",
+  "settingsClearCacheFinished": "Cache cleared.",
   "settingsUnableSpecifyConfiguration": "Could not specify until benchmarks is running or content is loading",
 
   "dialogTitleWarning": "Warning",
@@ -114,6 +116,10 @@
   "benchInfoLanguageProcessingDesc": "Question Answering finds the best answer to an input question based on a body of text, and is commonly employed in applications such as virtual assistants and chatbots. The reference model, MobileBERT, is evaluated on the Stanford Question Answering Dataset (SQUAD) v1.1 Dev-mini. The task requires a minimum F1-score of 87.4% (93% of FP32 F1-score of 93.08%).\n\nMobileBERT is a streamlined, mobile-optimized version of the larger BERT_LARGE network. It features bottleneck structures and a carefully designed balance between self-attention and feed-forward networks. While BERT is task-agnostic and can be applied to various downstream natural language processing tasks, the MobileBERT variant used in MLPerf is specifically fine-tuned for question answering.",
   "benchInfoSuperResolutionDesc": "Image Super Resolution (SR) upscales a lower resolution input into a higher resolution output image, enhancing the quality and detail. It is a common task in many mobile applications such as digital zoom. The reference model, EDSR F32B5, is a lightweight member of the Enhanced Deep Super Resolution (EDSR) family that is trained for 2X super resolution on the DIV2K dataset with bicubic downsampling and tested on the OpenSR test-set which comprises 25 selected 1920x1080 HDR images. The benchmark requires a minimum accuracy of 33 dB Peak Signal to Noise Ratio (PSNR) relative to a 33.58 dB accuracy with FP32.\n\nThe EDSR family of models demonstrated excellent performance by winning a super resolution challenge at CVPR 2017. The EDSR F32B5 reference model features five EDSR blocks, each with 32 feature maps. The EDSR block is a simple residual block consisting of a residual connection on one branch and a convolution-ReLU-convolution on the other branch. The final upsampling layer is a depth-to-space operator, which facilitates the x2 super resolution process.",
 
+  "resourceDownload": "Download",
+  "resourceClear": "Clear",
+  "resourceChecking": "Checking download status",
+  "resourceDownloading": "Downloading",
   "resourceErrorMessage": "Some resources failed to load.\nIf you didn't change config from default you can try clearing the cache.\nIf you use a custom configuration file ensure that it has correct structure or switch back to default config.",
   "resourceErrorSelectTaskFile": "Update task configuration",
   "resourceErrorCurrentConfig": "Current task config file: ",

flutter/lib/resources/cache_manager.dart

@@ -28,7 +28,7 @@ class CacheManager {
     return archiveFilePath;
   }
 
-  Future<void> deleteLoadedResources(List<String> nonRemovableResources,
+  Future<void> deleteLoadedResources(List<String> excludes,
       [int atLeastDaysOld = 0]) async {
     final directory = Directory(loadedResourcesDir);
 
@@ -40,20 +40,20 @@ class CacheManager {
       final relativePath = file.path
           .replaceAll('\\', '/')
           .substring(loadedResourcesDir.length + 1);
-      var nonRemovable = false;
-      for (var resource in nonRemovableResources) {
+      var keep = false;
+      for (var resource in excludes) {
         // relativePath.startsWith(resource): if we want to preserve a folder resource
         // resource.startsWith(relativePath): if we want to preserve a file resource
         //   for example:
         //   we are checking folder 'github.com'
         //   resource is 'github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/ade20k'
         if (relativePath.startsWith(resource) ||
             resource.startsWith(relativePath)) {
-          nonRemovable = true;
+          keep = true;
           break;
         }
       }
-      if (nonRemovable) continue;
+      if (keep) continue;
       if (atLeastDaysOld > 0) {
         var stat = await file.stat();
         if (DateTime.now().difference(stat.modified).inDays < atLeastDaysOld) {
@@ -81,8 +81,11 @@ class CacheManager {
     return deleteLoadedResources(currentResources, atLeastDaysOld);
   }
 
-  Future<void> cache(List<String> urls,
-      void Function(double, String) reportProgress, bool purgeOldCache) async {
+  Future<void> cache(
+      List<String> urls,
+      void Function(double, String) reportProgress,
+      bool purgeOldCache,
+      bool downloadMissing) async {
     final resourcesToDownload = <String>[];
     _resourcesMap = {};
 
@@ -106,8 +109,9 @@ class CacheManager {
 
       continue;
     }
-    await _download(resourcesToDownload, reportProgress);
-
+    if (downloadMissing) {
+      await _download(resourcesToDownload, reportProgress);
+    }
     if (purgeOldCache) {
       await purgeOutdatedCache(_oldFilesAgeInDays);
     }

flutter/lib/resources/resource_manager.dart

@@ -49,7 +49,8 @@ class ResourceManager {
       return resourceSystemPath;
     }
     if (isInternetResource(uri)) {
-      return cacheManager.get(uri)!;
+      final resourceSystemPath = cacheManager.get(uri);
+      return resourceSystemPath ?? '';
     }
     if (File(uri).isAbsolute) {
       return uri;
@@ -87,26 +88,16 @@ class ResourceManager {
     return _dataPrefix;
   }
 
-  Future<bool> isResourceExist(String? uri) async {
-    if (uri == null) return false;
-
-    final path = get(uri);
-
-    return path == '' ||
-        await File(path).exists() ||
-        await Directory(path).exists();
-  }
-
   Future<bool> isChecksumMatched(String filePath, String md5Checksum) async {
     var fileStream = File(filePath).openRead();
     final checksum = (await md5.bind(fileStream).first).toString();
     return checksum == md5Checksum;
   }
 
-  Future<void> handleResources(
-      List<Resource> resources, bool purgeOldCache) async {
+  Future<void> handleResources(List<Resource> resources, bool purgeOldCache,
+      bool downloadMissing) async {
     _loadingPath = '';
-    _loadingProgress = 0.0;
+    _loadingProgress = 0.001;
     _done = false;
     _onUpdate();
 
@@ -121,12 +112,16 @@ class ResourceManager {
     }
 
     final internetPaths = internetResources.map((e) => e.path).toList();
-    await cacheManager.cache(internetPaths,
-        (double currentProgress, String currentPath) {
-      _loadingProgress = currentProgress;
-      _loadingPath = currentPath;
-      _onUpdate();
-    }, purgeOldCache);
+    await cacheManager.cache(
+      internetPaths,
+      (double currentProgress, String currentPath) {
+        _loadingProgress = currentProgress;
+        _loadingPath = currentPath;
+        _onUpdate();
+      },
+      purgeOldCache,
+      downloadMissing,
+    );
 
     final checksumFailed = await validateResourcesChecksum(resources);
     if (checksumFailed.isNotEmpty) {
@@ -137,6 +132,8 @@ class ResourceManager {
     // delete downloaded archives to free up disk space
     await cacheManager.deleteArchives(internetPaths);
 
+    _loadingPath = '';
+    _loadingProgress = 1.0;
     _done = true;
     _onUpdate();
   }
@@ -172,15 +169,30 @@ class ResourceManager {
     resultManager = await ResultManager.create(applicationDirectory);
   }
 
-  Future<List<String>> validateResourcesExist(List<Resource> resources) async {
+  // Returns a map of { true: [existedResources], false: [missingResources] }
+  Future<Map<bool, List<String>>> validateResourcesExist(
+      List<Resource> resources) async {
     final missingResources = <String>[];
+    final existedResources = <String>[];
     for (var r in resources) {
-      if (!await isResourceExist(r.path)) {
-        final resolvedPath = get(r.path);
-        missingResources.add(resolvedPath);
+      final resolvedPath = get(r.path);
+      if (resolvedPath.isEmpty) {
+        missingResources.add(r.path);
+      } else {
+        final isResourceExist = await File(resolvedPath).exists() ||
+            await Directory(resolvedPath).exists();
+        if (isResourceExist) {
+          existedResources.add(r.path);
+        } else {
+          missingResources.add(r.path);
+        }
       }
     }
-    return missingResources;
+    final result = {
+      false: missingResources,
+      true: existedResources,
+    };
+    return result;
   }
 
   Future<List<Resource>> validateResourcesChecksum(

flutter/lib/resources/validation_helper.dart

@@ -18,6 +18,9 @@ class ValidationHelper {
     required this.selectedRunModes,
   });
 
+  List<Benchmark> get activeBenchmarks =>
+      benchmarkStore.benchmarks.where((e) => e.isActive).toList();
+
   Future<String> validateExternalResourcesDirectory(
       String errorDescription) async {
     final dataFolderPath = resourceManager.getDataFolder();
@@ -28,8 +31,11 @@ class ValidationHelper {
       return 'Data folder does not exist';
     }
     final resources = benchmarkStore.listResources(
-        modes: selectedRunModes, skipInactive: true);
-    final missing = await resourceManager.validateResourcesExist(resources);
+      modes: selectedRunModes,
+      benchmarks: activeBenchmarks,
+    );
+    final result = await resourceManager.validateResourcesExist(resources);
+    final missing = result[false] ?? [];
     if (missing.isEmpty) return '';
 
     return errorDescription +
@@ -38,7 +44,9 @@ class ValidationHelper {
 
   Future<String> validateOfflineMode(String errorDescription) async {
     final resources = benchmarkStore.listResources(
-        modes: selectedRunModes, skipInactive: true);
+      modes: selectedRunModes,
+      benchmarks: activeBenchmarks,
+    );
     final internetResources = filterInternetResources(resources);
     if (internetResources.isEmpty) return '';
 
@@ -47,4 +55,14 @@ class ValidationHelper {
             .mapIndexed((i, element) => '\n${i + 1}) $element')
             .join();
   }
+
+  Future<Map<bool, List<String>>> validateResourcesExist(
+      Benchmark benchmark, BenchmarkRunMode mode) async {
+    final resources = benchmarkStore.listResources(
+      modes: [mode],
+      benchmarks: [benchmark],
+    );
+    final result = await resourceManager.validateResourcesExist(resources);
+    return result;
+  }
 }