From 33670c674d59be0afcf5666c448404c3eab2da87 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 24 Sep 2024 13:41:36 +0700 Subject: [PATCH 1/5] Add TaskConfig.CustomConfig and pass them to backend --- flutter/assets/tasks.pbtxt | 8 ++++++++ flutter/cpp/proto/mlperf_task.proto | 13 ++++++++++++- flutter/lib/benchmark/benchmark.dart | 5 +++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index d9c05491a..b7e31935b 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -247,4 +247,12 @@ task { id: "stable_diffusion" name: "StableDiffusion" } + custom_config { + id: "stable_diffusion_seed" + value: "633994880" + } + custom_config { + id: "stable_diffusion_num_steps" + value: "20" + } } diff --git a/flutter/cpp/proto/mlperf_task.proto b/flutter/cpp/proto/mlperf_task.proto index 53e545e15..f9c2398b7 100644 --- a/flutter/cpp/proto/mlperf_task.proto +++ b/flutter/cpp/proto/mlperf_task.proto @@ -31,7 +31,7 @@ message MLPerfConfig { // Config of the mlperf tasks. // A task is basically a combination of models and a dataset. // -// Next ID: 11 +// Next ID: 12 message TaskConfig { // Must be unique in one task file. Ex: image_classification // used to match backend settings @@ -52,6 +52,7 @@ message TaskConfig { required string scenario = 7; required DatasetConfig datasets = 8; required ModelConfig model = 9; + repeated CustomConfig custom_config = 11; } // Datasets for a task @@ -107,3 +108,13 @@ message ModelConfig { // Number of detection classes if applicable optional int32 num_classes = 6; } + +// CustomConfig are task specific configuration. +// The TaskConfig.CustomConfig will be converted to BenchmarkSetting.CustomSetting and passed to the backend. +// To avoid name collision, the id should be prefixed with TaskConfig.id. +message CustomConfig { + // Id of this config. + required string id = 1; + // Value of this config. + required string value = 2; +} diff --git a/flutter/lib/benchmark/benchmark.dart b/flutter/lib/benchmark/benchmark.dart index 1ac2d00bd..511f45788 100644 --- a/flutter/lib/benchmark/benchmark.dart +++ b/flutter/lib/benchmark/benchmark.dart @@ -92,6 +92,11 @@ class Benchmark { setting: commonSettings, benchmarkSetting: benchmarkSettings, ); + // Convert TaskConfig.CustomConfig to BenchmarkSetting.CustomSetting + final customConfigs = taskConfig.customConfig + .map((e) => pb.CustomSetting(id: e.id, value: e.value)) + .toList(); + benchmarkSettings.customSetting.addAll(customConfigs); final uris = selectedDelegate.modelFile.map((e) => e.modelPath).toList(); final modelDirName = selectedDelegate.delegateName.replaceAll(' ', '_'); final backendModelPath = From e52d589cbc517e21d8c32162b56a7d5dac3a46e7 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 24 Sep 2024 16:37:10 +0700 Subject: [PATCH 2/5] Add CustomConfig for main.cc --- flutter/cpp/binary/main.cc | 10 +++--- flutter/cpp/proto/test.cc | 4 ++- flutter/cpp/utils.cc | 51 ++++++++++++++++++++++++---- flutter/cpp/utils.h | 5 +-- mobile_back_apple/dev-utils/Makefile | 1 + 5 files changed, 57 insertions(+), 14 deletions(-) diff --git a/flutter/cpp/binary/main.cc b/flutter/cpp/binary/main.cc index f2f3e51c9..89f5758ad 100644 --- a/flutter/cpp/binary/main.cc +++ b/flutter/cpp/binary/main.cc @@ -132,7 +132,7 @@ int Main(int argc, char *argv[]) { command_line += " " + backend_name + " " + benchmark_id; // Command Line Flags for mlperf. - std::string mode, scenario = "SingleStream", output_dir; + std::string mode, scenario = "SingleStream", output_dir, custom_config; int min_query_count = 100, min_duration_ms = 100, max_duration_ms = 10 * 60 * 1000, single_stream_expected_latency_ns = 1000000; @@ -157,8 +157,9 @@ int Main(int argc, char *argv[]) { "A hint used by the loadgen to pre-generate " "enough samples to meet the minimum test duration."), Flag::CreateFlag("output_dir", &output_dir, - "The output directory of mlperf.", Flag::kRequired)}); - + "The output directory of mlperf.", Flag::kRequired), + Flag::CreateFlag("custom_config", &custom_config, + "Custom config in form key1:val1,key2:val2.")}); // Command Line Flags for backend. std::unique_ptr backend; std::unique_ptr dataset; @@ -207,9 +208,8 @@ int Main(int argc, char *argv[]) { } } } - SettingList setting_list = - createSettingList(backend_setting, benchmark_id); + CreateSettingList(backend_setting, custom_config, benchmark_id); ExternalBackend *external_backend = new ExternalBackend( model_file_path, lib_path, setting_list, native_lib_path); diff --git a/flutter/cpp/proto/test.cc b/flutter/cpp/proto/test.cc index 07d4fd3fd..6a66969eb 100644 --- a/flutter/cpp/proto/test.cc +++ b/flutter/cpp/proto/test.cc @@ -139,9 +139,11 @@ int test_proto() { std::list benchmarks; benchmarks.push_back("image_classification"); benchmarks.push_back("image_classification_offline"); + std::string custom_config = "key1:val1,key2:val2"; for (auto benchmark_id : benchmarks) { // Convert to SettingList - SettingList setting_list = createSettingList(backend_setting, benchmark_id); + SettingList setting_list = + CreateSettingList(backend_setting, custom_config, benchmark_id); std::cout << "SettingList for " << benchmark_id << ":\n"; dumpSettingList(setting_list); diff --git a/flutter/cpp/utils.cc b/flutter/cpp/utils.cc index 965426f4d..7cea64c91 100644 --- a/flutter/cpp/utils.cc +++ b/flutter/cpp/utils.cc @@ -125,22 +125,61 @@ mlperf_backend_configuration_t CppToCSettings(const SettingList &settings) { return c_settings; } -SettingList createSettingList(const BackendSetting &backend_setting, - std::string benchmark_id) { +// Split the string by a given delimiter +std::vector _splitString(const std::string &str, char delimiter) { + std::vector tokens; + std::stringstream ss(str); + std::string token; + while (std::getline(ss, token, delimiter)) { + tokens.push_back(token); + } + return tokens; +} + +// Parse the key:value string list +std::unordered_map _parseKeyValueList( + const std::string &input) { + std::unordered_map keyValueMap; + std::vector pairs = _splitString(input, ','); // Split by comma + + for (const std::string &pair : pairs) { + std::vector keyValue = + _splitString(pair, ':'); // Split by colon + if (keyValue.size() == 2) { + keyValueMap[keyValue[0]] = keyValue[1]; + } else { + LOG(ERROR) << "Invalid key:value pair: " << pair; + } + } + return keyValueMap; +} + +// Create the setting list for backend +SettingList CreateSettingList(const BackendSetting &backend_setting, + const std::string &custom_config, + const std::string &benchmark_id) { SettingList setting_list; int setting_index = 0; - - for (auto setting : backend_setting.common_setting()) { + for (const auto &setting : backend_setting.common_setting()) { setting_list.add_setting(); (*setting_list.mutable_setting(setting_index)) = setting; setting_index++; } // Copy the benchmark specific settings - setting_index = 0; - for (auto bm_setting : backend_setting.benchmark_setting()) { + for (const auto &bm_setting : backend_setting.benchmark_setting()) { if (bm_setting.benchmark_id() == benchmark_id) { setting_list.mutable_benchmark_setting()->CopyFrom(bm_setting); + + auto parsed = _parseKeyValueList(custom_config); + for (const auto &kv : parsed) { + CustomSetting custom_setting = CustomSetting(); + custom_setting.set_id(kv.first); + custom_setting.set_value(kv.second); + setting_list.mutable_benchmark_setting()->mutable_custom_setting()->Add( + std::move(custom_setting)); + } + break; } } LOG(INFO) << "setting_list:" << std::endl << setting_list.DebugString(); diff --git a/flutter/cpp/utils.h b/flutter/cpp/utils.h index 08d7e86c8..17d590024 100644 --- a/flutter/cpp/utils.h +++ b/flutter/cpp/utils.h @@ -65,8 +65,9 @@ void DeleteBackendConfiguration(mlperf_backend_configuration_t *configs); mlperf_backend_configuration_t CppToCSettings(const SettingList &settings); -SettingList createSettingList(const BackendSetting &backend_setting, - std::string benchmark_id); +SettingList CreateSettingList(const BackendSetting &backend_setting, + const std::string &custom_config, + const std::string &benchmark_id); } // namespace mobile } // namespace mlperf diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 7ca5c27ca..a832d5b05 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -133,6 +133,7 @@ tflite-run-sd: --output_dir="${REPO_ROOT_DIR}/output" \ --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/sd-models" \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ + --custom_config="stable_diffusion_num_steps:20,stable_diffusion_seed:633994880" \ --input_tfrecord="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/coco_gen_full.tfrecord" \ --input_clip_model="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/clip_model_512x512.tflite" \ --min_query_count=5 From d84ec7a39918cf0ca931aa497b1ed9ad88c333b5 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 24 Sep 2024 16:56:09 +0700 Subject: [PATCH 3/5] Use seed and num_steps from CustomConfig for TFLite backend --- mobile_back_apple/dev-utils/Makefile | 2 +- .../cpp/backend_tflite/stable_diffusion_pipeline.cc | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index a832d5b05..32d196fb1 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -131,7 +131,7 @@ tflite-run-sd: bazel-bin/flutter/cpp/binary/main EXTERNAL stable_diffusion \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/sd-models" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/dynamic-sd-models" \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --custom_config="stable_diffusion_num_steps:20,stable_diffusion_seed:633994880" \ --input_tfrecord="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/stable_diffusion/coco_gen_full.tfrecord" \ diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc index 52d20b570..1003aaad3 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc @@ -64,6 +64,15 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create( SDBackendData* backend_data = new SDBackendData(); backendExists = true; + for (int i = 0; i < configs->count; ++i) { + if (strcmp(configs->keys[i], "stable_diffusion_seed") == 0) { + backend_data->seed = atoi(configs->values[i]); + } + if (strcmp(configs->keys[i], "stable_diffusion_num_steps") == 0) { + backend_data->num_steps = atoi(configs->values[i]); + } + } + // Load models from the provided directory path std::string text_encoder_path = std::string(model_path) + "/sd_text_encoder_dynamic.tflite"; From 685cf75d5964413529e1890af8777d00235307d5 Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 24 Sep 2024 17:02:22 +0700 Subject: [PATCH 4/5] Replace std::cout with LOG(INFO) --- .../cpp/backend_tflite/stable_diffusion_invoker.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc index 55ea8be07..94ac36ed2 100644 --- a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc +++ b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_invoker.cc @@ -24,15 +24,15 @@ StableDiffusionInvoker::StableDiffusionInvoker(SDBackendData* backend_data) : backend_data_(backend_data) {} std::vector StableDiffusionInvoker::invoke() { - std::cout << "Prompt encoding started" << std::endl; + LOG(INFO) << "Prompt encoding started"; auto encoded_text = encode_prompt(backend_data_->input_prompt_tokens); auto unconditional_encoded_text = encode_prompt(backend_data_->unconditional_tokens); - std::cout << "Diffusion process started" << std::endl; + LOG(INFO) << "Diffusion process started"; auto latent = diffusion_process(encoded_text, unconditional_encoded_text, backend_data_->num_steps, backend_data_->seed); - std::cout << "Image decoding started" << std::endl; + LOG(INFO) << "Image decoding started"; return decode_image(latent); } @@ -108,7 +108,7 @@ std::vector StableDiffusionInvoker::diffusion_process( auto alphas_prev = std::get<1>(alphas_tuple); for (int i = timesteps.size() - 1; i >= 0; --i) { - std::cout << "Step " << timesteps.size() - 1 - i << "\n"; + LOG(INFO) << "Step " << timesteps.size() - 1 - i; auto latent_prev = latent; auto t_emb = get_timestep_embedding(timesteps[i]); From 4482186966dcee8321e635ef182663d14bf72feb Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 24 Sep 2024 17:32:15 +0700 Subject: [PATCH 5/5] Format files --- flutter/cpp/proto/mlperf_task.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flutter/cpp/proto/mlperf_task.proto b/flutter/cpp/proto/mlperf_task.proto index f9c2398b7..5c1012490 100644 --- a/flutter/cpp/proto/mlperf_task.proto +++ b/flutter/cpp/proto/mlperf_task.proto @@ -110,7 +110,8 @@ message ModelConfig { } // CustomConfig are task specific configuration. -// The TaskConfig.CustomConfig will be converted to BenchmarkSetting.CustomSetting and passed to the backend. +// The TaskConfig.CustomConfig will be converted to +// BenchmarkSetting.CustomSetting and passed to the backend. // To avoid name collision, the id should be prefixed with TaskConfig.id. message CustomConfig { // Id of this config.