From d6dd9a06905466f1d5e7496912b653d8c737cd52 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Fri, 14 Aug 2026 22:01:31 +0800 Subject: [PATCH 1/8] feat(sdk): add MTP coverage to bench and expose accept-rate in JSON Wire the typical MTP user invocation (draft-mtp / gemma-4-26B target + RachidAR assistant draft / --draft-tokens 3) into the QDC bench matrix, gated to SC8480XP, and surface draft acceptance in the per-cell JSON so the aggregate step can pair each spec cell with its no-spec baseline and show accept% + decode-tps uplift in a standalone MTP table. Random-token prefill would collapse draft acceptance to ~0% on any real target/draft pair, so spec rows are forced through --prompt-file with a pre-existing per-ctx fixture (plus a new sample_prompt_8192.txt to cover the 8k ctx the MTP row exercises). Signed-off-by: Mengsheng Wu --- .github/workflows/bench.yml | 8 + sdk/benchmark/benchmark.c | 84 ++- sdk/benchmark/qdc/bench-models.json | 22 + sdk/benchmark/qdc/linux/run_linux.sh | 2 +- .../qdc/prompts/sample_prompt_8192.txt | 539 ++++++++++++++++++ sdk/benchmark/qdc/run_qdc_jobs.py | 109 +++- sdk/benchmark/qdc/tests/test_bench.py | 2 +- sdk/benchmark/qdc/windows/run_windows.ps1 | 41 +- 8 files changed, 771 insertions(+), 36 deletions(-) create mode 100644 sdk/benchmark/qdc/prompts/sample_prompt_8192.txt diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9493a9e61..f3754fa9e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -85,6 +85,14 @@ jobs: bench: name: ${{ matrix.device }} · ${{ matrix.model.plugin }} · ${{ matrix.model.name }} needs: [build-sdk, load-models] + # Catalog-only rows (empty devices, e.g. spec draft models) never run + # standalone — they're pulled by the target row via --draft-model. Spec + # rows currently only run on SC8480XP; other chipsets are skipped. + if: >- + ${{ + matrix.model.devices != null && toJson(matrix.model.devices) != '[]' + && (!matrix.model.spec || matrix.device == 'SC8480XP') + }} runs-on: ubuntu-latest timeout-minutes: 180 strategy: diff --git a/sdk/benchmark/benchmark.c b/sdk/benchmark/benchmark.c index 247629636..769ac7fc5 100644 --- a/sdk/benchmark/benchmark.c +++ b/sdk/benchmark/benchmark.c @@ -152,8 +152,10 @@ typedef struct { int64_t gen_tokens; double prefill_tps; double decode_tps; - const char* stop_reason; /* not freed; lifetime tied to SDK output */ - int32_t status; /* 0 ok */ + int64_t draft_n_total; /* 0 when spec-decoding is off */ + int64_t draft_n_accepted; /* 0 when spec-decoding is off */ + const char* stop_reason; /* not freed; lifetime tied to SDK output */ + int32_t status; /* 0 ok */ char err[256]; } run_result_t; @@ -1230,16 +1232,18 @@ static void run_llm(const options_t* o, const char* device_id, int32_t ngl, run_ if (!is_warmup) { run_result_t* r = &out[run_idx]; memset(r, 0, sizeof(*r)); - r->run_idx = run_idx; - r->ttft_us = gout.profile_data.ttft; - r->prompt_time_us = gout.profile_data.prompt_time; - r->decode_time_us = gout.profile_data.decode_time; - r->prompt_tokens = gout.profile_data.prompt_tokens; - r->gen_tokens = gout.profile_data.generated_tokens; - r->prefill_tps = gout.profile_data.prefill_speed; - r->decode_tps = gout.profile_data.decoding_speed; - r->stop_reason = gout.profile_data.stop_reason; - r->status = 0; + r->run_idx = run_idx; + r->ttft_us = gout.profile_data.ttft; + r->prompt_time_us = gout.profile_data.prompt_time; + r->decode_time_us = gout.profile_data.decode_time; + r->prompt_tokens = gout.profile_data.prompt_tokens; + r->gen_tokens = gout.profile_data.generated_tokens; + r->prefill_tps = gout.profile_data.prefill_speed; + r->decode_tps = gout.profile_data.decoding_speed; + r->draft_n_total = gout.profile_data.draft_n_total; + r->draft_n_accepted = gout.profile_data.draft_n_accepted; + r->stop_reason = gout.profile_data.stop_reason; + r->status = 0; normalize_prefill_metrics(r, o->plugin); } @@ -1383,16 +1387,18 @@ static void run_vlm(const options_t* o, const char* device_id, int32_t ngl, run_ if (!is_warmup) { run_result_t* r = &out[run_idx]; memset(r, 0, sizeof(*r)); - r->run_idx = run_idx; - r->ttft_us = gout.profile_data.ttft; - r->prompt_time_us = gout.profile_data.prompt_time; - r->decode_time_us = gout.profile_data.decode_time; - r->prompt_tokens = gout.profile_data.prompt_tokens; - r->gen_tokens = gout.profile_data.generated_tokens; - r->prefill_tps = gout.profile_data.prefill_speed; - r->decode_tps = gout.profile_data.decoding_speed; - r->stop_reason = gout.profile_data.stop_reason; - r->status = 0; + r->run_idx = run_idx; + r->ttft_us = gout.profile_data.ttft; + r->prompt_time_us = gout.profile_data.prompt_time; + r->decode_time_us = gout.profile_data.decode_time; + r->prompt_tokens = gout.profile_data.prompt_tokens; + r->gen_tokens = gout.profile_data.generated_tokens; + r->prefill_tps = gout.profile_data.prefill_speed; + r->decode_tps = gout.profile_data.decoding_speed; + r->draft_n_total = gout.profile_data.draft_n_total; + r->draft_n_accepted = gout.profile_data.draft_n_accepted; + r->stop_reason = gout.profile_data.stop_reason; + r->status = 0; normalize_prefill_metrics(r, o->plugin); } @@ -1529,7 +1535,7 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i fprintf(f, " \"params\": {\n"); fprintf(f, " \"warmup\": %d, \"repetitions\": %d, \"n_prompt\": %d, \"n_gen\": %d,\n" - " \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d\n", + " \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d", o->warmup, o->repeat, o->n_prompt, @@ -1539,14 +1545,25 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i o->n_ctx, o->n_threads, ngl); - fprintf(f, " },\n"); + if (o->spec_type) { + fprintf(f, ",\n \"spec_type\": "); + json_write_quoted(f, o->spec_type); + if (o->draft_model) { + fprintf(f, ",\n \"draft_model\": "); + json_write_quoted(f, o->draft_model); + } + fprintf(f, ",\n \"draft_tokens\": %d", o->draft_tokens); + } + fprintf(f, "\n },\n"); fprintf(f, " \"runs\": [\n"); for (int i = 0; i < o->repeat; ++i) { const run_result_t* r = &runs[i]; fprintf(f, " {\"run_idx\": %d, \"ttft_us\": %lld, \"prompt_tokens\": %lld, " "\"gen_tokens\": %lld, \"prefill_tps\": %.6f, \"decode_tps\": %.6f, " - "\"prompt_time_us\": %lld, \"decode_time_us\": %lld, \"stop_reason\": %s%s%s}%s\n", + "\"prompt_time_us\": %lld, \"decode_time_us\": %lld, " + "\"draft_n_total\": %lld, \"draft_n_accepted\": %lld, " + "\"stop_reason\": %s%s%s}%s\n", r->run_idx, (long long)r->ttft_us, (long long)r->prompt_tokens, @@ -1555,6 +1572,8 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i r->decode_tps, (long long)r->prompt_time_us, (long long)r->decode_time_us, + (long long)r->draft_n_total, + (long long)r->draft_n_accepted, r->stop_reason ? "\"" : "null", r->stop_reason ? r->stop_reason : "", r->stop_reason ? "\"" : "", @@ -1584,8 +1603,19 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i a->decode_mean, a->decode_sd); fprintf(f, " \"gen_tokens\": {\"median\": %.6f},\n", a->gen_tokens_med); - fprintf(f, " \"prompt_tokens\":{\"median\": %.6f}\n", a->prompt_tokens_med); - fprintf(f, " }\n"); + fprintf(f, " \"prompt_tokens\":{\"median\": %.6f}", a->prompt_tokens_med); + int64_t sum_draft_total = 0, sum_draft_accepted = 0; + for (int i = 0; i < o->repeat; ++i) { + sum_draft_total += runs[i].draft_n_total; + sum_draft_accepted += runs[i].draft_n_accepted; + } + if (sum_draft_total > 0) { + fprintf(f, ",\n \"draft_n_total\": {\"total\": %lld},\n", (long long)sum_draft_total); + fprintf(f, " \"draft_n_accepted\": {\"total\": %lld},\n", (long long)sum_draft_accepted); + fprintf( + f, " \"draft_accept_rate\": {\"value\": %.6f}", (double)sum_draft_accepted / (double)sum_draft_total); + } + fprintf(f, "\n }\n"); fprintf(f, "}\n"); fclose(f); /* keep static-analysis happy */ diff --git a/sdk/benchmark/qdc/bench-models.json b/sdk/benchmark/qdc/bench-models.json index 06a2d4769..af1599b5e 100644 --- a/sdk/benchmark/qdc/bench-models.json +++ b/sdk/benchmark/qdc/bench-models.json @@ -126,6 +126,28 @@ "vlm": true, "image": true }, + { + "name": "gemma-4-26B-A4B-it-assistant", + "plugin": "llama_cpp", + "devices": [], + "model_id": "RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf:Q4_0", + "hub": "hf", + "url": "https://huggingface.co/RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf/resolve/main/gemma-4-26b-A4B-it-assistant-Q4_0.gguf" + }, + { + "name": "gemma-4-26B-A4B-it-mtp", + "plugin": "llama_cpp", + "devices": ["npu", "hybrid"], + "model_id": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf:Q4_0", + "hub": "hf", + "url": "https://huggingface.co/google/gemma-4-26B-A4B-it-qat-q4_0-gguf/resolve/main/gemma-4-26B_q4_0-it.gguf", + "ctx": [512, 1024, 4096, 8192], + "spec": { + "type": "draft-mtp", + "draft": "gemma-4-26B-A4B-it-assistant", + "n_max": 3 + } + }, { "name": "Qwen3.5-2B", "plugin": "llama_cpp", diff --git a/sdk/benchmark/qdc/linux/run_linux.sh b/sdk/benchmark/qdc/linux/run_linux.sh index 0d8fc165e..7f85b2d87 100644 --- a/sdk/benchmark/qdc/linux/run_linux.sh +++ b/sdk/benchmark/qdc/linux/run_linux.sh @@ -59,7 +59,7 @@ for ctx in "${CTX_ARR[@]}"; do : > "/data/local/tmp/matrix-qairt-${ctx}.tsv" done -while IFS='|' read -r name plugin devs model_id vlm image; do +while IFS='|' read -r name plugin devs model_id vlm image _spec_type _draft_id _draft_tokens; do [ -z "$name" ] && continue echo "=== plan $name id=$model_id ===" case "$plugin" in diff --git a/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt b/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt new file mode 100644 index 000000000..55e6136fb --- /dev/null +++ b/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt @@ -0,0 +1,539 @@ +Please summarize the following comprehensive text in 2-3 sentences, capturing +the main themes and key points: + +The rapid evolution of artificial intelligence and machine learning +technologies has fundamentally transformed the landscape of modern +civilization, creating unprecedented opportunities for innovation while +simultaneously presenting complex challenges that demand careful consideration +and proactive management across multiple domains of human activity. In the +realm of healthcare and medical sciences, artificial intelligence systems have +achieved remarkable breakthroughs in diagnostic capabilities, enabling medical +professionals to analyze complex medical images with superhuman accuracy, +detect early-stage cancers and other life-threatening conditions with +unprecedented precision, and process vast amounts of patient data to predict +disease progression, optimize treatment protocols, and accelerate the +traditionally lengthy process of drug discovery and development. Pharmaceutical +companies worldwide are leveraging sophisticated AI algorithms to identify +promising drug compounds, predict molecular interactions with greater accuracy +than traditional methods, streamline clinical trial processes, and develop +personalized medicine approaches that tailor treatments to individual genetic +profiles, medical histories, and specific patient characteristics, potentially +reducing both the time and cost required to bring new medications to market +while significantly improving patient outcomes and quality of life. + +The financial services industry has undergone a revolutionary transformation +through the implementation of AI-powered systems that monitor millions of +financial transactions in real-time to detect fraudulent activities, execute +complex trading strategies based on sophisticated analysis of market patterns +and economic indicators, and provide personalized investment advice through +advanced robo-advisor platforms that analyze individual risk tolerance, +financial goals, market conditions, and economic trends to optimize portfolio +performance and maximize returns for investors. Banks, credit institutions, and +lending organizations are utilizing machine learning algorithms to assess +creditworthiness with greater accuracy than traditional scoring methods, while +insurance companies employ AI technologies to evaluate risk factors more +precisely, process claims more efficiently, develop innovative insurance +products that better serve customer needs, and reduce operational costs while +improving profitability and customer satisfaction. The transportation sector +has witnessed dramatic changes with the development of autonomous vehicles that +combine advanced computer vision systems, sophisticated sensor fusion +technologies, deep learning algorithms, and real-time decision-making +capabilities to navigate complex traffic scenarios safely and efficiently, +while ride-sharing platforms utilize AI to optimize driver-passenger matching +algorithms, implement dynamic pricing strategies based on demand patterns and +supply availability, and improve route efficiency to reduce travel times and +minimize environmental impact. + +Manufacturing industries have embraced the concept of Industry 4.0, where smart +factories equipped with AI-powered systems can predict equipment failures +before they occur, optimize production schedules based on sophisticated demand +forecasts, implement quality control measures that detect defects with greater +precision than human inspectors, and coordinate complex supply chain operations +to increase efficiency, reduce waste, improve product quality, and minimize +environmental impact. Educational institutions are implementing personalized +learning platforms that adapt to individual student learning styles, pace, +preferences, and cognitive abilities, providing customized content, +assessments, and feedback that help students achieve better educational +outcomes while enabling teachers to focus on higher-level instruction, student +interaction, and pedagogical innovation rather than administrative tasks and +routine assessments. Virtual assistants and chatbots powered by advanced +natural language processing technologies provide instant customer support +across various industries, handling routine inquiries, resolving common issues, +escalating complex problems to human agents when necessary, and improving +overall customer satisfaction while reducing operational costs and increasing +service availability. + +Social media platforms and content providers utilize AI algorithms to curate +personalized feeds, recommend relevant content based on user preferences and +behavior patterns, moderate user-generated content to maintain community +standards, and prevent harmful or inappropriate material from spreading across +their networks. The entertainment industry has been transformed by AI +technologies that can generate music, create visual effects, write scripts, +produce entire movies with minimal human intervention, and develop interactive +gaming experiences that adapt to player behavior and preferences, while +streaming services employ sophisticated recommendation algorithms to keep users +engaged and satisfied with personalized content suggestions that maximize +viewing time and subscriber retention. Retail and e-commerce companies employ +AI for inventory management, demand forecasting, price optimization, customer +service automation, and personalized marketing campaigns, creating more +efficient supply chains and better shopping experiences for consumers worldwide +while reducing costs and improving profitability. + +Agriculture has benefited significantly from AI-powered precision farming +techniques that optimize irrigation, fertilization, and pest control based on +real-time data from sensors, satellite imagery, weather forecasts, and soil +analysis, leading to increased crop yields, more sustainable farming practices, +reduced environmental impact, and improved food security for growing +populations. Energy companies utilize AI to optimize power grid operations, +predict equipment failures, integrate renewable energy sources more +effectively, manage energy storage systems, and develop smart grid technologies +that contribute to the transition toward cleaner, more efficient, and more +reliable energy systems. Environmental monitoring and climate research have +been enhanced by AI systems that can analyze vast amounts of satellite data, +weather patterns, atmospheric conditions, and environmental indicators to +predict natural disasters, track climate change impacts, monitor biodiversity, +and develop comprehensive strategies for environmental protection, +conservation, and sustainable development. + +However, these technological advances also present significant challenges that +society must address proactively, including concerns about job displacement as +automation and AI systems replace human workers in various sectors, requiring +comprehensive retraining programs, educational initiatives, and policy +interventions to help workers transition to new roles that complement rather +than compete with AI systems. Privacy protection has become increasingly +complex as AI systems require access to vast amounts of personal data to +function effectively, raising critical questions about data ownership, consent +mechanisms, security protocols, and the potential for misuse or unauthorized +access that could compromise individual privacy rights and personal security. +Algorithmic bias represents another critical concern, as AI systems trained on +historical data may perpetuate or amplify existing societal prejudices, +inequalities, and discriminatory practices, particularly in areas like hiring, +lending, criminal justice, healthcare access, and educational opportunities +where biased decisions could have serious consequences for individuals and +communities. + +The rapid pace of AI development has outpaced the creation of appropriate +regulatory frameworks, governance structures, and oversight mechanisms, leaving +significant gaps in accountability, transparency, and ethical oversight that +could lead to unintended consequences or misuse of powerful AI technologies. +International cooperation and coordination are essential for addressing global +challenges posed by AI, including issues of governance, standards development, +cybersecurity, intellectual property rights, and the equitable distribution of +benefits across different populations, regions, and socioeconomic groups. The +development of artificial general intelligence (AGI) remains a long-term goal +that could have profound implications for humanity, potentially solving complex +global challenges like climate change, disease eradication, poverty reduction, +and resource management while also raising new ethical, safety, and existential +concerns that require careful consideration, preparation, and international +collaboration. + +Cybersecurity threats have evolved alongside AI development, with malicious +actors using AI to create more sophisticated attacks, generate convincing +deepfakes, exploit vulnerabilities in AI systems themselves, and develop new +forms of cyber warfare that require constant vigilance, innovation in defensive +measures, and international cooperation to address effectively. The digital +divide between those who have access to AI technologies and those who do not +threatens to exacerbate existing inequalities, making it crucial to ensure that +AI benefits are distributed equitably across different socioeconomic groups, +geographic regions, and demographic populations. Ethical considerations around +AI development include questions about transparency, accountability, fairness, +human dignity, and the appropriate use of AI in sensitive applications like +military systems, surveillance technologies, autonomous weapons, and +decision-making processes that affect human lives, rights, and fundamental +freedoms. + +The environmental impact of AI systems, particularly large-scale machine +learning models that require enormous computational resources and energy +consumption, raises concerns about sustainability, carbon footprint, resource +depletion, and the long-term environmental consequences of AI development and +deployment. Intellectual property rights and ownership of AI-generated content +present complex legal challenges that courts, lawmakers, and international +bodies are still grappling with, particularly regarding copyright, patents, +trademarks, and the rights of AI systems themselves versus their human creators +and users. The concentration of AI expertise, resources, and capabilities in a +small number of large technology companies raises concerns about monopolistic +control, market dominance, and the need for more diverse, decentralized, and +democratized AI development approaches that promote competition, innovation, +and equitable access. + +Ensuring that AI development serves human interests and promotes global +prosperity will require ongoing dialogue, collaboration, and cooperation +between technologists, policymakers, ethicists, philosophers, social +scientists, and representatives from diverse communities, cultures, and +perspectives to create comprehensive frameworks that balance innovation with +responsibility, progress with protection, efficiency with equity, and +technological advancement with human flourishing in the age of artificial +intelligence. The future of AI will likely involve increased collaboration +between humans and machines, with augmented intelligence approaches that +enhance human capabilities rather than replacing them entirely, creating new +opportunities for creativity, problem-solving, artistic expression, scientific +discovery, and human flourishing in an AI-enhanced world that preserves human +agency, dignity, and purpose. + +Additionally, the integration of AI with emerging technologies like quantum +computing, blockchain, and the Internet of Things (IoT) is creating new +possibilities for innovation, efficiency, and transformative change across +multiple sectors and industries. Quantum computing has the potential to +dramatically accelerate AI computations, enable new types of algorithms that +could solve problems currently intractable for classical computers, and +revolutionize cryptography, optimization, and simulation capabilities that +could transform scientific research, financial modeling, and technological +development. Blockchain technology combined with AI is creating new +possibilities for secure, transparent, and decentralized applications in +finance, supply chain management, digital identity verification, voting +systems, and governance mechanisms that could enhance trust, reduce corruption, +and improve efficiency in various domains. + +The Internet of Things generates massive amounts of data that AI systems can +analyze to optimize processes, predict maintenance needs, improve +decision-making in real-time, and create smart environments across smart +cities, industrial facilities, consumer devices, and infrastructure systems. +Edge computing brings AI processing closer to data sources, enabling real-time +decision-making in applications like autonomous vehicles, industrial +automation, smart home systems, and mobile devices where latency and bandwidth +constraints make cloud-based processing impractical or inefficient. +Neuromorphic computing, which mimics the structure and function of the human +brain, may lead to more efficient and powerful AI systems that can process +information with lower energy consumption, faster response times, and greater +adaptability compared to traditional digital computers. + +Brain-computer interfaces represent another frontier where AI could enable +direct communication between human brains and computer systems, potentially +revolutionizing how we interact with technology, treat neurological conditions, +enhance cognitive abilities, and create new forms of human-machine symbiosis. +The development of explainable AI (XAI) is becoming increasingly important as +organizations seek to understand how AI systems make decisions, particularly in +high-stakes applications like healthcare, finance, criminal justice, and +autonomous systems where transparency, accountability, and interpretability are +essential for building trust, ensuring fair outcomes, and meeting regulatory +requirements. Federated learning approaches allow AI models to be trained on +data distributed across multiple devices or organizations without centralizing +sensitive information, addressing privacy concerns while still enabling +collaborative learning, model improvement, and knowledge sharing across +different entities and jurisdictions. + +The democratization of AI tools and platforms is making advanced machine +learning capabilities accessible to smaller organizations, individual +developers, researchers, and entrepreneurs, potentially leading to more +diverse, innovative, and creative applications across different industries, use +cases, and cultural contexts. However, this democratization also raises +concerns about the potential for misuse, as powerful AI capabilities become +available to individuals or groups who may not have the expertise, ethical +framework, or accountability mechanisms to use them responsibly and safely. The +development of AI safety research is becoming increasingly important as AI +systems become more powerful, autonomous, and capable, requiring careful +consideration of alignment problems, robustness issues, adversarial attacks, +and the potential for unintended consequences that could have serious negative +impacts on society, individuals, and global stability. + +International competition in AI development, particularly between major powers +like the United States, China, the European Union, and other nations, raises +concerns about technological arms races, national security implications, and +the potential for AI to be weaponized or used for surveillance, control, and +authoritarian purposes that could threaten democratic values, human rights, and +international peace and security. The need for international cooperation, +governance frameworks, and multilateral agreements becomes even more critical +as AI systems become more powerful and their impacts become more global in +scope, requiring coordinated efforts to ensure that AI development serves human +interests, promotes peace, prosperity, human dignity, and sustainable +development worldwide. The economic implications of AI adoption are complex and +multifaceted, with potential for significant productivity gains, economic +growth, and prosperity alongside concerns about job displacement, income +inequality, wealth concentration, and the distribution of benefits across +different segments of society. + +The transformation of labor markets will require comprehensive policy responses +including education reform, vocational training programs, social safety nets, +universal basic income experiments, and economic policies that ensure the +benefits of AI are shared broadly across society rather than accruing primarily +to a small elite of technology companies and wealthy individuals. The +development of AI ethics, responsible AI practices, and governance frameworks +is becoming increasingly important as organizations recognize the need to +ensure that their AI systems are fair, transparent, accountable, unbiased, and +aligned with human values, societal goals, and ethical principles. This +includes considerations of bias mitigation, privacy protection, algorithmic +transparency, human oversight, and the development of governance structures +that can oversee AI development and deployment in ways that promote beneficial +outcomes while minimizing risks, negative consequences, and unintended harm. + +The integration of AI into critical infrastructure systems like power grids, +transportation networks, communication systems, financial markets, and +healthcare systems raises important questions about resilience, security, +reliability, and the potential for cascading failures that could have serious +consequences for society, economy, and public safety. The development of AI +standards, certification processes, and regulatory frameworks is becoming +increasingly important to ensure that AI systems meet minimum requirements for +safety, reliability, ethical behavior, and human compatibility across different +applications, industries, and use cases. The role of government in AI +development and regulation is evolving as policymakers grapple with the +challenge of promoting innovation, economic competitiveness, and technological +leadership while protecting public interests, ensuring safety, and implementing +appropriate safeguards and oversight mechanisms. + +The future of work in an AI-driven economy will likely involve significant +changes in job requirements, skill demands, career paths, and the nature of +human-AI collaboration, requiring proactive efforts to prepare workers, +students, and society for these changes and ensure that the transition is +managed in ways that promote human flourishing, economic opportunity, social +mobility, and meaningful employment for all members of society. The development +of AI literacy, digital skills, and technological competence will become +increasingly important for individuals, organizations, and societies to +navigate the AI-driven world effectively, make informed decisions about AI +adoption, and participate meaningfully in shaping the future of AI development +and deployment. The ethical, social, and philosophical implications of AI +development require ongoing reflection, debate, and dialogue among diverse +stakeholders to ensure that AI technologies serve human values, promote human +flourishing, and contribute to a more just, equitable, and sustainable world +for future generations. + +The emergence of large language models and generative AI systems has created +new opportunities for human-computer interaction, creative expression, and +knowledge synthesis, while also raising questions about authenticity, +originality, and the nature of human creativity in an age where machines can +produce text, images, music, and other forms of content that rival or exceed +human capabilities. The development of multimodal AI systems that can process +and generate content across different modalities including text, images, audio, +and video represents a significant advancement that could revolutionize fields +like education, entertainment, communication, and creative industries, enabling +new forms of interactive experiences and personalized content creation that +adapt to individual preferences and learning styles. + +The intersection of AI with fields like psychology, neuroscience, and cognitive +science is providing new insights into human intelligence, learning processes, +and decision-making mechanisms, while also enabling the development of AI +systems that better understand and interact with human users in more natural +and intuitive ways. The application of AI in scientific research and discovery +is accelerating the pace of innovation across disciplines, from drug discovery +and materials science to astronomy and climate modeling, enabling researchers +to analyze complex datasets, identify patterns, generate hypotheses, and +conduct virtual experiments that would be impossible or impractical using +traditional methods. The development of AI-powered scientific writing +assistants, research tools, and knowledge management systems is transforming +how researchers collaborate, share information, and build upon each other's +work, potentially accelerating the pace of scientific discovery and innovation. + +The role of AI in addressing global challenges like climate change, food +security, healthcare access, and education equity is becoming increasingly +important as these systems demonstrate their potential to optimize resource +allocation, predict environmental changes, develop sustainable solutions, and +provide personalized services at scale. The development of AI systems that can +operate in resource-constrained environments, work with limited data, and adapt +to diverse cultural and linguistic contexts is crucial for ensuring that the +benefits of AI technology reach underserved populations and developing regions. +The integration of AI with mobile technologies, low-cost computing devices, and +emerging communication networks is creating new possibilities for delivering +AI-powered services to remote and rural communities, potentially reducing +inequalities and improving quality of life for millions of people worldwide. + +The evolution of AI-human collaboration patterns is reshaping organizational +structures, work processes, and decision-making frameworks across industries, +requiring new approaches to leadership, management, and organizational design +that can effectively leverage both human creativity and AI capabilities. The +development of AI systems that can learn from human feedback, adapt to +individual preferences, and collaborate effectively with human users represents +a significant advancement toward more intuitive and productive human-AI +partnerships. The emergence of AI-powered personal assistants, productivity +tools, and decision support systems is transforming how individuals work, +and learn, potentially enhancing human capabilities while +maintaining human agency and control over important choices and actions. + +The transportation sector stands at the threshold of a profound transformation +driven by the convergence of artificial intelligence, sensor technology, and +electrification, with autonomous vehicles progressing from controlled test +environments to real-world deployments in ride-hailing fleets, freight +corridors, last-mile delivery services, and public transit networks in major +metropolitan areas around the world. Advanced perception systems combining +lidar, radar, camera arrays, and high-definition mapping enable vehicles to +interpret complex urban scenes, anticipate the behavior of pedestrians and +other road users, and negotiate multi-agent intersections with a level of +consistency that begins to rival experienced human drivers under favorable +conditions. Behind the scenes, learning-based planners and prediction models +continuously ingest telemetry from vast fleets, allowing manufacturers to +identify long-tail edge cases, refine control policies, and push over-the-air +updates that improve safety margins without a physical service visit. +Meanwhile, AI-optimized traffic-signal control, dynamic congestion pricing, +and demand-responsive transit routing are reducing average commute times, +smoothing peak-hour bottlenecks, and cutting per-passenger emissions in cities +that have committed to data-driven mobility planning. + +Aviation, maritime shipping, and rail freight are undergoing parallel shifts, +with predictive-maintenance systems flagging component failures days or weeks +before they would strand a vessel, aircraft, or locomotive, while route- +optimization engines fold in weather forecasts, fuel prices, port congestion, +and regulatory constraints to shave hours off long-haul journeys and trim +substantial amounts off operator fuel bills. Air traffic controllers now +receive machine-learned recommendations for spacing and sequencing that +increase runway throughput without eroding separation minimums, and container +terminals rely on computer-vision systems to track stacks in real time, +coordinate straddle-carrier movements, and expose bottlenecks that used to +require weeks of manual observation to diagnose. The cumulative effect is a +supply chain that responds to disruption more quickly, wastes less capacity, +and creates new opportunities for smaller operators to compete with the +incumbent giants by renting analytics capabilities that once required a +dedicated internal data-science team. + +Education is being reshaped in equally consequential ways, with adaptive +learning platforms now capable of building individualized study paths that +respond to a learner's pace, misconceptions, and preferred modality, whether +that means additional worked examples, alternate explanations, or short +formative quizzes designed to catch shallow understanding before it hardens +into a persistent gap. Teachers, freed from significant portions of the +grading and lesson-preparation work that once consumed their evenings, can +spend more time on coaching, mentorship, and the socio-emotional aspects of +learning that machines are least equipped to handle. University researchers +are also using AI to accelerate literature review, generate plausible +experimental hypotheses, and simulate outcomes for interventions that would be +too expensive or ethically fraught to attempt directly, though these tools +have also introduced difficult new questions about attribution, academic +integrity, and the role that generative systems should play in student +assessment. Policy makers are still grappling with how to certify AI-assisted +curricula, how to protect student data, and how to ensure that adaptive +systems do not quietly reproduce or amplify existing inequities along lines +of race, socioeconomic status, first language, or learning difference. + +Environmental monitoring and climate science represent perhaps the most +consequential domain for near-term AI application, with machine-learning +emulators standing in for physically detailed climate simulations at a tiny +fraction of the computational cost, enabling exploration of many more +scenarios and thereby sharpening projections of regional impacts on +agriculture, water availability, and extreme-weather frequency. Satellite +imagery interpreted by deep-learning models supplies near real-time signals +on deforestation, illegal fishing, methane leaks from oil and gas +infrastructure, and shifts in land-use patterns that would otherwise take +years to appear in official statistics. In wildfire-prone regions, +AI-informed sensor networks detect ignitions within minutes and route +resources with less human coordination overhead, while in flood-vulnerable +watersheds, learned hydrological models produce probabilistic warnings that +give communities several additional hours of preparation time. On the +mitigation side, grid operators use forecasting models to balance rapidly +growing shares of solar and wind generation, dispatch battery storage at the +most valuable moments, and reduce the number of hours per year during which +carbon-intensive peaker plants must run. + +Agriculture, one of the oldest human activities, is being quietly rewired by +precision-farming platforms that interpret drone imagery, in-soil moisture +probes, and multispectral satellite passes to guide irrigation, fertilizer +placement, and integrated pest management with a spatial resolution far finer +than a single field. Growers who once relied on calendar-based spraying now +receive alerts that pinpoint specific rows or sections that warrant +intervention, cutting chemical inputs and reducing runoff into surrounding +watersheds. Robotic weeders, thinners, and harvesters increasingly handle +tasks that historically depended on seasonal labor shortages, though the +transition has raised legitimate concerns about the livelihoods of rural +workers and the concentration of decision-making power in the hands of a +small number of software vendors. In the developing world, low-cost mobile +advisory services translate weather forecasts, market prices, and agronomic +recommendations into local languages, giving smallholder farmers access to +insights that were previously available only to industrial-scale operations. + +Space exploration and Earth observation have entered a period of rapid +acceleration in which AI plays a central rather than peripheral role, with +autonomous spacecraft that can plan their own observation sequences, prioritize +downlink targets given limited bandwidth, and diagnose their own faults +without waiting for a round-trip conversation with mission control. Rover +missions to Mars and the Moon use learned obstacle-avoidance and traverse +planning to cover more ground per operational day than previous generations +could manage in a week. On the commercial side, an expanding fleet of small +imaging satellites captures the entire planet at daily or even sub-daily +cadence, and AI-based analytics extract signals ranging from car counts in +retail parking lots to soil moisture in agricultural regions to structural +displacement along fault lines and dam faces. These capabilities are quietly +reshaping insurance underwriting, commodity trading, disaster response, and +international treaty verification in ways that few observers anticipated only +a decade ago. + +The energy sector's transformation deserves particular attention, since it +sits at the intersection of climate policy, industrial strategy, and everyday +consumer experience. Utilities operating aging grids now rely on AI-based +fault localization to isolate outages faster and dispatch repair crews more +efficiently, while distribution planners use load-forecasting models that +account for electric-vehicle adoption curves, heat-pump uptake, and rooftop +solar penetration on a neighborhood-by-neighborhood basis. Fusion research +programs have begun to publish results in which reinforcement-learning +controllers stabilize plasma configurations that traditional control theory +could not, offering hope that long-elusive commercial fusion power might +arrive on a timeline that matters for climate goals. On the demand side, +building-management platforms coordinate HVAC, lighting, and plug loads to +minimize consumption during high-price hours, and residential thermostats +learn occupant preferences well enough to trim ten to twenty percent from +household heating and cooling bills without any perceptible loss of comfort. + +Manufacturing, long a testbed for automation of a mechanical rather than +cognitive variety, is now being reshaped by generative design systems that +propose thousands of candidate part geometries meeting a stated performance +envelope, computer-vision quality inspectors that detect surface defects far +smaller than a human operator can reliably see, and reinforcement-learning +robot controllers that adapt to variability in incoming stock and tool wear +without retooling. Small and medium-sized manufacturers, who once could not +afford the fixed cost of a full analytics team, increasingly consume these +capabilities as software services, narrowing the productivity gap that has +long separated them from multinational competitors. Additive manufacturing +combined with topology-optimized designs is enabling parts that would have +been impossible to produce by conventional means, unlocking weight savings in +aerospace, medical implants that better match the recipient's anatomy, and +spare-parts distribution models in which components are printed on demand +close to the point of use rather than shipped from a distant warehouse. + +The scientific enterprise as a whole is being augmented in ways that are +beginning to alter how research is planned, executed, and communicated. +Protein-structure prediction, once the crown jewel of an entire subfield, has +become a routine step in the workflow of any lab studying enzymes, membrane +receptors, or antibody engineering, and comparable transformations are under +way in materials discovery, catalysis, and small-molecule chemistry. +Automated laboratories combine liquid-handling robots, in-line analytics, and +active-learning experiment planners to run hundreds of experiments per day +under conditions that would be tedious and error-prone for human researchers, +while large language models tuned on scientific corpora assist with +literature synthesis, hypothesis generation, and the drafting of grant +proposals and manuscripts. These tools raise pressing questions about +authorship, reproducibility, and the appropriate level of human oversight, +but their productivity benefits are large enough that most funding agencies +and research institutions are actively investing in the infrastructure and +governance required to use them responsibly. + +Cybersecurity and digital trust have become correspondingly high-stakes +domains, with AI-driven attack tooling lowering the barrier to sophisticated +phishing, social engineering, and vulnerability discovery, and AI-driven +defense tooling detecting anomalous behavior in networks, endpoints, and +identity systems at a scale no human security operations center could match. +The arms-race dynamic between offense and defense is more visible than ever, +and organizations that once treated security as a compliance checkbox are +being forced to invest in continuous monitoring, red-team exercises, and +incident-response rehearsals that assume adversaries will use every automation +available. In parallel, national governments are experimenting with AI-based +content-provenance systems, deepfake-detection benchmarks, and coordinated +disclosure frameworks intended to preserve some measure of shared reality in +information ecosystems that generative models can otherwise flood with +plausible-looking synthetic material. + +Consumer experience is also being reshaped, sometimes in ways that draw +attention and sometimes so quietly that users only notice the improvement in +aggregate. Search engines, once purely keyword-driven, now interpret +questions in context and surface direct answers pulled from indexed sources +with citations that let the curious follow up. Streaming services and +retailers refine their recommendations continuously, translating browsing +signals into surprisingly accurate models of individual taste, and messaging +applications translate between languages, transcribe voice notes, and +summarize threads on demand. In the workplace, meeting assistants transcribe +conversations, extract action items, and populate task trackers, while +coding assistants draft, review, and refactor software at a level of fluency +that has already changed how many teams estimate feature timelines. These +gains are unevenly distributed, and questions about privacy, consent, and +the appropriate handling of sensitive conversational data remain very much +open, but the trajectory of adoption suggests they are now table stakes for +competitive products rather than optional differentiators. + +Taken together, the developments across all of these sectors point to a +world in which artificial intelligence has become a general-purpose +technology on the order of electrification or the internal combustion +engine, permeating industries and everyday life to a degree that few +observers fully appreciated even five years ago. The pace of change is +unlikely to slow in the near term, and the same institutions that must +harness AI for productivity, competitiveness, and scientific advance are +simultaneously responsible for anticipating harms, allocating benefits +fairly, and preserving the democratic accountability of decisions that +increasingly rely on opaque algorithmic components. How that dual mandate +is discharged in the coming decade will determine whether the promise of +this moment is broadly shared or captured by a narrow set of actors, and +whether public trust in the underlying technology grows in step with its +capabilities or erodes under the weight of avoidable missteps. diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index d5cb541b5..e234127b7 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -116,31 +116,57 @@ def resolve_model_url(m: dict, device: str) -> str | None: return m.get("url") +def _resolve_draft_model_id(models: list[dict], draft_name: str) -> str: + """Look up a draft model's model_id by name inside the same bench-models.json. + Draft models are declared as catalog-only entries (empty ``devices``) and + referenced from a target row's ``spec.draft`` field.""" + for m in models: + if m["name"] == draft_name: + return m["model_id"] + raise SystemExit(f"draft model {draft_name!r} not found in bench-models.json") + + def model_rows(models: list[dict], device: str) -> list[str]: """One pipe-delimited row per model, consumed by the device-side run scripts. Schema: name | plugin | csv_devices | model_id | vlm | image + | spec_type | draft_model_id | draft_tokens + + The trailing three fields are non-empty only for spec-decoding rows; + non-spec rows carry empty strings there so every script parses the + same column count. The host passes the chipset slug as a single shared --chipset flag to geniex-bench; the model-manager hub auto-routes "qualcomm/*" to AI Hub and everything else to HuggingFace, so per-row hub overrides aren't needed. mmproj/tokenizer paths come back from get_paths. + Entries with empty ``devices`` are catalog-only (e.g. spec draft + models) and are skipped here — they exist in bench-models.json so + other rows can reference them by name and the aggregate report can + resolve their download URL. + Rows for AI Hub models whose chipset isn't advertised are dropped upfront so the device doesn't waste time on a guaranteed-fail pull.""" rows = [] for m in models: if "model_id" not in m: raise SystemExit(f"{m['name']}: missing model_id in bench-models.json") + if not m["devices"]: + continue if m.get("hub") == "aihub" and not _aihub_chipset_supported(m, device): log.warning("no %s asset for %s, skipping", device, m["name"]) continue vlm = "1" if m.get("vlm") else "" image = "1" if m.get("image") else "" + spec = m.get("spec") or {} + spec_type = spec.get("type", "") + draft_id = _resolve_draft_model_id(models, spec["draft"]) if spec else "" + draft_tokens = str(spec.get("n_max", "")) if spec else "" rows.append( f"{m['name']}|{m['plugin']}|{','.join(m['devices'])}|{m['model_id']}" - f"|{vlm}|{image}" + f"|{vlm}|{image}|{spec_type}|{draft_id}|{draft_tokens}" ) return rows @@ -398,6 +424,79 @@ def _details_block( return lines +def _is_spec_cell(c: dict) -> bool: + return bool((c.get("params") or {}).get("spec_type")) + + +def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str]: + """Standalone MTP table pairing each spec cell with its no-spec baseline + on (target model_id, device, ctx). Emits nothing when no spec cells or + no matching baseline row is registered in bench-models.json.""" + if not models: + return [] + spec_entries = [m for m in models if m.get("spec")] + if not spec_entries: + return [] + by_name_key: dict[str, dict[tuple[str, int], dict]] = {} + for c in cells: + by_name_key.setdefault(_model_label(c), {})[ + (c["device"], _ctx_from_cell(c)) + ] = c + rows: list[str] = [] + for spec_m in spec_entries: + baseline = next( + ( + m + for m in models + if m["model_id"] == spec_m["model_id"] + and not m.get("spec") + and m.get("devices") + ), + None, + ) + draft_m = next( + (m for m in models if m["name"] == spec_m["spec"]["draft"]), None + ) + spec_cells = by_name_key.get(spec_m["name"], {}) + base_cells = by_name_key.get(baseline["name"], {}) if baseline else {} + for (dev, ctx), sc in sorted(spec_cells.items()): + agg = sc.get("agg") or {} + accept = (agg.get("draft_accept_rate") or {}).get("value") + accept_s = f"{100 * accept:.1f}%" if accept is not None else "-" + s_dec = (agg.get("decode_tps") or {}).get("median") + bc = base_cells.get((dev, ctx)) + b_dec = ( + ((bc.get("agg") or {}).get("decode_tps") or {}).get("median") + if bc + else None + ) + uplift = f"{s_dec / b_dec:.2f}x" if s_dec and b_dec else "-" + p_med = (agg.get("prompt_tokens") or {}).get("median") + g_med = (agg.get("gen_tokens") or {}).get("median") + test = ( + f"pp{int(p_med)}+tg{int(g_med)}" + if p_med is not None and g_med is not None + else "-" + ) + rows.append( + f"| {spec_m['name']} | {draft_m['name'] if draft_m else '-'} | " + f"{dev} | {ctx} | {test} | {accept_s} | " + f"{_fmt_med_sd(agg, 'decode_tps')} | " + f"{_fmt_med_sd((bc or {}).get('agg') or {}, 'decode_tps')} | " + f"{uplift} |" + ) + if not rows: + return [] + return [ + "", + "## MTP (speculative decoding)", + "", + "| Target | Draft | Device | Ctx | Test | Accept% | Decode (mtp) | Decode (baseline) | Uplift |", + "|--------|-------|--------|----:|------|--------:|-------------:|------------------:|-------:|", + *rows, + ] + + def render( cells: list[dict], device: str, @@ -412,6 +511,8 @@ def render( ] sort_key = lambda c: (_model_label(c), c["plugin"], c["device"], _ctx_from_cell(c)) # noqa: E731 for c in sorted(cells, key=sort_key): + if _is_spec_cell(c): + continue agg = c.get("agg") or {} params = c.get("params") or {} model = _model_label(c) @@ -431,6 +532,7 @@ def render( f"{_fmt_med_sd(agg, 'ttft_ms')} | {_fmt_med_sd(agg, 'prefill_tps')} | " f"{_fmt_med_sd(agg, 'decode_tps')} |" ) + lines += _render_mtp_table(cells, models) return "\n".join(lines) + "\n" @@ -541,7 +643,10 @@ def main() -> int: raise SystemExit( f"no model in {args.models_file} runs any of --compute={compute_pick}" ) - ctx_list, pp_list, tg_list = resolve_sweep(args.ctx, args.pp, args.tg) + ctx_arg = args.ctx + if not ctx_arg and len(models) == 1 and models[0].get("ctx"): + ctx_arg = ",".join(str(x) for x in models[0]["ctx"]) + ctx_list, pp_list, tg_list = resolve_sweep(ctx_arg, args.pp, args.tg) log.info("sweep: ctx=%s pp=%s tg=%s", ctx_list, pp_list, tg_list) client = _qdc.make_client(api_key) diff --git a/sdk/benchmark/qdc/tests/test_bench.py b/sdk/benchmark/qdc/tests/test_bench.py index 7eec38978..88c12e956 100644 --- a/sdk/benchmark/qdc/tests/test_bench.py +++ b/sdk/benchmark/qdc/tests/test_bench.py @@ -49,7 +49,7 @@ def test_bench(): (plugin, ctx): [] for plugin in ("llama_cpp", "qairt") for ctx in CTXS } for row in rows: - name, plugin, devs, model_id, vlm, image = row.split("|") + name, plugin, devs, model_id, vlm, image, *_spec = row.split("|") if plugin not in ("llama_cpp", "qairt"): continue if plugin != "qairt": diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index 171e32d53..f6838a6db 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -15,13 +15,18 @@ # >~7 GB GGUFs on X2 Elite). # # We sweep ctx in {512, 1024, 4096} per cell to align with test-llama.cpp's -# PERFORMANCE SESSION. Two prefill modes coexist: +# PERFORMANCE SESSION. Three prefill modes coexist: # - llama_cpp cells use random-ids prefill (`-p N`, mirrors llama-bench # `pp{N}`), so reported pp is exactly the ctx value; # - qairt cells go through prompt_utf8 (the plugin doesn't accept # pre-tokenized input_ids — see issue #1008), with a pre-trimmed # `sample_prompt_${ctx}.txt` per ctx so prompt length is bounded. -# Each plugin gets its own per-ctx TSV so the two invocations don't mix. +# - spec (llama_cpp speculative-decoding) cells force `--prompt-file` +# because random ids collapse draft acceptance to ~0% — the number +# would only reflect scheduling overhead, not MTP benefit. Spec cells +# also pass --spec-type/--draft-model/--draft-tokens as CLI-level +# flags, so each spec matrix invocation runs its own bench call. +# Each bucket gets its own per-ctx TSV so the invocations don't mix. $ErrorActionPreference = "Continue" @@ -63,17 +68,22 @@ $ctxList = @({CTX_LIST}) $ppList = @({PP_LIST}) $tgList = @({TG_LIST}) $tsvByPluginCtx = @{} -foreach ($plugin in @("llama", "qairt")) { +foreach ($plugin in @("llama", "qairt", "spec")) { foreach ($ctx in $ctxList) { $tsvByPluginCtx["$plugin-$ctx"] = "C:\Temp\matrix-$plugin-$ctx.tsv" Remove-Item $tsvByPluginCtx["$plugin-$ctx"] -ErrorAction SilentlyContinue } } +# Spec CLI parameters share one value across all cells inside a single +# bench invocation. This map holds them keyed by "$ctx" for the second +# pass below; every spec row is expected to agree on (type, draft, tokens) +# per ctx since we only ship one draft model per target today. +$specParamsByCtx = @{} foreach ($row in $rows) { - $name, $plugin, $devs, $model_id, $vlm, $image = $row -split '\|' + $name, $plugin, $devs, $model_id, $vlm, $image, $spec_type, $draft_model_id, $draft_tokens = $row -split '\|' Write-Output "=== plan $name id=$model_id ===" - $bucket = if ($plugin -eq "qairt") { "qairt" } elseif ($plugin -eq "llama_cpp") { "llama" } else { "" } + $bucket = if ($spec_type) { "spec" } elseif ($plugin -eq "qairt") { "qairt" } elseif ($plugin -eq "llama_cpp") { "llama" } else { "" } if (-not $bucket) { Write-Output "WARN: unknown plugin $plugin in $name, skipping" continue @@ -87,6 +97,13 @@ foreach ($row in $rows) { "{0}-{1}-{2}-c{3}`t{1}`t{2}`t{4}`t`t`t{5}`t{6}" -f ` $name, $plugin, $d, $ctx, $model_id, $imgpath, $vlm ` | Add-Content $tsvByPluginCtx["$bucket-$ctx"] + if ($bucket -eq "spec") { + $specParamsByCtx["$ctx"] = @{ + type = $spec_type + draft = $draft_model_id + tokens = $draft_tokens + } + } } } } @@ -115,6 +132,20 @@ for ($i = 0; $i -lt $ctxList.Count; $i++) { --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)" } + + $specTsv = $tsvByPluginCtx["spec-$ctx"] + if ((Test-Path $specTsv) -and ((Get-Item $specTsv).Length -gt 0)) { + $sp = $specParamsByCtx["$ctx"] + Write-Output "=== matrix spec ctx=$ctx tg=$tg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (prompt-file) ===" + Get-Content $specTsv + $extra = @() + if ($sp.tokens) { $extra += @("--draft-tokens", $sp.tokens) } + & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 3 ` + -c $ctx -n $tg --prompt-file "$PROMPTS\sample_prompt_$ctx.txt" ` + --spec-type $sp.type --draft-model $sp.draft @extra ` + --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" + Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)" + } } Write-Output "=== done ===" Stop-Transcript | Out-Null From 34504e811b2026fbb2014eb01284e3dc2badf058 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Fri, 14 Aug 2026 22:17:52 +0800 Subject: [PATCH 2/8] =?UTF-8?q?ci(bench):=20filter=20spec=C3=97non-SC8480X?= =?UTF-8?q?P=20via=20load-models=20excludes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit matrix.* is not addressable in a job-level if:, so GHA rejected the inline spec-device gate. Move the same logic into load-models so the bench matrix drops spec×non-SC8480XP up front and catalog-only rows never reach the strategy matrix at all. Signed-off-by: Mengsheng Wu --- .github/workflows/bench.yml | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f3754fa9e..ecc6cdfb8 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -45,6 +45,7 @@ jobs: outputs: matrix: ${{ steps.set.outputs.matrix }} devices: ${{ steps.set.outputs.devices }} + excludes: ${{ steps.set.outputs.excludes }} steps: - uses: actions/checkout@v7 - id: set @@ -64,35 +65,43 @@ jobs: fi echo "devices=$devices" >> "$GITHUB_OUTPUT" + # Catalog-only rows (empty `devices`, e.g. spec draft models) are + # never dispatched as their own bench cell — they exist so target + # rows can reference them by name via `spec.draft`. if [ -n "$PICK_MODEL" ]; then matrix=$(jq -c --arg s "$PICK_MODEL" ' ($s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))) as $pick | map(select(.name as $n | $pick | index($n))) + | map(select(.devices != [])) ' "$MODELS_FILE") picked_n=$(jq 'length' <<<"$matrix") want_n=$(jq -n --arg s "$PICK_MODEL" \ '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0)) | length') if [ "$picked_n" != "$want_n" ]; then - echo "::error::model input has unknown name(s): $PICK_MODEL" - echo "::error::available: $(jq -r '.[].name' "$MODELS_FILE" | paste -sd, -)" + echo "::error::model input has unknown or catalog-only name(s): $PICK_MODEL" + echo "::error::available: $(jq -r '.[] | select(.devices != []) | .name' "$MODELS_FILE" | paste -sd, -)" exit 1 fi else - matrix=$(jq -c . "$MODELS_FILE") + matrix=$(jq -c '[.[] | select(.devices != [])]' "$MODELS_FILE") fi echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + # Spec-decoding rows only run on SC8480XP today (target+draft mem + # footprint eliminates the other chipsets). Emit dynamic exclude + # combos so the bench matrix drops spec × non-SC8480XP without + # burning a runner slot per skip. + excludes=$(jq -c -n --argjson devices "$devices" --argjson models "$matrix" ' + [ $devices[] as $d | $models[] as $m + | select(($m.spec // null) != null and $d != "SC8480XP") + | {device: $d, model: {name: $m.name}} + ] + ') + echo "excludes=$excludes" >> "$GITHUB_OUTPUT" + bench: name: ${{ matrix.device }} · ${{ matrix.model.plugin }} · ${{ matrix.model.name }} needs: [build-sdk, load-models] - # Catalog-only rows (empty devices, e.g. spec draft models) never run - # standalone — they're pulled by the target row via --draft-model. Spec - # rows currently only run on SC8480XP; other chipsets are skipped. - if: >- - ${{ - matrix.model.devices != null && toJson(matrix.model.devices) != '[]' - && (!matrix.model.spec || matrix.device == 'SC8480XP') - }} runs-on: ubuntu-latest timeout-minutes: 180 strategy: @@ -101,6 +110,7 @@ jobs: matrix: device: ${{ fromJson(needs.load-models.outputs.devices) }} model: ${{ fromJson(needs.load-models.outputs.matrix) }} + exclude: ${{ fromJson(needs.load-models.outputs.excludes) }} env: QDC_API_KEY: ${{ secrets.QDC_API_KEY }} steps: From 4e92f25a881a8fe145e63ce480139bca2c20aafc Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Fri, 14 Aug 2026 22:38:44 +0800 Subject: [PATCH 3/8] fix(sdk): preserve spec.draft deps when --model-name trims models When --model-name filters the model list to a single spec row, the draft entry it references gets dropped, so _resolve_draft_model_id fails. Pull the referenced draft back in from the full catalog after filtering. Signed-off-by: Mengsheng Wu --- sdk/benchmark/qdc/run_qdc_jobs.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index e234127b7..f8547f172 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -618,11 +618,21 @@ def main() -> int: if platform not in BUILDERS: raise SystemExit(f"{platform} not implemented yet") - models = json.loads(args.models_file.read_text()) + all_models = json.loads(args.models_file.read_text()) if args.model_name: - models = [m for m in models if m["name"] == args.model_name] + models = [m for m in all_models if m["name"] == args.model_name] if not models: raise SystemExit(f"model {args.model_name!r} not in {args.models_file}") + # Pull in every spec.draft dependency so _resolve_draft_model_id can + # still find it after --model-name has trimmed the list to one row. + needed = {m["spec"]["draft"] for m in models if m.get("spec")} + for name in needed - {m["name"] for m in models}: + entry = next((m for m in all_models if m["name"] == name), None) + if entry is None: + raise SystemExit(f"draft {name!r} not in {args.models_file}") + models.append(entry) + else: + models = all_models compute_pick = [c.strip() for c in args.compute.split(",") if c.strip()] if compute_pick: From fba3cabaf3db0896df3b3777facb79883f7f5f23 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Fri, 14 Aug 2026 23:56:46 +0800 Subject: [PATCH 4/8] fix(sdk): resolve --draft-model via model-manager; drop unused accept-rate JSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without model-manager resolution the plugin gets a raw id like org/repo:Q4_0 for --draft-model, fails to open it as a file, and silently falls back to non-speculative decode — draft_n_total stays 0 and the MTP bench measures nothing spec-specific. Also drop the accept-rate additions to per-cell JSON + the MTP table's Accept% column: the bench only needs to prove the spec code path runs at random-token throughput, matching the plain llama_cpp bucket, so the prompt-file requirement + accept-rate reporting are unnecessary. Delete sample_prompt_8192.txt (spec cells now share random-ids prefill). Signed-off-by: Mengsheng Wu --- sdk/benchmark/benchmark.c | 155 +++-- .../qdc/prompts/sample_prompt_8192.txt | 539 ------------------ sdk/benchmark/qdc/run_qdc_jobs.py | 8 +- sdk/benchmark/qdc/windows/run_windows.ps1 | 15 +- 4 files changed, 117 insertions(+), 600 deletions(-) delete mode 100644 sdk/benchmark/qdc/prompts/sample_prompt_8192.txt diff --git a/sdk/benchmark/benchmark.c b/sdk/benchmark/benchmark.c index 769ac7fc5..1ac1e5277 100644 --- a/sdk/benchmark/benchmark.c +++ b/sdk/benchmark/benchmark.c @@ -86,9 +86,13 @@ typedef struct { const char* mmproj_path; /* Heap-owned copies populated when the model is resolved through the * model manager; freed at the end of run_one_cell. */ - char* mm_model_path; - char* mm_mmproj; - char* mm_tokenizer; + char* mm_model_path; + char* mm_mmproj; + char* mm_tokenizer; + /* Heap-owned local path for the speculative-decoding draft when + * --draft-model is a model-manager id rather than a filesystem path; + * kept for cleanup and shadows draft_model for the plugin call. */ + char* mm_draft_model; bool force_vlm; /* run VLM path even without an mmproj (QAIRT bundles) */ bool mm_is_vlm; /* manager classified the resolved model as VLM (geniex_ModelType) */ const char* image_paths[MAX_PATHS]; @@ -152,10 +156,8 @@ typedef struct { int64_t gen_tokens; double prefill_tps; double decode_tps; - int64_t draft_n_total; /* 0 when spec-decoding is off */ - int64_t draft_n_accepted; /* 0 when spec-decoding is off */ - const char* stop_reason; /* not freed; lifetime tied to SDK output */ - int32_t status; /* 0 ok */ + const char* stop_reason; /* not freed; lifetime tied to SDK output */ + int32_t status; /* 0 ok */ char err[256]; } run_result_t; @@ -516,6 +518,67 @@ static int resolve_via_mm(options_t* o, const char* id_in) { return 0; } +/* Resolve --draft-model when it looks like a model-manager id (not a + * filesystem path). Populates o->mm_draft_model (heap-owned, freed by + * run_one_cell) and rewrites o->draft_model to point at it. Without this + * the llama_cpp plugin gets a raw id like "org/repo:Q4_0", fails to open + * it as a file, and silently falls back to non-speculative decode — + * draft_n_total stays 0 and the whole spec bench is meaningless. */ +static int resolve_draft_via_mm(options_t* o) { + if (!o->draft_model || looks_like_path(o->draft_model)) return 0; + if (!g_mm_inited) { + int32_t rc = geniex_model_init(o->mm_data_dir); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_init: %s (%d)\n", m ? m : "?", rc); + return 1; + } + g_mm_inited = true; + } + size_t n = strlen(o->draft_model); + char* buf = (char*)malloc(n + 1); + if (!buf) return 1; + memcpy(buf, o->draft_model, n + 1); + const char* name; + const char* quant; + split_id(buf, &name, &quant); + geniex_ModelPaths paths; + memset(&paths, 0, sizeof(paths)); + int32_t rc = geniex_model_get_paths(o->draft_model, &paths); + if (rc != GENIEX_SUCCESS) { + geniex_ModelPullInput in; + memset(&in, 0, sizeof(in)); + in.struct_size = (uint32_t)sizeof(in); + in.model_name = name; + in.quant = quant; + in.hub = parse_hub(o->mm_hub); + in.chipset = o->mm_chipset; + in.model_type = GENIEX_MODEL_TYPE_AUTO; + fprintf(stderr, "[mm ] pulling draft %s%s%s ...\n", name, quant ? ":" : "", quant ? quant : ""); + rc = geniex_model_pull(&in); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_pull(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc); + free(buf); + return 1; + } + rc = geniex_model_get_paths(o->draft_model, &paths); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_get_paths(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc); + free(buf); + return 1; + } + } + free(buf); + o->mm_draft_model = paths.model_path; + paths.model_path = NULL; + geniex_model_paths_free(&paths); + o->draft_model = o->mm_draft_model; + fprintf(stderr, "[mm ] resolved draft %s -> %s\n", o->draft_model, o->mm_draft_model); + return 0; +} + /* If `path` is a directory, return a heap-allocated path to a regular file * inside it (preferring `tokenizer.json`, otherwise the lexicographically * first regular file). The SDK derives the model dir via `parent_path()`, @@ -660,6 +723,7 @@ static void parse_args(int argc, char** argv, options_t* o) { o->mm_model_path = NULL; o->mm_mmproj = NULL; o->mm_tokenizer = NULL; + o->mm_draft_model = NULL; o->force_vlm = false; o->mm_is_vlm = false; o->image_count = 0; @@ -1232,18 +1296,16 @@ static void run_llm(const options_t* o, const char* device_id, int32_t ngl, run_ if (!is_warmup) { run_result_t* r = &out[run_idx]; memset(r, 0, sizeof(*r)); - r->run_idx = run_idx; - r->ttft_us = gout.profile_data.ttft; - r->prompt_time_us = gout.profile_data.prompt_time; - r->decode_time_us = gout.profile_data.decode_time; - r->prompt_tokens = gout.profile_data.prompt_tokens; - r->gen_tokens = gout.profile_data.generated_tokens; - r->prefill_tps = gout.profile_data.prefill_speed; - r->decode_tps = gout.profile_data.decoding_speed; - r->draft_n_total = gout.profile_data.draft_n_total; - r->draft_n_accepted = gout.profile_data.draft_n_accepted; - r->stop_reason = gout.profile_data.stop_reason; - r->status = 0; + r->run_idx = run_idx; + r->ttft_us = gout.profile_data.ttft; + r->prompt_time_us = gout.profile_data.prompt_time; + r->decode_time_us = gout.profile_data.decode_time; + r->prompt_tokens = gout.profile_data.prompt_tokens; + r->gen_tokens = gout.profile_data.generated_tokens; + r->prefill_tps = gout.profile_data.prefill_speed; + r->decode_tps = gout.profile_data.decoding_speed; + r->stop_reason = gout.profile_data.stop_reason; + r->status = 0; normalize_prefill_metrics(r, o->plugin); } @@ -1387,18 +1449,16 @@ static void run_vlm(const options_t* o, const char* device_id, int32_t ngl, run_ if (!is_warmup) { run_result_t* r = &out[run_idx]; memset(r, 0, sizeof(*r)); - r->run_idx = run_idx; - r->ttft_us = gout.profile_data.ttft; - r->prompt_time_us = gout.profile_data.prompt_time; - r->decode_time_us = gout.profile_data.decode_time; - r->prompt_tokens = gout.profile_data.prompt_tokens; - r->gen_tokens = gout.profile_data.generated_tokens; - r->prefill_tps = gout.profile_data.prefill_speed; - r->decode_tps = gout.profile_data.decoding_speed; - r->draft_n_total = gout.profile_data.draft_n_total; - r->draft_n_accepted = gout.profile_data.draft_n_accepted; - r->stop_reason = gout.profile_data.stop_reason; - r->status = 0; + r->run_idx = run_idx; + r->ttft_us = gout.profile_data.ttft; + r->prompt_time_us = gout.profile_data.prompt_time; + r->decode_time_us = gout.profile_data.decode_time; + r->prompt_tokens = gout.profile_data.prompt_tokens; + r->gen_tokens = gout.profile_data.generated_tokens; + r->prefill_tps = gout.profile_data.prefill_speed; + r->decode_tps = gout.profile_data.decoding_speed; + r->stop_reason = gout.profile_data.stop_reason; + r->status = 0; normalize_prefill_metrics(r, o->plugin); } @@ -1561,9 +1621,7 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i fprintf(f, " {\"run_idx\": %d, \"ttft_us\": %lld, \"prompt_tokens\": %lld, " "\"gen_tokens\": %lld, \"prefill_tps\": %.6f, \"decode_tps\": %.6f, " - "\"prompt_time_us\": %lld, \"decode_time_us\": %lld, " - "\"draft_n_total\": %lld, \"draft_n_accepted\": %lld, " - "\"stop_reason\": %s%s%s}%s\n", + "\"prompt_time_us\": %lld, \"decode_time_us\": %lld, \"stop_reason\": %s%s%s}%s\n", r->run_idx, (long long)r->ttft_us, (long long)r->prompt_tokens, @@ -1572,8 +1630,6 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i r->decode_tps, (long long)r->prompt_time_us, (long long)r->decode_time_us, - (long long)r->draft_n_total, - (long long)r->draft_n_accepted, r->stop_reason ? "\"" : "null", r->stop_reason ? r->stop_reason : "", r->stop_reason ? "\"" : "", @@ -1603,19 +1659,8 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i a->decode_mean, a->decode_sd); fprintf(f, " \"gen_tokens\": {\"median\": %.6f},\n", a->gen_tokens_med); - fprintf(f, " \"prompt_tokens\":{\"median\": %.6f}", a->prompt_tokens_med); - int64_t sum_draft_total = 0, sum_draft_accepted = 0; - for (int i = 0; i < o->repeat; ++i) { - sum_draft_total += runs[i].draft_n_total; - sum_draft_accepted += runs[i].draft_n_accepted; - } - if (sum_draft_total > 0) { - fprintf(f, ",\n \"draft_n_total\": {\"total\": %lld},\n", (long long)sum_draft_total); - fprintf(f, " \"draft_n_accepted\": {\"total\": %lld},\n", (long long)sum_draft_accepted); - fprintf( - f, " \"draft_accept_rate\": {\"value\": %.6f}", (double)sum_draft_accepted / (double)sum_draft_total); - } - fprintf(f, "\n }\n"); + fprintf(f, " \"prompt_tokens\":{\"median\": %.6f}\n", a->prompt_tokens_med); + fprintf(f, " }\n"); fprintf(f, "}\n"); fclose(f); /* keep static-analysis happy */ @@ -1876,6 +1921,10 @@ static int run_one_cell(options_t* o) { o->force_vlm = true; } + if (resolve_draft_via_mm(o) != 0) { + return 1; + } + char* anchored = resolve_local_anchor(o->model_path); if (anchored) { fprintf(stderr, "[info] resolved model dir to anchor: %s\n", anchored); @@ -1942,6 +1991,10 @@ static int run_one_cell(options_t* o) { geniex_free(o->mm_tokenizer); o->mm_tokenizer = NULL; } + if (o->mm_draft_model) { + geniex_free(o->mm_draft_model); + o->mm_draft_model = NULL; + } return 0; } @@ -1985,6 +2038,10 @@ static int run_one_cell(options_t* o) { geniex_free(o->mm_tokenizer); o->mm_tokenizer = NULL; } + if (o->mm_draft_model) { + geniex_free(o->mm_draft_model); + o->mm_draft_model = NULL; + } return 0; } diff --git a/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt b/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt deleted file mode 100644 index 55e6136fb..000000000 --- a/sdk/benchmark/qdc/prompts/sample_prompt_8192.txt +++ /dev/null @@ -1,539 +0,0 @@ -Please summarize the following comprehensive text in 2-3 sentences, capturing -the main themes and key points: - -The rapid evolution of artificial intelligence and machine learning -technologies has fundamentally transformed the landscape of modern -civilization, creating unprecedented opportunities for innovation while -simultaneously presenting complex challenges that demand careful consideration -and proactive management across multiple domains of human activity. In the -realm of healthcare and medical sciences, artificial intelligence systems have -achieved remarkable breakthroughs in diagnostic capabilities, enabling medical -professionals to analyze complex medical images with superhuman accuracy, -detect early-stage cancers and other life-threatening conditions with -unprecedented precision, and process vast amounts of patient data to predict -disease progression, optimize treatment protocols, and accelerate the -traditionally lengthy process of drug discovery and development. Pharmaceutical -companies worldwide are leveraging sophisticated AI algorithms to identify -promising drug compounds, predict molecular interactions with greater accuracy -than traditional methods, streamline clinical trial processes, and develop -personalized medicine approaches that tailor treatments to individual genetic -profiles, medical histories, and specific patient characteristics, potentially -reducing both the time and cost required to bring new medications to market -while significantly improving patient outcomes and quality of life. - -The financial services industry has undergone a revolutionary transformation -through the implementation of AI-powered systems that monitor millions of -financial transactions in real-time to detect fraudulent activities, execute -complex trading strategies based on sophisticated analysis of market patterns -and economic indicators, and provide personalized investment advice through -advanced robo-advisor platforms that analyze individual risk tolerance, -financial goals, market conditions, and economic trends to optimize portfolio -performance and maximize returns for investors. Banks, credit institutions, and -lending organizations are utilizing machine learning algorithms to assess -creditworthiness with greater accuracy than traditional scoring methods, while -insurance companies employ AI technologies to evaluate risk factors more -precisely, process claims more efficiently, develop innovative insurance -products that better serve customer needs, and reduce operational costs while -improving profitability and customer satisfaction. The transportation sector -has witnessed dramatic changes with the development of autonomous vehicles that -combine advanced computer vision systems, sophisticated sensor fusion -technologies, deep learning algorithms, and real-time decision-making -capabilities to navigate complex traffic scenarios safely and efficiently, -while ride-sharing platforms utilize AI to optimize driver-passenger matching -algorithms, implement dynamic pricing strategies based on demand patterns and -supply availability, and improve route efficiency to reduce travel times and -minimize environmental impact. - -Manufacturing industries have embraced the concept of Industry 4.0, where smart -factories equipped with AI-powered systems can predict equipment failures -before they occur, optimize production schedules based on sophisticated demand -forecasts, implement quality control measures that detect defects with greater -precision than human inspectors, and coordinate complex supply chain operations -to increase efficiency, reduce waste, improve product quality, and minimize -environmental impact. Educational institutions are implementing personalized -learning platforms that adapt to individual student learning styles, pace, -preferences, and cognitive abilities, providing customized content, -assessments, and feedback that help students achieve better educational -outcomes while enabling teachers to focus on higher-level instruction, student -interaction, and pedagogical innovation rather than administrative tasks and -routine assessments. Virtual assistants and chatbots powered by advanced -natural language processing technologies provide instant customer support -across various industries, handling routine inquiries, resolving common issues, -escalating complex problems to human agents when necessary, and improving -overall customer satisfaction while reducing operational costs and increasing -service availability. - -Social media platforms and content providers utilize AI algorithms to curate -personalized feeds, recommend relevant content based on user preferences and -behavior patterns, moderate user-generated content to maintain community -standards, and prevent harmful or inappropriate material from spreading across -their networks. The entertainment industry has been transformed by AI -technologies that can generate music, create visual effects, write scripts, -produce entire movies with minimal human intervention, and develop interactive -gaming experiences that adapt to player behavior and preferences, while -streaming services employ sophisticated recommendation algorithms to keep users -engaged and satisfied with personalized content suggestions that maximize -viewing time and subscriber retention. Retail and e-commerce companies employ -AI for inventory management, demand forecasting, price optimization, customer -service automation, and personalized marketing campaigns, creating more -efficient supply chains and better shopping experiences for consumers worldwide -while reducing costs and improving profitability. - -Agriculture has benefited significantly from AI-powered precision farming -techniques that optimize irrigation, fertilization, and pest control based on -real-time data from sensors, satellite imagery, weather forecasts, and soil -analysis, leading to increased crop yields, more sustainable farming practices, -reduced environmental impact, and improved food security for growing -populations. Energy companies utilize AI to optimize power grid operations, -predict equipment failures, integrate renewable energy sources more -effectively, manage energy storage systems, and develop smart grid technologies -that contribute to the transition toward cleaner, more efficient, and more -reliable energy systems. Environmental monitoring and climate research have -been enhanced by AI systems that can analyze vast amounts of satellite data, -weather patterns, atmospheric conditions, and environmental indicators to -predict natural disasters, track climate change impacts, monitor biodiversity, -and develop comprehensive strategies for environmental protection, -conservation, and sustainable development. - -However, these technological advances also present significant challenges that -society must address proactively, including concerns about job displacement as -automation and AI systems replace human workers in various sectors, requiring -comprehensive retraining programs, educational initiatives, and policy -interventions to help workers transition to new roles that complement rather -than compete with AI systems. Privacy protection has become increasingly -complex as AI systems require access to vast amounts of personal data to -function effectively, raising critical questions about data ownership, consent -mechanisms, security protocols, and the potential for misuse or unauthorized -access that could compromise individual privacy rights and personal security. -Algorithmic bias represents another critical concern, as AI systems trained on -historical data may perpetuate or amplify existing societal prejudices, -inequalities, and discriminatory practices, particularly in areas like hiring, -lending, criminal justice, healthcare access, and educational opportunities -where biased decisions could have serious consequences for individuals and -communities. - -The rapid pace of AI development has outpaced the creation of appropriate -regulatory frameworks, governance structures, and oversight mechanisms, leaving -significant gaps in accountability, transparency, and ethical oversight that -could lead to unintended consequences or misuse of powerful AI technologies. -International cooperation and coordination are essential for addressing global -challenges posed by AI, including issues of governance, standards development, -cybersecurity, intellectual property rights, and the equitable distribution of -benefits across different populations, regions, and socioeconomic groups. The -development of artificial general intelligence (AGI) remains a long-term goal -that could have profound implications for humanity, potentially solving complex -global challenges like climate change, disease eradication, poverty reduction, -and resource management while also raising new ethical, safety, and existential -concerns that require careful consideration, preparation, and international -collaboration. - -Cybersecurity threats have evolved alongside AI development, with malicious -actors using AI to create more sophisticated attacks, generate convincing -deepfakes, exploit vulnerabilities in AI systems themselves, and develop new -forms of cyber warfare that require constant vigilance, innovation in defensive -measures, and international cooperation to address effectively. The digital -divide between those who have access to AI technologies and those who do not -threatens to exacerbate existing inequalities, making it crucial to ensure that -AI benefits are distributed equitably across different socioeconomic groups, -geographic regions, and demographic populations. Ethical considerations around -AI development include questions about transparency, accountability, fairness, -human dignity, and the appropriate use of AI in sensitive applications like -military systems, surveillance technologies, autonomous weapons, and -decision-making processes that affect human lives, rights, and fundamental -freedoms. - -The environmental impact of AI systems, particularly large-scale machine -learning models that require enormous computational resources and energy -consumption, raises concerns about sustainability, carbon footprint, resource -depletion, and the long-term environmental consequences of AI development and -deployment. Intellectual property rights and ownership of AI-generated content -present complex legal challenges that courts, lawmakers, and international -bodies are still grappling with, particularly regarding copyright, patents, -trademarks, and the rights of AI systems themselves versus their human creators -and users. The concentration of AI expertise, resources, and capabilities in a -small number of large technology companies raises concerns about monopolistic -control, market dominance, and the need for more diverse, decentralized, and -democratized AI development approaches that promote competition, innovation, -and equitable access. - -Ensuring that AI development serves human interests and promotes global -prosperity will require ongoing dialogue, collaboration, and cooperation -between technologists, policymakers, ethicists, philosophers, social -scientists, and representatives from diverse communities, cultures, and -perspectives to create comprehensive frameworks that balance innovation with -responsibility, progress with protection, efficiency with equity, and -technological advancement with human flourishing in the age of artificial -intelligence. The future of AI will likely involve increased collaboration -between humans and machines, with augmented intelligence approaches that -enhance human capabilities rather than replacing them entirely, creating new -opportunities for creativity, problem-solving, artistic expression, scientific -discovery, and human flourishing in an AI-enhanced world that preserves human -agency, dignity, and purpose. - -Additionally, the integration of AI with emerging technologies like quantum -computing, blockchain, and the Internet of Things (IoT) is creating new -possibilities for innovation, efficiency, and transformative change across -multiple sectors and industries. Quantum computing has the potential to -dramatically accelerate AI computations, enable new types of algorithms that -could solve problems currently intractable for classical computers, and -revolutionize cryptography, optimization, and simulation capabilities that -could transform scientific research, financial modeling, and technological -development. Blockchain technology combined with AI is creating new -possibilities for secure, transparent, and decentralized applications in -finance, supply chain management, digital identity verification, voting -systems, and governance mechanisms that could enhance trust, reduce corruption, -and improve efficiency in various domains. - -The Internet of Things generates massive amounts of data that AI systems can -analyze to optimize processes, predict maintenance needs, improve -decision-making in real-time, and create smart environments across smart -cities, industrial facilities, consumer devices, and infrastructure systems. -Edge computing brings AI processing closer to data sources, enabling real-time -decision-making in applications like autonomous vehicles, industrial -automation, smart home systems, and mobile devices where latency and bandwidth -constraints make cloud-based processing impractical or inefficient. -Neuromorphic computing, which mimics the structure and function of the human -brain, may lead to more efficient and powerful AI systems that can process -information with lower energy consumption, faster response times, and greater -adaptability compared to traditional digital computers. - -Brain-computer interfaces represent another frontier where AI could enable -direct communication between human brains and computer systems, potentially -revolutionizing how we interact with technology, treat neurological conditions, -enhance cognitive abilities, and create new forms of human-machine symbiosis. -The development of explainable AI (XAI) is becoming increasingly important as -organizations seek to understand how AI systems make decisions, particularly in -high-stakes applications like healthcare, finance, criminal justice, and -autonomous systems where transparency, accountability, and interpretability are -essential for building trust, ensuring fair outcomes, and meeting regulatory -requirements. Federated learning approaches allow AI models to be trained on -data distributed across multiple devices or organizations without centralizing -sensitive information, addressing privacy concerns while still enabling -collaborative learning, model improvement, and knowledge sharing across -different entities and jurisdictions. - -The democratization of AI tools and platforms is making advanced machine -learning capabilities accessible to smaller organizations, individual -developers, researchers, and entrepreneurs, potentially leading to more -diverse, innovative, and creative applications across different industries, use -cases, and cultural contexts. However, this democratization also raises -concerns about the potential for misuse, as powerful AI capabilities become -available to individuals or groups who may not have the expertise, ethical -framework, or accountability mechanisms to use them responsibly and safely. The -development of AI safety research is becoming increasingly important as AI -systems become more powerful, autonomous, and capable, requiring careful -consideration of alignment problems, robustness issues, adversarial attacks, -and the potential for unintended consequences that could have serious negative -impacts on society, individuals, and global stability. - -International competition in AI development, particularly between major powers -like the United States, China, the European Union, and other nations, raises -concerns about technological arms races, national security implications, and -the potential for AI to be weaponized or used for surveillance, control, and -authoritarian purposes that could threaten democratic values, human rights, and -international peace and security. The need for international cooperation, -governance frameworks, and multilateral agreements becomes even more critical -as AI systems become more powerful and their impacts become more global in -scope, requiring coordinated efforts to ensure that AI development serves human -interests, promotes peace, prosperity, human dignity, and sustainable -development worldwide. The economic implications of AI adoption are complex and -multifaceted, with potential for significant productivity gains, economic -growth, and prosperity alongside concerns about job displacement, income -inequality, wealth concentration, and the distribution of benefits across -different segments of society. - -The transformation of labor markets will require comprehensive policy responses -including education reform, vocational training programs, social safety nets, -universal basic income experiments, and economic policies that ensure the -benefits of AI are shared broadly across society rather than accruing primarily -to a small elite of technology companies and wealthy individuals. The -development of AI ethics, responsible AI practices, and governance frameworks -is becoming increasingly important as organizations recognize the need to -ensure that their AI systems are fair, transparent, accountable, unbiased, and -aligned with human values, societal goals, and ethical principles. This -includes considerations of bias mitigation, privacy protection, algorithmic -transparency, human oversight, and the development of governance structures -that can oversee AI development and deployment in ways that promote beneficial -outcomes while minimizing risks, negative consequences, and unintended harm. - -The integration of AI into critical infrastructure systems like power grids, -transportation networks, communication systems, financial markets, and -healthcare systems raises important questions about resilience, security, -reliability, and the potential for cascading failures that could have serious -consequences for society, economy, and public safety. The development of AI -standards, certification processes, and regulatory frameworks is becoming -increasingly important to ensure that AI systems meet minimum requirements for -safety, reliability, ethical behavior, and human compatibility across different -applications, industries, and use cases. The role of government in AI -development and regulation is evolving as policymakers grapple with the -challenge of promoting innovation, economic competitiveness, and technological -leadership while protecting public interests, ensuring safety, and implementing -appropriate safeguards and oversight mechanisms. - -The future of work in an AI-driven economy will likely involve significant -changes in job requirements, skill demands, career paths, and the nature of -human-AI collaboration, requiring proactive efforts to prepare workers, -students, and society for these changes and ensure that the transition is -managed in ways that promote human flourishing, economic opportunity, social -mobility, and meaningful employment for all members of society. The development -of AI literacy, digital skills, and technological competence will become -increasingly important for individuals, organizations, and societies to -navigate the AI-driven world effectively, make informed decisions about AI -adoption, and participate meaningfully in shaping the future of AI development -and deployment. The ethical, social, and philosophical implications of AI -development require ongoing reflection, debate, and dialogue among diverse -stakeholders to ensure that AI technologies serve human values, promote human -flourishing, and contribute to a more just, equitable, and sustainable world -for future generations. - -The emergence of large language models and generative AI systems has created -new opportunities for human-computer interaction, creative expression, and -knowledge synthesis, while also raising questions about authenticity, -originality, and the nature of human creativity in an age where machines can -produce text, images, music, and other forms of content that rival or exceed -human capabilities. The development of multimodal AI systems that can process -and generate content across different modalities including text, images, audio, -and video represents a significant advancement that could revolutionize fields -like education, entertainment, communication, and creative industries, enabling -new forms of interactive experiences and personalized content creation that -adapt to individual preferences and learning styles. - -The intersection of AI with fields like psychology, neuroscience, and cognitive -science is providing new insights into human intelligence, learning processes, -and decision-making mechanisms, while also enabling the development of AI -systems that better understand and interact with human users in more natural -and intuitive ways. The application of AI in scientific research and discovery -is accelerating the pace of innovation across disciplines, from drug discovery -and materials science to astronomy and climate modeling, enabling researchers -to analyze complex datasets, identify patterns, generate hypotheses, and -conduct virtual experiments that would be impossible or impractical using -traditional methods. The development of AI-powered scientific writing -assistants, research tools, and knowledge management systems is transforming -how researchers collaborate, share information, and build upon each other's -work, potentially accelerating the pace of scientific discovery and innovation. - -The role of AI in addressing global challenges like climate change, food -security, healthcare access, and education equity is becoming increasingly -important as these systems demonstrate their potential to optimize resource -allocation, predict environmental changes, develop sustainable solutions, and -provide personalized services at scale. The development of AI systems that can -operate in resource-constrained environments, work with limited data, and adapt -to diverse cultural and linguistic contexts is crucial for ensuring that the -benefits of AI technology reach underserved populations and developing regions. -The integration of AI with mobile technologies, low-cost computing devices, and -emerging communication networks is creating new possibilities for delivering -AI-powered services to remote and rural communities, potentially reducing -inequalities and improving quality of life for millions of people worldwide. - -The evolution of AI-human collaboration patterns is reshaping organizational -structures, work processes, and decision-making frameworks across industries, -requiring new approaches to leadership, management, and organizational design -that can effectively leverage both human creativity and AI capabilities. The -development of AI systems that can learn from human feedback, adapt to -individual preferences, and collaborate effectively with human users represents -a significant advancement toward more intuitive and productive human-AI -partnerships. The emergence of AI-powered personal assistants, productivity -tools, and decision support systems is transforming how individuals work, -and learn, potentially enhancing human capabilities while -maintaining human agency and control over important choices and actions. - -The transportation sector stands at the threshold of a profound transformation -driven by the convergence of artificial intelligence, sensor technology, and -electrification, with autonomous vehicles progressing from controlled test -environments to real-world deployments in ride-hailing fleets, freight -corridors, last-mile delivery services, and public transit networks in major -metropolitan areas around the world. Advanced perception systems combining -lidar, radar, camera arrays, and high-definition mapping enable vehicles to -interpret complex urban scenes, anticipate the behavior of pedestrians and -other road users, and negotiate multi-agent intersections with a level of -consistency that begins to rival experienced human drivers under favorable -conditions. Behind the scenes, learning-based planners and prediction models -continuously ingest telemetry from vast fleets, allowing manufacturers to -identify long-tail edge cases, refine control policies, and push over-the-air -updates that improve safety margins without a physical service visit. -Meanwhile, AI-optimized traffic-signal control, dynamic congestion pricing, -and demand-responsive transit routing are reducing average commute times, -smoothing peak-hour bottlenecks, and cutting per-passenger emissions in cities -that have committed to data-driven mobility planning. - -Aviation, maritime shipping, and rail freight are undergoing parallel shifts, -with predictive-maintenance systems flagging component failures days or weeks -before they would strand a vessel, aircraft, or locomotive, while route- -optimization engines fold in weather forecasts, fuel prices, port congestion, -and regulatory constraints to shave hours off long-haul journeys and trim -substantial amounts off operator fuel bills. Air traffic controllers now -receive machine-learned recommendations for spacing and sequencing that -increase runway throughput without eroding separation minimums, and container -terminals rely on computer-vision systems to track stacks in real time, -coordinate straddle-carrier movements, and expose bottlenecks that used to -require weeks of manual observation to diagnose. The cumulative effect is a -supply chain that responds to disruption more quickly, wastes less capacity, -and creates new opportunities for smaller operators to compete with the -incumbent giants by renting analytics capabilities that once required a -dedicated internal data-science team. - -Education is being reshaped in equally consequential ways, with adaptive -learning platforms now capable of building individualized study paths that -respond to a learner's pace, misconceptions, and preferred modality, whether -that means additional worked examples, alternate explanations, or short -formative quizzes designed to catch shallow understanding before it hardens -into a persistent gap. Teachers, freed from significant portions of the -grading and lesson-preparation work that once consumed their evenings, can -spend more time on coaching, mentorship, and the socio-emotional aspects of -learning that machines are least equipped to handle. University researchers -are also using AI to accelerate literature review, generate plausible -experimental hypotheses, and simulate outcomes for interventions that would be -too expensive or ethically fraught to attempt directly, though these tools -have also introduced difficult new questions about attribution, academic -integrity, and the role that generative systems should play in student -assessment. Policy makers are still grappling with how to certify AI-assisted -curricula, how to protect student data, and how to ensure that adaptive -systems do not quietly reproduce or amplify existing inequities along lines -of race, socioeconomic status, first language, or learning difference. - -Environmental monitoring and climate science represent perhaps the most -consequential domain for near-term AI application, with machine-learning -emulators standing in for physically detailed climate simulations at a tiny -fraction of the computational cost, enabling exploration of many more -scenarios and thereby sharpening projections of regional impacts on -agriculture, water availability, and extreme-weather frequency. Satellite -imagery interpreted by deep-learning models supplies near real-time signals -on deforestation, illegal fishing, methane leaks from oil and gas -infrastructure, and shifts in land-use patterns that would otherwise take -years to appear in official statistics. In wildfire-prone regions, -AI-informed sensor networks detect ignitions within minutes and route -resources with less human coordination overhead, while in flood-vulnerable -watersheds, learned hydrological models produce probabilistic warnings that -give communities several additional hours of preparation time. On the -mitigation side, grid operators use forecasting models to balance rapidly -growing shares of solar and wind generation, dispatch battery storage at the -most valuable moments, and reduce the number of hours per year during which -carbon-intensive peaker plants must run. - -Agriculture, one of the oldest human activities, is being quietly rewired by -precision-farming platforms that interpret drone imagery, in-soil moisture -probes, and multispectral satellite passes to guide irrigation, fertilizer -placement, and integrated pest management with a spatial resolution far finer -than a single field. Growers who once relied on calendar-based spraying now -receive alerts that pinpoint specific rows or sections that warrant -intervention, cutting chemical inputs and reducing runoff into surrounding -watersheds. Robotic weeders, thinners, and harvesters increasingly handle -tasks that historically depended on seasonal labor shortages, though the -transition has raised legitimate concerns about the livelihoods of rural -workers and the concentration of decision-making power in the hands of a -small number of software vendors. In the developing world, low-cost mobile -advisory services translate weather forecasts, market prices, and agronomic -recommendations into local languages, giving smallholder farmers access to -insights that were previously available only to industrial-scale operations. - -Space exploration and Earth observation have entered a period of rapid -acceleration in which AI plays a central rather than peripheral role, with -autonomous spacecraft that can plan their own observation sequences, prioritize -downlink targets given limited bandwidth, and diagnose their own faults -without waiting for a round-trip conversation with mission control. Rover -missions to Mars and the Moon use learned obstacle-avoidance and traverse -planning to cover more ground per operational day than previous generations -could manage in a week. On the commercial side, an expanding fleet of small -imaging satellites captures the entire planet at daily or even sub-daily -cadence, and AI-based analytics extract signals ranging from car counts in -retail parking lots to soil moisture in agricultural regions to structural -displacement along fault lines and dam faces. These capabilities are quietly -reshaping insurance underwriting, commodity trading, disaster response, and -international treaty verification in ways that few observers anticipated only -a decade ago. - -The energy sector's transformation deserves particular attention, since it -sits at the intersection of climate policy, industrial strategy, and everyday -consumer experience. Utilities operating aging grids now rely on AI-based -fault localization to isolate outages faster and dispatch repair crews more -efficiently, while distribution planners use load-forecasting models that -account for electric-vehicle adoption curves, heat-pump uptake, and rooftop -solar penetration on a neighborhood-by-neighborhood basis. Fusion research -programs have begun to publish results in which reinforcement-learning -controllers stabilize plasma configurations that traditional control theory -could not, offering hope that long-elusive commercial fusion power might -arrive on a timeline that matters for climate goals. On the demand side, -building-management platforms coordinate HVAC, lighting, and plug loads to -minimize consumption during high-price hours, and residential thermostats -learn occupant preferences well enough to trim ten to twenty percent from -household heating and cooling bills without any perceptible loss of comfort. - -Manufacturing, long a testbed for automation of a mechanical rather than -cognitive variety, is now being reshaped by generative design systems that -propose thousands of candidate part geometries meeting a stated performance -envelope, computer-vision quality inspectors that detect surface defects far -smaller than a human operator can reliably see, and reinforcement-learning -robot controllers that adapt to variability in incoming stock and tool wear -without retooling. Small and medium-sized manufacturers, who once could not -afford the fixed cost of a full analytics team, increasingly consume these -capabilities as software services, narrowing the productivity gap that has -long separated them from multinational competitors. Additive manufacturing -combined with topology-optimized designs is enabling parts that would have -been impossible to produce by conventional means, unlocking weight savings in -aerospace, medical implants that better match the recipient's anatomy, and -spare-parts distribution models in which components are printed on demand -close to the point of use rather than shipped from a distant warehouse. - -The scientific enterprise as a whole is being augmented in ways that are -beginning to alter how research is planned, executed, and communicated. -Protein-structure prediction, once the crown jewel of an entire subfield, has -become a routine step in the workflow of any lab studying enzymes, membrane -receptors, or antibody engineering, and comparable transformations are under -way in materials discovery, catalysis, and small-molecule chemistry. -Automated laboratories combine liquid-handling robots, in-line analytics, and -active-learning experiment planners to run hundreds of experiments per day -under conditions that would be tedious and error-prone for human researchers, -while large language models tuned on scientific corpora assist with -literature synthesis, hypothesis generation, and the drafting of grant -proposals and manuscripts. These tools raise pressing questions about -authorship, reproducibility, and the appropriate level of human oversight, -but their productivity benefits are large enough that most funding agencies -and research institutions are actively investing in the infrastructure and -governance required to use them responsibly. - -Cybersecurity and digital trust have become correspondingly high-stakes -domains, with AI-driven attack tooling lowering the barrier to sophisticated -phishing, social engineering, and vulnerability discovery, and AI-driven -defense tooling detecting anomalous behavior in networks, endpoints, and -identity systems at a scale no human security operations center could match. -The arms-race dynamic between offense and defense is more visible than ever, -and organizations that once treated security as a compliance checkbox are -being forced to invest in continuous monitoring, red-team exercises, and -incident-response rehearsals that assume adversaries will use every automation -available. In parallel, national governments are experimenting with AI-based -content-provenance systems, deepfake-detection benchmarks, and coordinated -disclosure frameworks intended to preserve some measure of shared reality in -information ecosystems that generative models can otherwise flood with -plausible-looking synthetic material. - -Consumer experience is also being reshaped, sometimes in ways that draw -attention and sometimes so quietly that users only notice the improvement in -aggregate. Search engines, once purely keyword-driven, now interpret -questions in context and surface direct answers pulled from indexed sources -with citations that let the curious follow up. Streaming services and -retailers refine their recommendations continuously, translating browsing -signals into surprisingly accurate models of individual taste, and messaging -applications translate between languages, transcribe voice notes, and -summarize threads on demand. In the workplace, meeting assistants transcribe -conversations, extract action items, and populate task trackers, while -coding assistants draft, review, and refactor software at a level of fluency -that has already changed how many teams estimate feature timelines. These -gains are unevenly distributed, and questions about privacy, consent, and -the appropriate handling of sensitive conversational data remain very much -open, but the trajectory of adoption suggests they are now table stakes for -competitive products rather than optional differentiators. - -Taken together, the developments across all of these sectors point to a -world in which artificial intelligence has become a general-purpose -technology on the order of electrification or the internal combustion -engine, permeating industries and everyday life to a degree that few -observers fully appreciated even five years ago. The pace of change is -unlikely to slow in the near term, and the same institutions that must -harness AI for productivity, competitiveness, and scientific advance are -simultaneously responsible for anticipating harms, allocating benefits -fairly, and preserving the democratic accountability of decisions that -increasingly rely on opaque algorithmic components. How that dual mandate -is discharged in the coming decade will determine whether the promise of -this moment is broadly shared or captured by a narrow set of actors, and -whether public trust in the underlying technology grows in step with its -capabilities or erodes under the weight of avoidable missteps. diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index f8547f172..8671499e1 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -461,8 +461,6 @@ def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str] base_cells = by_name_key.get(baseline["name"], {}) if baseline else {} for (dev, ctx), sc in sorted(spec_cells.items()): agg = sc.get("agg") or {} - accept = (agg.get("draft_accept_rate") or {}).get("value") - accept_s = f"{100 * accept:.1f}%" if accept is not None else "-" s_dec = (agg.get("decode_tps") or {}).get("median") bc = base_cells.get((dev, ctx)) b_dec = ( @@ -480,7 +478,7 @@ def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str] ) rows.append( f"| {spec_m['name']} | {draft_m['name'] if draft_m else '-'} | " - f"{dev} | {ctx} | {test} | {accept_s} | " + f"{dev} | {ctx} | {test} | " f"{_fmt_med_sd(agg, 'decode_tps')} | " f"{_fmt_med_sd((bc or {}).get('agg') or {}, 'decode_tps')} | " f"{uplift} |" @@ -491,8 +489,8 @@ def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str] "", "## MTP (speculative decoding)", "", - "| Target | Draft | Device | Ctx | Test | Accept% | Decode (mtp) | Decode (baseline) | Uplift |", - "|--------|-------|--------|----:|------|--------:|-------------:|------------------:|-------:|", + "| Target | Draft | Device | Ctx | Test | Decode (mtp) | Decode (baseline) | Uplift |", + "|--------|-------|--------|----:|------|-------------:|------------------:|-------:|", *rows, ] diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index f6838a6db..54cd8e6af 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -21,11 +21,12 @@ # - qairt cells go through prompt_utf8 (the plugin doesn't accept # pre-tokenized input_ids — see issue #1008), with a pre-trimmed # `sample_prompt_${ctx}.txt` per ctx so prompt length is bounded. -# - spec (llama_cpp speculative-decoding) cells force `--prompt-file` -# because random ids collapse draft acceptance to ~0% — the number -# would only reflect scheduling overhead, not MTP benefit. Spec cells -# also pass --spec-type/--draft-model/--draft-tokens as CLI-level -# flags, so each spec matrix invocation runs its own bench call. +# - spec (llama_cpp speculative-decoding) cells share the random-ids +# prefill of the plain llama_cpp bucket — we only care about mechanical +# decode throughput with the spec path enabled, not real-world draft +# acceptance. Spec cells additionally pass --spec-type/--draft-model/ +# --draft-tokens as CLI-level flags, so each spec matrix invocation +# runs its own bench call. # Each bucket gets its own per-ctx TSV so the invocations don't mix. $ErrorActionPreference = "Continue" @@ -136,12 +137,12 @@ for ($i = 0; $i -lt $ctxList.Count; $i++) { $specTsv = $tsvByPluginCtx["spec-$ctx"] if ((Test-Path $specTsv) -and ((Get-Item $specTsv).Length -gt 0)) { $sp = $specParamsByCtx["$ctx"] - Write-Output "=== matrix spec ctx=$ctx tg=$tg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (prompt-file) ===" + Write-Output "=== matrix spec ctx=$ctx pp=$pp tg=$tg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (random-ids prefill) ===" Get-Content $specTsv $extra = @() if ($sp.tokens) { $extra += @("--draft-tokens", $sp.tokens) } & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 3 ` - -c $ctx -n $tg --prompt-file "$PROMPTS\sample_prompt_$ctx.txt" ` + -c $ctx -p $pp -n $tg ` --spec-type $sp.type --draft-model $sp.draft @extra ` --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)" From 19a2a67cc67eb399bd459196ac1bb5d50f57c261 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Sat, 15 Aug 2026 00:01:37 +0800 Subject: [PATCH 5/8] chore(sdk): trim over-verbose comments in the MTP bench diff Signed-off-by: Mengsheng Wu --- .github/workflows/bench.yml | 9 ++------ sdk/benchmark/benchmark.c | 17 +++++---------- sdk/benchmark/qdc/run_qdc_jobs.py | 25 +++++----------------- sdk/benchmark/qdc/windows/run_windows.ps1 | 26 ++++++++--------------- 4 files changed, 21 insertions(+), 56 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index ecc6cdfb8..8fa74ca60 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -65,9 +65,7 @@ jobs: fi echo "devices=$devices" >> "$GITHUB_OUTPUT" - # Catalog-only rows (empty `devices`, e.g. spec draft models) are - # never dispatched as their own bench cell — they exist so target - # rows can reference them by name via `spec.draft`. + # Catalog-only rows (empty `devices`) are never dispatched. if [ -n "$PICK_MODEL" ]; then matrix=$(jq -c --arg s "$PICK_MODEL" ' ($s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))) as $pick @@ -87,10 +85,7 @@ jobs: fi echo "matrix=$matrix" >> "$GITHUB_OUTPUT" - # Spec-decoding rows only run on SC8480XP today (target+draft mem - # footprint eliminates the other chipsets). Emit dynamic exclude - # combos so the bench matrix drops spec × non-SC8480XP without - # burning a runner slot per skip. + # Spec-decoding rows only run on SC8480XP; drop the other combos. excludes=$(jq -c -n --argjson devices "$devices" --argjson models "$matrix" ' [ $devices[] as $d | $models[] as $m | select(($m.spec // null) != null and $d != "SC8480XP") diff --git a/sdk/benchmark/benchmark.c b/sdk/benchmark/benchmark.c index 1ac1e5277..51305078e 100644 --- a/sdk/benchmark/benchmark.c +++ b/sdk/benchmark/benchmark.c @@ -86,12 +86,9 @@ typedef struct { const char* mmproj_path; /* Heap-owned copies populated when the model is resolved through the * model manager; freed at the end of run_one_cell. */ - char* mm_model_path; - char* mm_mmproj; - char* mm_tokenizer; - /* Heap-owned local path for the speculative-decoding draft when - * --draft-model is a model-manager id rather than a filesystem path; - * kept for cleanup and shadows draft_model for the plugin call. */ + char* mm_model_path; + char* mm_mmproj; + char* mm_tokenizer; char* mm_draft_model; bool force_vlm; /* run VLM path even without an mmproj (QAIRT bundles) */ bool mm_is_vlm; /* manager classified the resolved model as VLM (geniex_ModelType) */ @@ -518,12 +515,8 @@ static int resolve_via_mm(options_t* o, const char* id_in) { return 0; } -/* Resolve --draft-model when it looks like a model-manager id (not a - * filesystem path). Populates o->mm_draft_model (heap-owned, freed by - * run_one_cell) and rewrites o->draft_model to point at it. Without this - * the llama_cpp plugin gets a raw id like "org/repo:Q4_0", fails to open - * it as a file, and silently falls back to non-speculative decode — - * draft_n_total stays 0 and the whole spec bench is meaningless. */ +/* Without this the plugin gets a raw id like "org/repo:Q4_0", fails to + * open it as a file, and silently falls back to non-speculative decode. */ static int resolve_draft_via_mm(options_t* o) { if (!o->draft_model || looks_like_path(o->draft_model)) return 0; if (!g_mm_inited) { diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index 8671499e1..3274f787c 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -117,9 +117,6 @@ def resolve_model_url(m: dict, device: str) -> str | None: def _resolve_draft_model_id(models: list[dict], draft_name: str) -> str: - """Look up a draft model's model_id by name inside the same bench-models.json. - Draft models are declared as catalog-only entries (empty ``devices``) and - referenced from a target row's ``spec.draft`` field.""" for m in models: if m["name"] == draft_name: return m["model_id"] @@ -133,20 +130,9 @@ def model_rows(models: list[dict], device: str) -> list[str]: name | plugin | csv_devices | model_id | vlm | image | spec_type | draft_model_id | draft_tokens - The trailing three fields are non-empty only for spec-decoding rows; - non-spec rows carry empty strings there so every script parses the - same column count. - - The host passes the chipset slug as a single shared --chipset flag to - geniex-bench; the model-manager hub auto-routes "qualcomm/*" to - AI Hub and everything else to HuggingFace, so per-row hub overrides - aren't needed. mmproj/tokenizer paths come back from get_paths. - - Entries with empty ``devices`` are catalog-only (e.g. spec draft - models) and are skipped here — they exist in bench-models.json so - other rows can reference them by name and the aggregate report can - resolve their download URL. - + Trailing three fields carry spec-decoding params or empty strings so + every script parses the same column count. Entries with empty + ``devices`` are catalog-only (e.g. spec draft models) and skipped. Rows for AI Hub models whose chipset isn't advertised are dropped upfront so the device doesn't waste time on a guaranteed-fail pull.""" rows = [] @@ -429,9 +415,8 @@ def _is_spec_cell(c: dict) -> bool: def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str]: - """Standalone MTP table pairing each spec cell with its no-spec baseline - on (target model_id, device, ctx). Emits nothing when no spec cells or - no matching baseline row is registered in bench-models.json.""" + """Pair each spec cell with its no-spec baseline on (target model_id, + device, ctx). Emits nothing when no spec cells are present.""" if not models: return [] spec_entries = [m for m in models if m.get("spec")] diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index 54cd8e6af..242db15bb 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -15,19 +15,14 @@ # >~7 GB GGUFs on X2 Elite). # # We sweep ctx in {512, 1024, 4096} per cell to align with test-llama.cpp's -# PERFORMANCE SESSION. Three prefill modes coexist: -# - llama_cpp cells use random-ids prefill (`-p N`, mirrors llama-bench -# `pp{N}`), so reported pp is exactly the ctx value; -# - qairt cells go through prompt_utf8 (the plugin doesn't accept -# pre-tokenized input_ids — see issue #1008), with a pre-trimmed -# `sample_prompt_${ctx}.txt` per ctx so prompt length is bounded. -# - spec (llama_cpp speculative-decoding) cells share the random-ids -# prefill of the plain llama_cpp bucket — we only care about mechanical -# decode throughput with the spec path enabled, not real-world draft -# acceptance. Spec cells additionally pass --spec-type/--draft-model/ -# --draft-tokens as CLI-level flags, so each spec matrix invocation -# runs its own bench call. -# Each bucket gets its own per-ctx TSV so the invocations don't mix. +# PERFORMANCE SESSION. Three buckets, each with its own per-ctx TSV so +# their invocations don't mix: +# - llama_cpp cells use random-ids prefill (`-p N`); +# - qairt cells go through prompt_utf8 with `sample_prompt_${ctx}.txt` +# because the plugin doesn't accept pre-tokenized input_ids (#1008); +# - spec (llama_cpp speculative-decoding) cells share random-ids +# prefill and additionally pass --spec-type/--draft-model/--draft-tokens +# as CLI-level flags per matrix invocation. $ErrorActionPreference = "Continue" @@ -75,10 +70,7 @@ foreach ($plugin in @("llama", "qairt", "spec")) { Remove-Item $tsvByPluginCtx["$plugin-$ctx"] -ErrorAction SilentlyContinue } } -# Spec CLI parameters share one value across all cells inside a single -# bench invocation. This map holds them keyed by "$ctx" for the second -# pass below; every spec row is expected to agree on (type, draft, tokens) -# per ctx since we only ship one draft model per target today. +# Spec CLI params are per-invocation, not per-cell. Keyed by "$ctx". $specParamsByCtx = @{} foreach ($row in $rows) { From 203ac6d8499e15206f2431cfce7de463a5fcbf8f Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Sat, 15 Aug 2026 00:41:30 +0800 Subject: [PATCH 6/8] chore(sdk): dump QDC logs when no cells recovered Diagnostic aid: when download_cells returns empty, pull .log/.stdout/.txt members from the QDC log archive and print them so the failure cause (usually a device-side stderr) is visible in the GH Actions log instead of being locked inside QDC's archive. Signed-off-by: Mengsheng Wu --- sdk/benchmark/qdc/run_qdc_jobs.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index 3274f787c..7e87e2f8a 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -662,6 +662,15 @@ def main() -> int: cells = download_cells( client, job_id, tmp, model_names=[m["name"] for m in models] ) + if not cells: + for name, data in _qdc.download_log_members( + client, job_id, tmp, lambda n: n.endswith((".log", ".stdout", ".txt")) + ): + print(f"===== QDC log: {name} =====") + try: + print(data.decode("utf-8", errors="replace")) + except Exception as e: + print(f"[decode failed: {e}]") if args.cells_out: args.cells_out.write_text(json.dumps(cells)) From 1abbfda65cc1998de40c68bed1fa1d427e851d8a Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Sat, 15 Aug 2026 01:15:16 +0800 Subject: [PATCH 7/8] fix(sdk): make MTP bench survive spec-KV bug + fix ctx=8192 sweep Two issues found from the diag dump: 1. Spec-decoding accept-rate is fine (~70%) but llama.cpp fails to alloc a KV memory slot for the 4-token (target+3 draft) batch on the 2nd measured run, and bench.c exit(1)s on any generate failure -- so 0 cell JSONs were written for the entire spec matrix. Work around by forcing '-r 1 --no-warmup' on spec invocations; single measured run avoids the KV re-use path entirely. 2. The per-row ctx override in main() checked 'len(models) == 1', which silently disabled the 8192 sweep the moment resolve_via_mm pulled the draft-dep entry into 'models'. Filter to entries with non-empty devices before the count check. Signed-off-by: Mengsheng Wu --- sdk/benchmark/qdc/run_qdc_jobs.py | 6 ++++-- sdk/benchmark/qdc/windows/run_windows.ps1 | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index 7e87e2f8a..a92067ae8 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -637,8 +637,10 @@ def main() -> int: f"no model in {args.models_file} runs any of --compute={compute_pick}" ) ctx_arg = args.ctx - if not ctx_arg and len(models) == 1 and models[0].get("ctx"): - ctx_arg = ",".join(str(x) for x in models[0]["ctx"]) + if not ctx_arg: + active = [m for m in models if m.get("devices")] + if len(active) == 1 and active[0].get("ctx"): + ctx_arg = ",".join(str(x) for x in active[0]["ctx"]) ctx_list, pp_list, tg_list = resolve_sweep(ctx_arg, args.pp, args.tg) log.info("sweep: ctx=%s pp=%s tg=%s", ctx_list, pp_list, tg_list) diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index 242db15bb..27f4f2bae 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -133,7 +133,11 @@ for ($i = 0; $i -lt $ctxList.Count; $i++) { Get-Content $specTsv $extra = @() if ($sp.tokens) { $extra += @("--draft-tokens", $sp.tokens) } - & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 3 ` + # -r 1 --no-warmup: llama.cpp spec-decoding leaks KV between runs + # (batch-of-4 draft+target trips 'no memory slot' on the 2nd run), + # and bench.c exit(1)s on generate failure — so a multi-run spec + # cell writes 0 JSON. Single measured run keeps the JSON coming. + & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 1 --no-warmup ` -c $ctx -p $pp -n $tg ` --spec-type $sp.type --draft-model $sp.draft @extra ` --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" From b591400c67b2c4d7576b1047a5991ddd62742d47 Mon Sep 17 00:00:00 2001 From: Mengsheng Wu Date: Sat, 15 Aug 2026 02:02:13 +0800 Subject: [PATCH 8/8] fix(sdk): reserve KV headroom for spec draft in bench Bench defaults pp+tg = ctx exactly, but spec-decoding needs draft_tokens+1 extra KV slots on the last decode step (target + N drafted tokens), or llama.cpp trips 'decode: failed to find a memory slot for batch of size N+1' and bench exits with no JSON. Trim tg by that margin for spec cells; ctx=512 stays a 384-prompt cell but decodes 124 tokens instead of 128. Signed-off-by: Mengsheng Wu --- sdk/benchmark/qdc/windows/run_windows.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index 27f4f2bae..b4dcb61aa 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -129,16 +129,18 @@ for ($i = 0; $i -lt $ctxList.Count; $i++) { $specTsv = $tsvByPluginCtx["spec-$ctx"] if ((Test-Path $specTsv) -and ((Get-Item $specTsv).Length -gt 0)) { $sp = $specParamsByCtx["$ctx"] - Write-Output "=== matrix spec ctx=$ctx pp=$pp tg=$tg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (random-ids prefill) ===" + # Spec-decoding needs `--draft-tokens` extra KV slots on the last + # decode step (target + draft), or llama.cpp trips + # "decode: failed to find a memory slot for batch of size N+1". + # Bench defaults pp+tg = ctx exactly, so trim tg by that margin. + $draftHeadroom = if ($sp.tokens) { [int]$sp.tokens + 1 } else { 4 } + $specTg = [Math]::Max(1, [int]$tg - $draftHeadroom) + Write-Output "=== matrix spec ctx=$ctx pp=$pp tg=$specTg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (random-ids prefill) ===" Get-Content $specTsv $extra = @() if ($sp.tokens) { $extra += @("--draft-tokens", $sp.tokens) } - # -r 1 --no-warmup: llama.cpp spec-decoding leaks KV between runs - # (batch-of-4 draft+target trips 'no memory slot' on the 2nd run), - # and bench.c exit(1)s on generate failure — so a multi-run spec - # cell writes 0 JSON. Single measured run keeps the JSON coming. & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 1 --no-warmup ` - -c $ctx -p $pp -n $tg ` + -c $ctx -p $pp -n $specTg ` --spec-type $sp.type --draft-model $sp.draft @extra ` --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)"