[LLM:Feature] Add Agentic Flow to MNN LLM Chat

.gitignore

@@ -17,6 +17,7 @@
 
 # Java class files
 *.class
+**/.cxx
 
 # Generated files
 bin/

apps/Android/MnnLlmChat/README.md

@@ -35,6 +35,34 @@ This is our full multimodal language model (LLM) Android app
 
  !!!warning!!! This version has been tested exclusively on the OnePlus 13 and Xiaomi 14 Ultra, Due to the demanding performance requirements of large language models (LLMs), many budget or low-spec devices may experience issues such as slow inference speeds, application instability, or even failure to run entirely. and its stability on other devices cannot be guaranteed. If you encounter any issues, please feel free to open an issue for assistance.
 
+## Agentic Capability
+
+The Android app includes an optional session-level Agent mode inspired by the ActMe project:
+
++ 核心技术来源 / Core technical source: `https://github.com/huangzhengxiang/ActMe.git`
++ Normal chat and Agent chat are different conversation modes. The mode is selected when a conversation is created and is persisted with that session.
++ Existing history created before this feature is treated as normal chat.
++ Agent mode runs a lightweight local agentic loop: model planning, `system_calls` parsing, visible tool execution, tool observations, and final answer generation.
++ Current MnnLlmChat tool scope is adapted from ActMe but excludes ADB: `get_current_time`, `web_search`, `browser_url`, and `python_exec`.
++ Built-in Python includes Excel helpers plus `numpy`, `pandas`, and `openpyxl` for local data analysis.
++ Agent mode persists lightweight Memory and Skill updates in the MnnLlmChat chat database and injects them into future Agent prompts.
++ Local ChatActivity conversations rely on the live native MNN session for KV/prompt-cache reuse. Each turn submits only new input; stored chat history is for UI display. Reloading an old session does not currently restore a persisted prompt cache.
+
+<p align="center">
+  <img width="20%" alt="Icon"  src="https://meta.alicdn.com/data/mnn/assets/Agent.gif" style="margin: 0 10px;">
+</p>
+
+Design notes:
+
++ [Agentic loop design](./docs/agentic_design.md)
++ [Built-in browser design](./docs/builtin_browser_design.md)
++ [Built-in Python design](./docs/builtin_python_design.md)
++ [Skill and memory design](./docs/skill_memory_design.md)
++ [Development notes](./docs/development_notes.md)
++ [Iteration roadmap](./docs/iteration_roadmap.md)
+
+ADB control is intentionally not included in this MnnLlmChat port.
+
 
 # Development
 + Prepare

apps/Android/MnnLlmChat/app/.gitignore

@@ -1,4 +1,6 @@
 /build
+/.cxx
+/standard
 google-services.json
 src/standard/google-services.json
 

apps/Android/MnnLlmChat/app/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id 'com.android.application'
     id 'org.jetbrains.kotlin.android'
     id 'org.jetbrains.kotlin.plugin.serialization' version '2.1.21'
+    id 'com.chaquo.python'
 }
 
 def enableFirebase = (project.findProperty("ENABLE_FIREBASE")?.toString()?.toBoolean() ?: false)
@@ -127,7 +128,7 @@ android {
             }
         }
     }
-    
+
     productFlavors {
         standard {
             dimension "store"
@@ -138,6 +139,11 @@ android {
             buildConfigField "boolean", "IS_GOOGLE_PLAY_BUILD", "true"
             versionNameSuffix ".gp"
         }
+        agent {
+            dimension "store"
+            buildConfigField "boolean", "IS_GOOGLE_PLAY_BUILD", "false"
+            versionNameSuffix ".agent"
+        }
     }
 
 
@@ -183,6 +189,21 @@ android {
     }
 }
 
+chaquopy {
+    defaultConfig {
+        version "3.11"
+    }
+    productFlavors {
+        getByName("agent") {
+            pip {
+                install "openpyxl==3.1.5"
+                install "numpy"
+                install "pandas"
+            }
+        }
+    }
+}
+
 dependencies {
 
 //    https://developer.android.com/jetpack/androidx/releases/compose

apps/Android/MnnLlmChat/app/src/main/AndroidManifest.xml

@@ -46,6 +46,7 @@
             android:name="com.alibaba.mnnllm.android.chat.ChatActivity"
             android:configChanges="orientation|screenSize"
             android:exported="true"
+            android:screenOrientation="portrait"
             android:windowSoftInputMode="adjustResize">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
@@ -75,7 +76,8 @@
         <activity
             android:name=".main.MainActivity"
             android:configChanges="orientation|screenSize"
-            android:exported="true">
+            android:exported="true"
+            android:screenOrientation="portrait">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
@@ -86,7 +88,8 @@
             android:name=".mainsettings.MainSettingsActivity"
             android:configChanges="orientation|screenSize"
             android:exported="true"
-            android:label="@string/settings">
+            android:label="@string/settings"
+            android:screenOrientation="portrait">
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.DEFAULT" />
@@ -97,7 +100,8 @@
             android:name=".mainsettings.StorageManagementActivity"
             android:configChanges="orientation|screenSize"
             android:exported="false"
-            android:label="@string/storage_management" />
+            android:label="@string/storage_management"
+            android:screenOrientation="portrait" />
 
 <!--        <activity-->
 <!--            android:name=".modelsettings.SettingsActivity"-->

apps/Android/MnnLlmChat/app/src/main/cpp/llm_mnn_jni.cpp

@@ -167,6 +167,7 @@ JNIEXPORT jobject JNICALL Java_com_alibaba_mnnllm_android_llm_LlmSession_submitN
     if (!onProgressMethod) {
         MNN_DEBUG("ProgressListener onProgress method not found.");
     }
+    int64_t input_len = llm->CountInputTokens(input_str);
     auto *context = llm->Response(input_str, [&, progressListener, onProgressMethod](
             const std::string &response, bool is_eop) {
         if (progressListener && onProgressMethod) {
@@ -185,6 +186,16 @@ JNIEXPORT jobject JNICALL Java_com_alibaba_mnnllm_android_llm_LlmSession_submitN
     int64_t audio_time = 0;
     int64_t prefill_time = 0;
     int64_t decode_time = 0;
+    if (context == nullptr) {
+        env->ReleaseStringUTFChars(inputStr, input_str);
+        jclass hashMapClass = env->FindClass("java/util/HashMap");
+        jmethodID hashMapInit = env->GetMethodID(hashMapClass, "<init>", "()V");
+        jmethodID putMethod =
+            env->GetMethodID(hashMapClass, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
+        jobject hashMap = env->NewObject(hashMapClass, hashMapInit);
+        env->CallObjectMethod(hashMap, putMethod, env->NewStringUTF("error"), env->NewStringUTF("Generation failed"));
+        return hashMap;
+    }
     prompt_len += context->prompt_len;
     decode_len += context->gen_seq_len;
     vision_time += context->vision_us;
@@ -198,6 +209,10 @@ JNIEXPORT jobject JNICALL Java_com_alibaba_mnnllm_android_llm_LlmSession_submitN
     jobject hashMap = env->NewObject(hashMapClass, hashMapInit);
 
     // Add metrics to the HashMap
+    env->CallObjectMethod(hashMap, putMethod, env->NewStringUTF("input_len"),
+                          env->NewObject(env->FindClass("java/lang/Long"),
+                                         env->GetMethodID(env->FindClass("java/lang/Long"), "<init>", "(J)V"),
+                                         input_len));
     env->CallObjectMethod(hashMap, putMethod, env->NewStringUTF("prompt_len"),
                           env->NewObject(env->FindClass("java/lang/Long"),
                                          env->GetMethodID(env->FindClass("java/lang/Long"),
@@ -222,6 +237,7 @@ JNIEXPORT jobject JNICALL Java_com_alibaba_mnnllm_android_llm_LlmSession_submitN
                           env->NewObject(env->FindClass("java/lang/Long"),
                                          env->GetMethodID(env->FindClass("java/lang/Long"),
                                                           "<init>", "(J)V"), decode_time));
+    env->ReleaseStringUTFChars(inputStr, input_str);
     return hashMap;
 }
 

apps/Android/MnnLlmChat/app/src/main/cpp/llm_session.cpp

@@ -289,16 +289,22 @@ const MNN::Transformer::LlmContext * LlmSession::Response(const std::string &pro
     history_.emplace_back("user", getUserString(prompt.c_str(), false, is_r1_));
 #endif
     MNN_DEBUG("submitNative history count %zu", history_.size());
-
-    // Generate full prompt string using Prompt::applyTemplate
+
+    prompt_string_for_debug.clear();
+    response_string_for_debug.clear();
+
+    // Generate a logical prompt preview for debugging. The engine may still use
+    // prompt cache internally and prefill only the delta; see the PERF log below.
     std::string full_prompt_text;
     for (auto & it : history_) {
         full_prompt_text += it.second;
-        prompt_string_for_debug += it.second;
     }
-
-    MNN_DEBUG("submitNative prompt_string_for_debug count %s max_new_tokens_:%d", prompt_string_for_debug.c_str(), max_new_tokens_);
-
+    prompt_string_for_debug = full_prompt_text;
+
+    std::string prompt_preview = full_prompt_text.substr(0, 2048);
+    MNN_DEBUG("submitNative logical_prompt chars=%zu preview=%s max_new_tokens_:%d", full_prompt_text.size(),
+              prompt_preview.c_str(), max_new_tokens_);
+
     // Check for multimodal content in the full prompt
     auto multimodal_result = processMultimodalPrompt(full_prompt_text);
     restoreAndroidSteppingStatusIfNeeded(llm_);
@@ -361,10 +367,12 @@ const MNN::Transformer::LlmContext * LlmSession::Response(const std::string &pro
     float decode_s = context->decode_us / 1e6f;
     float prefill_tps = (prefill_s > 0) ? context->prompt_len / prefill_s : 0;
     float decode_tps = (decode_s > 0) ? context->gen_seq_len / decode_s : 0;
-    MNN_DEBUG("PERF | prefill: %d tok in %.2fs (%.1f t/s) | decode: %d tok in %.2fs (%.1f t/s) | history: %zu msgs",
-              context->prompt_len, prefill_s, prefill_tps,
-              context->gen_seq_len, decode_s, decode_tps,
-              history_.size());
+    bool prompt_cache_enabled = current_config_.value("prompt_cache", false);
+    MNN_DEBUG(
+        "PERF | prompt_cache: %d | actual_prefill: %d tok in %.2fs (%.1f t/s) | decode: %d tok in %.2fs (%.1f t/s) | "
+        "history: %zu msgs",
+        prompt_cache_enabled ? 1 : 0, context->prompt_len, prefill_s, prefill_tps, context->gen_seq_len, decode_s,
+        decode_tps, history_.size());
     return context;
 }
 
@@ -403,6 +411,13 @@ void LlmSession::SetMaxNewTokens(int i) {
     max_new_tokens_ = i;
 }
 
+int LlmSession::CountInputTokens(const std::string& prompt) const {
+    if (llm_ == nullptr) {
+        return 0;
+    }
+    return static_cast<int>(llm_->tokenizer_encode(prompt).size());
+}
+
 void LlmSession::setSystemPrompt(std::string system_prompt) {
     system_prompt_= std::move(system_prompt);
     if (history_.size() > 1) {

apps/Android/MnnLlmChat/app/src/main/cpp/llm_session.h

@@ -39,6 +39,8 @@ class LlmSession {
     Response(const std::string &prompt, const std::function<bool(const std::string &, bool is_eop)> &on_progress);
     void SetMaxNewTokens(int i);
 
+    int CountInputTokens(const std::string& prompt) const;
+
     void setSystemPrompt(std::string system_prompt);
 
     void SetAssistantPrompt(const std::string& assistant_prompt);

apps/Android/MnnLlmChat/app/src/main/java/com/alibaba/mnnllm/android/MnnLlmApplication.kt

@@ -6,6 +6,7 @@ import android.app.Application
 import com.alibaba.mls.api.ApplicationProvider
 import com.alibaba.mls.api.download.ModelDownloadManager
 import com.alibaba.mnnllm.android.update.UpdateChecker
+import com.alibaba.mnnllm.android.agent.AgenticPythonEngine
 import com.alibaba.mnnllm.android.utils.CrashUtil
 import com.alibaba.mnnllm.android.utils.CurrentActivityTracker
 import com.alibaba.mnnllm.android.utils.TimberConfig
@@ -21,6 +22,7 @@ class MnnLlmApplication : Application() {
     override fun onCreate() {
         super.onCreate()
         ApplicationProvider.set(this)
+        AgenticPythonEngine.initialize(this)
         UpdateChecker.registerDownloadReceiver(applicationContext)
         CrashUtil.init(this)
         instance = this