Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package com.studyos.studyos_agent

import android.content.Context
import android.content.Intent
import android.content.pm.PackageManager
import android.provider.Settings
import com.example.studyOS.offline.Tools

class AndroidNativeToolExecutor(context: Context) {
private val appContext = context.applicationContext
private val tools: Tools by lazy { Tools(appContext) }

fun canControlFlashlight(): Boolean {
return appContext.packageManager.hasSystemFeature(PackageManager.FEATURE_CAMERA_FLASH)
}

fun capabilities(): List<Map<String, Any?>> {
return listOf(
supported("get_device_status"),
supported(
"set_flashlight",
canControlFlashlight(),
"This device does not report a camera flash.",
),
supported("open_installed_app"),
supported("search_youtube"),
supported("open_system_setting"),
mapOf(
"name" to "create_reminder",
"supported" to false,
"reason" to "Reminder scheduling is reserved for the reminder PR.",
),
)
}

fun execute(name: String, arguments: Map<*, *>): String {
return when (name) {
"get_device_status" -> tools.getDeviceStatus()
"set_flashlight" -> tools.toggleFlashlight(booleanArgument(arguments, "enabled"))
"open_installed_app" -> {
val appName = stringArgument(arguments, "name")
tools.openApp(appName)
}
"search_youtube" -> {
val query = stringArgument(arguments, "query")
tools.searchYoutube(query)
"Opened YouTube search for '$query'."
}
"open_system_setting" -> openSystemSetting(stringArgument(arguments, "setting"))
else -> throw IllegalArgumentException("Native tool is not available: $name")
}
}

private fun openSystemSetting(setting: String): String {
val normalized = setting.trim().lowercase()
val action = when (normalized) {
"wifi" -> Settings.ACTION_WIFI_SETTINGS
"bluetooth" -> Settings.ACTION_BLUETOOTH_SETTINGS
"location" -> Settings.ACTION_LOCATION_SOURCE_SETTINGS
"mobile_data" -> Settings.ACTION_DATA_ROAMING_SETTINGS
else -> throw IllegalArgumentException(
"Unsupported setting '$setting'. Use wifi, bluetooth, location, or mobile_data.",
)
}
val intent = Intent(action).setFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
appContext.startActivity(intent)
return "Opened $normalized settings. Direct toggles are controlled by Android."
}

private fun supported(
name: String,
supported: Boolean = true,
reasonWhenUnsupported: String? = null,
): Map<String, Any?> {
return mapOf(
"name" to name,
"supported" to supported,
"reason" to if (supported) null else reasonWhenUnsupported,
)
}

private fun stringArgument(arguments: Map<*, *>, key: String): String {
val value = arguments[key]?.toString()?.trim().orEmpty()
if (value.isBlank()) {
throw IllegalArgumentException("Missing required '$key' argument.")
}
return value
}

private fun booleanArgument(arguments: Map<*, *>, key: String): Boolean {
return when (val value = arguments[key]) {
is Boolean -> value
is String -> value.equals("true", ignoreCase = true)
else -> throw IllegalArgumentException("Missing required boolean '$key' argument.")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class MainActivity : FlutterActivity() {
private var localPromptClient: AndroidLocalPromptClient? = null
private var localModelStore: AndroidLocalModelStore? = null
private var liteRtToolExecutor: AndroidLiteRtToolExecutor? = null
private var nativeToolExecutor: AndroidNativeToolExecutor? = null
private lateinit var intentBridge: AndroidIntentBridge

override fun onCreate(savedInstanceState: Bundle?) {
Expand Down Expand Up @@ -67,6 +68,8 @@ class MainActivity : FlutterActivity() {
"initialize" -> result.success(initializeNativeLayer())
"getWorldState" -> result.success(worldStateMap())
"getCapabilities" -> result.success(capabilities())
"getNativeToolCapabilities" -> result.success(nativeToolCapabilities())
"executeNativeTool" -> executeNativeTool(call, result)
"listLocalModels" -> result.success(localModelStore().listModels())
"downloadLocalModel" -> downloadLocalModel(call, result)
"cancelLocalModelDownload" -> {
Expand Down Expand Up @@ -296,6 +299,32 @@ class MainActivity : FlutterActivity() {
}
}

private fun nativeToolExecutor(): AndroidNativeToolExecutor {
val existing = nativeToolExecutor
if (existing != null) return existing
return AndroidNativeToolExecutor(applicationContext).also {
nativeToolExecutor = it
}
}

private fun executeNativeTool(call: MethodCall, result: MethodChannel.Result) {
val name = call.argument<String>("name")?.trim().orEmpty()
val arguments = call.argument<Map<*, *>>("arguments") ?: emptyMap<String, Any?>()
if (name.isBlank()) {
result.error("native_tool_missing_name", "Native tool name is required.", null)
return
}
try {
result.success(nativeToolExecutor().execute(name, arguments))
} catch (error: Throwable) {
result.error(
"native_tool_failed",
error.message ?: "Native tool failed.",
null,
)
}
}

private fun downloadLocalModel(call: MethodCall, result: MethodChannel.Result) {
val id = call.argument<String>("id").orEmpty()
val label = call.argument<String>("label").orEmpty()
Expand Down Expand Up @@ -394,14 +423,24 @@ class MainActivity : FlutterActivity() {
"canUseOfflineLiteRtModel" to true,
"canManageDownloadedLiteRtModels" to true,
"canUseAndroidGeminiNanoPrompt" to true,
"canControlFlashlight" to true,
"canControlFlashlight" to nativeToolExecutor().canControlFlashlight(),
"canStartPhoneCall" to hasPermission(Manifest.permission.CALL_PHONE),
"nativeToolContractVersion" to 1,
"nativeTools" to nativeToolExecutor().capabilities(),
"androidAssistantSnapshot" to intentBridge.snapshotStatus(),
"iosParity" to "limited by iOS background execution and app-control policies",
"webDesktopParity" to "limited shell only until adapters are implemented",
) + localPromptClient().capabilities()
}

private fun nativeToolCapabilities(): Map<String, Any?> {
return mapOf(
"platform" to "android",
"nativeToolContractVersion" to 1,
"nativeTools" to nativeToolExecutor().capabilities(),
)
}

private fun hasLocationPermission(): Boolean {
return hasPermission(Manifest.permission.ACCESS_FINE_LOCATION) ||
hasPermission(Manifest.permission.ACCESS_COARSE_LOCATION)
Expand Down
44 changes: 43 additions & 1 deletion flutter_app/ios/Runner/StudyOSNativeBridge.swift
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ final class StudyOSNativeBridge: NSObject, FlutterStreamHandler, CLLocationManag
result(worldState())
case "getCapabilities":
result(capabilities())
case "getNativeToolCapabilities":
result(nativeToolCapabilityMap())
case "executeNativeTool":
executeNativeTool(call: call, result: result)
case "publishIntentSnapshot":
publishIntentSnapshot(call: call, result: result)
case "consumePendingIntentPrompt":
Expand Down Expand Up @@ -209,6 +213,22 @@ final class StudyOSNativeBridge: NSObject, FlutterStreamHandler, CLLocationManag
result("iOS speech started.")
}

private func executeNativeTool(call: FlutterMethodCall, result: FlutterResult) {
guard
let args = call.arguments as? [String: Any],
let name = args["name"] as? String
else {
result(FlutterError(code: "native_tool_missing_name", message: "Native tool name is required.", details: nil))
return
}

result(FlutterError(
code: "native_tool_unsupported",
message: "Native tool is not supported on iOS in this build: \(name).",
details: nil
))
}

private func worldState() -> [String: Any] {
var state: [String: Any] = [
"platform": "ios",
Expand Down Expand Up @@ -241,13 +261,15 @@ final class StudyOSNativeBridge: NSObject, FlutterStreamHandler, CLLocationManag
"canOpenInstalledApps": false,
"canReadCalendar": false,
"canUseOfflineLiteRtModel": false,
"canControlFlashlight": true,
"canControlFlashlight": false,
"canStartPhoneCall": true,
"canUseSpeechRecognition": SFSpeechRecognizer(locale: Locale.current)?.isAvailable ?? false,
"canUseTextToSpeech": true,
"canCreateLocalNotificationReminder": true,
"canUseAppIntents": canUseAppIntents()
]
values["nativeToolContractVersion"] = 1
values["nativeTools"] = nativeToolCapabilities()

#if canImport(FoundationModels)
if #available(iOS 26.0, *) {
Expand All @@ -264,6 +286,26 @@ final class StudyOSNativeBridge: NSObject, FlutterStreamHandler, CLLocationManag
return values
}

private func nativeToolCapabilities() -> [[String: Any]] {
let iosControlReason = "This native control is Android-only in this build."
return [
["name": "get_device_status", "supported": false, "reason": iosControlReason],
["name": "set_flashlight", "supported": false, "reason": iosControlReason],
["name": "open_installed_app", "supported": false, "reason": "iOS does not support arbitrary installed-app launching from this app."],
["name": "search_youtube", "supported": false, "reason": iosControlReason],
["name": "open_system_setting", "supported": false, "reason": iosControlReason],
["name": "create_reminder", "supported": false, "reason": "Reminder tool exposure is reserved for the reminder PR."]
]
}

private func nativeToolCapabilityMap() -> [String: Any] {
return [
"platform": "ios",
"nativeToolContractVersion": 1,
"nativeTools": nativeToolCapabilities()
]
}

private func canUseAppIntents() -> Bool {
#if canImport(AppIntents)
if #available(iOS 16.0, *) {
Expand Down
16 changes: 13 additions & 3 deletions flutter_app/lib/src/agent_llm_provider.dart
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import 'mail_tools.dart';
import 'memory_store.dart';
import 'models.dart';
import 'native_bridge.dart';
import 'native_tool_router.dart';
import 'prompt_context.dart';
import 'studyos_tool_catalog.dart';
import 'studyos_tool_executor.dart';
Expand Down Expand Up @@ -104,7 +105,12 @@ class LocalNativeLlmProvider implements AgentLlmProvider {

@override
Future<String> send(AgentLlmRequest request) async {
final systemPrompt = _localSystemPrompt(request.context.systemPrompt());
final nativeTools = NativeToolRouter(_bridge);
final supportedNativeToolNames = await nativeTools.supportedToolNames();
final systemPrompt = _localSystemPrompt(
request.context.systemPrompt(),
supportedNativeToolNames,
);
var response = await _bridge.sendMessage(
request.userText,
systemPrompt: systemPrompt,
Expand All @@ -117,6 +123,7 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
readMemory: () async => request.memoryText,
readSchedule: request.readSchedule,
mailTools: request.mailTools,
nativeTools: nativeTools,
);

for (var round = 0; round < _maxToolRounds; round += 1) {
Expand Down Expand Up @@ -163,7 +170,10 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
return response;
}

String _localSystemPrompt(String basePrompt) {
String _localSystemPrompt(
String basePrompt,
Set<String> supportedNativeToolNames,
) {
final buffer = StringBuffer()
..writeln(basePrompt)
..writeln()
Expand All @@ -181,7 +191,7 @@ class LocalNativeLlmProvider implements AgentLlmProvider {
)
..writeln('After tool results are returned, answer naturally.')
..writeln('Available StudyOS tools:');
for (final tool in studyOsTools) {
for (final tool in studyOsToolsForNativeSupport(supportedNativeToolNames)) {
final args = tool.required.isEmpty
? '{}'
: '{${tool.required.map((name) => '"$name":"..."').join(',')}}';
Expand Down
5 changes: 4 additions & 1 deletion flutter_app/lib/src/agent_request_runner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import 'mail_tools.dart';
import 'memory_store.dart';
import 'models.dart';
import 'native_bridge.dart';
import 'native_tool_router.dart';
import 'prompt_context.dart';

class AgentRequestRunner {
Expand All @@ -23,7 +24,9 @@ class AgentRequestRunner {
configStore: configStore,
memoryStore: memoryStore,
appendMemory: appendMemory,
cloudClient: cloudClient ?? CloudAgentClient(),
cloudClient:
cloudClient ??
CloudAgentClient(nativeTools: NativeToolRouter(bridge)),
);

final NativeBridge bridge;
Expand Down
23 changes: 19 additions & 4 deletions flutter_app/lib/src/cloud_agent_client.dart
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,25 @@ import 'package:http/http.dart' as http;
import 'cloud_tool_definitions.dart';
import 'mail_tools.dart';
import 'models.dart';
import 'native_tool_router.dart';
import 'prompt_context.dart';
import 'studyos_tool_catalog.dart';
import 'studyos_tool_executor.dart';

class CloudAgentClient {
CloudAgentClient({http.Client? httpClient, StudyOsToolExecutor? toolExecutor})
: _httpClient = httpClient ?? http.Client(),
_toolExecutor = toolExecutor ?? const StudyOsToolExecutor();
CloudAgentClient({
http.Client? httpClient,
StudyOsToolExecutor? toolExecutor,
NativeToolRunner? nativeTools,
}) : _httpClient = httpClient ?? http.Client(),
_toolExecutor = toolExecutor ?? const StudyOsToolExecutor(),
// Keep the public constructor parameter named `nativeTools`.
// ignore: prefer_initializing_formals
_nativeTools = nativeTools;

final http.Client _httpClient;
final StudyOsToolExecutor _toolExecutor;
final NativeToolRunner? _nativeTools;

Future<String> sendMessage({
required AgentConfig config,
Expand All @@ -40,11 +48,14 @@ class CloudAgentClient {
throw const CloudAgentException('API key is required.');
}

final supportedNativeToolNames =
await _nativeTools?.supportedToolNames() ?? const <String>{};
final request = _requestBody(
config: config,
history: history,
userText: userText,
context: context,
supportedNativeToolNames: supportedNativeToolNames,
);
final response = await _post(endpoint, apiKey, request);
final decoded = _decodeResponse(response);
Expand All @@ -61,6 +72,7 @@ class CloudAgentClient {
readMemory: readMemory,
readSchedule: readSchedule,
mailTools: mailTools,
nativeTools: _nativeTools,
);
for (final call in toolCalls) {
onToolTrace?.call(_traceForCall(call, 'running'));
Expand Down Expand Up @@ -116,6 +128,7 @@ class CloudAgentClient {
required List<ChatMessage> history,
required String userText,
required PromptContext context,
required Set<String> supportedNativeToolNames,
}) {
final historyWithoutCurrent =
history.isNotEmpty &&
Expand All @@ -135,7 +148,9 @@ class CloudAgentClient {
},
<String, Object?>{'role': 'user', 'content': userText},
],
'tools': cloudToolDefinitions(),
'tools': cloudToolDefinitions(
supportedNativeToolNames: supportedNativeToolNames,
),
'tool_choice': 'auto',
};
}
Expand Down
Loading
Loading