Add project documentation and reference materials
Include Luna AI Assistant design docs covering channels, configuration, core architecture, memory, scheduler, and skills. Add reference docs from OpenClaw and ZeroClaw projects, plus Mistral and OpenAI API specs.
This commit is contained in:
Vendored
+1
@@ -0,0 +1 @@
|
||||
{}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"file-explorer": true,
|
||||
"global-search": true,
|
||||
"switcher": true,
|
||||
"graph": true,
|
||||
"backlink": true,
|
||||
"canvas": true,
|
||||
"outgoing-link": true,
|
||||
"tag-pane": true,
|
||||
"footnotes": false,
|
||||
"properties": true,
|
||||
"page-preview": true,
|
||||
"daily-notes": true,
|
||||
"templates": true,
|
||||
"note-composer": true,
|
||||
"command-palette": true,
|
||||
"slash-command": false,
|
||||
"editor-status": true,
|
||||
"bookmarks": true,
|
||||
"markdown-importer": false,
|
||||
"zk-prefixer": false,
|
||||
"random-note": false,
|
||||
"outline": true,
|
||||
"word-count": true,
|
||||
"slides": false,
|
||||
"audio-recorder": false,
|
||||
"workspaces": false,
|
||||
"file-recovery": true,
|
||||
"publish": false,
|
||||
"sync": true,
|
||||
"bases": true,
|
||||
"webviewer": false
|
||||
}
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
{
|
||||
"main": {
|
||||
"id": "127e4477a3357592",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "4586ccd2645dafd5",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "0f41b448f12ad909",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "Luna AI Assistant/Channels.md",
|
||||
"mode": "source",
|
||||
"source": false
|
||||
},
|
||||
"icon": "lucide-file",
|
||||
"title": "Channels"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "vertical"
|
||||
},
|
||||
"left": {
|
||||
"id": "7ee2740363116488",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "6d1750ac051219d1",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "6f7065cc7dffd180",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "file-explorer",
|
||||
"state": {
|
||||
"sortOrder": "alphabetical",
|
||||
"autoReveal": false
|
||||
},
|
||||
"icon": "lucide-folder-closed",
|
||||
"title": "Files"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "d20183e68c8d07ef",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "search",
|
||||
"state": {
|
||||
"query": "",
|
||||
"matchingCase": false,
|
||||
"explainSearch": false,
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical"
|
||||
},
|
||||
"icon": "lucide-search",
|
||||
"title": "Search"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "8e3e15922e0dd412",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "bookmarks",
|
||||
"state": {},
|
||||
"icon": "lucide-bookmark",
|
||||
"title": "Bookmarks"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"right": {
|
||||
"id": "2ab570a7ae13ec78",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "99c3f35707cad904",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "8575bcc7d3c3ca6a",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "backlink",
|
||||
"state": {
|
||||
"file": "Luna AI Assistant/Core.md",
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical",
|
||||
"showSearch": false,
|
||||
"searchQuery": "",
|
||||
"backlinkCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
},
|
||||
"icon": "links-coming-in",
|
||||
"title": "Backlinks for Core"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1d27ee4ebd4aa48b",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outgoing-link",
|
||||
"state": {
|
||||
"file": "Luna AI Assistant/Core.md",
|
||||
"linksCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
},
|
||||
"icon": "links-going-out",
|
||||
"title": "Outgoing links from Core"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "635fcf8053d67540",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "tag",
|
||||
"state": {
|
||||
"sortOrder": "frequency",
|
||||
"useHierarchy": true,
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-tags",
|
||||
"title": "Tags"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "fa0c1e82da200838",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "all-properties",
|
||||
"state": {
|
||||
"sortOrder": "frequency",
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-archive",
|
||||
"title": "All properties"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "aebb6ab1e3be46a1",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outline",
|
||||
"state": {
|
||||
"file": "Luna AI Assistant/Core.md",
|
||||
"followCursor": false,
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-list",
|
||||
"title": "Outline of Core"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"left-ribbon": {
|
||||
"hiddenItems": {
|
||||
"switcher:Open quick switcher": false,
|
||||
"graph:Open graph view": false,
|
||||
"canvas:Create new canvas": false,
|
||||
"daily-notes:Open today's daily note": false,
|
||||
"templates:Insert template": false,
|
||||
"command-palette:Open command palette": false,
|
||||
"bases:Create new base": false
|
||||
}
|
||||
},
|
||||
"active": "0f41b448f12ad909",
|
||||
"lastOpenFiles": [
|
||||
"References/OpenClaw/Compaction Strategy.md",
|
||||
"References/ZeroClaw/Agent Loop & Tools.md",
|
||||
"References/ZeroClaw/Compaction Strategy.md",
|
||||
"References/ZeroClaw/Memory.md",
|
||||
"References/OpenClaw/Memory SDK.md",
|
||||
"References/ZeroClaw/Channel Messages.md",
|
||||
"Luna AI Assistant/Project Plan.md",
|
||||
"References/ZeroClaw/Skills.md",
|
||||
"References/ZeroClaw/Configuration.md",
|
||||
"References/ZeroClaw/Runtime Adapters.md",
|
||||
"References/ZeroClaw/Sessions.md",
|
||||
"References/OpenClaw/Channel System.md",
|
||||
"References/ZeroClaw/Security.md",
|
||||
"References/ZeroClaw.md",
|
||||
"References/ZeroClaw/ReliableProvider.md",
|
||||
"References/OpenClaw/Security.md",
|
||||
"References/OpenClaw/Plugin Architecture.md",
|
||||
"References/OpenClaw/Session Management.md",
|
||||
"References/ZeroClaw/Overview.md",
|
||||
"References/OpenClaw/Overview.md",
|
||||
"References/ZeroClaw",
|
||||
"References/OpenClaw",
|
||||
"References/OpenClaw.md",
|
||||
"Luna AI Assistant/Core.md",
|
||||
"Luna AI Assistant/Multi-Agent Architecture.md",
|
||||
"Luna AI Assistant/Memory.md",
|
||||
"References/mistral.openapi.yaml",
|
||||
"References/openai.openapi.yaml",
|
||||
"References/openapi.yaml",
|
||||
"References/openai.yaml",
|
||||
"References"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"file-explorer": true,
|
||||
"global-search": true,
|
||||
"switcher": true,
|
||||
"graph": true,
|
||||
"backlink": true,
|
||||
"canvas": true,
|
||||
"outgoing-link": true,
|
||||
"tag-pane": true,
|
||||
"footnotes": false,
|
||||
"properties": true,
|
||||
"page-preview": true,
|
||||
"daily-notes": true,
|
||||
"templates": true,
|
||||
"note-composer": true,
|
||||
"command-palette": true,
|
||||
"slash-command": false,
|
||||
"editor-status": true,
|
||||
"bookmarks": true,
|
||||
"markdown-importer": false,
|
||||
"zk-prefixer": false,
|
||||
"random-note": false,
|
||||
"outline": true,
|
||||
"word-count": true,
|
||||
"slides": false,
|
||||
"audio-recorder": false,
|
||||
"workspaces": false,
|
||||
"file-recovery": true,
|
||||
"publish": false,
|
||||
"sync": true,
|
||||
"bases": true,
|
||||
"webviewer": false
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"collapse-filter": true,
|
||||
"search": "",
|
||||
"showTags": false,
|
||||
"showAttachments": false,
|
||||
"hideUnresolved": false,
|
||||
"showOrphans": true,
|
||||
"collapse-color-groups": true,
|
||||
"colorGroups": [],
|
||||
"collapse-display": true,
|
||||
"showArrow": false,
|
||||
"textFadeMultiplier": 0,
|
||||
"nodeSizeMultiplier": 1,
|
||||
"lineSizeMultiplier": 1,
|
||||
"collapse-forces": true,
|
||||
"centerStrength": 0.518713248970312,
|
||||
"repelStrength": 10,
|
||||
"linkStrength": 1,
|
||||
"linkDistance": 250,
|
||||
"scale": 1,
|
||||
"close": true
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
{
|
||||
"main": {
|
||||
"id": "b89020bafd288606",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "3b084bb6aea7bf06",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "27fd26dcd453c61f",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "Channels/Web Interface Channel.md",
|
||||
"mode": "source",
|
||||
"source": false
|
||||
},
|
||||
"icon": "lucide-file",
|
||||
"title": "Web Interface Channel"
|
||||
},
|
||||
"group": "47a210897f386c25"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "7b841bf992949f81",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "5056061ce0a62aac",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "Channels/Web Interface Channel.md",
|
||||
"mode": "source",
|
||||
"source": false
|
||||
},
|
||||
"icon": "lucide-file",
|
||||
"title": "Web Interface Channel"
|
||||
},
|
||||
"group": "47a210897f386c25"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "vertical"
|
||||
},
|
||||
"left": {
|
||||
"id": "d65ed31b4cf140a8",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "cb79094e37f1bfbf",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "fcd6318fb0efd8c3",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "file-explorer",
|
||||
"state": {
|
||||
"sortOrder": "alphabetical",
|
||||
"autoReveal": false
|
||||
},
|
||||
"icon": "lucide-folder-closed",
|
||||
"title": "Files"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "1bbb51a5f3e6fe42",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "search",
|
||||
"state": {
|
||||
"query": "",
|
||||
"matchingCase": false,
|
||||
"explainSearch": false,
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical"
|
||||
},
|
||||
"icon": "lucide-search",
|
||||
"title": "Search"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bb387a4f03db1101",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "bookmarks",
|
||||
"state": {},
|
||||
"icon": "lucide-bookmark",
|
||||
"title": "Bookmarks"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300
|
||||
},
|
||||
"right": {
|
||||
"id": "be153589260b27a6",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "b9a0a82b097773e2",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "99660be9c41b8d54",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "backlink",
|
||||
"state": {
|
||||
"file": "Channels/Web Interface Channel.md",
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical",
|
||||
"showSearch": false,
|
||||
"searchQuery": "",
|
||||
"backlinkCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
},
|
||||
"icon": "links-coming-in",
|
||||
"title": "Backlinks for Web Interface Channel"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "e5acfa574174fe3a",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outgoing-link",
|
||||
"state": {
|
||||
"file": "Core.md",
|
||||
"linksCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
},
|
||||
"icon": "links-going-out",
|
||||
"title": "Outgoing links from Core"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "3693f3ae811b7b80",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "tag",
|
||||
"state": {
|
||||
"sortOrder": "frequency",
|
||||
"useHierarchy": true,
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-tags",
|
||||
"title": "Tags"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "018d06694edd9d01",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "all-properties",
|
||||
"state": {
|
||||
"sortOrder": "frequency",
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-archive",
|
||||
"title": "All properties"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "b9b058ee8cd7d7a6",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "outline",
|
||||
"state": {
|
||||
"file": "Core.md",
|
||||
"followCursor": false,
|
||||
"showSearch": false,
|
||||
"searchQuery": ""
|
||||
},
|
||||
"icon": "lucide-list",
|
||||
"title": "Outline of Core"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"left-ribbon": {
|
||||
"hiddenItems": {
|
||||
"switcher:Open quick switcher": false,
|
||||
"graph:Open graph view": false,
|
||||
"canvas:Create new canvas": false,
|
||||
"daily-notes:Open today's daily note": false,
|
||||
"templates:Insert template": false,
|
||||
"command-palette:Open command palette": false,
|
||||
"bases:Create new base": false
|
||||
}
|
||||
},
|
||||
"active": "5056061ce0a62aac",
|
||||
"lastOpenFiles": [
|
||||
"Core.md",
|
||||
"Channels/Web Interface Channel.md",
|
||||
"Channels.md",
|
||||
"Channels/Telegram Channel.md",
|
||||
"Channels/CLI Channel.md",
|
||||
"Channels/Channels.md",
|
||||
"Channels",
|
||||
"Skills.md",
|
||||
"Configuration.md",
|
||||
"Scheduler.md",
|
||||
"Memory.md",
|
||||
"Project Plan.md",
|
||||
"Welcome.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# CLI Channel
|
||||
|
||||
The CLI connects to Luna via SignalR. `CliChannel` is a server-side adapter that bridges the SignalR hub to the `IChannel` interface defined in [[Channels]].
|
||||
|
||||
## How It Works
|
||||
|
||||
`CliChannel` wraps `IChatHubContext` from [[Core]] to send streaming responses back to the connected SignalR client. The flow:
|
||||
|
||||
1. The `ChatHub` receives a message from the CLI client over SignalR.
|
||||
2. `ChatHub` calls `RaiseMessageReceived` on the `CliChannel` instance.
|
||||
3. The [[Channels|ChannelManager]] picks up the event and routes it through `SessionManager.RouteMessagesAsync`.
|
||||
4. The resulting `IAsyncEnumerable<ChatStreamUpdate>` is passed back to `CliChannel.SendStreamingMessageAsync`.
|
||||
5. `CliChannel` forwards the stream to the SignalR client via `IChatHubContext`.
|
||||
|
||||
## Key Characteristics
|
||||
|
||||
- **Server-side adapter**: `CliChannel` lives on the server; the actual CLI is a separate SignalR client.
|
||||
- **Single connection**: One `CliChannel` instance maps to the SignalR hub connection.
|
||||
- **No polling**: Unlike [[Telegram Channel]], the CLI uses persistent WebSocket connections via SignalR.
|
||||
|
||||
## Namespace
|
||||
|
||||
`Luna.Channels.Cli`
|
||||
@@ -0,0 +1,101 @@
|
||||
# Channels
|
||||
|
||||
The Channels module provides a unified abstraction for communication platforms like the CLI and Telegram. It handles message transport, while [[Core]] manages AI context and session state.
|
||||
|
||||
## Luna.Channels.Abstractions
|
||||
|
||||
This project defines the contracts and event models for all channel implementations.
|
||||
|
||||
### IChannel
|
||||
The primary interface for any communication transport. There is no abstract base class; `IChannel` is the complete contract.
|
||||
|
||||
```csharp
|
||||
public interface IChannel : IDisposable
|
||||
{
|
||||
string ChannelId { get; }
|
||||
string ChannelType { get; }
|
||||
string DisplayName { get; }
|
||||
bool IsConnected { get; }
|
||||
|
||||
event ChannelMessageReceivedEventHandler? MessageReceived;
|
||||
event EventHandler<ChannelConnectionEventArgs>? ConnectionStateChanged;
|
||||
|
||||
Task SendMessageAsync(ChannelMessage message, CancellationToken ct = default);
|
||||
Task SendStreamingMessageAsync(IAsyncEnumerable<ChatStreamUpdate> messageStream, CancellationToken ct = default);
|
||||
}
|
||||
```
|
||||
|
||||
### IChannelManager
|
||||
Coordinates registered channels and handles message routing.
|
||||
|
||||
```csharp
|
||||
public interface IChannelManager
|
||||
{
|
||||
void RegisterChannel(IChannel channel);
|
||||
void UnregisterChannel(string channelId);
|
||||
IChannel? GetChannel(string channelId);
|
||||
IReadOnlyList<IChannel> GetAllChannels();
|
||||
event EventHandler<ChannelMessageReceivedEventArgs>? MessageRouted;
|
||||
}
|
||||
```
|
||||
|
||||
### Supporting Types
|
||||
- **ChannelType**: A static class providing constants like `cli` and `telegram`.
|
||||
- **ChannelMessageReceivedEventHandler**: Delegate for handling incoming messages.
|
||||
`Task ChannelMessageReceivedEventHandler(object? sender, ChannelMessageReceivedEventArgs e)`
|
||||
- **ChannelMessageReceivedEventArgs**: Contains the `Channel`, `Message`, and an `IsHandled` flag.
|
||||
- **ChannelConnectionEventArgs**: Contains the `Channel` and `IsConnected` status.
|
||||
|
||||
## Luna.Channels
|
||||
|
||||
This project contains the concrete management logic and specific channel implementations.
|
||||
|
||||
### ChannelManager
|
||||
The `ChannelManager` implements `IChannelManager` and acts as the central hub for message traffic. It injects `ISessionManager` from [[Core]] and `ILogger<ChannelManager>`.
|
||||
|
||||
When a channel is registered via `RegisterChannel`, the manager subscribes to its `MessageReceived` event. The `OnMessageReceivedAsync` handler performs the following:
|
||||
1. Calls `sessionManager.RouteMessagesAsync(message.Content, message.ConversationId, channel.ChannelId)`.
|
||||
2. Pipes the resulting `IAsyncEnumerable<ChatStreamUpdate>` back to the channel via `channel.SendStreamingMessageAsync`.
|
||||
|
||||
### Channel Implementations
|
||||
- [[CLI Channel]] — SignalR-based adapter for the command-line interface.
|
||||
- [[Telegram Channel]] — Telegram Bot API adapter with message splitting and streaming.
|
||||
- [[Web Interface Channel]] — Browser-based chat UI (not yet implemented).
|
||||
|
||||
## Routing Flow
|
||||
|
||||
1. `Channel.MessageReceived` event triggers.
|
||||
2. `ChannelManager` catches the event and identifies the sender.
|
||||
3. Manager calls `SessionManager.RouteMessagesAsync`.
|
||||
4. `SessionManager` returns an `IAsyncEnumerable<ChatStreamUpdate>`.
|
||||
5. `ChannelManager` passes this stream to `Channel.SendStreamingMessageAsync`.
|
||||
6. The channel implementation handles the physical transport of the stream.
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
- **Separation of Concerns**: `ChannelManager` handles transport and routing. `SessionManager` handles AI context and conversation logic.
|
||||
- **Conversation Scoping**: Sessions are scoped to conversations, not specific channels. This allows for potential cross-channel persistence.
|
||||
- **Unified DTOs**: All message data uses the `ChannelMessage` DTO from `Luna.Shared`.
|
||||
- **Registration**: Channels are registered via the `AddChannels()` DI extension method, following the [[Configuration]] patterns.
|
||||
- **Options Pattern**: Implementations use `IOptions<TOptions>` (e.g., `TelegramOptions`) for configuration.
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Project References
|
||||
- `Luna.Channels.Abstractions`
|
||||
- `Luna.Configuration`
|
||||
- `Luna.Core.Abstractions`
|
||||
- `Luna.Shared`
|
||||
|
||||
### NuGet Packages
|
||||
- `Telegram.Bot`
|
||||
- `Microsoft.Extensions.Hosting.Abstractions`
|
||||
- `Microsoft.Extensions.Logging.Abstractions`
|
||||
|
||||
## Adding a New Channel
|
||||
|
||||
To implement a new channel:
|
||||
1. Create a class implementing `IChannel`.
|
||||
2. Ensure it handles both `SendMessageAsync` and `SendStreamingMessageAsync`.
|
||||
3. Raise `MessageReceived` when the external platform sends a message.
|
||||
4. Register the channel with `IChannelManager` during startup or via a background adapter (like `TelegramAdapter`).
|
||||
@@ -0,0 +1,39 @@
|
||||
# Telegram Channel
|
||||
|
||||
The Telegram integration consists of two classes: `TelegramChannel` (the `IChannel` implementation) and `TelegramAdapter` (the hosted service that manages channel lifecycles). Both live in the `Luna.Channels.Telegram` namespace.
|
||||
|
||||
## TelegramChannel
|
||||
|
||||
Handles interaction with the Telegram Bot API, implementing the `IChannel` interface defined in [[Channels]].
|
||||
|
||||
### Message Splitting
|
||||
Telegram enforces a 4096-character limit per message. `TelegramChannel` automatically splits long responses into sequential chunks that respect this limit.
|
||||
|
||||
### Streaming
|
||||
Rather than forwarding every `ChatStreamUpdate` individually (which would hit Telegram's rate limits), the channel accumulates streaming updates and sends them in periodic batches.
|
||||
|
||||
### Typing Indicators
|
||||
Uses `KeepTypingAsync` to maintain a "typing..." indicator in the Telegram chat while the AI generates a response. This runs as a background loop until the response completes.
|
||||
|
||||
### User Filtering
|
||||
`RaiseMessageReceived` filters incoming updates by `AllowedUserIds` (configured via `TelegramOptions` in [[Configuration]]). Messages from unauthorized users or with empty content are silently dropped.
|
||||
|
||||
## TelegramAdapter
|
||||
|
||||
An `IHostedService` and `IUpdateHandler` that polls Telegram for updates using long polling.
|
||||
|
||||
### Lifecycle
|
||||
1. On startup, begins polling the Telegram Bot API.
|
||||
2. For each incoming update, identifies the chat ID.
|
||||
3. Creates a new `TelegramChannel` instance for each unique chat (if one doesn't already exist).
|
||||
4. Registers the channel with `IChannelManager` from [[Channels]].
|
||||
5. Routes the update to the appropriate `TelegramChannel`.
|
||||
|
||||
### Configuration
|
||||
Configured via `TelegramOptions` (see [[Configuration]]), which includes:
|
||||
- `BotToken` — Telegram Bot API token.
|
||||
- `AllowedUserIds` — Whitelist of Telegram user IDs permitted to interact with Luna.
|
||||
|
||||
## Namespace
|
||||
|
||||
`Luna.Channels.Telegram`
|
||||
@@ -0,0 +1,41 @@
|
||||
# Web Interface Channel
|
||||
|
||||
> [!info] Status: Not Yet Implemented
|
||||
|
||||
The Web Interface channel will provide a browser-based chat UI for interacting with Luna directly, without requiring the CLI or Telegram. It implements the `IChannel` interface defined in [[Channels]].
|
||||
|
||||
## Planned Approach
|
||||
|
||||
### WebInterfaceChannel
|
||||
|
||||
A server-side `IChannel` implementation that bridges the web frontend to Luna's channel system. Similar to [[CLI Channel]], it will likely use SignalR for real-time bidirectional communication.
|
||||
|
||||
**Key responsibilities:**
|
||||
- Accept messages from authenticated browser sessions.
|
||||
- Stream `ChatStreamUpdate` responses back to the frontend in real time.
|
||||
- Manage connection lifecycle (connect, disconnect, reconnect).
|
||||
|
||||
### WebInterfaceAdapter
|
||||
|
||||
An `IHostedService` (similar to `TelegramAdapter` in [[Telegram Channel]]) responsible for:
|
||||
- Registering `WebInterfaceChannel` instances with `IChannelManager` from [[Channels]].
|
||||
- Managing per-user or per-session channel lifecycle.
|
||||
|
||||
## Configuration
|
||||
|
||||
Will follow the existing [[Configuration]] options pattern with a `WebInterfaceOptions` class containing settings such as:
|
||||
- Authentication / authorization settings.
|
||||
- CORS policy.
|
||||
- Session timeout.
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Expected Project References
|
||||
- `Luna.Channels.Abstractions`
|
||||
- `Luna.Configuration`
|
||||
- `Luna.Core.Abstractions`
|
||||
- `Luna.Shared`
|
||||
|
||||
## Namespace
|
||||
|
||||
`Luna.Channels.Web`
|
||||
@@ -0,0 +1,112 @@
|
||||
# Configuration
|
||||
|
||||
## Overview
|
||||
Luna uses the standard .NET Options pattern for managing application settings. The configuration system relies on `IOptions<T>` and `IOptionsMonitor<T>` to provide typed access to settings defined in `appsettings.json`. This approach ensures type safety and allows for easy validation at startup.
|
||||
|
||||
The configuration is centralized in the `Luna.Configuration` project. Settings are bound during application startup using the `.BindConfiguration().ValidateDataAnnotations().ValidateOnStart()` pattern.
|
||||
|
||||
## Options Classes
|
||||
|
||||
### AgentOptions
|
||||
These settings define the behavior and identity of AI agents like the [[Core]] agent or the Librarian. Agents retrieve their specific configuration using `IOptionsMonitor<AgentOptions>.Get(name)`.
|
||||
|
||||
```csharp
|
||||
public class AgentOptions
|
||||
{
|
||||
public required string Name { get; set; }
|
||||
public string? DisplayName { get; init; }
|
||||
public string? Description { get; init; }
|
||||
public required string Provider { get; init; }
|
||||
public required string ModelId { get; init; }
|
||||
public required string Instructions { get; init; }
|
||||
public required int MaxContextTokens { get; init; }
|
||||
}
|
||||
```
|
||||
|
||||
### ProviderOptions
|
||||
Configures the connection details for AI model providers such as Mistral or OpenAI.
|
||||
|
||||
```csharp
|
||||
public class ProviderOptions
|
||||
{
|
||||
public required string ApiKey { get; init; }
|
||||
public required string ApiUrl { get; init; }
|
||||
public required string[] Models { get; init; }
|
||||
}
|
||||
```
|
||||
|
||||
### SessionOptions
|
||||
Controls how [[Memory]] and conversation sessions are managed. It specifically dictates when the session context should be compacted to save tokens.
|
||||
|
||||
```csharp
|
||||
public class SessionOptions
|
||||
{
|
||||
public required float ContextTokenThreshold { get; init; }
|
||||
public required int RetainedMessagesAfterCompacting { get; init; }
|
||||
}
|
||||
```
|
||||
|
||||
### TelegramOptions
|
||||
Specific settings for the Telegram [[Channels]] adapter, including bot authentication and user access control.
|
||||
|
||||
```csharp
|
||||
public class TelegramOptions
|
||||
{
|
||||
public required string BotToken { get; init; }
|
||||
public string? WebhookUrl { get; init; }
|
||||
public int PollingTimeoutSeconds { get; init; } = 30;
|
||||
public string[] AllowedUserIds { get; init; } = [];
|
||||
}
|
||||
```
|
||||
|
||||
## Example Configuration
|
||||
|
||||
The following `appsettings.json` structure demonstrates how these options are populated:
|
||||
|
||||
```json
|
||||
{
|
||||
"Agents": {
|
||||
"Core": {
|
||||
"Name": "Core",
|
||||
"Provider": "Mistral",
|
||||
"ModelId": "mistral-small-latest",
|
||||
"Instructions": "You are Luna, a helpful AI assistant.",
|
||||
"MaxContextTokens": 8192
|
||||
},
|
||||
"Librarian": {
|
||||
"Name": "Librarian",
|
||||
"Provider": "Mistral",
|
||||
"ModelId": "mistral-small-latest",
|
||||
"Instructions": "Summarize conversations concisely.",
|
||||
"MaxContextTokens": 4096
|
||||
}
|
||||
},
|
||||
"Providers": {
|
||||
"Mistral": {
|
||||
"ApiKey": "YOUR_API_KEY",
|
||||
"ApiUrl": "https://api.mistral.ai",
|
||||
"Models": ["mistral-small-latest"]
|
||||
}
|
||||
},
|
||||
"Session": {
|
||||
"ContextTokenThreshold": 0.7,
|
||||
"RetainedMessagesAfterCompacting": 5
|
||||
},
|
||||
"Channels": {
|
||||
"Telegram": {
|
||||
"BotToken": "YOUR_BOT_TOKEN",
|
||||
"PollingTimeoutSeconds": 30,
|
||||
"AllowedUserIds": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
The configuration module depends on the following NuGet packages:
|
||||
|
||||
- Microsoft.Extensions.DependencyInjection.Abstractions
|
||||
- Microsoft.Extensions.FileProviders.Embedded
|
||||
- Microsoft.Extensions.Options
|
||||
- Tomlyn (for parsing embedded TOML resources)
|
||||
@@ -0,0 +1,106 @@
|
||||
# Context Compaction Research
|
||||
|
||||
This document captures a research analysis comparing Claude Code's compaction engine patterns against Luna's current implementation, with recommendations for adoption. Reference source: https://barazany.dev/blog/claude-codes-compaction-engine
|
||||
|
||||
## Luna's Current Compaction Architecture
|
||||
|
||||
| Component | Current Approach |
|
||||
|---|---|
|
||||
| Trigger | Token threshold: `session.TokenAmount > MaxContextTokens * 0.75` |
|
||||
| Token estimation | Naive `text.Length / 4` heuristic (`TokenEstimator`) |
|
||||
| Compaction method | Single LLM call via `LibrarianAgent` (Mistral Small) — plain bullet-point summary |
|
||||
| Retained context | Last N messages (default: 2) carried forward |
|
||||
| Post-compaction reconstruction | Summary wrapped in `MEMORY BEGIN/END` markers as an Assistant message, then retained messages appended |
|
||||
| Persistent memory | File-based `MemoryStore` — raw conversation logs to disk, last 10 read back as system message |
|
||||
| Tiers | None — only full LLM summarization |
|
||||
| Cache awareness | None |
|
||||
| Tool result management | None — tool outputs accumulate until full compaction |
|
||||
|
||||
## Claude Code's Three-Tier Pattern
|
||||
|
||||
Claude Code employs a tiered strategy to manage context while maximizing cache efficiency:
|
||||
|
||||
- **Tier 1**: Lightweight deterministic cleanup before every API call. This process clears old tool results (retaining only the last 5) and replaces them with placeholders. No LLM is involved at this stage.
|
||||
- **Tier 2**: API-level server-side strategies for token management using Anthropic-specific infrastructure.
|
||||
- **Tier 3**: Full LLM summarization as a last resort. This involves a structured 9-section summary with a chain-of-thought scratchpad. Post-compaction reconstruction includes a boundary marker, the summary, the 5 most recently read files (capped at 50K tokens), re-injected skills, tool definitions, and session hooks.
|
||||
|
||||
The key architectural insight is that cache economics drive every decision. This includes using `cache_edits` for surgical server-side deletions and ensuring the summarization call reuses the same cache key.
|
||||
|
||||
## Pattern Evaluation
|
||||
|
||||
### 1. Tier 1 — Deterministic Tool Result Cleanup
|
||||
**Gap**: Luna currently has zero tool result management.
|
||||
**Recommendation**: ADOPT.
|
||||
**Effort**: Low.
|
||||
**Details**: Implement a pre-call sanitizer that trims old tool results before each API call to prevent context bloating from large tool outputs.
|
||||
|
||||
### 2. Structured Compaction Prompt
|
||||
**Gap**: Luna uses a simple "max 12 bullet points" prompt.
|
||||
**Recommendation**: ADOPT.
|
||||
**Effort**: Low.
|
||||
**Details**: Replace the current prompt with a structured template that forces categorized output, including user intent, key decisions, unresolved tasks, relevant facts, and technical context.
|
||||
|
||||
### 3. Tiered Compaction (Delay LLM Summarization)
|
||||
**Gap**: Luna only supports full LLM summarization.
|
||||
**Recommendation**: ADOPT.
|
||||
**Effort**: Moderate.
|
||||
**Details**: Implement a 2-tier system. Tier 1 performs deterministic cleanup on every call, while Tier 2 triggers LLM summarization only when Tier 1 is insufficient. The Anthropic-specific server-side tier will be skipped.
|
||||
|
||||
### 4. Post-Compaction Reconstruction
|
||||
**Gap**: Luna's reconstruction logic is basic and uses Assistant messages for summaries.
|
||||
**Recommendation**: PARTIALLY ADOPT.
|
||||
**Effort**: Moderate.
|
||||
**Details**: Place the summary as a System message instead of an Assistant message. Re-inject agent instructions and add a boundary marker with metadata (timestamp, pre-compaction message count). Include a continuation message so the agent does not treat the summary as something to respond to.
|
||||
|
||||
### 5. Improved Token Estimation
|
||||
**Gap**: Luna relies on a `text.Length / 4` heuristic.
|
||||
**Recommendation**: ADOPT.
|
||||
**Effort**: Low.
|
||||
**Details**: Replace the current heuristic with a proper tokenizer, such as `Microsoft.ML.Tokenizers`, or a significantly improved heuristic.
|
||||
|
||||
### 6. Autonomous Continuation After Compaction
|
||||
**Gap**: Compaction can disrupt the conversation flow.
|
||||
**Recommendation**: ADOPT LIGHTLY.
|
||||
**Effort**: Trivial.
|
||||
**Details**: Prepend a brief system message after compaction, such as "Context was compacted. Continue naturally."
|
||||
|
||||
### 7. Cache-Aware `cache_edits`
|
||||
**Gap**: This is specific to Anthropic's API.
|
||||
**Recommendation**: NOT APPLICABLE.
|
||||
**Effort**: N/A.
|
||||
**Details**: This could be revisited if an Anthropic provider is added to `IProvider` in the future.
|
||||
|
||||
### 8. Same-Cache-Key Summarization
|
||||
**Gap**: Luna uses a cheaper model (Mistral Small) for compaction.
|
||||
**Recommendation**: NOT APPLICABLE.
|
||||
**Effort**: N/A.
|
||||
**Details**: Luna's current approach is effective when prompt caching is not a primary factor.
|
||||
|
||||
## Priority Adoption Matrix
|
||||
|
||||
| Priority | Pattern | Effort | Impact |
|
||||
|---|---|---|---|
|
||||
| P0 | Tier 1 — Deterministic tool result cleanup | Low | High |
|
||||
| P0 | Structured compaction prompt | Low | High |
|
||||
| P1 | Tiered compaction (delay LLM summarization) | Moderate | High |
|
||||
| P1 | Post-compaction reconstruction improvements | Moderate | Medium |
|
||||
| P1 | Improved token estimation | Low | Medium |
|
||||
| P2 | Continuation message after compaction | Trivial | Low-Medium |
|
||||
| N/A | Cache-aware `cache_edits` | — | Not applicable (Mistral) |
|
||||
| N/A | Same-cache-key summarization | — | Not applicable |
|
||||
|
||||
## Known Issues Found During Analysis
|
||||
|
||||
There is a bug in `SessionManager.SaveSessionLogAsync()` at line 97:
|
||||
|
||||
```csharp
|
||||
.Where(m => m.Role != ChatRole.System || m.Role != ChatRole.Tool)
|
||||
```
|
||||
|
||||
This condition is always true due to De Morgan's law. It should use `&&` instead of `||` to correctly filter out system and tool messages.
|
||||
|
||||
## Cross-References
|
||||
|
||||
- [[Core]] — SessionManager, compaction flow, token estimation
|
||||
- [[Memory]] — MemoryStore, persistent conversation logs
|
||||
- [[Configuration]] — SessionOptions (ContextTokenThreshold, RetainedMessagesAfterCompacting)
|
||||
@@ -0,0 +1,137 @@
|
||||
# Core Module
|
||||
|
||||
The Core module serves as the central orchestration engine for Luna, managing sessions, message routing, context compaction, and tool integration. It is split into `Luna.Core.Abstractions` for interfaces and `Luna.Core` for the primary implementation.
|
||||
|
||||
## Luna.Core.Abstractions
|
||||
|
||||
This project defines the contracts used by the Core and other modules.
|
||||
|
||||
### ISessionManager
|
||||
The primary interface for managing chat sessions and routing messages.
|
||||
```csharp
|
||||
public interface ISessionManager
|
||||
{
|
||||
IAsyncEnumerable<ChatStreamUpdate> RouteMessagesAsync(string content, string conversationId, string connectionId, CancellationToken ct);
|
||||
Task ClientDisconnectedAsync(string connectionId);
|
||||
}
|
||||
```
|
||||
|
||||
### IChatHubContext
|
||||
Interface for sending responses back to clients via SignalR.
|
||||
```csharp
|
||||
public interface IChatHubContext
|
||||
{
|
||||
Task SendResponseAsync(string connectionId, ChannelMessage message, CancellationToken ct);
|
||||
Task SendStreamingResponseAsync(string connectionId, IAsyncEnumerable<ChatStreamUpdate> messageStream, CancellationToken ct);
|
||||
}
|
||||
```
|
||||
|
||||
## Luna.Core
|
||||
|
||||
The main implementation project containing the session logic and SignalR hubs.
|
||||
|
||||
### Session
|
||||
The `Session` class manages the state of an active conversation.
|
||||
```csharp
|
||||
public class Session
|
||||
{
|
||||
public string SessionId { get; set; }
|
||||
public List<ChatMessage> Messages { get; } = new();
|
||||
public int TokenAmount => TokenEstimator.EstimateTokens(Messages);
|
||||
}
|
||||
```
|
||||
|
||||
### SessionManager
|
||||
Implements `ISessionManager`. It coordinates between agents, memory, and the core LLM processing.
|
||||
- **Injected Services**:
|
||||
- `[FromKeyedServices("Core")] IAgent coreAgent`
|
||||
- `[FromKeyedServices("Librarian")] IAgent librarianAgent`
|
||||
- `IOptions<SessionOptions> options`
|
||||
- `IChatStreamUpdateBuilder streamUpdateBuilder`
|
||||
- `IMemoryStore memoryStore`
|
||||
- **State Management**: Maintains an in-memory `Dictionary<string, Session>` for sessions and a `ConcurrentDictionary<string, string>` for mapping connection IDs to session IDs.
|
||||
|
||||
#### Message Routing Flow
|
||||
1. **Session Initialization**: Creates a new session if one does not exist and loads existing [[Memory]] via `memoryStore.GetMemoriesAsync()`.
|
||||
2. **Token Check**: Evaluates current token usage against `MaxContextTokens * ContextTokenThreshold`.
|
||||
3. **Compaction**: If the threshold is exceeded, triggers `CompactSessionAsync`.
|
||||
4. **Processing**: Appends the user message, streams the response from the `coreAgent` via `ProcessStreamingAsync`, and converts AI content to `ChatStreamUpdate` using the `IChatStreamUpdateBuilder`.
|
||||
5. **Persistence**: Appends the assistant response to the session history.
|
||||
|
||||
#### Compaction Strategy
|
||||
When a session exceeds the token threshold, the system:
|
||||
1. Retains the last $N$ messages (defined by `SessionOptions.RetainedMessagesAfterCompacting`).
|
||||
2. Sends all older messages to the `librarianAgent` for summarization.
|
||||
3. Wraps the resulting summary in `<---- MEMORY BEGIN ---->` and `<---- MEMORY END ---->` markers and inserts it at the beginning of the message list.
|
||||
|
||||
#### Disconnect Flow
|
||||
When `ClientDisconnectedAsync` is called:
|
||||
1. The conversation log is saved to the `IMemoryStore`.
|
||||
2. The session and connection mappings are cleaned up.
|
||||
|
||||
### Hubs and Contexts
|
||||
|
||||
#### ChatHub
|
||||
A SignalR hub that serves as the entry point for real-time communication.
|
||||
- **OnConnected**: Creates a `CliChannel` and registers it with the `IChannelManager`.
|
||||
- **OnDisconnected**: Unregisters the channel.
|
||||
- **OnMessageReceived**: Delegates message handling to the `CliChannel.RaiseMessageReceived`.
|
||||
|
||||
#### ChatHubContext
|
||||
Implements `IChatHubContext` using `IHubContext<ChatHub>`. It handles the actual transmission of data to SignalR clients, supporting both discrete and streaming responses.
|
||||
|
||||
### Tools System
|
||||
|
||||
#### IToolbox
|
||||
Provides a mechanism for discovering and exposing tools to the AI.
|
||||
- **Implementation**: Uses reflection to find methods decorated with `[ToolAttribute]`.
|
||||
- **Function Creation**: Generates `AITool` instances using `AIFunctionFactory.Create`.
|
||||
|
||||
#### IToolsProvider
|
||||
Exposes the collection of discovered tools.
|
||||
```csharp
|
||||
public interface IToolsProvider
|
||||
{
|
||||
IEnumerable<AITool> GetTools();
|
||||
}
|
||||
```
|
||||
|
||||
### Token Estimation
|
||||
The `TokenEstimator` provides a heuristic-based token count:
|
||||
- **Calculation**: Number of characters divided by 4.
|
||||
|
||||
## Architecture Flow
|
||||
|
||||
The following flow describes how a message moves through the Core module:
|
||||
|
||||
```text
|
||||
Channel.MessageReceived → ChannelManager → SessionManager.RouteMessagesAsync
|
||||
→ Token Check → Compaction if needed (LibrarianAgent)
|
||||
→ CoreAgent.ProcessStreamingAsync → IChatClient Streaming
|
||||
→ ChatStreamUpdateBuilder → IAsyncEnumerable<ChatStreamUpdate>
|
||||
→ Channel.SendStreamingMessageAsync → Client
|
||||
```
|
||||
|
||||
## Cross-References
|
||||
- [[Channels]]: Management of communication pathways.
|
||||
- [[Memory]]: Long-term and short-term state persistence.
|
||||
- [[Configuration]]: `SessionOptions` and system settings.
|
||||
- [[Skills]]: Integration of specialized capabilities.
|
||||
- [[Scheduler]]: Task timing and execution.
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Project References
|
||||
- Luna.Agents.Abstractions
|
||||
- Luna.Channels
|
||||
- Luna.Channels.Abstractions
|
||||
- [[Configuration]] (Luna.Configuration)
|
||||
- Luna.Core.Abstractions
|
||||
- [[Memory]] (Luna.Memory)
|
||||
- Luna.Providers
|
||||
- Luna.Providers.Abstractions
|
||||
- Luna.Shared
|
||||
|
||||
### NuGet Packages
|
||||
- Microsoft.AspNetCore.OpenApi
|
||||
- Microsoft.Extensions.AI
|
||||
@@ -0,0 +1,84 @@
|
||||
# Memory Module
|
||||
|
||||
## Overview
|
||||
|
||||
The Memory module provides persistent conversation storage for the Luna AI Assistant. It uses a simple **filesystem-based** approach — conversation logs are written as plain text files and read back to seed new sessions with prior context.
|
||||
|
||||
The module consists of a single project: `Luna.Memory`.
|
||||
|
||||
---
|
||||
|
||||
## Interface
|
||||
|
||||
```csharp
|
||||
public interface IMemoryStore
|
||||
{
|
||||
Task AddMemoryAsync(string memory);
|
||||
Task<string> GetMemoriesAsync();
|
||||
}
|
||||
```
|
||||
|
||||
- `AddMemoryAsync` — persists a conversation log or compaction summary to storage.
|
||||
- `GetMemoriesAsync` — retrieves recent memories as a single concatenated string, used to seed new conversations.
|
||||
|
||||
---
|
||||
|
||||
## Implementation
|
||||
|
||||
### MemoryStore
|
||||
|
||||
`MemoryStore` is the concrete `IMemoryStore` implementation. It stores conversation logs as individual files on disk.
|
||||
|
||||
**Storage path**: `~/.luna/memory/conversations/`
|
||||
|
||||
**File naming**: `Luna_Conversation_Log_{yyyyMMddHHmmss}`
|
||||
|
||||
```csharp
|
||||
public class MemoryStore : IMemoryStore
|
||||
```
|
||||
|
||||
| Method | Behavior |
|
||||
|--------|----------|
|
||||
| `AddMemoryAsync` | Creates the storage directory if it does not exist, then writes the memory string to a new timestamped file. |
|
||||
| `GetMemoriesAsync` | Reads the **last 10 files** (ordered by timestamp descending, parsed from the filename), concatenates their contents, and returns the result. Returns an empty string if no files exist. |
|
||||
|
||||
There is no database, no Redis, and no SQLite involved — persistence is purely filesystem-based.
|
||||
|
||||
---
|
||||
|
||||
## Integration with the System
|
||||
|
||||
The `SessionManager` in [[Core]] injects `IMemoryStore` and uses it at three points:
|
||||
|
||||
1. **Session creation** — On the first message in a new conversation, `SessionManager` calls `GetMemoriesAsync()` and, if non-empty, prepends the result as a `ChatMessage` with `ChatRole.System`. This gives the agent prior conversational context.
|
||||
|
||||
2. **Client disconnect** — When a client disconnects, `SessionManager.ClientDisconnectedAsync` triggers `SaveSessionLogAsync`, which formats all user and assistant messages from the session and calls `AddMemoryAsync` to persist the log.
|
||||
|
||||
3. **Compaction** — When the session's token count exceeds the configured threshold (see [[Configuration]] `SessionOptions`), `SessionManager` uses the Librarian agent to summarize older messages. The summary is retained in-session as an assistant message wrapped in:
|
||||
```
|
||||
<---- MEMORY BEGIN ---->
|
||||
[Meta] Conversation Recorded at: {timestamp}
|
||||
{summary}
|
||||
<---- MEMORY END ---->
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## DI Registration
|
||||
|
||||
Memory services are registered via the `AddMemory()` extension method on `IServiceCollection`.
|
||||
|
||||
---
|
||||
|
||||
## Dependencies
|
||||
|
||||
**Project references**: None (standalone module).
|
||||
|
||||
**NuGet packages**:
|
||||
- `Microsoft.Extensions.Caching.Memory`
|
||||
- `Microsoft.Extensions.DependencyInjection.Abstractions`
|
||||
- `Microsoft.Extensions.Logging.Abstractions`
|
||||
- `Microsoft.Extensions.Options`
|
||||
- `StackExchange.Redis` (present in csproj, currently unused)
|
||||
- `Microsoft.Data.Sqlite` (present in csproj, currently unused)
|
||||
- `Microsoft.EntityFrameworkCore.Sqlite` (present in csproj, currently unused)
|
||||
@@ -0,0 +1,24 @@
|
||||
# Scheduler
|
||||
|
||||
> [!info] Status: Not Yet Implemented
|
||||
> This module is planned for a future development phase.
|
||||
|
||||
The Scheduler provides a mechanism for Luna to handle time-based operations and periodic tasks. It acts as a temporal bridge between [[Core]] and [[Memory]], ensuring that scheduled actions are executed reliably and within the specified context.
|
||||
|
||||
## Purpose
|
||||
The primary role of the Scheduler is to periodically scan [[Memory]] for pending tasks and coordinate their execution. It manages the lifecycle of long-running or deferred operations, ensuring they are handed off to the appropriate Agents at the right time.
|
||||
|
||||
## Planned Features
|
||||
- **Recurring Tasks**: Support for daily, weekly, or interval-based execution patterns.
|
||||
- **Task Dependencies**: Ability to chain tasks so that one starts only after another completes successfully.
|
||||
- **Priority Queue**: Management of task urgency to ensure critical system operations take precedence.
|
||||
- **Cron Scheduling**: Standardized string-based scheduling for precise control over execution times.
|
||||
|
||||
## Architecture & Integration
|
||||
The Scheduler is expected to integrate deeply with the following modules:
|
||||
- **[[Core]]**: For task execution logic and agent coordination.
|
||||
- **[[Memory]]**: To persist task states and schedules.
|
||||
|
||||
## Likely Dependencies
|
||||
To ensure robust background execution, the module will likely utilize:
|
||||
- `Microsoft.Extensions.Hosting`: Leveraging `IHostedService` or `BackgroundService` for long-lived process management within the .NET ecosystem.
|
||||
@@ -0,0 +1,23 @@
|
||||
# Skills
|
||||
|
||||
> [!info] Status: Not Yet Implemented
|
||||
> This module is planned for a future development phase.
|
||||
|
||||
The Skills module is designed as a plugin system to extend Luna with third-party integrations and specialized tools. It provides a modular framework for adding new capabilities without modifying the core system.
|
||||
|
||||
## Existing Tools System
|
||||
It is important to note that a basic tools system already exists in [[Core]]. This existing system includes:
|
||||
- `IToolbox`: A registry for managing available tools.
|
||||
- `IToolsProvider`: An interface for supplying tools to the assistant.
|
||||
- `ToolAttribute`: Used for reflective discovery of tools within the codebase.
|
||||
|
||||
The Skills module will build upon or complement this system by allowing for more complex, external integrations.
|
||||
|
||||
## Planned Features
|
||||
- **Dynamic Skill Loading**: Support for loading and unloading skills at runtime without requiring a system restart.
|
||||
- **Skill Discovery**: Automatic detection of new skills within designated plugin directories.
|
||||
- **Capability-Based Security**: A granular permission model where skills must declare and be granted specific capabilities (e.g., network access, file system access).
|
||||
- **Third-Party Integrations**: A standardized interface for connecting Luna to external services and APIs.
|
||||
|
||||
## Integration
|
||||
- **[[Core]]**: The Skills module will interface with the existing tools system in the Core to expose its capabilities to the AI Agents.
|
||||
@@ -0,0 +1,70 @@
|
||||
# Channel System
|
||||
|
||||
The Channel System in OpenClaw is a highly decoupled messaging platform integration layer. It allows OpenClaw to interface with diverse services like WhatsApp, Telegram, Slack, and Discord through a unified set of interfaces while preserving platform-specific capabilities.
|
||||
|
||||
***
|
||||
|
||||
## Plugin-Per-Platform Pattern
|
||||
|
||||
OpenClaw employs a strict plugin architecture where every supported messaging platform is a self-contained module located under `src/channels/plugins/`. This directory contains over 100 files, with each platform adapter implementing a standard set of interfaces defined in `types.plugin.ts` and `types.adapters.ts`.
|
||||
|
||||
Key aspects of this pattern include:
|
||||
* **Isolation**: Each plugin maintains its own dependencies and platform-specific logic (e.g., the Telegram plugin handles its own bot API calls).
|
||||
* **Standardized Lifecycle**: Plugins implement `ChannelLifecycleAdapter` to manage startup, shutdown, and health checks.
|
||||
* **Feature-Based Opt-in**: Plugins advertise their capabilities (e.g., `threads`, `reactions`, `media`) via a `ChannelCapabilities` object, allowing the core to gracefully degrade or enhance features per-channel.
|
||||
|
||||
***
|
||||
|
||||
## Shared Registry & Helpers
|
||||
|
||||
A centralized `registry.ts` manages all active channel plugins. Instead of hardcoding platform logic into the core, the registry provides a discovery mechanism for the system to interact with whatever plugins are currently loaded.
|
||||
|
||||
### Message Normalization
|
||||
Incoming raw messages from various platforms are normalized into a standard `MsgContext` before reaching the agent or session logic. This normalization ensures consistent handling of:
|
||||
* **Sender Identification**: Mapping platform-specific IDs to a common structure containing `SenderId`, `SenderName`, and `SenderUsername`.
|
||||
* **Thread Tracking**: Normalizing `ThreadId` and `ReplyToId` so the core can track conversations across platforms that represent threads differently (e.g., Slack's thread timestamps vs. Telegram's reply-to message IDs).
|
||||
* **Channel Metadata**: Attaching `Channel` (platform name) and `ChatType` (direct, group, or channel) to every inbound payload.
|
||||
|
||||
### Shared Utilities
|
||||
OpenClaw provides several helper modules that plugins use to reduce boilerplate:
|
||||
* `sender-identity.ts`: Validates and sanitizes sender metadata.
|
||||
* `chat-meta.ts`: Manages channel-level metadata like labels, blurbs, and documentation links.
|
||||
* `session-envelope.ts`: Handles the wrapping of messages for persistent storage.
|
||||
|
||||
***
|
||||
|
||||
## Access Control & Routing
|
||||
|
||||
OpenClaw enforces security and session boundaries at the channel level.
|
||||
|
||||
### Allowlist-based Access
|
||||
The system uses an allowlist-based policy for each channel. The `allowlist-match.ts` and `allow-from.ts` utilities provide logic to verify if a specific sender or group is permitted to interact with the agent. This ties directly into [[Security]] policies, preventing unauthorized access before a session is even initialized.
|
||||
|
||||
### Session Routing
|
||||
Incoming messages are routed to specific sessions based on their origin:
|
||||
* **Direct Messages (DMs)**: Usually routed to a per-sender session.
|
||||
* **Group Chats**: Messages are routed to a session keyed by the group ID.
|
||||
* **Thread Binding**: `thread-bindings-policy.ts` determines if a message should stay within an existing session or trigger the creation of a child session.
|
||||
|
||||
***
|
||||
|
||||
## Channel-Specific Auth
|
||||
|
||||
Authentication is delegated to the individual plugins through the `ChannelAuthAdapter`. Each platform handles its own credential requirements:
|
||||
* **Bot Tokens**: Simple token-based auth (Telegram, Discord).
|
||||
* **OAuth**: Flow-based authentication for user-level access (Slack, Matrix).
|
||||
* **QR Code Pairing**: Used by platforms like WhatsApp or Signal to link an existing account.
|
||||
|
||||
The `ChannelSetupAdapter` provides a standardized `ChannelSetupInput` bag containing fields for `botToken`, `appToken`, `privateKey`, and more, which are then stored in the system's secure configuration.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
The OpenClaw channel system provides a blueprint for expanding Luna beyond its current CLI-only interface.
|
||||
|
||||
Key lessons for [[Channels]]:
|
||||
* **Establish a Rich Normalized Type**: Before adding a second channel, Luna should define a robust internal message format. This prevents the core logic from becoming littered with "if platform is Discord" checks.
|
||||
* **Abstract Outbound Routing**: By defining a common `ChannelOutboundAdapter`, Luna can send messages to any platform using a unified API, even if the underlying delivery mechanism (WebSocket, HTTP POST, CLI print) varies wildly.
|
||||
|
||||
Cross-references: [[Core]], [[Security]]
|
||||
@@ -0,0 +1,142 @@
|
||||
# Compaction Strategy
|
||||
|
||||
OpenClaw employs a multi-layered conversation compaction system to manage long-running sessions within finite model context windows. Unlike basic truncation, this system uses LLM-driven summarization, token-budget gating, and quality safeguards to preserve critical state, identifiers, and recent context.
|
||||
|
||||
***
|
||||
|
||||
## Trigger & Token Gating
|
||||
|
||||
Compaction is primarily triggered by "preflight" checks before processing a new turn, ensuring the model has sufficient headroom for its next response. This process is managed in `auto-reply/reply/agent-runner-memory.ts` and `memory-flush.ts`.
|
||||
|
||||
### Gating Logic
|
||||
The decision to compact is based on a projected token count compared against a computed threshold:
|
||||
- **Threshold Formula**: `contextWindowTokens - reserveTokensFloor - softThresholdTokens`
|
||||
- **Reserve Floor**: Defaults to 20,000 tokens (`reserveTokensFloor`), providing a safety buffer for tool outputs and reasoning.
|
||||
- **Soft Threshold**: An additional 4,000 token buffer (`softThresholdTokens`) to prevent compaction oscillations at the exact limit.
|
||||
- **Token Estimation**: Performed by `estimateMessagesTokens()`, which strips `toolResult.details` for security and applies a `SAFETY_MARGIN` of 1.2 to account for estimation inaccuracies.
|
||||
|
||||
### Execution Hooks
|
||||
- **shouldRunPreflightCompaction**: Runs before a turn to ensure the input fits the budget.
|
||||
- **shouldRunMemoryFlush**: Evaluates if the session should be "flushed" to long-term memory based on token pressure.
|
||||
- **Deduplication**: `hasAlreadyFlushedForCurrentCompaction` prevents redundant flushes within a single compaction cycle.
|
||||
- **Manual Trigger**: The `/compact` command allows users to manually force a compaction cycle.
|
||||
|
||||
***
|
||||
|
||||
## Summarization Core
|
||||
|
||||
The core summarization engine in `agents/compaction.ts` orchestrates the transformation of raw history into structured summaries.
|
||||
|
||||
### Resource Allocation
|
||||
- **SUMMARIZATION_OVERHEAD_TOKENS**: 4,096 tokens are reserved for the summarization prompt, system instructions, and previous summaries.
|
||||
- **Adaptive Chunking**: `computeAdaptiveChunkRatio` shrinks chunk sizes as the average message size increases. If a single message exceeds 50% of the context window, it is flagged as `isOversizedForSummary` and handled via fallback mechanisms.
|
||||
|
||||
### Orchestration Pipeline
|
||||
1. **summarizeChunks**: Splits the history into chunks based on `maxChunkTokens`.
|
||||
2. **summarizeWithFallback**: Attempts a full summary. On failure, it separates "small" messages from "oversized" ones, summarizing the small messages and annotating the oversized ones (e.g., `[Large message (~15K tokens) omitted from summary]`).
|
||||
3. **summarizeInStages**: For very large histories, it generates partial summaries and then merges them using `MERGE_SUMMARIES_INSTRUCTIONS`.
|
||||
|
||||
### Preservation Priorities
|
||||
The system uses `MERGE_SUMMARIES_INSTRUCTIONS` to ensure the model retains:
|
||||
- Active tasks and current status (in-progress, blocked, pending).
|
||||
- Commitments, decisions, and their rationale.
|
||||
- Unresolved user asks and key factual identifiers.
|
||||
- Recent context over older history.
|
||||
|
||||
***
|
||||
|
||||
## Safeguard Extension
|
||||
|
||||
The `compaction-safeguard.ts` hook acts as a safety layer, registering on `session_before_compact` to manage context sharing and content preservation.
|
||||
|
||||
### Context Preservation
|
||||
- **Recent Turns**: Preserves `DEFAULT_RECENT_TURNS_PRESERVE = 3` turns verbatim to maintain immediate conversational flow.
|
||||
- **History Pruning**: `pruneHistoryForContextShare` drops older chunks if the new content consumes too much of the history budget. Dropped messages are summarized and prepended as a `previousSummary`.
|
||||
- **Suffix Protection**: Critical metadata is appended to a protected suffix that survives truncation:
|
||||
- Tool failures (capped at 8 failures).
|
||||
- File operations (read/modified lists).
|
||||
- Workspace rules (extracted from `AGENTS.md`).
|
||||
|
||||
### Length Constraints
|
||||
- **MAX_COMPACTION_SUMMARY_CHARS**: 16,000 characters cap for the total summary.
|
||||
- **MAX_FILE_OPS_SECTION_CHARS**: 2,000 characters for file operation logs.
|
||||
- **MAX_FILE_OPS_LIST_CHARS**: 900 characters for the list of files.
|
||||
|
||||
***
|
||||
|
||||
## Quality Guard
|
||||
|
||||
The Quality Guard (`compaction-safeguard-quality.ts`) ensures the LLM-generated summary meets strict structural and content requirements.
|
||||
|
||||
### Required Sections
|
||||
Every summary must contain these exact Markdown headings:
|
||||
- `## Decisions`
|
||||
- `## Open TODOs`
|
||||
- `## Constraints/Rules`
|
||||
- `## Pending user asks`
|
||||
- `## Exact identifiers`
|
||||
|
||||
### Identifier Preservation
|
||||
The system extracts opaque identifiers (URLs, file paths, hex IDs, ports) using regex and enforces their preservation.
|
||||
- **Strict Policy**: If `identifierPolicy` is set to `strict`, the guard validates that all extracted identifiers appear in the final summary.
|
||||
- **Audit Loop**: `auditSummaryQuality` checks for section presence and identifier integrity. If checks fail, the system triggers a regeneration with `qualityFeedbackInstruction`.
|
||||
|
||||
***
|
||||
|
||||
## Default Instructions
|
||||
|
||||
Default behavior is governed by `compaction-instructions.ts`, which merges user-defined, runtime, and system-level instructions.
|
||||
|
||||
```typescript
|
||||
export const DEFAULT_COMPACTION_INSTRUCTIONS =
|
||||
"Write the summary body in the primary language used in the conversation.\n" +
|
||||
"Focus on factual content: what was discussed, decisions made, and current state.\n" +
|
||||
"Keep the required summary structure and section headers unchanged.\n" +
|
||||
"Do not translate or alter code, file paths, identifiers, or error messages.";
|
||||
```
|
||||
|
||||
Instructions are capped at `MAX_INSTRUCTION_LENGTH = 800` characters to prevent prompt bloat.
|
||||
|
||||
***
|
||||
|
||||
## Runtime Execution & Truncation
|
||||
|
||||
The `compact.ts` runner provides the entry point for both automated and manual compaction.
|
||||
|
||||
### Execution Flow
|
||||
1. **Preparation**: Opens the session, sanitizes history, and runs `before_compaction` hooks.
|
||||
2. **Safety Timeout**: Wraps the LLM call in `compactWithSafetyTimeout` to prevent hanging processes.
|
||||
3. **Post-Processing**: Runs `after_compaction` hooks and estimates the resulting token count.
|
||||
|
||||
### Session Truncation
|
||||
If enabled via `config.agents.defaults.compaction.truncateAfterCompaction`, the system physically rewrites the session JSONL file using `session-truncation.ts`.
|
||||
- **Removal**: Deletes message entries that were summarized.
|
||||
- **Re-parenting**: Re-parents orphaned entries to the nearest kept ancestor to maintain the integrity of the session tree.
|
||||
- **Archiving**: Optionally creates an archive of the original session file before truncation.
|
||||
|
||||
***
|
||||
|
||||
## Configuration Knobs
|
||||
|
||||
Compaction behavior can be tuned via `OpenClawConfig`:
|
||||
- `config.agents.defaults.compaction.model`: Override the model used for summarization.
|
||||
- `config.agents.defaults.compaction.reserveTokensFloor`: Minimum buffer (default ~20,000).
|
||||
- `config.agents.defaults.compaction.timeoutSeconds`: Max time allowed for a summarization call.
|
||||
- `config.agents.defaults.compaction.truncateAfterCompaction`: Boolean to enable physical file cleanup.
|
||||
- `memoryFlush`: Configuration for soft thresholds and forced flush triggers.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna currently uses a basic `LibrarianAgent` for compaction with no token-budget gating, no quality guards, and no structured instruction sets. To achieve OpenClaw-level reliability, Luna should adopt:
|
||||
|
||||
- **Token-Budget Gating**: Triggering compaction based on projected context usage rather than arbitrary turn counts.
|
||||
- **Structured Sections**: Enforcing a specific Markdown schema in summaries to ensure critical state is never lost.
|
||||
- **Identifier Preservation**: Using regex extraction and quality audits to protect file paths and IDs.
|
||||
- **Quality Audit Loop**: Implementing a verification step that can re-trigger summarization if requirements are missed.
|
||||
- **Session Truncation**: Physically cleaning up on-disk history files to prevent unbounded growth.
|
||||
|
||||
Patterns like the `compaction-safeguard` provide a more resilient approach for long-term project management in Luna by ensuring the project state, goals, and critical constraints are always prioritized in the model's working memory.
|
||||
|
||||
See also: [[Core]], [[Configuration]], [[Session Management]]
|
||||
@@ -0,0 +1,161 @@
|
||||
# Memory SDK
|
||||
|
||||
OpenClaw's memory system lives in a standalone package (`packages/memory-host-sdk`) that exposes composable engines for storage, embeddings, query processing, and foundation utilities. The SDK defines a clear host-engine boundary: the host application consumes the SDK surface, while the SDK encapsulates all storage, indexing, and retrieval logic behind exported contracts.
|
||||
|
||||
***
|
||||
|
||||
## Package Structure
|
||||
|
||||
The SDK entry point (`engine.ts`) re-exports four focused engine modules:
|
||||
|
||||
```typescript
|
||||
export * from "./engine-foundation.js";
|
||||
export * from "./engine-storage.js";
|
||||
export * from "./engine-embeddings.js";
|
||||
export * from "./engine-qmd.js";
|
||||
```
|
||||
|
||||
Each engine is a self-contained barrel export covering one concern. New code is directed to the focused subpaths rather than the aggregate surface.
|
||||
|
||||
***
|
||||
|
||||
## Engine Storage
|
||||
|
||||
The storage engine (`engine-storage.ts`) provides the persistence and retrieval primitives built on SQLite with the `sqlite-vec` vector extension.
|
||||
|
||||
### Key Exports
|
||||
|
||||
- **`MemoryChunk`** and **`MemoryFileEntry`** — the core data types. A chunk is an indexed segment of content with metadata; a file entry represents a source document on disk.
|
||||
- **`buildFileEntry`** / **`buildMultimodalChunkForIndexing`** — constructors for creating index-ready records.
|
||||
- **`chunkMarkdown`** — splits markdown content into semantically coherent chunks for embedding.
|
||||
- **`cosineSimilarity`** — vector distance computation for retrieval ranking.
|
||||
- **`parseEmbedding`** — deserializes stored embedding vectors.
|
||||
- **`ensureMemoryIndexSchema`** — creates or migrates the SQLite schema (tables, FTS indexes, vector columns).
|
||||
- **`loadSqliteVecExtension`** — loads the `sqlite-vec` native extension into the SQLite connection.
|
||||
- **`requireNodeSqlite`** — resolves the Node.js SQLite driver.
|
||||
- **`readMemoryFile`** — reads a memory file from disk with safety checks.
|
||||
- **`resolveMemoryBackendConfig`** — resolves backend-specific configuration (SQLite paths, vector dimensions, etc.).
|
||||
|
||||
### Storage Architecture
|
||||
|
||||
The storage layer uses SQLite as a single-file database with the `sqlite-vec` extension providing vector column support. Memory chunks are stored alongside their embeddings in the same database, enabling hybrid queries that combine full-text search with cosine vector similarity in a single SQL statement. This mirrors the approach used by ZeroClaw's SQLite backend (see [[ZeroClaw/Memory]]).
|
||||
|
||||
```typescript
|
||||
// Types exported for consumers
|
||||
type MemoryChunk = { /* chunk content, metadata, embedding vector */ };
|
||||
type MemoryFileEntry = { /* source file path, hash, modification time */ };
|
||||
type MemorySearchResult = { /* ranked results with scores and source info */ };
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Engine Embeddings
|
||||
|
||||
The embeddings engine (`engine-embeddings.ts`) provides a pluggable embedding provider system with six built-in backends and batch processing utilities.
|
||||
|
||||
### Embedding Providers
|
||||
|
||||
| Provider | Default Model | Module |
|
||||
|----------|--------------|--------|
|
||||
| Local | `DEFAULT_LOCAL_MODEL` | `embeddings.js` |
|
||||
| Gemini | `DEFAULT_GEMINI_EMBEDDING_MODEL` | `embeddings-gemini.js` |
|
||||
| Mistral | `DEFAULT_MISTRAL_EMBEDDING_MODEL` | `embeddings-mistral.js` |
|
||||
| Ollama | `DEFAULT_OLLAMA_EMBEDDING_MODEL` | `embeddings-ollama.js` |
|
||||
| OpenAI | `DEFAULT_OPENAI_EMBEDDING_MODEL` | `embeddings-openai.js` |
|
||||
| Voyage | `DEFAULT_VOYAGE_EMBEDDING_MODEL` | `embeddings-voyage.js` |
|
||||
|
||||
### Provider Interface
|
||||
|
||||
```typescript
|
||||
// Core provider contract
|
||||
type MemoryEmbeddingProvider = {
|
||||
name: string;
|
||||
// ... embedding computation methods
|
||||
};
|
||||
|
||||
type MemoryEmbeddingProviderAdapter = {
|
||||
// Adapts raw provider to the SDK contract
|
||||
};
|
||||
|
||||
// Factory and registry
|
||||
function getMemoryEmbeddingProvider(name: string): MemoryEmbeddingProvider;
|
||||
function listMemoryEmbeddingProviders(): string[];
|
||||
```
|
||||
|
||||
### Batch Processing
|
||||
|
||||
Three batch helpers handle high-throughput embedding jobs for initial indexing or re-indexing:
|
||||
|
||||
- **`runGeminiEmbeddingBatches`** — Gemini-specific batching with rate limit handling.
|
||||
- **`runOpenAiEmbeddingBatches`** — OpenAI batch API integration (`OPENAI_BATCH_ENDPOINT`).
|
||||
- **`runVoyageEmbeddingBatches`** — Voyage AI batch processing.
|
||||
|
||||
### Safety Utilities
|
||||
|
||||
- **`enforceEmbeddingMaxInputTokens`** — truncates input to provider-specific token limits.
|
||||
- **`estimateStructuredEmbeddingInputBytes`** / **`estimateUtf8Bytes`** — byte estimation for payload sizing.
|
||||
- **`hasNonTextEmbeddingParts`** — detects multimodal content that requires special handling.
|
||||
|
||||
***
|
||||
|
||||
## Engine Foundation
|
||||
|
||||
The foundation engine (`engine-foundation.ts`) re-exports core utilities from the main OpenClaw application that the memory system depends on. Rather than duplicating logic, the SDK imports these through relative paths to the monorepo `src/` directory.
|
||||
|
||||
### Key Capabilities
|
||||
|
||||
- **Agent Scope Resolution**: `resolveAgentDir`, `resolveAgentWorkspaceDir`, `resolveDefaultAgentId`, `resolveSessionAgentId` — determines which agent's memory space to operate on.
|
||||
- **Memory Search Configuration**: `resolveMemorySearchConfig` with `ResolvedMemorySearchConfig` type — controls search behavior (result limits, relevance thresholds).
|
||||
- **Configuration Loading**: `loadConfig`, `resolveStateDir` — reads OpenClaw configuration and state directories.
|
||||
- **Session Integration**: `resolveSessionTranscriptsDirForAgent`, `onSessionTranscriptUpdate` — watches for new conversation transcripts to index into memory.
|
||||
- **File Safety**: `writeFileWithinRoot` — prevents path traversal when writing memory files.
|
||||
- **Secrets**: `hasConfiguredSecretInput`, `normalizeResolvedSecretInputString` — resolves API keys for embedding providers.
|
||||
|
||||
***
|
||||
|
||||
## Engine QMD (Query Processing)
|
||||
|
||||
The QMD engine (`engine-qmd.ts`) handles query decomposition, keyword extraction, and scope filtering before executing memory searches.
|
||||
|
||||
### Query Pipeline
|
||||
|
||||
- **`extractKeywords`** — pulls meaningful terms from a natural language query, filtering stop words via `isQueryStopWordToken`.
|
||||
- **`parseQmdQueryJson`** — parses structured query results (type `QmdQueryResult`) from the QMD binary.
|
||||
- **Scope Filtering**: `deriveQmdScopeChannel`, `deriveQmdScopeChatType`, `isQmdScopeAllowed` — restricts search results to the appropriate channel and conversation type.
|
||||
- **Session Files**: `buildSessionEntry`, `listSessionFilesForAgent`, `sessionPathForFile` — manages the session transcript files that feed into memory indexing.
|
||||
- **CLI Integration**: `checkQmdBinaryAvailability`, `resolveCliSpawnInvocation`, `runCliCommand` — interfaces with an external QMD binary for query processing.
|
||||
|
||||
***
|
||||
|
||||
## Integration Pattern
|
||||
|
||||
The SDK is consumed by the host application through the aggregate `engine.ts` export. The boundary is clear:
|
||||
|
||||
- **Host responsibility**: Decides when to index, when to search, which agent scope to use, and how to present results.
|
||||
- **SDK responsibility**: Handles all storage I/O, embedding computation, vector indexing, query processing, and schema management.
|
||||
|
||||
The typical flow:
|
||||
|
||||
1. **Initialization**: Host calls `resolveMemoryBackendConfig` to set up the storage backend.
|
||||
2. **Indexing**: Host uses `engine-embeddings` to generate vectors and `engine-storage` to persist chunks.
|
||||
3. **Retrieval**: Host uses `engine-qmd` to process the query, then performs a similarity search via `engine-storage`.
|
||||
|
||||
This separation means the memory engine can be tested independently, and the host can swap backends (e.g., different SQLite configurations, different embedding providers) without changing application logic.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna's [[Memory]] module defines an `IMemoryStore` interface but the current file-based implementation only supports writing — there is no recall or search capability.
|
||||
|
||||
### Patterns to Adopt
|
||||
|
||||
- **Composable Engine Architecture**: Separating storage, embeddings, and query processing into distinct layers is a clean pattern for Luna. Rather than building a monolithic memory service, Luna could define `IMemoryStorage`, `IEmbeddingProvider`, and `IQueryProcessor` interfaces that compose into a full memory system.
|
||||
- **SQLite + Vector Extension**: Both OpenClaw and ZeroClaw use SQLite with vector extensions for hybrid search. Luna already references SQLite packages — finishing `SqliteMemoryStore` with this architecture is the natural path.
|
||||
- **Pluggable Embedding Providers**: Luna should support multiple embedding backends from the start. The factory pattern (`getMemoryEmbeddingProvider(name)`) maps to C# service registration.
|
||||
- **Batch Processing**: For initial indexing or re-indexing of conversation history, batch embedding helpers prevent rate-limiting issues.
|
||||
- **Query Preprocessing**: Keyword extraction and scope filtering before search improves relevance. Luna should implement query expansion as a pipeline step before hitting the vector store.
|
||||
|
||||
### Key Difference
|
||||
|
||||
OpenClaw's SDK is a package boundary within a monorepo — it re-exports from the main `src/` tree through barrel files. Luna, as a single compiled application, would implement this as a set of interfaces within a `Memory` namespace rather than a separate package. The architectural principle (composable engines with clear contracts) translates directly to C# DI registration.
|
||||
@@ -0,0 +1,94 @@
|
||||
# OpenClaw
|
||||
|
||||
## Overview
|
||||
OpenClaw is a mature TypeScript/Node.js AI assistant platform (~342k stars, ~24k commits). Local-first gateway control plane with plugin-SDK architecture. Monorepo with packages. Repo: https://github.com/openclaw/openclaw
|
||||
|
||||
## Architecture
|
||||
- Local-first WebSocket gateway as central control plane
|
||||
- Plugin-SDK architecture — features are installable packages, not inline code
|
||||
- Monorepo: `src/` for core, `packages/` for SDKs
|
||||
|
||||
## Agent Runtime
|
||||
- Pi agent runtime with RPC mode
|
||||
- Tool streaming and block streaming
|
||||
- Multi-agent routing
|
||||
- Agent compaction for long conversations
|
||||
|
||||
## Providers
|
||||
- Multiple AI providers via plugin-sdk
|
||||
- OAuth subscription auth (OpenAI Codex, Claude Code)
|
||||
|
||||
## Channels
|
||||
24+ messaging platforms via plugin-per-platform pattern with shared registry/helpers:
|
||||
WhatsApp, Telegram, Slack, Discord, Signal, iMessage, Matrix, IRC, Teams, LINE, Nostr, WeChat, and more.
|
||||
Each channel plugin: channel-specific auth, message normalization, allowlist-based access control.
|
||||
|
||||
## Memory
|
||||
- memory-host-sdk package with three engines:
|
||||
- engine-embeddings (vector embedding computation/storage)
|
||||
- engine-foundation (core memory primitives)
|
||||
- engine-storage (pluggable storage backends)
|
||||
- Pluggable vector DB support for semantic search
|
||||
|
||||
## Tools
|
||||
Rich tool ecosystem:
|
||||
- Browser: Managed Chromium via CDP
|
||||
- Canvas: A2UI visual workspace
|
||||
- Cron: Scheduled tasks
|
||||
- Nodes: Device-local (camera, screen, notifications)
|
||||
- Sessions: Session management tools
|
||||
- Web Fetch: HTTP with SSRF guards and visibility rules
|
||||
- PDF, TTS, Image tools
|
||||
|
||||
## Sessions
|
||||
- JSON-persisted session store
|
||||
- Pruning, capping, rotation, archiving
|
||||
- Per-session sandbox policies
|
||||
- Activation modes, group routing
|
||||
- Session lifecycle events
|
||||
|
||||
## Security
|
||||
- DM Pairing: 6-digit codes + per-channel allowlists
|
||||
- Per-session Sandboxing: Docker/SSH backends
|
||||
- Tool Allow/Deny Lists
|
||||
- SSRF Guards on web fetch
|
||||
- Web Fetch Visibility Rules
|
||||
|
||||
## Skills & Plugins
|
||||
Full plugin lifecycle:
|
||||
- Manifest Registry with capability declarations
|
||||
- Install/enable/disable/uninstall lifecycle
|
||||
- Workspace Skills via AGENTS.md and SKILL.md conventions
|
||||
- Bundled plugins + external ClawHub package registry
|
||||
|
||||
## Voice & Apps
|
||||
- Wake word detection, Talk Mode, TTS (ElevenLabs + system fallback)
|
||||
- macOS menu bar app, iOS/Android nodes, WebChat browser client
|
||||
|
||||
***
|
||||
|
||||
## In-Depth Reference Pages
|
||||
|
||||
- [[Session Management]] — SessionEntry type, store maintenance (pruning/capping/rotation), atomic writes, lifecycle events
|
||||
- [[Plugin Architecture]] — Plugin type system, 40+ provider hooks, tool factories, config schemas, channel handlers
|
||||
- [[Channel System]] — Plugin-per-platform pattern, shared registry, message normalization, access control
|
||||
- [[Security]] — DM/group access decisions, pairing, allowlists, per-session sandboxing, SSRF guards
|
||||
- [[Memory SDK]] — Composable engine architecture (storage, embeddings, QMD), SQLite + sqlite-vec, 6 embedding providers
|
||||
- [[Compaction Strategy]] — Token-budget gating, staged summarization, quality guards, identifier preservation, session truncation
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
### Patterns Worth Adopting
|
||||
- **Session Management**: Pruning/capping/rotation is much more mature than Luna's basic SessionManager. Configurable max message counts, session lifecycle events, group routing. See [[Session Management]] for details. Link to [[Core]].
|
||||
- **Plugin Architecture**: Manifest-based registry with install/enable/disable lifecycle is a good reference for Luna's [[Skills]] module. See [[Plugin Architecture]] for the hook system.
|
||||
- **Channel Normalization**: Plugin-per-platform with shared registry validates Luna's [[Channels]] design. Key lesson: establish a rich normalized message type before adding the second channel. See [[Channel System]].
|
||||
- **Security Layers**: DM pairing and per-session sandboxing are practical patterns Luna should adopt before exposing tool execution. See [[Security]] for the access decision system.
|
||||
- **Compaction**: Token-budget driven summarization with quality guards and structured output sections. Far more sophisticated than Luna's basic LibrarianAgent compaction. See [[Compaction Strategy]].
|
||||
|
||||
### Key Differences from Luna
|
||||
- OpenClaw is a platform for third-party plugin development; Luna is a standalone assistant
|
||||
- TypeScript vs .NET/C#
|
||||
- Features are installable packages vs compiled modules
|
||||
- Mature multi-app distribution (macOS, iOS, Android, WebChat) vs CLI only
|
||||
@@ -0,0 +1,131 @@
|
||||
# Plugin Architecture
|
||||
|
||||
OpenClaw employs a multi-tiered plugin architecture designed for high extensibility across model providers, messaging channels, and agent capabilities. The system is built on a massive type system that allows plugins to hook into almost every stage of the AI lifecycle, from initial message receipt to final inference and tool execution.
|
||||
|
||||
***
|
||||
|
||||
## Plugin Type System
|
||||
|
||||
The core of the architecture is defined in `src/plugins/types.ts`. It provides a unified contract for different plugin flavors: `ProviderPlugin`, `WebSearchProviderPlugin`, `SpeechProviderPlugin`, and the general `OpenClawPluginDefinition`.
|
||||
|
||||
### Provider Plugin Hooks
|
||||
|
||||
The `ProviderPlugin` is the most complex type, featuring over 40 hooks for deep integration with LLM providers. These are categorized into distinct functional areas:
|
||||
|
||||
* **Catalog & Discovery**: Hooks like `catalog` and `augmentModelCatalog` allow plugins to publish model definitions dynamically.
|
||||
* **Auth & Credentials**: `prepareRuntimeAuth` and `resolveSyntheticAuth` handle the exchange of source credentials for runtime tokens.
|
||||
* **Model Resolution**: `resolveDynamicModel` and `normalizeResolvedModel` provide last-mile control over how model IDs are mapped to API endpoints.
|
||||
* **Streaming & Transport**: `createStreamFn` and `wrapStreamFn` allow plugins to replace or wrap the default transport layer with provider-specific logic.
|
||||
* **Usage & Billing**: `resolveUsageAuth` and `fetchUsageSnapshot` enable integrated quota tracking.
|
||||
|
||||
```typescript
|
||||
export type ProviderPlugin = {
|
||||
id: string;
|
||||
label: string;
|
||||
auth: ProviderAuthMethod[];
|
||||
catalog?: ProviderPluginCatalog;
|
||||
resolveDynamicModel?: (ctx: ProviderResolveDynamicModelContext) => ProviderRuntimeModel | null | undefined;
|
||||
prepareRuntimeAuth?: (ctx: ProviderPrepareRuntimeAuthContext) => Promise<ProviderPreparedRuntimeAuth | null | undefined>;
|
||||
createStreamFn?: (ctx: ProviderCreateStreamFnContext) => StreamFn | null | undefined;
|
||||
fetchUsageSnapshot?: (ctx: ProviderFetchUsageSnapshotContext) => Promise<ProviderUsageSnapshot | null | undefined>;
|
||||
// ... and 30+ more hooks
|
||||
};
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Tool Factories
|
||||
|
||||
Plugins expose agent-level capabilities through `OpenClawPluginToolFactory`. This pattern allows tools to be instantiated with a trusted execution context that includes workspace paths, session identifiers, and security boundaries.
|
||||
|
||||
```typescript
|
||||
export type OpenClawPluginToolContext = {
|
||||
config?: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
agentId?: string;
|
||||
sessionId?: string;
|
||||
deliveryContext?: DeliveryContext;
|
||||
senderIsOwner?: boolean;
|
||||
};
|
||||
|
||||
export type OpenClawPluginToolFactory = (
|
||||
ctx: OpenClawPluginToolContext,
|
||||
) => AnyAgentTool | AnyAgentTool[] | null | undefined;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Config Schema
|
||||
|
||||
Plugins declare their configuration requirements using a Zod-like validation system. The `OpenClawPluginConfigSchema` allows the host to validate plugin settings, generate UI forms, and provide help text without loading the plugin's full implementation.
|
||||
|
||||
```typescript
|
||||
export type OpenClawPluginConfigSchema = {
|
||||
safeParse?: (value: unknown) => {
|
||||
success: boolean;
|
||||
data?: unknown;
|
||||
error?: { issues?: Array<{ path: Array<string | number>; message: string }> };
|
||||
};
|
||||
uiHints?: Record<string, PluginConfigUiHint>;
|
||||
jsonSchema?: Record<string, unknown>;
|
||||
};
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Channel Handlers
|
||||
|
||||
Interactive platform support (Telegram, Discord, Slack) is handled via specialized interactive handlers. These allow plugins to respond to platform-specific events like button clicks or modal submissions through a unified context.
|
||||
|
||||
```typescript
|
||||
export type PluginInteractiveTelegramHandlerContext = {
|
||||
channel: "telegram";
|
||||
callback: { data: string; namespace: string; payload: string };
|
||||
respond: {
|
||||
reply: (params: { text: string; buttons?: PluginInteractiveButtons }) => Promise<void>;
|
||||
editMessage: (params: { text: string; buttons?: PluginInteractiveButtons }) => Promise<void>;
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Conversation Bindings
|
||||
|
||||
Plugins can request "Conversation Bindings" to take over the message flow for a specific thread or user. This is used for interactive wizards or stateful interactions that bypass the general LLM dispatcher.
|
||||
|
||||
* `requestConversationBinding`: Attaches a plugin to the current conversation.
|
||||
* `detachConversationBinding`: Releases the conversation back to the general agent.
|
||||
* `getCurrentConversationBinding`: Checks if the conversation is currently owned by a plugin.
|
||||
|
||||
***
|
||||
|
||||
## Plugin Lifecycle
|
||||
|
||||
The lifecycle is managed through several stages, primarily defined in the `OpenClawPluginApi` provided during registration:
|
||||
|
||||
1. **Discovery**: Plugins are found in `bundled`, `global`, or `workspace` directories.
|
||||
2. **Registration**: `register(api)` is called. The plugin registers its tools, hooks, and services.
|
||||
3. **Activation**: `activate(api)` is called when the plugin is enabled and ready for service.
|
||||
4. **Runtime**: The plugin responds to lifecycle hooks such as `before_agent_start`, `llm_input`, and `agent_end`.
|
||||
5. **Uninstallation**: Managed via the `ClawHub` registry or local file deletion.
|
||||
|
||||
***
|
||||
|
||||
## Specialized Plugin Types
|
||||
|
||||
### Web Search Providers
|
||||
`WebSearchProviderPlugin` defines how the agent interacts with search engines. It requires specific credential resolution logic and a `createTool` factory that returns a standardized search tool definition.
|
||||
|
||||
### Speech Providers
|
||||
`SpeechProviderPlugin` handles text-to-speech (TTS) capabilities. It includes hooks for voice listing, directive parsing (e.g., for custom SSML or tokens), and synthesis for both standard and telephony audio formats.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
While Luna does not require the full overhead of OpenClaw's provider ecosystem, it can adopt several key patterns:
|
||||
|
||||
* **Manifest-Based Registration**: Luna could use a simplified version of `OpenClawPluginDefinition` to allow for [[Skills]] that declare their dependencies and config schemas upfront.
|
||||
* **Hook Interceptors**: Adopting the `before_prompt_build` and `llm_output` hook patterns would allow Luna's [[Core]] to support middleware for logging, safety filtering, or memory injection.
|
||||
* **Trusted Contexts**: Passing a `ToolContext` similar to OpenClaw's would ensure that Luna's tools have consistent access to session and workspace state without relying on global variables.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Security
|
||||
|
||||
OpenClaw implements a multi-layered security model designed to isolate tool execution and strictly control access via Direct Messages (DMs) and Groups. This model ensures that only authorized users can trigger sensitive operations while protecting the host system from potentially malicious tool output or SSRF attacks.
|
||||
|
||||
***
|
||||
|
||||
## DM/Group Access Decision System
|
||||
|
||||
The core logic for determining if a message should be processed resides in `src/security/dm-policy-shared.ts`. The system evaluates the `dmPolicy` and `groupPolicy` against the sender's identity.
|
||||
|
||||
### Access Decision Types
|
||||
|
||||
The `DmGroupAccessDecision` type defines the three possible outcomes of an access check:
|
||||
|
||||
```typescript
|
||||
export type DmGroupAccessDecision = "allow" | "block" | "pairing";
|
||||
```
|
||||
|
||||
### Access Reason Codes
|
||||
|
||||
The system provides granular reason codes for every decision, enabling precise logging and user feedback:
|
||||
|
||||
```typescript
|
||||
export const DM_GROUP_ACCESS_REASON = {
|
||||
GROUP_POLICY_ALLOWED: "group_policy_allowed",
|
||||
GROUP_POLICY_DISABLED: "group_policy_disabled",
|
||||
GROUP_POLICY_EMPTY_ALLOWLIST: "group_policy_empty_allowlist",
|
||||
GROUP_POLICY_NOT_ALLOWLISTED: "group_policy_not_allowlisted",
|
||||
DM_POLICY_OPEN: "dm_policy_open",
|
||||
DM_POLICY_DISABLED: "dm_policy_disabled",
|
||||
DM_POLICY_ALLOWLISTED: "dm_policy_allowlisted",
|
||||
DM_POLICY_PAIRING_REQUIRED: "dm_policy_pairing_required",
|
||||
DM_POLICY_NOT_ALLOWLISTED: "dm_policy_not_allowlisted",
|
||||
} as const;
|
||||
```
|
||||
|
||||
### Resolution Logic
|
||||
|
||||
The `resolveDmGroupAccessDecision()` function implements the policy evaluation:
|
||||
|
||||
```typescript
|
||||
export function resolveDmGroupAccessDecision(params: {
|
||||
isGroup: boolean;
|
||||
dmPolicy?: string | null;
|
||||
groupPolicy?: string | null;
|
||||
effectiveAllowFrom: Array<string | number>;
|
||||
effectiveGroupAllowFrom: Array<string | number>;
|
||||
isSenderAllowed: (allowFrom: string[]) => boolean;
|
||||
}) {
|
||||
const dmPolicy = params.dmPolicy ?? "pairing";
|
||||
const groupPolicy = params.groupPolicy ?? "allowlist";
|
||||
|
||||
if (params.isGroup) {
|
||||
// Group logic: evaluate against groupPolicy (open/disabled/allowlist)
|
||||
// Returns 'allow' if policy is open or user is in allowlist
|
||||
// Returns 'block' if policy is disabled or user not in allowlist
|
||||
} else {
|
||||
// DM logic: evaluate against dmPolicy
|
||||
if (dmPolicy === "disabled") return { decision: "block", ... };
|
||||
if (dmPolicy === "open") return { decision: "allow", ... };
|
||||
|
||||
if (params.isSenderAllowed(effectiveAllowFrom)) {
|
||||
return { decision: "allow", reasonCode: "dm_policy_allowlisted" };
|
||||
}
|
||||
|
||||
if (dmPolicy === "pairing") {
|
||||
return { decision: "pairing", reasonCode: "dm_policy_pairing_required" };
|
||||
}
|
||||
|
||||
return { decision: "block", reasonCode: "dm_policy_not_allowlisted" };
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## DM Pairing
|
||||
|
||||
The Pairing system allows users to prove their identity when `dmPolicy` is set to `pairing`.
|
||||
|
||||
1. **Initiation**: When an unknown user messages the bot, the system returns a `pairing` decision.
|
||||
2. **Challenge**: The bot provides a 6-digit pairing code (often via console or a side-channel).
|
||||
3. **Verification**: Once the user provides the correct code, their unique account ID is added to the **Store-backed Allowlist**.
|
||||
4. **Persistence**: Subsequent messages from this account ID are automatically allowed as they now appear in the resolved allowlist.
|
||||
|
||||
***
|
||||
|
||||
## Access Policies
|
||||
|
||||
Policies are configured per-channel and define the default security posture.
|
||||
|
||||
### dmPolicy Options
|
||||
- `disabled`: The bot will not respond to any DMs.
|
||||
- `open`: Anyone can message the bot (high risk).
|
||||
- `pairing`: Users must undergo the 6-digit pairing flow to gain access.
|
||||
- `allowlist`: Only users explicitly listed in the configuration file can message the bot.
|
||||
|
||||
### groupPolicy Options
|
||||
- `open`: Any user in the group can trigger the bot.
|
||||
- `disabled`: The bot is effectively silent in group settings.
|
||||
- `allowlist`: Only specific users within the group can trigger commands.
|
||||
|
||||
***
|
||||
|
||||
## Allowlist Management
|
||||
|
||||
OpenClaw uses a hybrid allowlist system:
|
||||
- **Config Allowlist**: Static entries defined in the `config.yaml` or environment variables.
|
||||
- **Store Allowlist**: Dynamic entries stored in a local database (e.g., `pairing-store.js`), primarily populated by successful pairings.
|
||||
|
||||
The `resolveEffectiveAllowFromLists()` function merges these sources, ensuring that per-channel scoping is respected. Wildcards (`*`) can be used in the config to grant broad access, though this is discouraged for production environments.
|
||||
|
||||
***
|
||||
|
||||
## Per-Session Sandboxing
|
||||
|
||||
To prevent tool execution from compromising the host, OpenClaw supports isolated backends:
|
||||
- **Docker Backend**: Each session spawns a transient container. Tools execute inside this container with limited CPU, memory, and no access to the host filesystem.
|
||||
- **SSH Backend**: Tools are executed on a remote machine or VM, isolating the main bot process from the execution environment.
|
||||
|
||||
***
|
||||
|
||||
## Tool Allow/Deny Lists
|
||||
|
||||
Security is further tightened by restricting which tools are available to which users or channels.
|
||||
- **Global Denylist**: Prevents high-risk tools (like `shell_execute`) from being loaded.
|
||||
- **Per-User Permissions**: Can restrict specific tools to "Admin" users only.
|
||||
|
||||
***
|
||||
|
||||
## SSRF Guards
|
||||
|
||||
The `web_fetch` tool and other network-touching skills include SSRF (Server-Side Request Forgery) protection. This includes:
|
||||
- **IP Blocklists**: Preventing requests to `localhost`, `127.0.0.1`, and internal private IP ranges (10.0.0.0/8, etc.).
|
||||
- **Protocol Restriction**: Only allowing `http` and `https`.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna currently lacks a security layer, which is a critical blocker for exposing tool execution to external interfaces.
|
||||
|
||||
### Priority Patterns to Adopt:
|
||||
1. **DM Pairing**: Implement the 6-digit pairing flow before allowing any tool interaction in DMs.
|
||||
2. **Access Decision Logic**: Port the `resolveDmGroupAccessDecision` pattern to [[Channels]] to centralize authorization.
|
||||
3. **SSRF Guards**: Ensure any "web search" or "fetch" skills implemented in [[Skills]] cannot hit internal Luna metadata services.
|
||||
4. **Docker Isolation**: Before enabling `shell_execute`, Luna must implement the Docker-based sandboxing seen in OpenClaw.
|
||||
|
||||
***
|
||||
|
||||
Cross-refs: [[Core]], [[Skills]], [[Channels]]
|
||||
@@ -0,0 +1,144 @@
|
||||
# Session Management
|
||||
|
||||
OpenClaw employs a centralized, file-based session management system that handles persistence, lifecycle events, and automated store maintenance. Unlike simple memory-only stores, this system is designed for high-concurrency environments with atomic write guarantees and strict disk budget enforcement.
|
||||
|
||||
***
|
||||
|
||||
## SessionEntry type
|
||||
|
||||
The `SessionEntry` structure in `src/config/sessions/types.ts` is the core data model, containing nearly 80 fields that track everything from model overrides to granular token usage and skill snapshots.
|
||||
|
||||
```typescript
|
||||
export type SessionEntry = {
|
||||
sessionId: string;
|
||||
updatedAt: number;
|
||||
sessionFile?: string;
|
||||
|
||||
// Model and Provider Overrides
|
||||
modelProvider?: string;
|
||||
model?: string;
|
||||
providerOverride?: string;
|
||||
modelOverride?: string;
|
||||
thinkingLevel?: string;
|
||||
|
||||
// Token Tracking and Context
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
totalTokensFresh?: boolean;
|
||||
compactionCount?: number;
|
||||
|
||||
// Delivery and Origin
|
||||
deliveryContext?: DeliveryContext;
|
||||
lastChannel?: SessionChannelId;
|
||||
origin?: SessionOrigin;
|
||||
|
||||
// Meta and Snapshots
|
||||
skillsSnapshot?: SessionSkillSnapshot;
|
||||
systemPromptReport?: SessionSystemPromptReport;
|
||||
acp?: SessionAcpMeta; // Agent Control Plane metadata
|
||||
queueMode?: "steer" | "followup" | "collect" | "queue" | "interrupt";
|
||||
};
|
||||
```
|
||||
|
||||
Key features of the type system:
|
||||
* **Model Overrides**: Allows per-session binding to specific models or providers, bypassing global defaults.
|
||||
* **Token Intelligence**: Tracks `totalTokensFresh` to determine if usage displays need a refresh.
|
||||
* **ACP Meta**: Preserves state for the Agent Control Plane, ensuring persistent agent behaviors across turns.
|
||||
* **Skill Snapshots**: Captures the state of available [[Skills]] at the time of the session turn.
|
||||
|
||||
***
|
||||
|
||||
## Store Maintenance
|
||||
|
||||
Automated maintenance is handled in `src/config/sessions/store-maintenance.ts` to prevent unbounded growth of the session file and ensure performance.
|
||||
|
||||
### Pruning and Capping
|
||||
* **`pruneStaleEntries()`**: Removes sessions older than a configured threshold (default: 30 days). It iterates through the store and deletes entries where `updatedAt` is before the cutoff.
|
||||
* **`capEntryCount()`**: Enforces a maximum number of sessions (default: 500). It sorts sessions by `updatedAt` descending and keeps only the most recent $N$ entries.
|
||||
|
||||
### File Rotation
|
||||
The `rotateSessionFile()` function monitors the `sessions.json` file size. If it exceeds the limit (default: 10MB), the system:
|
||||
1. Renames the current file to `sessions.json.bak.{timestamp}`.
|
||||
2. Maintains a maximum of 3 rotation backups, unlinking older ones.
|
||||
|
||||
### Maintenance Configuration
|
||||
The `resolveMaintenanceConfig()` function aggregates parameters from [[Configuration]]:
|
||||
* `pruneAfterMs`: Max age before eviction.
|
||||
* `maxEntries`: Hard limit on session count.
|
||||
* `rotateBytes`: File size trigger for rotation.
|
||||
* `maxDiskBytes`: Total disk budget for session transcripts.
|
||||
|
||||
***
|
||||
|
||||
## Session Store
|
||||
|
||||
The `SessionStore` in `src/config/sessions/store.ts` manages the I/O layer with a focus on reliability and cross-platform compatibility.
|
||||
|
||||
### Key Features
|
||||
* **Atomic Writes**: Uses `writeTextAtomic` to prevent file corruption during crashes.
|
||||
* **Write Locks**: Implements a `withSessionStoreLock` mechanism using a file-based lock to prevent race conditions between concurrent agent turns.
|
||||
* **Normalization**: Session keys are case-insensitive and trimmed via `normalizeStoreSessionKey()`. Legacy keys are automatically migrated to the normalized format on load.
|
||||
* **Windows Retry Semantics**: Includes specialized retry logic for Windows to handle transient file locks during the "truncate and write" phase.
|
||||
* **ACP Metadata Preservation**: Specifically protects `acp` metadata during updates, ensuring agent state isn't lost if a partial patch is applied.
|
||||
|
||||
```typescript
|
||||
export async function updateSessionStore<T>(
|
||||
storePath: string,
|
||||
mutator: (store: Record<string, SessionEntry>) => Promise<T> | T,
|
||||
opts?: SaveSessionStoreOptions,
|
||||
): Promise<T> {
|
||||
return await withSessionStoreLock(storePath, async () => {
|
||||
const store = loadSessionStore(storePath, { skipCache: true });
|
||||
const previousAcpByKey = collectAcpMetadataSnapshot(store);
|
||||
const result = await mutator(store);
|
||||
preserveExistingAcpMetadata({
|
||||
previousAcpByKey,
|
||||
nextStore: store,
|
||||
});
|
||||
await saveSessionStoreUnlocked(storePath, store, opts);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Session Lifecycle Events
|
||||
|
||||
A lightweight pub/sub system in `src/sessions/session-lifecycle-events.ts` allows other subsystems to react to session changes.
|
||||
|
||||
```typescript
|
||||
export type SessionLifecycleEvent = {
|
||||
sessionKey: string;
|
||||
reason: string;
|
||||
parentSessionKey?: string;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export function onSessionLifecycleEvent(listener: SessionLifecycleListener): () => void {
|
||||
SESSION_LIFECYCLE_LISTENERS.add(listener);
|
||||
return () => { SESSION_LIFECYCLE_LISTENERS.delete(listener); };
|
||||
}
|
||||
|
||||
export function emitSessionLifecycleEvent(event: SessionLifecycleEvent): void {
|
||||
for (const listener of SESSION_LIFECYCLE_LISTENERS) {
|
||||
listener(event);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This system is used to trigger [[Memory]] indexing, update [[Channels]] status, or log audit trails when sessions are created or deleted.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna's current `SessionManager` is a basic dictionary-backed store that lacks persistence and maintenance. To reach parity with OpenClaw, Luna should adopt several patterns:
|
||||
|
||||
1. **Atomic Persistence**: Implement a background saver for [[Core]] sessions that uses atomic file swaps to prevent data loss.
|
||||
2. **Maintenance Tasks**: Add a background service to prune old sessions and cap the total count to prevent memory leaks in the .NET runtime.
|
||||
3. **Locking**: Use a `SemaphoreSlim` or file-system lock for session updates to support concurrent requests from different [[Channels]].
|
||||
4. **Detailed Metadata**: Expand Luna's session objects to include token tracking and model overrides, allowing for better cost management and user customization.
|
||||
|
||||
Refer to [[Core]] for the current implementation status of Luna's session handling.
|
||||
@@ -0,0 +1,152 @@
|
||||
# Agent Loop & Tools
|
||||
|
||||
The ZeroClaw agent architecture is built around a recursive execution loop that allows an LLM to invoke external capabilities, observe their results, and refine its response until a final answer is reached.
|
||||
|
||||
***
|
||||
|
||||
## Tool Trait
|
||||
|
||||
The `Tool` trait in `tools/traits.rs` defines the interface for any capability exposed to the agent. It provides both the metadata needed for LLM registration and the execution logic for the tool itself.
|
||||
|
||||
```rust
|
||||
/// Result of a tool execution
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ToolResult {
|
||||
pub success: bool,
|
||||
pub output: String,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Description of a tool for the LLM
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ToolSpec {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub parameters: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Core tool trait — implement for any capability
|
||||
#[async_trait]
|
||||
pub trait Tool: Send + Sync {
|
||||
/// Tool name (used in LLM function calling)
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Human-readable description
|
||||
fn description(&self) -> &str;
|
||||
|
||||
/// JSON schema for parameters
|
||||
fn parameters_schema(&self) -> serde_json::Value;
|
||||
|
||||
/// Execute the tool with given arguments
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<ToolResult>;
|
||||
|
||||
/// Get the full spec for LLM registration
|
||||
fn spec(&self) -> ToolSpec {
|
||||
ToolSpec {
|
||||
name: self.name().to_string(),
|
||||
description: self.description().to_string(),
|
||||
parameters: self.parameters_schema(),
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Agent Loop
|
||||
|
||||
The core logic resides in `agent/loop_.rs` within `run_tool_call_loop`. This function orchestrates the interaction between the provider and the tool registry.
|
||||
|
||||
The process follows a repeating cycle:
|
||||
1. Send the current conversation history and tool definitions to the LLM.
|
||||
2. Receive a response.
|
||||
3. Parse the response for tool calls (native or prompt-guided).
|
||||
4. If tool calls exist, execute them and append the results to the history, then repeat.
|
||||
5. If no tool calls exist, return the final text response.
|
||||
|
||||
### Iteration Safety
|
||||
To prevent infinite loops or runaway execution, the system enforces a strict iteration cap:
|
||||
* `DEFAULT_MAX_TOOL_ITERATIONS` = 10
|
||||
|
||||
### Streaming and Progress
|
||||
Streaming responses are handled by accumulating chunks before relaying them to the draft channel to minimize noise:
|
||||
* `STREAM_CHUNK_MIN_CHARS` = 80
|
||||
* `PROGRESS_MIN_INTERVAL_MS` = 500 (minimum time between progress updates)
|
||||
* `DRAFT_CLEAR_SENTINEL` = `\x00CLEAR\x00` (used to clear progress lines before showing the final answer)
|
||||
|
||||
***
|
||||
|
||||
## Auto-Compaction
|
||||
|
||||
Conversation history is managed through an auto-compaction mechanism to stay within context window limits while preserving essential information.
|
||||
|
||||
* **Threshold**: Triggered when non-system message count exceeds 50 (`DEFAULT_MAX_HISTORY_MESSAGES`).
|
||||
* **Retention**: Keeps the 20 most recent messages (`COMPACTION_KEEP_RECENT_MESSAGES`).
|
||||
* **Summarization**: Older messages are summarized by the LLM.
|
||||
* **Caps**: Summarization source is capped at 12,000 characters; the resulting summary is capped at 2,000 characters.
|
||||
|
||||
***
|
||||
|
||||
## Security & Pattern Matching
|
||||
|
||||
### Credential Scrubbing
|
||||
ZeroClaw proactively redacts sensitive information from tool outputs before they are fed back into the LLM history.
|
||||
|
||||
```rust
|
||||
static SENSITIVE_KV_REGEX: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r#"(?i)(token|api[_-]?key|password|secret|user[_-]?key|bearer|credential)["']?\s*[:=]\s*(?:"([^"]{8,})"|'([^']{8,})'|([a-zA-Z0-9_\-\.]{8,}))"#).unwrap()
|
||||
});
|
||||
```
|
||||
The scrubbing logic preserves the first 4 characters for context then appends `*[REDACTED]`.
|
||||
|
||||
### Autosave
|
||||
Meaningful exchanges are automatically persisted to memory based on message length:
|
||||
* `AUTOSAVE_MIN_MESSAGE_CHARS` = 20
|
||||
|
||||
***
|
||||
|
||||
## Tool Serialization & Formats
|
||||
|
||||
Tools are converted into the appropriate format for the specific LLM provider.
|
||||
|
||||
### tools_to_openai_format()
|
||||
This function serializes the tool registry into the standard OpenAI function-calling schema used by many providers.
|
||||
|
||||
```rust
|
||||
fn tools_to_openai_format(tools_registry: &[Box<dyn Tool>]) -> Vec<serde_json::Value> {
|
||||
tools_registry
|
||||
.iter()
|
||||
.map(|tool| {
|
||||
serde_json::json!({
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool.name(),
|
||||
"description": tool.description(),
|
||||
"parameters": tool.parameters_schema()
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
```
|
||||
|
||||
### Parsing Priority
|
||||
The system handles multiple response formats to support different providers:
|
||||
1. Native structured JSON (OpenAI style)
|
||||
2. XML tags (`<tool_call>`, `<invoke>`)
|
||||
3. Markdown code blocks with `tool_call` identifiers
|
||||
4. Provider-specific shortened formats (GLM, etc.)
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna currently operates as a pure chat relay, streaming LLM responses without any capability to interact with the system or external APIs. This lack of a tool execution layer is the primary bottleneck preventing Luna from becoming an autonomous assistant.
|
||||
|
||||
### Bridging the Gap
|
||||
Implementing the agent loop pattern in Luna would involve:
|
||||
* Defining an `ITool` interface in C# that mirrors the Rust `Tool` trait.
|
||||
* Transitioning from a single request-response model to a recursive loop in the [[Core]] engine.
|
||||
* Integrating the tool loop with [[Skills]] to allow dynamic capability discovery.
|
||||
|
||||
The implementation of this protocol will turn Luna from a passive interface into an active participant capable of executing commands and managing its own [[Memory]].
|
||||
@@ -0,0 +1,132 @@
|
||||
# Channel Messages
|
||||
|
||||
The ZeroClaw messaging protocol is built around the `Channel` trait and a unified message format. This allows the system to treat disparate platforms—from Discord to Slack to SMS—as a consistent stream of `ChannelMessage` objects.
|
||||
|
||||
***
|
||||
|
||||
## Channel Trait
|
||||
|
||||
The `Channel` trait is the core abstraction for all platform implementations. It defines how the system interacts with a specific messaging service.
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait Channel: Send + Sync {
|
||||
/// Human-readable channel name
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Send a message through this channel
|
||||
async fn send(&self, message: &SendMessage) -> anyhow::Result<()>;
|
||||
|
||||
/// Start listening for incoming messages (long-running)
|
||||
async fn listen(&self, tx: tokio::sync::mpsc::Sender<ChannelMessage>) -> anyhow::Result<()>;
|
||||
|
||||
/// Check if channel is healthy
|
||||
async fn health_check(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Signal that the bot is processing a response (e.g. "typing" indicator).
|
||||
async fn start_typing(&self, _recipient: &str) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop any active typing indicator.
|
||||
async fn stop_typing(&self, _recipient: &str) -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ... draft and reaction methods
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Draft Update Protocol
|
||||
|
||||
ZeroClaw implements a progressive "Draft" protocol designed for streaming LLM responses. Instead of sending multiple message fragments, platforms that support editing (Telegram, Discord, Slack) can update a single message in place as the response is generated.
|
||||
|
||||
### Protocol Methods
|
||||
|
||||
* `supports_draft_updates()`: Returns true if the platform allows message editing.
|
||||
* `send_draft(&SendMessage)`: Sends the initial message and returns a platform-specific `message_id`.
|
||||
* `update_draft(recipient, message_id, text)`: Appends or replaces the content of the existing message.
|
||||
* `finalize_draft(recipient, message_id, text)`: Performs a final update, often used to apply markdown formatting or remove "typing" statuses.
|
||||
* `cancel_draft(recipient, message_id)`: Deletes the draft if the generation is aborted.
|
||||
|
||||
This protocol significantly reduces notification noise on user devices and provides a much smoother "typing" experience during long generations.
|
||||
|
||||
***
|
||||
|
||||
## Reactions and Pinning
|
||||
|
||||
ZeroClaw supports standard interactive elements across most platforms.
|
||||
|
||||
### Reactions
|
||||
* `add_reaction(channel_id, message_id, emoji)`: Adds a Unicode emoji reaction.
|
||||
* `remove_reaction(channel_id, message_id, emoji)`: Removes a previously added reaction.
|
||||
|
||||
### Pinning
|
||||
* `pin_message(channel_id, message_id)`: Pins a message to the channel.
|
||||
* `unpin_message(channel_id, message_id)`: Unpins a message.
|
||||
|
||||
***
|
||||
|
||||
## ChannelMessage Struct
|
||||
|
||||
The `ChannelMessage` is the Data Transfer Object (DTO) for all incoming and outgoing communication.
|
||||
|
||||
```rust
|
||||
pub struct ChannelMessage {
|
||||
pub id: String,
|
||||
pub sender: String,
|
||||
pub reply_target: String,
|
||||
pub content: String,
|
||||
pub channel: String,
|
||||
pub timestamp: u64,
|
||||
/// Platform thread identifier (e.g. Slack `ts`, Discord thread ID).
|
||||
/// When set, replies should be posted as threaded responses.
|
||||
pub thread_ts: Option<String>,
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## SendMessage Builder
|
||||
|
||||
Sending messages uses a builder pattern to handle optional fields like subjects and threading context.
|
||||
|
||||
```rust
|
||||
pub struct SendMessage {
|
||||
pub content: String,
|
||||
pub recipient: String,
|
||||
pub subject: Option<String>,
|
||||
pub thread_ts: Option<String>,
|
||||
}
|
||||
|
||||
impl SendMessage {
|
||||
pub fn new(content: impl Into<String>, recipient: impl Into<String>) -> Self;
|
||||
pub fn with_subject(content: impl Into<String>, recipient: impl Into<String>, subject: impl Into<String>) -> Self;
|
||||
pub fn in_thread(mut self, thread_ts: Option<String>) -> Self;
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Platform Implementations
|
||||
|
||||
ZeroClaw includes implementations for 26 platforms, including:
|
||||
* Slack, Discord, Telegram, Microsoft Teams
|
||||
* WhatsApp (Twilio/Meta), Signal, Matrix
|
||||
* Twilio SMS, SendGrid Email, Postmark
|
||||
* IRC, XMPP, Mattermost, Rocket.Chat
|
||||
* Custom Webhooks and WebSocket adapters
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna should adopt the `ChannelMessage` structure as its baseline message DTO in [[Core]].
|
||||
|
||||
The **Draft Update Protocol** is particularly critical for Luna's SignalR implementation. Rather than streaming raw text chunks to the frontend and letting the client manage the state, the SignalR hub can follow the `send_draft` / `update_draft` / `finalize_draft` flow. This ensures consistency between web clients and external messaging channels documented in [[Channels]].
|
||||
|
||||
The `SendMessage` builder pattern provides a clean API for Luna services to dispatch notifications without manually constructing complex JSON payloads.
|
||||
@@ -0,0 +1,213 @@
|
||||
# Compaction Strategy
|
||||
|
||||
ZeroClaw employs a dual-path conversation compaction system to manage long-running sessions. This strategy ensures the LLM maintains relevant context while staying within performance and token limits. Unlike basic history trimming, ZeroClaw uses proactive summarization to preserve state and reactive truncation to recover from context overflows.
|
||||
|
||||
***
|
||||
|
||||
## Constants
|
||||
|
||||
The following constants are defined in `agent/loop_.rs` and govern the default behavior of the compaction engine:
|
||||
|
||||
```rust
|
||||
/// Default trigger for auto-compaction when non-system message count exceeds this threshold.
|
||||
const DEFAULT_MAX_HISTORY_MESSAGES: usize = 50;
|
||||
|
||||
/// Keep this many most-recent non-system messages after compaction.
|
||||
const COMPACTION_KEEP_RECENT_MESSAGES: usize = 20;
|
||||
|
||||
/// Safety cap for compaction source transcript passed to the summarizer.
|
||||
const COMPACTION_MAX_SOURCE_CHARS: usize = 12_000;
|
||||
|
||||
/// Max characters retained in stored compaction summary.
|
||||
const COMPACTION_MAX_SUMMARY_CHARS: usize = 2_000;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Two Compaction Triggers
|
||||
|
||||
ZeroClaw implements both agent-side proactive compaction and channel-side reactive recovery.
|
||||
|
||||
### 1. Proactive (Agent-Side)
|
||||
|
||||
The `auto_compact_history()` function in `agent/loop_.rs` is called after each turn in the interactive agent loop. It operates based on message count thresholds rather than token counts.
|
||||
|
||||
**Process:**
|
||||
1. **Threshold Check**: It counts non-system messages. If the count is less than or equal to `max_history` (default 50), it exits.
|
||||
2. **Preservation**: It identifies the system prompt (the first message if the role is "system") to ensure it is never compacted.
|
||||
3. **Range Calculation**: It determines how many messages to keep (`COMPACTION_KEEP_RECENT_MESSAGES`) and how many to compact.
|
||||
4. **Transcript Generation**: Older messages are formatted into a plain-text transcript via `build_compaction_transcript()`.
|
||||
5. **LLM Summarization**: The transcript is sent to a summarizer model with a temperature of 0.2.
|
||||
6. **Injection**: The resulting summary is inserted as an assistant message prefixed with `[Compaction summary]`.
|
||||
7. **Safety Fallback**: If summarization fails, it falls back to deterministic truncation via `truncate_with_ellipsis()`.
|
||||
8. **Hard Trim**: `trim_history()` runs afterward to ensure the history length is strictly enforced.
|
||||
|
||||
### 2. Reactive (Channel-Side)
|
||||
|
||||
The `compact_sender_history()` function in `channels/mod.rs` provides a fallback mechanism when the LLM returns a context-window overflow error.
|
||||
|
||||
**Process:**
|
||||
1. **Error Detection**: `is_context_window_overflow_error()` checks for 8 keyword patterns in the error string:
|
||||
- "exceeds the context window"
|
||||
- "context window of this model"
|
||||
- "maximum context length"
|
||||
- "context length exceeded"
|
||||
- "too many tokens"
|
||||
- "token limit exceeded"
|
||||
- "prompt is too long"
|
||||
- "input is too long"
|
||||
2. **Aggressive Truncation**: It keeps only the last `CHANNEL_HISTORY_COMPACT_KEEP_MESSAGES` messages.
|
||||
3. **Content Cap**: Each retained message is truncated to `CHANNEL_HISTORY_COMPACT_CONTENT_CHARS`.
|
||||
4. **User Recovery**: The system typically asks the user to resend their last message after the history has been thinned.
|
||||
|
||||
***
|
||||
|
||||
## Summarization Prompt
|
||||
|
||||
The summarizer uses a specific system and user prompt pair to ensure the output is useful for future context.
|
||||
|
||||
**System Prompt:**
|
||||
> You are a conversation compaction engine. Summarize older chat history into concise context for future turns. Preserve: user preferences, commitments, decisions, unresolved tasks, key facts. Omit: filler, repeated chit-chat, verbose tool logs. Output plain text bullet points only.
|
||||
|
||||
**User Prompt:**
|
||||
> Summarize the following conversation history for context preservation. Keep it short (max 12 bullet points).
|
||||
>
|
||||
> {transcript}
|
||||
|
||||
**Temperature:** 0.2
|
||||
|
||||
***
|
||||
|
||||
## Helper Functions
|
||||
|
||||
### trim_history()
|
||||
|
||||
Located in `agent/loop_.rs`, this function performs a hard drain on the oldest non-system messages.
|
||||
|
||||
```rust
|
||||
fn trim_history(history: &mut Vec<ChatMessage>, max_history: usize) {
|
||||
// Nothing to trim if within limit
|
||||
let has_system = history.first().map_or(false, |m| m.role == "system");
|
||||
let non_system_count = if has_system {
|
||||
history.len() - 1
|
||||
} else {
|
||||
history.len()
|
||||
};
|
||||
|
||||
if non_system_count <= max_history {
|
||||
return;
|
||||
}
|
||||
|
||||
let start = if has_system { 1 } else { 0 };
|
||||
let to_remove = non_system_count - max_history;
|
||||
history.drain(start..start + to_remove);
|
||||
}
|
||||
```
|
||||
|
||||
### build_compaction_transcript()
|
||||
|
||||
Formats the range of messages targeted for compaction into a single string for the summarizer.
|
||||
|
||||
```rust
|
||||
fn build_compaction_transcript(messages: &[ChatMessage]) -> String {
|
||||
let mut transcript = String::new();
|
||||
for msg in messages {
|
||||
let role = msg.role.to_uppercase();
|
||||
let _ = writeln!(transcript, "{role}: {}", msg.content.trim());
|
||||
}
|
||||
|
||||
if transcript.chars().count() > COMPACTION_MAX_SOURCE_CHARS {
|
||||
truncate_with_ellipsis(&transcript, COMPACTION_MAX_SOURCE_CHARS)
|
||||
} else {
|
||||
transcript
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### apply_compaction_summary()
|
||||
|
||||
Splices the generated summary into the conversation history, replacing the original messages.
|
||||
|
||||
```rust
|
||||
fn apply_compaction_summary(
|
||||
history: &mut Vec<ChatMessage>,
|
||||
start: usize,
|
||||
compact_end: usize,
|
||||
summary: &str,
|
||||
) {
|
||||
let summary_msg = ChatMessage::assistant(format!("[Compaction summary]\n{}", summary.trim()));
|
||||
history.splice(start..compact_end, std::iter::once(summary_msg));
|
||||
}
|
||||
```
|
||||
|
||||
### Agent::trim_history()
|
||||
|
||||
The high-level implementation in `agent/agent.rs` handles the separation and reassembly of system messages during a trim.
|
||||
|
||||
```rust
|
||||
fn trim_history(&mut self) {
|
||||
let max = self.config.max_history_messages;
|
||||
if self.history.len() <= max {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut system_messages = Vec::new();
|
||||
let mut other_messages = Vec::new();
|
||||
|
||||
for msg in self.history.drain(..) {
|
||||
match &msg {
|
||||
ConversationMessage::Chat(chat) if chat.role == "system" => {
|
||||
system_messages.push(msg);
|
||||
}
|
||||
_ => other_messages.push(msg),
|
||||
}
|
||||
}
|
||||
|
||||
if other_messages.len() > max {
|
||||
let drop_count = other_messages.len() - max;
|
||||
other_messages.drain(0..drop_count);
|
||||
}
|
||||
|
||||
self.history = system_messages;
|
||||
self.history.extend(other_messages);
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Injection Format
|
||||
|
||||
When compaction occurs, the summary is injected back into the history as a single assistant message. This ensures that the model sees the previous context as its own earlier summary of events.
|
||||
|
||||
**Format:**
|
||||
```text
|
||||
assistant: [Compaction summary]
|
||||
- Bullet point 1
|
||||
- Bullet point 2
|
||||
...
|
||||
```
|
||||
|
||||
The summary replaces the exact range of messages that were sent to the summarizer, maintaining chronological integrity between the system prompt and the recent "live" messages.
|
||||
|
||||
***
|
||||
|
||||
## Configuration
|
||||
|
||||
The compaction behavior is influenced by the following configuration settings:
|
||||
|
||||
* `config.agent.max_history_messages`: The primary threshold (default 50).
|
||||
* `compact_context`: A boolean flag to enable or disable the summarization logic.
|
||||
* Note: Internal constants like `COMPACTION_KEEP_RECENT_MESSAGES` are currently hardcoded in `loop_.rs`.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
The ZeroClaw compaction strategy provides a significant upgrade over the basic LibrarianAgent compaction currently used in Luna. Implementing this approach in Luna would address several limitations:
|
||||
|
||||
* **Dual-Path Robustness**: Combining proactive message-count triggers with reactive overflow recovery ensures the agent never hits an unrecoverable "context too long" state.
|
||||
* **Structured Context**: Using a specific summarization prompt preserves critical state (decisions, preferences) that is often lost in simple truncation.
|
||||
* **Identifiable Injection**: The `[Compaction summary]` prefix allows the model (and debugging tools) to distinguish between raw history and summarized context.
|
||||
* **Threshold Management**: Moving toward the ZeroClaw constants would provide more predictable behavior in high-volume tool-call loops.
|
||||
|
||||
The summarization prompt text and the `apply_compaction_summary` pattern are directly compatible with Luna's [[Core]] architecture.
|
||||
@@ -0,0 +1,137 @@
|
||||
# Configuration
|
||||
|
||||
ZeroClaw utilizes a TOML-based configuration system designed for modularity, security, and extensibility. The configuration is primarily managed through `config.toml`, which is loaded from the workspace directory or a fallback legacy location.
|
||||
|
||||
***
|
||||
|
||||
## Config Struct
|
||||
|
||||
The core of the configuration system is defined in `config/schema.rs` via the `Config` struct. It contains over 30 sections, each governing a specific subsystem.
|
||||
|
||||
### Global Settings
|
||||
* `workspace_dir`: The root directory for ZeroClaw data (computed at runtime).
|
||||
* `config_path`: The path to the active `config.toml`.
|
||||
* `api_key`: Global API key for the default provider.
|
||||
* `api_url`: Base URL override for provider APIs.
|
||||
* `default_provider`: The primary provider ID (e.g., "anthropic", "ollama").
|
||||
* `default_model`: The default model used for queries.
|
||||
* `default_temperature`: Model temperature (0.0 to 2.0, default 0.7).
|
||||
|
||||
### Subsystem Sections
|
||||
* `observability`: Tracing and metrics configuration.
|
||||
* `autonomy`: Security policies and autonomy levels.
|
||||
* `security`: Secret management and sandbox settings.
|
||||
* `runtime`: Native vs Docker execution modes.
|
||||
* `reliability`: Retry logic and fallback providers.
|
||||
* `scheduler`: Periodic task execution settings.
|
||||
* `agent`: Orchestration parameters (history size, tool iterations).
|
||||
* `skills`: Skill loading and prompt injection modes.
|
||||
* `model_routes`: Routing specific hints to provider/model pairs.
|
||||
* `embedding_routes`: Routing for vector embedding models.
|
||||
* `channels_config`: Configuration for Telegram, Discord, Slack, etc.
|
||||
* `memory`: Backend settings for SQLite or vector storage.
|
||||
* `storage`: Persistent file storage provider configuration.
|
||||
* `secrets`: Encryption settings for credentials.
|
||||
* `browser`: Browser automation and "computer-use" sidecar.
|
||||
* `identity`: AIEOS or OpenClaw format identity documents.
|
||||
* `cost`: Budget enforcement and price tracking.
|
||||
* `hardware`: Physical world interaction (serial/probe) settings.
|
||||
* `hooks`: Lifecycle and built-in hook toggles.
|
||||
|
||||
***
|
||||
|
||||
## ModelProviderConfig
|
||||
|
||||
ZeroClaw supports multiple provider profiles via the `model_providers` HashMap.
|
||||
|
||||
```rust
|
||||
pub struct ModelProviderConfig {
|
||||
pub name: Option<String>,
|
||||
pub base_url: Option<String>,
|
||||
pub wire_api: Option<String>,
|
||||
pub requires_openai_auth: bool,
|
||||
pub azure_openai_resource: Option<String>,
|
||||
pub azure_openai_deployment: Option<String>,
|
||||
pub azure_openai_api_version: Option<String>,
|
||||
}
|
||||
```
|
||||
|
||||
This allows for configuring multiple instances of the same provider (e.g., local Ollama vs. remote Ollama) or complex Azure OpenAI deployments with distinct resource names and versions.
|
||||
|
||||
***
|
||||
|
||||
## DelegateAgentConfig
|
||||
|
||||
Sub-agents used by the `delegate` tool are configured separately to allow for specialized behavior.
|
||||
|
||||
```rust
|
||||
pub struct DelegateAgentConfig {
|
||||
pub provider: String,
|
||||
pub model: String,
|
||||
pub system_prompt: Option<String>,
|
||||
pub api_key: Option<String>,
|
||||
pub temperature: Option<f64>,
|
||||
pub max_depth: u32,
|
||||
pub agentic: bool,
|
||||
pub allowed_tools: Vec<String>,
|
||||
pub max_iterations: usize,
|
||||
}
|
||||
```
|
||||
|
||||
Sub-agents can be limited to specific toolsets and have a `max_depth` to prevent infinite delegation loops.
|
||||
|
||||
***
|
||||
|
||||
## TOML Structure
|
||||
|
||||
Example `config.toml` demonstrating provider setup and security routing:
|
||||
|
||||
```toml
|
||||
default_provider = "anthropic"
|
||||
default_model = "claude-3-5-sonnet"
|
||||
|
||||
[model_providers.ollama_local]
|
||||
name = "ollama"
|
||||
base_url = "http://localhost:11434"
|
||||
|
||||
[[model_routes]]
|
||||
hint = "fast"
|
||||
provider = "ollama_local"
|
||||
model = "llama3"
|
||||
|
||||
[security]
|
||||
encrypt = true
|
||||
|
||||
[autonomy]
|
||||
level = "high"
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Hot-Reload
|
||||
|
||||
ZeroClaw supports hot-reloading for specific runtime fields without requiring a full service restart.
|
||||
|
||||
### Supported Fields
|
||||
* `api_key` / `api_url`
|
||||
* `default_provider`
|
||||
* `default_model`
|
||||
* `default_temperature`
|
||||
* `reliability` settings
|
||||
|
||||
### Mechanism
|
||||
The system utilizes a file watcher (via `config_file_stamp`) that monitors the `config.toml` modification time. When a change is detected, the `maybe_apply_runtime_config_update` function reloads the file, decrypts any secrets using the `SecretStore`, and updates the runtime provider cache.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
ZeroClaw's configuration architecture serves as the target state for [[Configuration]] in Luna.
|
||||
|
||||
* **Subsystem Isolation**: Luna plans to migrate from `appsettings.json` to a per-subsystem TOML structure to improve modularity, similar to ZeroClaw's section pattern.
|
||||
* **Security First**: The pattern of separating `autonomy` and `security` configs is a requirement for Luna's [[Core]].
|
||||
* **Hot-Reload**: ZeroClaw's implementation of lightweight file watching for key LLM parameters is a "nice-to-have" feature Luna aims to adopt during the migration.
|
||||
|
||||
***
|
||||
|
||||
[[Configuration]] | [[Core]]
|
||||
@@ -0,0 +1,187 @@
|
||||
# Memory
|
||||
|
||||
ZeroClaw implements a hybrid memory system that combines traditional keyword search with modern vector similarity. This approach ensures that exact matches (like function names or specific terminology) and semantic matches (concepts and related ideas) are both discoverable.
|
||||
|
||||
***
|
||||
|
||||
## Memory Trait
|
||||
|
||||
The foundation of the memory system is the `Memory` trait. Any backend implementation must satisfy this interface to be used by the system.
|
||||
|
||||
```rust
|
||||
pub struct MemoryEntry {
|
||||
pub id: String,
|
||||
pub key: String,
|
||||
pub content: String,
|
||||
pub category: MemoryCategory,
|
||||
pub timestamp: String,
|
||||
pub session_id: Option<String>,
|
||||
pub score: Option<f64>,
|
||||
}
|
||||
|
||||
pub enum MemoryCategory {
|
||||
Core,
|
||||
Daily,
|
||||
Conversation,
|
||||
Custom(String),
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Memory: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
|
||||
async fn store(
|
||||
&self,
|
||||
key: &str,
|
||||
content: &str,
|
||||
category: MemoryCategory,
|
||||
session_id: Option<&str>,
|
||||
) -> anyhow::Result<()>;
|
||||
|
||||
async fn recall(
|
||||
&self,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
session_id: Option<&str>,
|
||||
) -> anyhow::Result<Vec<MemoryEntry>>;
|
||||
|
||||
async fn get(&self, key: &str) -> anyhow::Result<Option<MemoryEntry>>;
|
||||
|
||||
async fn list(
|
||||
&self,
|
||||
category: Option<&MemoryCategory>,
|
||||
session_id: Option<&str>,
|
||||
) -> anyhow::Result<Vec<MemoryEntry>>;
|
||||
|
||||
async fn forget(&self, key: &str) -> anyhow::Result<bool>;
|
||||
|
||||
async fn count(&self) -> anyhow::Result<usize>;
|
||||
|
||||
async fn health_check(&self) -> bool;
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## SQLite Backend
|
||||
|
||||
The `SqliteMemory` implementation serves as the primary persistent backend. It uses `rusqlite` and is tuned for high performance and reliability in a local environment.
|
||||
|
||||
### Performance Tuning
|
||||
|
||||
The backend initializes with specific PRAGMAs to optimize for concurrent access and speed:
|
||||
|
||||
```rust
|
||||
conn.execute_batch(
|
||||
"PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA mmap_size = 8388608;
|
||||
PRAGMA cache_size = -2000;
|
||||
PRAGMA temp_store = MEMORY;",
|
||||
)?;
|
||||
```
|
||||
|
||||
- **WAL mode**: Enables concurrent reads even during write operations.
|
||||
- **mmap (8MB)**: Allows the OS to handle hot reads through memory mapping.
|
||||
- **temp_store MEMORY**: Ensures temporary tables never touch the disk.
|
||||
|
||||
***
|
||||
|
||||
## Schema Design
|
||||
|
||||
The system maintains a relational table for core data and a virtual table for full-text search (FTS5). Triggers keep these in sync automatically.
|
||||
|
||||
```sql
|
||||
-- Core memories table
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
key TEXT NOT NULL UNIQUE,
|
||||
content TEXT NOT NULL,
|
||||
category TEXT NOT NULL DEFAULT 'core',
|
||||
embedding BLOB,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
session_id TEXT
|
||||
);
|
||||
|
||||
-- FTS5 virtual table for keyword search
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
|
||||
key, content, content=memories, content_rowid=rowid
|
||||
);
|
||||
|
||||
-- Sync triggers
|
||||
CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
|
||||
INSERT INTO memories_fts(rowid, key, content)
|
||||
VALUES (new.rowid, new.key, new.content);
|
||||
END;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Hybrid Search
|
||||
|
||||
The `recall` method executes a hybrid search strategy. It merges results from two distinct search mechanisms:
|
||||
|
||||
1. **Keyword Search**: Uses FTS5 BM25 scoring for exact word matches.
|
||||
2. **Vector Search**: Computes cosine similarity between query embeddings and stored memory embeddings.
|
||||
|
||||
### Score Fusion
|
||||
|
||||
Results are combined using a weighted average. The defaults are typically:
|
||||
- **Vector Weight**: 0.7
|
||||
- **Keyword Weight**: 0.3
|
||||
|
||||
If vector results are unavailable (e.g., if embeddings are disabled), the system falls back to keyword-only search. If both high-level search mechanisms return no results, it uses a final `LIKE %query%` fallback to ensure maximum recall.
|
||||
|
||||
***
|
||||
|
||||
## Embedding Cache
|
||||
|
||||
To avoid redundant API calls to embedding providers, ZeroClaw uses an internal LRU (Least Recently Used) cache stored in SQLite.
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS embedding_cache (
|
||||
content_hash TEXT PRIMARY KEY,
|
||||
embedding BLOB NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
accessed_at TEXT NOT NULL
|
||||
);
|
||||
```
|
||||
|
||||
Whenever a memory is stored or a query is processed, the system checks this cache first using a deterministic content hash. Eviction occurs once the cache reaches its configured limit (default 10,000 entries).
|
||||
|
||||
***
|
||||
|
||||
## Backend Implementations
|
||||
|
||||
ZeroClaw supports several backend types configured via [[Configuration]]:
|
||||
|
||||
1. **SQLite**: The standard persistent local backend using FTS5 and BLOB embeddings.
|
||||
2. **PostgreSQL**: Used for distributed or cloud-hosted deployments (requires `pgvector`).
|
||||
3. **Lucid**: A bridge backend that synchronizes local SQLite memory with remote services.
|
||||
4. **Markdown**: A simple file-based implementation that stores memories as `.md` files in the workspace.
|
||||
5. **Qdrant**: A dedicated vector database backend for high-scale semantic search.
|
||||
6. **None**: An explicit no-op backend that disables all memory persistence.
|
||||
|
||||
***
|
||||
|
||||
## Timeout Guards
|
||||
|
||||
Database operations are protected by timeout guards. Specifically, opening a SQLite connection is capped at 300 seconds to prevent the system from hanging on locked or slow filesystems.
|
||||
|
||||
```rust
|
||||
const SQLITE_OPEN_TIMEOUT_CAP_SECS: u64 = 300;
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
ZeroClaw's memory architecture serves as the blueprint for [[Memory]]. While Luna currently has an `IMemoryStore` interface, the existing file-based implementation lacks sophisticated recall.
|
||||
|
||||
Adopting the ZeroClaw pattern provides:
|
||||
- **Structured Categories**: Using `MemoryCategory` (Core, Daily, Conversation) for better context management.
|
||||
- **Hybrid Recall**: Moving beyond simple file reading to a ranked keyword+vector search.
|
||||
- **SQLite Reference**: Luna can adopt the `SqliteMemory` implementation directly to replace the current unstructured storage.
|
||||
|
||||
Cross-links: [[Core]], [[Configuration]]
|
||||
@@ -0,0 +1,193 @@
|
||||
# ZeroClaw
|
||||
|
||||
## Overview
|
||||
ZeroClaw is a lean, trait-driven Rust AI assistant runtime (~1.7k stars, ~1.6k commits). Single binary, <5MB RAM, <10ms startup. Everything is a swappable trait. Repo: https://github.com/openagen/zeroclaw
|
||||
|
||||
## Architecture
|
||||
- Single-binary Rust runtime — no microservices, no plugins to install
|
||||
- Trait-driven design: every subsystem (Provider, Channel, Tool, Memory, Runtime, Security) is a trait you swap
|
||||
- Minimal footprint: <5MB RAM, <10ms cold start
|
||||
- All features compiled in-process, zero external dependencies for core functionality
|
||||
|
||||
## Providers
|
||||
15+ provider implementations behind a `Provider` trait:
|
||||
OpenAI, Anthropic, Gemini, Ollama, OpenRouter, Azure OpenAI, AWS Bedrock, GitHub Copilot, GLM, Telnyx, llama.cpp, vLLM, custom endpoints.
|
||||
|
||||
Key pattern — `ReliableProvider` wrapper: wraps any provider with automatic fallback chains and retry logic. If primary fails, falls through to secondary, tertiary, etc.
|
||||
|
||||
```rust
|
||||
// Provider trait (src/providers/traits.rs)
|
||||
pub trait Provider: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
async fn chat(&self, request: ChatRequest) -> Result<ChatResponse>;
|
||||
async fn stream(&self, request: ChatRequest) -> Result<Box<dyn Stream<Item = Result<ChatChunk>>>>;
|
||||
}
|
||||
```
|
||||
|
||||
## Agent Loop
|
||||
- Full tool-calling protocol: LLM requests tool call → agent executes tool → feeds result back → loops until LLM stops requesting tools
|
||||
- `DelegateTool`: spawns a sub-agent with a scoped task for multi-agent delegation
|
||||
- Memory injection into agent context
|
||||
- Identity/persona system supporting AIEOS v1.1 JSON format and OpenClaw markdown format
|
||||
|
||||
## Channels
|
||||
26 channel implementations behind a `Channel` trait:
|
||||
CLI, Telegram, Discord, Slack, Mattermost, iMessage, Matrix, Signal, WhatsApp (Web + Business API), Email, IRC, Lark, DingTalk, QQ, Nostr, MQTT, Webhook, and more.
|
||||
|
||||
```rust
|
||||
// Channel message types (src/channels/traits.rs)
|
||||
pub struct ChannelMessage {
|
||||
pub id: String,
|
||||
pub sender: String,
|
||||
pub reply_target: Option<String>,
|
||||
pub content: String,
|
||||
pub channel: String,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub thread_ts: Option<String>,
|
||||
}
|
||||
```
|
||||
|
||||
## Memory
|
||||
6 backends behind a `Memory` trait:
|
||||
- SQLite: hybrid FTS5 full-text + cosine vector similarity in a single query, zero external deps
|
||||
- PostgreSQL
|
||||
- Lucid bridge
|
||||
- Markdown (file-based)
|
||||
- Qdrant (vector DB)
|
||||
- Explicit none
|
||||
|
||||
```rust (src/memory/traits.rs)
|
||||
pub trait Memory: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
async fn store(&self, entry: MemoryEntry) -> Result<()>;
|
||||
async fn recall(&self, query: &str, limit: usize) -> Result<Vec<MemoryEntry>>;
|
||||
}
|
||||
|
||||
pub struct MemoryEntry {
|
||||
pub id: String,
|
||||
pub key: String,
|
||||
pub content: String,
|
||||
pub category: MemoryCategory,
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub session_id: Option<String>,
|
||||
pub score: Option<f64>,
|
||||
}
|
||||
|
||||
pub enum MemoryCategory {
|
||||
Core,
|
||||
Daily,
|
||||
Conversation,
|
||||
Custom(String),
|
||||
}
|
||||
```
|
||||
|
||||
Custom embedding provider routing — memory backends can use different embedding models.
|
||||
|
||||
## Tools
|
||||
42+ tool implementations behind a `Tool` trait:
|
||||
|
||||
```rust
|
||||
// Tool trait (src/tools/traits.rs)
|
||||
pub trait Tool: Send + Sync {
|
||||
fn name(&self) -> &str;
|
||||
fn description(&self) -> &str;
|
||||
fn parameters_schema(&self) -> serde_json::Value;
|
||||
async fn execute(&self, args: serde_json::Value) -> Result<ToolResult>;
|
||||
}
|
||||
```
|
||||
|
||||
Tool categories:
|
||||
- **Shell & Files**: shell execution, file read/write/edit, glob search, content search
|
||||
- **Browser & Web**: browser automation, HTTP requests, web fetch/search, screenshots
|
||||
- **Documents**: PDF read, image info
|
||||
- **Git**: git operations (status, diff, commit, log, etc.)
|
||||
- **Scheduling**: cron add/list/remove/update/run
|
||||
- **Memory**: store/recall/forget (tools that interact with the Memory trait)
|
||||
- **Delegation**: DelegateTool for sub-agent spawning
|
||||
- **Hardware**: USB discover/introspect, robot-kit integration
|
||||
- **Integration**: Composio (1000+ OAuth apps), SOP tools, Pushover notifications
|
||||
|
||||
## Sessions
|
||||
- Per-sender conversation history
|
||||
- Compaction (summarization of old messages)
|
||||
- Autosave thresholds (every N messages, not just on shutdown)
|
||||
- Per-sender provider/model overrides (each conversation can use a different model)
|
||||
|
||||
## Security
|
||||
Comprehensive security stack (16 modules):
|
||||
- **SecurityPolicy**: Autonomy levels — readonly, supervised, full
|
||||
- **Workspace Scoping**: Operations restricted to declared workspace paths
|
||||
- **Filesystem Guards**: 14 system directories blocked, symlink escape detection, null byte injection blocked
|
||||
- **Sandboxing**: Landlock, Bubblewrap, Docker, Firejail — multiple isolation backends
|
||||
- **Gateway Pairing**: 6-digit codes + bearer tokens for channel authentication
|
||||
- **Encrypted Secrets**: SecretStore for API keys and credentials
|
||||
- **Audit Logging**: All security-relevant actions logged
|
||||
- **Emergency Stop**: EstopManager for immediate agent shutdown
|
||||
- **Prompt Guard**: Detection of prompt injection attempts
|
||||
- **Leak Detector**: Prevents secrets from appearing in agent output
|
||||
- **OTP**: One-time password support
|
||||
|
||||
## Runtime
|
||||
`RuntimeAdapter` trait with two implementations:
|
||||
- `NativeRuntime`: Direct execution on host OS
|
||||
- `DockerRuntime`: Isolated execution in Docker containers
|
||||
Tool execution can be routed through either runtime for isolation.
|
||||
|
||||
## Skills
|
||||
- TOML manifests (`SKILL.toml`) + `SKILL.md` instructions
|
||||
- open-skills sync for community skill repositories
|
||||
- Security audit on install: blocks symlinks, scripts, unsafe patterns
|
||||
|
||||
## Configuration
|
||||
- TOML-based config at `~/.zeroclaw/config.toml`
|
||||
- Hot-reload on key fields without restart
|
||||
- Per-channel, per-provider, per-tool configuration sections
|
||||
|
||||
## Tunnel
|
||||
Trait-based tunneling for exposing local instance to the internet:
|
||||
- Cloudflare Tunnel
|
||||
- Tailscale
|
||||
- ngrok
|
||||
- Custom tunnel implementations
|
||||
|
||||
***
|
||||
|
||||
## In-Depth Reference Pages
|
||||
|
||||
- [[ReliableProvider]] — Three-level failover (model chain, provider chain, retry loop), error classification, API key rotation
|
||||
- [[Agent Loop & Tools]] — Tool trait, agent execution loop, tool serialization, credential scrubbing
|
||||
- [[Memory]] — Memory trait, SQLite hybrid FTS5+vector search, embedding cache, 6 backends
|
||||
- [[Channel Messages]] — Channel trait, draft update protocol, ChannelMessage struct, SendMessage builder
|
||||
- [[Security]] — SecurityPolicy, autonomy levels, command allowlist, path validation, sandboxing, secret store, e-stop
|
||||
- [[Sessions]] — Per-sender storage, compaction logic, autosave thresholds, per-sender overrides
|
||||
- [[Configuration]] — TOML config struct (30+ sections), ModelProviderConfig, DelegateAgentConfig, hot-reload
|
||||
- [[Runtime Adapters]] — RuntimeAdapter trait, NativeRuntime vs DockerRuntime, capability querying
|
||||
- [[Skills]] — SKILL.toml manifest, security audit on install, open-skills sync, prompt injection
|
||||
- [[Compaction Strategy]] — Proactive agent-side and reactive channel-side compaction, summarization prompts, helper functions
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
### Patterns Worth Adopting
|
||||
|
||||
ZeroClaw is architecturally closest to Luna (monolithic binary, trait/interface-driven design). These patterns map directly:
|
||||
|
||||
- **ReliableProvider wrapper**: Luna's `IProvider` returning `IChatClient` is clean. Add a decorator that wraps any `IProvider` with fallback chains and retry logic. See [[ReliableProvider]] for the three-level failover pattern. Link to [[Core]].
|
||||
- **Tool trait + Agent loop**: This is Luna's #1 gap. The `Tool` trait maps cleanly to a C# `ITool` interface. The agent loop is the missing piece that turns Luna from a chat relay into an assistant. See [[Agent Loop & Tools]]. Link to [[Skills]].
|
||||
- **Memory with hybrid search**: Luna's `IMemoryStore` exists but has no recall. ZeroClaw's SQLite memory does hybrid FTS5 + cosine vector similarity in one query. See [[Memory]]. Link to Luna's [[Memory]].
|
||||
- **Channel message normalization**: The `ChannelMessage` struct should be the baseline for Luna's DTO. The draft update protocol maps to SignalR. See [[Channel Messages]]. Link to [[Channels]].
|
||||
- **Session autosave + per-sender overrides**: Autosave every N messages for crash resilience. Per-sender model overrides are cheap to add. See [[Sessions]]. Link to [[Core]].
|
||||
- **Compaction**: Dual-path approach (proactive message-count + reactive overflow recovery) with structured summarization prompts. See [[Compaction Strategy]].
|
||||
- **Security fundamentals**: Before tool execution, Luna needs filesystem guards, workspace scoping, and autonomy levels. See [[Security]]. ZeroClaw's `SecurityPolicy` is the reference.
|
||||
- **TOML config with hot-reload**: Luna's [[Configuration]] module plans this. ZeroClaw's per-subsystem section pattern is the target. See [[Configuration]].
|
||||
- **Runtime adapters**: When tool execution is added, a similar `IRuntimeAdapter` allows switching between native and Docker. See [[Runtime Adapters]].
|
||||
- **Skills system**: SKILL.toml manifest with security audit on install. See [[Skills]].
|
||||
|
||||
### Key Differences from Luna
|
||||
- Rust vs .NET/C# — different ecosystem, but the trait-driven architecture maps well to C# interfaces
|
||||
- ZeroClaw has 42+ tools; Luna has 0 — tool system is the critical gap
|
||||
- ZeroClaw has 15+ providers; Luna has 2 (Mistral, Ollama)
|
||||
- ZeroClaw has 26 channels; Luna has 1 (CLI)
|
||||
- ZeroClaw has comprehensive security (16 modules); Luna has none
|
||||
- ZeroClaw runs as a single <5MB binary; Luna's .NET runtime is heavier but offers richer DI/middleware ecosystem
|
||||
@@ -0,0 +1,152 @@
|
||||
# ReliableProvider
|
||||
|
||||
The `ReliableProvider` is a resilient decorator in ZeroClaw that wraps one or more standard providers to implement a three-level failover strategy. It ensures high availability by managing model fallbacks, provider redundancy, and intelligent retry logic with backoff.
|
||||
|
||||
***
|
||||
|
||||
## Provider Trait
|
||||
|
||||
All LLM integrations in ZeroClaw implement the `Provider` trait. This trait defines a unified interface for chat interactions, tool calling, and capability discovery.
|
||||
|
||||
```rust
|
||||
#[async_trait]
|
||||
pub trait Provider: Send + Sync {
|
||||
fn capabilities(&self) -> ProviderCapabilities {
|
||||
ProviderCapabilities::default()
|
||||
}
|
||||
|
||||
async fn chat_with_system(
|
||||
&self,
|
||||
system_prompt: Option<&str>,
|
||||
message: &str,
|
||||
model: &str,
|
||||
temperature: f64,
|
||||
) -> anyhow::Result<String>;
|
||||
|
||||
async fn chat_with_history(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
model: &str,
|
||||
temperature: f64,
|
||||
) -> anyhow::Result<String>;
|
||||
|
||||
async fn chat(
|
||||
&self,
|
||||
request: ChatRequest<'_>,
|
||||
model: &str,
|
||||
temperature: f64,
|
||||
) -> anyhow::Result<ChatResponse>;
|
||||
|
||||
async fn chat_with_tools(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
_tools: &[serde_json::Value],
|
||||
model: &str,
|
||||
temperature: f64,
|
||||
) -> anyhow::Result<ChatResponse>;
|
||||
}
|
||||
```
|
||||
|
||||
### ProviderCapabilities
|
||||
The `ProviderCapabilities` struct allows providers to declare their feature set, which `ReliableProvider` uses to route requests correctly:
|
||||
- `native_tool_calling`: Support for API-native function calling (Gemini, Anthropic, OpenAI).
|
||||
- `vision`: Support for multimodal image inputs.
|
||||
|
||||
### ToolsPayload
|
||||
When tools are used, `Provider::convert_tools` returns a `ToolsPayload` enum to handle varying API requirements:
|
||||
- `Gemini { function_declarations }`
|
||||
- `Anthropic { tools }`
|
||||
- `OpenAI { tools }`
|
||||
- `PromptGuided { instructions }`: Textual fallback for providers without native support.
|
||||
|
||||
***
|
||||
|
||||
## Three-Level Failover
|
||||
|
||||
The `ReliableProvider` implements a nested loop structure to exhaust all possibilities before returning an error.
|
||||
|
||||
```rust
|
||||
pub struct ReliableProvider {
|
||||
providers: Vec<(String, Box<dyn Provider>)>,
|
||||
max_retries: u32,
|
||||
base_backoff_ms: u64,
|
||||
api_keys: Vec<String>,
|
||||
key_index: AtomicUsize,
|
||||
model_fallbacks: HashMap<String, Vec<String>>,
|
||||
}
|
||||
```
|
||||
|
||||
1. **Level 1: Model Chain**: Iterates through the primary model and its configured fallbacks (e.g., try `claude-3-5-sonnet`, fallback to `claude-3-haiku`).
|
||||
2. **Level 2: Provider Chain**: For each model, iterates through registered providers in priority order (e.g., try Anthropic direct, fallback to OpenRouter).
|
||||
3. **Level 3: Retry Loop**: For a specific (provider, model) pair, retries transient failures with exponential backoff.
|
||||
|
||||
***
|
||||
|
||||
## Error Classification
|
||||
|
||||
Intelligent failure handling depends on distinguishing transient issues from permanent ones. ZeroClaw uses heuristics and status codes for this classification.
|
||||
|
||||
### Non-Retryable Errors
|
||||
The `is_non_retryable()` function identifies errors that won't resolve with retries, such as:
|
||||
- **Client Errors**: HTTP 4xx (except 429/408).
|
||||
- **Authentication**: Key-word matching for "invalid api key", "unauthorized", or "permission denied".
|
||||
- **Model Availability**: Heuristics for "model not found", "unknown model", or "unsupported".
|
||||
|
||||
### Rate Limiting and Quota
|
||||
- `is_rate_limited()`: Specifically detects HTTP 429 errors.
|
||||
- `is_non_retryable_rate_limit()`: Detects business-level failures returned as 429s, such as "insufficient balance", "quota exhausted", or "plan does not include requested model". These trigger a provider fallback immediately.
|
||||
|
||||
### Context Window Exceeded
|
||||
The system short-circuits when the context window is exceeded to avoid useless retries:
|
||||
```rust
|
||||
fn is_context_window_exceeded(err: &anyhow::Error) -> bool {
|
||||
let hints = [
|
||||
"exceeds the context window",
|
||||
"maximum context length",
|
||||
"token limit exceeded",
|
||||
"prompt is too long",
|
||||
];
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Exponential Backoff
|
||||
|
||||
Retries use an exponential backoff strategy starting from `base_backoff_ms`.
|
||||
- **Retry-After Parsing**: The system parses the `Retry-After` header or error body.
|
||||
- **Capping**: Backoff is capped at 30 seconds to prevent indefinite stalls.
|
||||
- **Jitter**: While the core logic doubles the wait time, it ensures the wait respects provider-suggested intervals.
|
||||
|
||||
***
|
||||
|
||||
## API Key Rotation
|
||||
|
||||
ZeroClaw supports round-robin API key rotation for the same provider to maximize throughput:
|
||||
```rust
|
||||
fn rotate_key(&self) -> Option<&str> {
|
||||
if self.api_keys.is_empty() { return None; }
|
||||
let idx = self.key_index.fetch_add(1, Ordering::Relaxed) % self.api_keys.len();
|
||||
Some(&self.api_keys[idx])
|
||||
}
|
||||
```
|
||||
When a 429 is encountered (and it's not a quota error), the system cycles to the next key for the subsequent retry attempt.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Currently, Luna's [[Core]] implementation of `IProvider` is simple and lacks resilience. If a provider is down, the request fails.
|
||||
|
||||
By implementing a `ReliableProvider` decorator for Luna's `IProvider`, we can:
|
||||
- Wrap multiple `IChatClient` instances.
|
||||
- Implement per-sender model overrides to allow specific users to access higher-tier models with fallback to cheaper ones.
|
||||
- Integrate the ZeroClaw error classification logic to handle common LLM API failures gracefully.
|
||||
|
||||
This pattern is essential for moving from a prototype to a production-grade system where provider stability cannot be guaranteed.
|
||||
|
||||
***
|
||||
|
||||
[[Core]]
|
||||
[[Configuration]]
|
||||
@@ -0,0 +1,103 @@
|
||||
# Runtime Adapters
|
||||
|
||||
ZeroClaw uses the `RuntimeAdapter` trait to abstract the execution environment from the core agent logic. This abstraction allows the same agent code to run natively on a host machine, inside a Docker container, or on restricted edge runtimes without modification to the core loop.
|
||||
|
||||
***
|
||||
|
||||
## RuntimeAdapter Trait
|
||||
|
||||
The `RuntimeAdapter` trait defines the capabilities and interface for any environment ZeroClaw operates within. It is designed to be `Send + Sync` to allow safe sharing across asynchronous tasks.
|
||||
|
||||
```rust
|
||||
pub trait RuntimeAdapter: Send + Sync {
|
||||
/// Return the human-readable name of this runtime environment.
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Report whether this runtime supports shell command execution.
|
||||
fn has_shell_access(&self) -> bool;
|
||||
|
||||
/// Report whether this runtime supports filesystem read/write.
|
||||
fn has_filesystem_access(&self) -> bool;
|
||||
|
||||
/// Return the base directory for persistent storage on this runtime.
|
||||
fn storage_path(&self) -> PathBuf;
|
||||
|
||||
/// Report whether this runtime supports long-running background processes.
|
||||
fn supports_long_running(&self) -> bool;
|
||||
|
||||
/// Return the maximum memory budget in bytes for this runtime.
|
||||
fn memory_budget(&self) -> u64 {
|
||||
0
|
||||
}
|
||||
|
||||
/// Build a shell command process configured for this runtime.
|
||||
fn build_shell_command(
|
||||
&self,
|
||||
command: &str,
|
||||
workspace_dir: &Path,
|
||||
) -> anyhow::Result<tokio::process::Command>;
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## NativeRuntime
|
||||
|
||||
The `NativeRuntime` implementation provides direct execution on the host operating system. It represents the least restrictive environment and is typically used for local development or trusted server environments.
|
||||
|
||||
* **Shell Access**: Returns `true`. Commands are executed directly via the system shell.
|
||||
* **Filesystem Access**: Returns `true`. The agent can interact with any path permitted by the user's OS permissions.
|
||||
* **Memory Limits**: Typically returns `0` (unlimited), relying on the OS to manage process resources.
|
||||
* **Command Building**: Spawns `tokio::process::Command` directly with the requested command string.
|
||||
|
||||
***
|
||||
|
||||
## DockerRuntime
|
||||
|
||||
The `DockerRuntime` provides isolated execution by wrapping operations in Docker containers. This is the preferred runtime for untrusted code execution or when strict environment reproducibility is required.
|
||||
|
||||
* **Shell Access**: Configurable, but generally `true`. Commands are wrapped in `docker exec` calls targeting a specific container.
|
||||
* **Filesystem Access**: Restricted to the volumes and mounts defined in the container configuration.
|
||||
* **Memory Budgets**: Returns the memory limits defined for the container, allowing the agent to adapt its cache and buffer sizes.
|
||||
* **Command Building**: Instead of direct execution, it constructs a command that executes inside the container namespace, often involving complex argument escaping and environment variable injection.
|
||||
|
||||
***
|
||||
|
||||
## Capability Querying
|
||||
|
||||
The agent loop queries the `RuntimeAdapter` before attempting to execute tools or background tasks. This ensures that the agent fails gracefully or skips unavailable functionality based on its environment.
|
||||
|
||||
1. **Tool Pre-flight**: Before executing a shell tool, the orchestrator checks `has_shell_access()`.
|
||||
2. **Persistence Check**: Before initializing disk-based state, the agent checks `has_filesystem_access()`.
|
||||
3. **Background Services**: The heartbeat loop and gateway server only start if `supports_long_running()` returns `true`.
|
||||
|
||||
This pattern prevents runtime errors by verifying environmental support at the logic gate rather than deep within the execution stack.
|
||||
|
||||
***
|
||||
|
||||
## Command Building
|
||||
|
||||
The `build_shell_command` method is the primary bridge between the agent and the OS. It takes a raw command string and a workspace directory, returning a configured `tokio::process::Command`.
|
||||
|
||||
Runtimes use this to:
|
||||
* Prepend sandbox wrappers (e.g., `firejail` or `sudo -u limited`).
|
||||
* Set environment variables specific to the runtime (e.g., `PATH` or `HOME`).
|
||||
* Handle working directory mapping (especially important in Docker where host paths and container paths differ).
|
||||
|
||||
***
|
||||
|
||||
## Tool Routing
|
||||
|
||||
Tool execution can be routed through specific runtimes based on security requirements. A "Security Router" might send filesystem operations to a native runtime for speed while routing unknown shell scripts to a Docker runtime for isolation. This routing logic relies on the standardized interface provided by the `RuntimeAdapter`.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna currently operates with implicit native execution. As Luna evolves to support tool execution and autonomous tasks, implementing an `IRuntimeAdapter` interface will be critical for safety.
|
||||
|
||||
* **Isolation**: Adopting a pattern similar to ZeroClaw's `DockerRuntime` would allow Luna to run generated code in a sandbox without risking the host system.
|
||||
* **Cross-Platform**: A runtime abstraction simplifies porting Luna to different OSs or containerized environments.
|
||||
* **Security Integration**: This pairs with [[Security]] patterns to ensure that capabilities are not just checked by the runtime, but also verified against user-defined security policies.
|
||||
|
||||
Cross-links: [[Core]], [[Skills]], [[Configuration]].
|
||||
@@ -0,0 +1,214 @@
|
||||
# Security
|
||||
|
||||
ZeroClaw implements a multi-layered security stack designed to provide defense-in-depth for autonomous agent operations. This system ensures that agents operate within defined boundaries, protect sensitive credentials, and provide a verifiable audit trail of all actions.
|
||||
|
||||
***
|
||||
|
||||
## SecurityPolicy
|
||||
|
||||
The `SecurityPolicy` is the central enforcement mechanism for all tool and command executions. It defines the agent's level of independence and the specific constraints on its operating environment.
|
||||
|
||||
### Autonomy Levels
|
||||
|
||||
Autonomy is categorized into three distinct levels, controlling the baseline behavior of the agent:
|
||||
|
||||
```rust
|
||||
pub enum AutonomyLevel {
|
||||
/// Read-only: can observe but not act
|
||||
ReadOnly,
|
||||
/// Supervised: acts but requires approval for risky operations
|
||||
Supervised,
|
||||
/// Full: autonomous execution within policy bounds
|
||||
Full,
|
||||
}
|
||||
```
|
||||
|
||||
### Risk Classification
|
||||
|
||||
Commands and operations are classified by their potential impact:
|
||||
|
||||
```rust
|
||||
pub enum CommandRiskLevel {
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
}
|
||||
|
||||
pub enum ToolOperation {
|
||||
Read,
|
||||
Act,
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Action Tracker
|
||||
|
||||
ZeroClaw uses a sliding-window rate limiter to prevent automated abuse or runaway processes. It maintains a 1-hour window of all side-effecting actions.
|
||||
|
||||
```rust
|
||||
pub struct ActionTracker {
|
||||
/// Timestamps of recent actions (kept within the last hour).
|
||||
actions: Mutex<Vec<Instant>>,
|
||||
}
|
||||
|
||||
impl ActionTracker {
|
||||
pub fn record(&self) -> usize {
|
||||
let mut actions = self.actions.lock();
|
||||
let cutoff = Instant::now()
|
||||
.checked_sub(std::time::Duration::from_secs(3600))
|
||||
.unwrap_or_else(Instant::now);
|
||||
actions.retain(|t| *t > cutoff);
|
||||
actions.push(Instant::now());
|
||||
actions.len()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Command Allowlist
|
||||
|
||||
ZeroClaw employs a deny-by-default strategy for shell execution. Only explicitly listed commands are permitted, and they are subjected to rigorous parsing to prevent bypasses.
|
||||
|
||||
### Default Allowed Commands
|
||||
`git`, `npm`, `cargo`, `ls`, `cat`, `grep`, `find`, `echo`, `pwd`, `wc`, `head`, `tail`, `date`.
|
||||
|
||||
### Default Forbidden Paths
|
||||
The system blocks access to 14 system directories (e.g., `/etc`, `/root`, `/usr`, `/bin`, `/proc`) and 4 sensitive dotfile locations (`~/.ssh`, `~/.gnupg`, `~/.aws`, `~/.config`) even if workspace confinement is disabled.
|
||||
|
||||
### Shell Parsing Logic
|
||||
To prevent command injection via chained operators or environment variables, ZeroClaw uses a quote-aware shell segment splitter and environment assignment stripper:
|
||||
|
||||
```rust
|
||||
fn skip_env_assignments(s: &str) -> &str {
|
||||
// Strips 'FOO=bar' from 'FOO=bar cmd args'
|
||||
}
|
||||
|
||||
fn split_unquoted_segments(command: &str) -> Vec<String> {
|
||||
// Splits on ';', '|', '&&', '||', and newlines
|
||||
// Respects quotes to allow literal separators in arguments
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Path Validation
|
||||
|
||||
The `is_path_allowed()` function implements multiple layers of protection to enforce workspace confinement and prevent directory traversal.
|
||||
|
||||
### Validation Logic
|
||||
|
||||
```rust
|
||||
pub fn is_path_allowed(&self, path: &str) -> bool {
|
||||
// 1. Null-byte injection guard
|
||||
if path.contains('\0') { return false; }
|
||||
|
||||
// 2. Directory traversal detection
|
||||
if Path::new(path).components().any(|c| matches!(c, Component::ParentDir)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. URL-encoding detection (..%2f)
|
||||
let lower = path.to_lowercase();
|
||||
if lower.contains("..%2f") || lower.contains("%2f..") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Tilde expansion guard (blocks ~user forms)
|
||||
if path.starts_with('~') && path != "~" && !path.starts_with("~/") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 5. Absolute path block and forbidden prefix match
|
||||
let expanded = expand_user_path(path);
|
||||
if self.workspace_only && expanded.is_absolute() { return false; }
|
||||
// ... (forbidden path checks)
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Sandboxing
|
||||
|
||||
ZeroClaw supports multiple isolation backends via a unified `Sandbox` trait, allowing for varied levels of process and filesystem isolation.
|
||||
|
||||
- **Landlock**: Linux-native LSM for fine-grained filesystem restriction.
|
||||
- **Bubblewrap**: Unprivileged sandboxing utility (used by Flatpak).
|
||||
- **Docker**: Containerized isolation for high-risk environments.
|
||||
- **Firejail**: SUID-based sandbox for easy desktop application isolation.
|
||||
|
||||
### Landlock Implementation snippet:
|
||||
|
||||
```rust
|
||||
fn apply_restrictions(&self) -> std::io::Result<()> {
|
||||
let mut ruleset = Ruleset::default()
|
||||
.handle_access(AccessFs::ReadFile | AccessFs::WriteFile | AccessFs::ReadDir | ...)
|
||||
.and_then(|ruleset| ruleset.create())?;
|
||||
|
||||
// Grant access only to workspace and necessary system paths (/usr, /bin)
|
||||
if let Some(ref workspace) = self.workspace_dir {
|
||||
ruleset = ruleset.add_rule(PathBeneath::new(workspace_fd, ...))?;
|
||||
}
|
||||
ruleset.restrict_self()
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Secret Management
|
||||
|
||||
The `SecretStore` protects API keys and credentials using authenticated encryption (ChaCha20-Poly1305).
|
||||
|
||||
```rust
|
||||
pub struct SecretStore {
|
||||
key_path: PathBuf, // ~/.zeroclaw/.secret_key (mode 0600)
|
||||
enabled: bool,
|
||||
}
|
||||
```
|
||||
|
||||
Encryption ensures that secrets are never stored in plaintext in configuration files, preventing accidental exposure via `grep`, `git log`, or file sharing.
|
||||
|
||||
***
|
||||
|
||||
## Audit Logging
|
||||
|
||||
Every security-relevant event is recorded in a structured JSONL format. This provides a forensic trail of what was executed, by whom, and whether it was permitted by policy.
|
||||
|
||||
```rust
|
||||
pub struct AuditEvent {
|
||||
pub timestamp: DateTime<Utc>,
|
||||
pub event_type: AuditEventType,
|
||||
pub actor: Option<Actor>,
|
||||
pub action: Option<Action>,
|
||||
pub result: Option<ExecutionResult>,
|
||||
pub security: SecurityContext,
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Emergency Stop (E-Stop)
|
||||
|
||||
The `EstopManager` provides a "big red button" capability to immediately halt or restrict agent activity.
|
||||
|
||||
- **Levels**: `KillAll`, `NetworkKill`, `DomainBlock`, `ToolFreeze`.
|
||||
- **Fail-Closed**: If the state file is corrupt or unreadable, the system defaults to `KillAll`.
|
||||
- **OTP Requirement**: Resuming operations after an E-Stop engagement can be configured to require a One-Time Password (TOTP) to ensure human authorization.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna currently lacks a security framework. Adopting ZeroClaw's security stack is critical before enabling tool execution in non-trusted environments.
|
||||
|
||||
### Prioritized Adoption Order
|
||||
|
||||
1. **Filesystem Guards + Workspace Scoping**: Implement `is_path_allowed()` to prevent Luna from escaping its designated project directory.
|
||||
2. **Command Allowlist**: Restrict shell tools to a known-safe subset with defined autonomy levels.
|
||||
3. **Path Validation**: Integrate component-aware path checking in all file-handling tools.
|
||||
4. **Secret Store**: Migrate Luna's configuration to use ChaCha20-Poly1305 for API keys.
|
||||
5. **Audit Logging**: Implement structured event logging for all tool invocations.
|
||||
6. **Sandboxing**: Introduce Landlock or Docker backends for executing untrusted code.
|
||||
|
||||
See also: [[Core]], [[Skills]], [[Configuration]].
|
||||
@@ -0,0 +1,65 @@
|
||||
# Sessions
|
||||
|
||||
ZeroClaw implements a robust session management system designed for multi-user environments and crash resilience. Unlike simple stateless request-response loops, ZeroClaw treats every interaction as part of a persistent, per-sender conversation that survives process restarts and handles context growth through automated compaction.
|
||||
|
||||
***
|
||||
|
||||
## Per-Sender Storage
|
||||
|
||||
Each conversation is isolated and keyed by a combination of the sender ID and the communication channel. This ensures that users on different platforms (e.g., Discord vs. Slack) or multiple users within the same platform maintain independent histories.
|
||||
|
||||
- **Session Keying**: Sessions are typically indexed by `(channel_id, sender_id)`.
|
||||
- **Isolation**: Memory and history from one sender never bleed into another's session unless explicitly shared via global memory.
|
||||
- **Persistence**: Sessions are serialized and stored as JSON files, allowing the agent to resume exactly where it left off after a restart.
|
||||
|
||||
***
|
||||
|
||||
## Compaction Logic
|
||||
|
||||
To prevent context window overflow and maintain performance over long conversations, ZeroClaw uses an automated compaction mechanism defined in `agent/loop_.rs`. When the history grows beyond a specific threshold, old messages are summarized and replaced by a concise context block.
|
||||
|
||||
- **Threshold**: Compaction triggers when the non-system message count exceeds `DEFAULT_MAX_HISTORY_MESSAGES = 50`.
|
||||
- **Retention**: ZeroClaw keeps the `COMPACTION_KEEP_RECENT_MESSAGES = 20` most recent messages in their original form.
|
||||
- **Source Cap**: The transcript passed to the summarizer is limited to `COMPACTION_MAX_SOURCE_CHARS = 12_000` to avoid overwhelming the summarization model.
|
||||
- **Summary Cap**: The resulting summary is truncated to `COMPACTION_MAX_SUMMARY_CHARS = 2_000`.
|
||||
|
||||
The summarization process is handled by a recursive call to the provider, instructing it to preserve key facts, user preferences, and unresolved tasks while omitting filler and verbose tool logs.
|
||||
|
||||
***
|
||||
|
||||
## Autosave Thresholds
|
||||
|
||||
ZeroClaw prioritizes data integrity through aggressive autosaving. Sessions are not just saved on graceful shutdown; they are updated after every significant interaction.
|
||||
|
||||
- **Trigger**: `AUTOSAVE_MIN_MESSAGE_CHARS = 20`.
|
||||
- **Behavior**: If a user message exceeds this length, it is considered "meaningful" enough to trigger an immediate save of the session state.
|
||||
- **Crash Resilience**: This granular saving ensures that even in the event of a sudden crash or hardware failure, the agent loses at most the very last exchange.
|
||||
|
||||
***
|
||||
|
||||
## Per-Sender Overrides
|
||||
|
||||
Each session can carry its own configuration, allowing for dynamic behavior based on the specific user or channel requirements.
|
||||
|
||||
- **Model Overrides**: A specific conversation can be configured to use a different provider or model than the global default.
|
||||
- **Config Storage**: These overrides are stored alongside the session JSON, ensuring consistency across restarts.
|
||||
|
||||
***
|
||||
|
||||
## Session Lifecycle
|
||||
|
||||
1. **Creation**: A new session is initialized when a message is received from a previously unknown `(channel, sender)` pair.
|
||||
2. **Loading**: Upon receiving a message, ZeroClaw checks the local storage for an existing session file matching the sender's key.
|
||||
3. **Processing**: Messages are appended to the history, and compaction is checked before sending the request to the LLM.
|
||||
4. **Saving**: The session is written to disk after the LLM responds or when the autosave threshold is met.
|
||||
5. **Cleanup**: Older sessions can be archived or deleted based on global retention policies, though ZeroClaw typically favors long-term persistence.
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna's SessionManager currently implements basic compaction via the LibrarianAgent, but lacks several of ZeroClaw's more robust features. Integrating these would significantly improve reliability and flexibility:
|
||||
|
||||
- **Autosave**: Implementing the `AUTOSAVE_MIN_MESSAGE_CHARS` logic to move away from shutdown-only saves. This is critical for [[Core]] stability.
|
||||
- **Per-Sender Overrides**: Enabling users to select specific models for their sessions, a feature currently missing from Luna's [[Configuration]].
|
||||
- **Configurable Thresholds**: Exposing compaction constants (like 50/20 messages) as configurable parameters rather than hardcoded values.
|
||||
@@ -0,0 +1,167 @@
|
||||
# Skills
|
||||
|
||||
The ZeroClaw skills system enables the extension of agent capabilities through modular, audited packages. Each skill can define custom prompts, tools (shell, HTTP, or scripts), and metadata. This system serves as the reference architecture for [[Skills]] in Luna.
|
||||
|
||||
***
|
||||
|
||||
## Skill Struct
|
||||
|
||||
ZeroClaw defines skills and their associated tools using robust Rust structures. A `Skill` is the top-level container, while `SkillTool` defines specific executable capabilities.
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Skill {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub version: String,
|
||||
#[serde(default)]
|
||||
pub author: Option<String>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub tools: Vec<SkillTool>,
|
||||
#[serde(default)]
|
||||
pub prompts: Vec<String>,
|
||||
#[serde(skip)]
|
||||
pub location: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SkillTool {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
/// "shell", "http", "script"
|
||||
pub kind: String,
|
||||
/// The command/URL/script to execute
|
||||
pub command: String,
|
||||
#[serde(default)]
|
||||
pub args: HashMap<String, String>,
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## SKILL.toml Manifest
|
||||
|
||||
The preferred way to define a skill is via a `SKILL.toml` manifest. ZeroClaw also supports a legacy `SKILL.md` fallback for prompt-only skills.
|
||||
|
||||
```rust
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct SkillManifest {
|
||||
skill: SkillMeta,
|
||||
#[serde(default)]
|
||||
tools: Vec<SkillTool>,
|
||||
#[serde(default)]
|
||||
prompts: Vec<String>,
|
||||
}
|
||||
```
|
||||
|
||||
### Example SKILL.toml
|
||||
```toml
|
||||
[skill]
|
||||
name = "weather"
|
||||
description = "Fetch weather forecasts"
|
||||
version = "0.1.0"
|
||||
author = "Luna-Team"
|
||||
tags = ["utility", "api"]
|
||||
|
||||
[[tools]]
|
||||
name = "get_weather"
|
||||
description = "Fetch forecast from wttr.in"
|
||||
kind = "shell"
|
||||
command = "curl -s wttr.in/$CITY"
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Security Audit on Install
|
||||
|
||||
Security is a first-class citizen in ZeroClaw. The `load_skills()` function enforces a security gate via `audit::audit_skill_directory()`.
|
||||
|
||||
What gets blocked:
|
||||
- Symlinks (both for the directory itself and files within)
|
||||
- Unsafe path patterns (traversal attempts)
|
||||
- Insecure script patterns
|
||||
|
||||
The `copy_dir_recursive_secure` function ensures that no symlinks are introduced during the installation process:
|
||||
|
||||
```rust
|
||||
fn copy_dir_recursive_secure(src: &Path, dest: &Path) -> Result<()> {
|
||||
let src_meta = std::fs::symlink_metadata(src)?;
|
||||
if src_meta.file_type().is_symlink() {
|
||||
anyhow::bail!("Refusing to copy symlinked skill source path: {}", src.display());
|
||||
}
|
||||
// ... recursive copy logic ...
|
||||
if metadata.file_type().is_symlink() {
|
||||
anyhow::bail!("Refusing to copy symlink within skill source: {}", src_path.display());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## Open-Skills Repository Sync
|
||||
|
||||
ZeroClaw supports a community-driven repository of skills. The system performs a weekly shallow clone to discover and update these capabilities.
|
||||
|
||||
- **URL**: `https://github.com/besoeasy/open-skills`
|
||||
- **Sync Interval**: 7 days (`60 * 60 * 24 * 7` seconds)
|
||||
- **Mechanism**: `git clone --depth 1` for initialization, followed by periodic `git pull --ff-only`.
|
||||
|
||||
The `ensure_open_skills_repo` function manages this lifecycle, checking a `.zeroclaw-open-skills-sync` marker file to determine if a sync is required.
|
||||
|
||||
***
|
||||
|
||||
## Skill Prompt Injection
|
||||
|
||||
Skills are surfaced to the LLM through XML-structured injection in the system prompt. This is handled by `skills_to_prompt_with_mode()`.
|
||||
|
||||
### Full vs Compact Mode
|
||||
- **Full**: Injects name, description, location, all instructions, and tool metadata.
|
||||
- **Compact**: Injects only name, description, and location. Instructions and tools are loaded on demand by the LLM reading the file.
|
||||
|
||||
```rust
|
||||
pub fn skills_to_prompt_with_mode(
|
||||
skills: &[Skill],
|
||||
workspace_dir: &Path,
|
||||
mode: crate::config::SkillsPromptInjectionMode,
|
||||
) -> String {
|
||||
// ...
|
||||
for skill in skills {
|
||||
let _ = writeln!(prompt, " <skill>");
|
||||
write_xml_text_element(&mut prompt, 4, "name", &skill.name);
|
||||
write_xml_text_element(&mut prompt, 4, "description", &skill.description);
|
||||
// ...
|
||||
if matches!(mode, crate::config::SkillsPromptInjectionMode::Full) {
|
||||
// Injects <instructions> and <tools> tags
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
***
|
||||
|
||||
## CLI Management
|
||||
|
||||
The `handle_command()` function provides a CLI interface for managing the skills lifecycle:
|
||||
- `list`: Displays installed skills, versions, tools, and tags.
|
||||
- `audit`: Manually runs the security audit on a local or installed skill.
|
||||
- `install`: Clones from git or copies from a local path, followed by a mandatory audit.
|
||||
- `remove`: Securely deletes a skill directory, preventing path traversal.
|
||||
|
||||
***
|
||||
|
||||
## Secure Install
|
||||
|
||||
When installing a skill, ZeroClaw uses `install_git_skill_source` or `install_local_skill_source`. Both paths terminate in a mandatory `enforce_skill_security_audit()` call. If the audit fails, the installed files are immediately rolled back (deleted).
|
||||
|
||||
***
|
||||
|
||||
## Relevance to Luna
|
||||
|
||||
Luna's skills system is a planned feature that will adopt the ZeroClaw architecture:
|
||||
- **Manifest**: Adopt the `SKILL.toml` format for interoperability.
|
||||
- **Security**: Implement the same mandatory audit gate and symlink rejection policy.
|
||||
- **Injection**: Use the XML-structured prompt injection pattern to give the LLM clear boundaries for skill usage.
|
||||
|
||||
Cross-links: [[Skills]], [[Core]], [[Configuration]].
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user