diff --git a/Makefile b/Makefile index 2be77f7e..b4d04d13 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ build: deps dev-build: deps @echo "$(OK_COLOR)==> Building the application for Linux...$(NO_COLOR)" - @GOOS=linux go build -v -ldflags="-s -w -X main.Version=$(or $(tag),dev-$(shell git describe --tags --abbrev=0))" -o "$(BUILD_DIR)/$(NAME)" "$(BUILD_SRC)" + @GOOS=linux GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-linux-musl-gcc go build -v -ldflags="-s -w -X main.Version=$(or $(tag),dev-$(shell git describe --tags --abbrev=0))" -o "$(BUILD_DIR)/$(NAME)" "$(BUILD_SRC)" clean: @rm -rf ./bin diff --git a/README.md b/README.md index 5764e5a2..603f8fcf 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Kdeps is loaded with features to streamline AI app development: - 🔄 Use [reusable AI agents](https://kdeps.com/getting-started/resources/remix.html) for flexible workflows. - 🖥️ Run [shell scripts](https://kdeps.com/getting-started/resources/exec.html) seamlessly. - 🌍 Make [API calls](https://kdeps.com/getting-started/resources/client.html) directly from configuration. +- 💾 Manage state with [memory operations](https://kdeps.com/getting-started/resources/memory.html) to store, retrieve, and clear persistent data. - 📊 Generate [structured outputs](https://kdeps.com/getting-started/resources/llm.html#chat-block) from LLMs. - 📦 Install [Ubuntu packages](https://kdeps.com/getting-started/configuration/workflow.html#ubuntu-packages) via configuration. - 📜 Define [Ubuntu repositories or PPAs](https://kdeps.com/getting-started/configuration/workflow.html#ubuntu-repositories). diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 45f00aba..1cb792ea 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -174,6 +174,10 @@ export default defineConfig({ text: "Data Folder", link: "/getting-started/resources/global-functions#data-folder-functions", }, + { + text: "Memory Operations", + link: "/getting-started/resources/global-functions#memory-operation-functions", + }, { text: "JSON Document Parser", link: "/getting-started/resources/global-functions#document-json-parsers", @@ -211,6 +215,8 @@ export default defineConfig({ text: "Preflight Validations", link: "/getting-started/resources/validations", }, + { text: "Memory Operations", link: "/getting-started/resources/memory" }, + { text: "Expr Block", link: "/getting-started/resources/expr" }, { text: "Data Folder", link: "/getting-started/resources/data" }, { text: "File Uploads", link: "/getting-started/tutorials/files" }, { diff --git a/docs/getting-started/resources/expr.md b/docs/getting-started/resources/expr.md new file mode 100644 index 00000000..5728a052 --- /dev/null +++ b/docs/getting-started/resources/expr.md @@ -0,0 +1,45 @@ +--- +outline: deep +--- + +# Expr Block + +The `expr` is resource block for evaluating standard PKL expressions. It is primarily used to execute +expressions that produce side effects, such as updating resources or triggering actions, but also supports +general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and scripting within +a configuration. + +## Overview of the `expr` Block + +The `expr` block is designed to evaluate PKL expressions in a straightforward manner. Its key uses include: + +- **Side-Effecting Operations**: Executing functions like `memory.setItem` that modify resources or state without + returning significant values. + +- **Inline Scripting**: Evaluating arbitrary PKL expressions to implement logic, assignments, or procedural tasks + directly within a configuration. + +The `expr` block simplifies the execution of side-effecting operations that does not makes sense to output it's results. + +## Syntax and Usage + +The `expr` block is defined as follows: + +```apl +expr { + // Valid PKL expression(s) +} +``` + +Each expression within the block is evaluated in sequence, allowing multiple expressions to form a procedural sequence if needed. + +The `expr` block is well-suited for operations that update state, such as setting memory items. + +```apl +expr { + "@(memory.setItem("status", "active"))" +} +``` + +In this example, the memory store is updated to indicate an active status. The `memory.setItem` function is executed as +a side effect, and no return value is required. This also applies to `memory.clear()`. diff --git a/docs/getting-started/resources/global-functions.md b/docs/getting-started/resources/global-functions.md index 76e68911..42e943ee 100644 --- a/docs/getting-started/resources/global-functions.md +++ b/docs/getting-started/resources/global-functions.md @@ -32,6 +32,18 @@ Below is a list of the global functions available for each resource: | request.path() | Retrieves the URI path of the API request. | | request.method() | Retrieves the HTTP method (e.g., GET, POST) of the API request. | +## Memory Operation Functions + +| **Function** | **Description** | +|:-------------------------------|:----------------------------------------------------------| +| memory.getItem("key") | Fetches the value of key from persistent AI Agent memory | +| memory.setItem("key", "value") | Stores the value of key to the persistent AI Agent memory | +| memory.clear() | Clears all persistent memory (CAUTION!) | + +> *Note:* The `setItem` and `clear` are side-effecting functions—it performs an action but doesn't return a +> meaningful value. That is why it is recommended to placed them inside an `expr` block: to ensure the expression is +> evaluated for its effect. + ## Data Folder Functions | **Function** | **Description** | diff --git a/docs/getting-started/resources/memory.md b/docs/getting-started/resources/memory.md new file mode 100644 index 00000000..e6ece7fc --- /dev/null +++ b/docs/getting-started/resources/memory.md @@ -0,0 +1,78 @@ +--- +outline: deep +--- + +# Memory Operations + +Memory operations provide a way to store, retrieve, and clear key-value pairs in a persistent memory store. These +operations are useful for managing state or caching data across different executions or sessions. + +The memory operations include `getItem`, `setItem`, and `clear`, which allow you to interact with the memory store +efficiently. + +## Memory Operation Functions + +Below are the available memory operation functions, their purposes, and how to use them. + +### `getItem(id: String): String` + +Retrieves the textual content of a memory item by its identifier. + +- **Parameters**: + - `id`: The identifier of the memory item. +- **Returns**: The textual content of the memory entry, or an empty string if not found. + +#### Example: Retrieving a Stored Value + +```apl +local taskContext = "@(memory.getItem("task_123_context"))" +``` + +In this example, the `getItem` function retrieves the `task_123_context` record item. +> **Note:** Because Apple PKL uses late binding, the taskContext expression won’t be evaluated until it is actually +> accessed—for example, when included in a response output or passed into an LLM prompt. + +### `setItem(id: String, value: String): String` + +Sets or updates a memory item with a new value. + +- **Parameters**: + - `id`: The identifier of the memory item. + - `value`: The value to store. +- **Returns**: The set value as confirmation. + +#### Example: Storing a Value + +```apl +expr { + taskId = "task_123" + result = "completed_successfully" + "@(memory.setItem(taskId, result))" +} +``` + +In this example, `memory.setItem` stores the value `"completed_successfully"` under the key `"task_123"` in memory. + +We use the `exp`r block because `setItem` is a side-effecting function—it performs an action but doesn't return a +meaningful value. That is why it's placed inside an `expr` block: to ensure the expression is evaluated for its effect +rather than for a result that would otherwise be ignored. + +### `clear(): String` + +Clears all memory items in the store. + +- **Returns**: A confirmation message. + +#### Example: Resetting All Stored Data + +```apl +clear() +``` + +This example clears all memory items, resetting the memory store to an empty state. A confirmation message is returned. + +## Notes + +- The `getItem` and `setItem` functions operate on string-based key-value pairs. +- The `clear` function removes all stored items, so use it cautiously to avoid unintended data loss. +- Memory operations are synchronous and return immediately with the result or confirmation. diff --git a/docs/index.md b/docs/index.md index a363e6f3..44203240 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,6 +25,7 @@ Kdeps is loaded with features to streamline AI app development: - 🔄 Use [reusable AI agents](/getting-started/resources/remix.md) for flexible workflows. - 🖥️ Run [shell scripts](/getting-started/resources/exec.md) seamlessly. - 🌍 Make [API calls](/getting-started/resources/client.md) directly from configuration. +- 💾 Manage state with [memory operations](/getting-started/resources/memory.md) to store, retrieve, and clear persistent data. - 📊 Generate [structured outputs](/getting-started/resources/llm.md#chat-block) from LLMs. - 📦 Install [Ubuntu packages](/getting-started/configuration/workflow.md#ubuntu-packages) via configuration. - 📜 Define [Ubuntu repositories or PPAs](/getting-started/configuration/workflow.md#ubuntu-repositories). diff --git a/go.mod b/go.mod index 503881a3..491d80ce 100644 --- a/go.mod +++ b/go.mod @@ -23,8 +23,9 @@ require ( github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 github.com/kdeps/kartographer v0.0.0-20240808015651-b2afd5d97715 - github.com/kdeps/schema v0.2.20 + github.com/kdeps/schema v0.2.23 github.com/kr/pretty v0.3.1 + github.com/mattn/go-sqlite3 v1.14.28 github.com/spf13/afero v1.14.0 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 @@ -39,11 +40,11 @@ require ( github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/charmbracelet/bubbles v0.21.0 // indirect - github.com/charmbracelet/bubbletea v1.3.4 // indirect + github.com/charmbracelet/bubbletea v1.3.5 // indirect github.com/charmbracelet/colorprofile v0.3.1 // indirect - github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/ansi v0.9.2 // indirect github.com/charmbracelet/x/cellbuf v0.0.13 // indirect - github.com/charmbracelet/x/exp/strings v0.0.0-20250424195755-e256bf9b4ee5 // indirect + github.com/charmbracelet/x/exp/strings v0.0.0-20250505150409-97991a1f17d1 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cloudwego/base64x v0.1.5 // indirect github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect @@ -104,13 +105,13 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.29.0 // indirect go.opentelemetry.io/otel/metric v1.35.0 // indirect go.opentelemetry.io/otel/trace v1.35.0 // indirect - golang.org/x/arch v0.16.0 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/arch v0.17.0 // indirect + golang.org/x/crypto v0.38.0 // indirect + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.1 // indirect diff --git a/go.sum b/go.sum index 710ca589..bb5ec38a 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs= github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg= -github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI= -github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo= +github.com/charmbracelet/bubbletea v1.3.5 h1:JAMNLTbqMOhSwoELIr0qyP4VidFq72/6E9j7HHmRKQc= +github.com/charmbracelet/bubbletea v1.3.5/go.mod h1:TkCnmH+aBd4LrXhXcqrKiYwRs7qyQx5rBgH5fVY3v54= github.com/charmbracelet/colorprofile v0.3.1 h1:k8dTHMd7fgw4bnFd7jXTLZrSU/CQrKnL3m+AxCzDz40= github.com/charmbracelet/colorprofile v0.3.1/go.mod h1:/GkGusxNs8VB/RSOh3fu0TJmQ4ICMMPApIIVn0KszZ0= github.com/charmbracelet/huh v0.7.0 h1:W8S1uyGETgj9Tuda3/JdVkc3x7DBLZYPZc4c+/rnRdc= @@ -40,8 +40,8 @@ github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoF github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= github.com/charmbracelet/log v0.4.1 h1:6AYnoHKADkghm/vt4neaNEXkxcXLSV2g1rdyFDOpTyk= github.com/charmbracelet/log v0.4.1/go.mod h1:pXgyTsqsVu4N9hGdHmQ0xEA4RsXof402LX9ZgiITn2I= -github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= -github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/ansi v0.9.2 h1:92AGsQmNTRMzuzHEYfCdjQeUzTrgE1vfO5/7fEVoXdY= +github.com/charmbracelet/x/ansi v0.9.2/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= @@ -52,8 +52,8 @@ github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9 github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/exp/strings v0.0.0-20250424195755-e256bf9b4ee5 h1:1kNgPePfZiKvvZYGDl7T4T4RqOJLUYrNXvYZAbl1jkg= -github.com/charmbracelet/x/exp/strings v0.0.0-20250424195755-e256bf9b4ee5/go.mod h1:Rgw3/F+xlcUc5XygUtimVSxAqCOsqyvJjqF5UHRvc5k= +github.com/charmbracelet/x/exp/strings v0.0.0-20250505150409-97991a1f17d1 h1:EFaxes9CfsRKhJZeQHBhTYu7sZubVs9RO5roVdPZlDc= +github.com/charmbracelet/x/exp/strings v0.0.0-20250505150409-97991a1f17d1/go.mod h1:Rgw3/F+xlcUc5XygUtimVSxAqCOsqyvJjqF5UHRvc5k= github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= @@ -152,8 +152,10 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kdeps/kartographer v0.0.0-20240808015651-b2afd5d97715 h1:CxUIVGV6VdgZo62Q84pOVJwUa0ONNqJIH3/rvWsAiUs= github.com/kdeps/kartographer v0.0.0-20240808015651-b2afd5d97715/go.mod h1:DYSCAer2OsX5F3Jne82p4P1LCIu42DQFfL5ypZYcUbk= -github.com/kdeps/schema v0.2.20 h1:BgjP4HBq0bUV7CfsP/c4K2FDYXha3qokX2grDKtItec= -github.com/kdeps/schema v0.2.20/go.mod h1:jcI+1Q8GAor+pW+RxPG9EJDM5Ji+GUORirTCSslfH0M= +github.com/kdeps/schema v0.2.22 h1:UIOfmy5gIgMInvi5snhsfifzjfHc6KoBwY7sJq2yEvA= +github.com/kdeps/schema v0.2.22/go.mod h1:jcI+1Q8GAor+pW+RxPG9EJDM5Ji+GUORirTCSslfH0M= +github.com/kdeps/schema v0.2.23 h1:NbwlJL5tEEOVcCnYWL6yiTe/ZjULrWWluOB0/0zLjpA= +github.com/kdeps/schema v0.2.23/go.mod h1:jcI+1Q8GAor+pW+RxPG9EJDM5Ji+GUORirTCSslfH0M= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -177,6 +179,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -275,39 +279,39 @@ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U= -golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= +golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/docker/image.go b/pkg/docker/image.go index 4fc98322..c2abe0c5 100644 --- a/pkg/docker/image.go +++ b/pkg/docker/image.go @@ -221,8 +221,8 @@ ENV TZ=%s RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \ bzip2 ca-certificates git subversion mercurial libglib2.0-0 \ libsm6 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxinerama1 libxrandr2 libxrender1 \ - gpg-agent openssh-client procps software-properties-common wget curl nano jq python3 python3-pip - + gpg-agent openssh-client procps software-properties-common wget curl nano jq python3 python3-pip musl musl-dev \ + musl-tools `) if useLatest { diff --git a/pkg/memory/memory.go b/pkg/memory/memory.go new file mode 100644 index 00000000..e62f8bc2 --- /dev/null +++ b/pkg/memory/memory.go @@ -0,0 +1,221 @@ +package memory + +import ( + "database/sql" + "errors" + "fmt" + "log" + "net/url" + "strings" + "time" + + "github.com/apple/pkl-go/pkl" + _ "github.com/mattn/go-sqlite3" +) + +// PklResourceReader implements the pkl.ResourceReader interface for SQLite. +type PklResourceReader struct { + DB *sql.DB + DBPath string // Store dbPath for reinitialization +} + +// Scheme returns the URI scheme for this reader. +func (r *PklResourceReader) Scheme() string { + return "memory" +} + +// IsGlobbable indicates whether the reader supports globbing (not needed here). +func (r *PklResourceReader) IsGlobbable() bool { + return false +} + +// HasHierarchicalUris indicates whether URIs are hierarchical (not needed here). +func (r *PklResourceReader) HasHierarchicalUris() bool { + return false +} + +// ListElements is not used in this implementation. +func (r *PklResourceReader) ListElements(_ url.URL) ([]pkl.PathElement, error) { + return nil, nil +} + +// Read retrieves, sets, or clears items in the SQLite database based on the URI. +func (r *PklResourceReader) Read(uri url.URL) ([]byte, error) { + // Check if receiver is nil and initialize with fixed DBPath + if r == nil { + log.Printf("Warning: PklResourceReader is nil for URI: %s, initializing with DBPath", uri.String()) + newReader, err := InitializeMemory(r.DBPath) + if err != nil { + log.Printf("Failed to initialize PklResourceReader in Read: %v", err) + return nil, fmt.Errorf("failed to initialize PklResourceReader: %w", err) + } + r = newReader + log.Printf("Initialized PklResourceReader with DBPath") + } + + // Check if db is nil and initialize with retries + if r.DB == nil { + log.Printf("Database connection is nil, attempting to initialize with path: %s", r.DBPath) + maxAttempts := 5 + for attempt := 1; attempt <= maxAttempts; attempt++ { + db, err := InitializeDatabase(r.DBPath) + if err == nil { + r.DB = db + log.Printf("Database initialized successfully in Read on attempt %d", attempt) + break + } + log.Printf("Attempt %d: Failed to initialize database in Read: %v", attempt, err) + if attempt == maxAttempts { + return nil, fmt.Errorf("failed to initialize database after %d attempts: %w", maxAttempts, err) + } + time.Sleep(1 * time.Second) + } + } + + id := strings.TrimPrefix(uri.Path, "/") + query := uri.Query() + operation := query.Get("op") + + log.Printf("Read called with URI: %s, operation: %s", uri.String(), operation) + + switch operation { + case "set": + if id == "" { + log.Printf("setItem failed: no item ID provided") + return nil, errors.New("invalid URI: no item ID provided for set operation") + } + newValue := query.Get("value") + if newValue == "" { + log.Printf("setItem failed: no value provided") + return nil, errors.New("set operation requires a value parameter") + } + + log.Printf("setItem processing id: %s, value: %s", id, newValue) + + result, err := r.DB.Exec( + "INSERT OR REPLACE INTO items (id, value) VALUES (?, ?)", + id, newValue, + ) + if err != nil { + log.Printf("setItem failed to execute SQL: %v", err) + return nil, fmt.Errorf("failed to set item: %w", err) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + log.Printf("setItem failed to check result: %v", err) + return nil, fmt.Errorf("failed to check set result: %w", err) + } + if rowsAffected == 0 { + log.Printf("setItem: no item set for ID %s", id) + return nil, fmt.Errorf("no item set for ID %s", id) + } + + log.Printf("setItem succeeded for id: %s, value: %s", id, newValue) + return []byte(newValue), nil + + case "clear": + if id != "_" { + log.Printf("clear failed: invalid path, expected '/_'") + return nil, errors.New("invalid URI: clear operation requires path '/_'") + } + + log.Printf("clear processing") + + result, err := r.DB.Exec("DELETE FROM items") + if err != nil { + log.Printf("clear failed to execute SQL: %v", err) + return nil, fmt.Errorf("failed to clear items: %w", err) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + log.Printf("clear failed to check result: %v", err) + return nil, fmt.Errorf("failed to check clear result: %w", err) + } + + log.Printf("clear succeeded, removed %d items", rowsAffected) + return []byte(fmt.Sprintf("Cleared %d items", rowsAffected)), nil + + default: // getItem (no operation specified) + if id == "" { + log.Printf("getItem failed: no item ID provided") + return nil, errors.New("invalid URI: no item ID provided") + } + + log.Printf("getItem processing id: %s", id) + + var value string + err := r.DB.QueryRow("SELECT value FROM items WHERE id = ?", id).Scan(&value) + if err == sql.ErrNoRows { + log.Printf("getItem: no item found for id: %s", id) + return []byte(""), nil // Return empty string for not found + } + if err != nil { + log.Printf("getItem failed to read item for id: %s, error: %v", id, err) + return nil, fmt.Errorf("failed to read item: %w", err) + } + + log.Printf("getItem succeeded for id: %s, value: %s", id, value) + return []byte(value), nil + } +} + +// InitializeDatabase sets up the SQLite database and creates the items table with retries. +func InitializeDatabase(dbPath string) (*sql.DB, error) { + const maxAttempts = 5 + for attempt := 1; attempt <= maxAttempts; attempt++ { + log.Printf("Attempt %d: Initializing SQLite database at %s", attempt, dbPath) + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + log.Printf("Attempt %d: Failed to open database: %v", attempt, err) + if attempt == maxAttempts { + return nil, fmt.Errorf("failed to open database after %d attempts: %w", maxAttempts, err) + } + time.Sleep(1 * time.Second) + continue + } + + // Verify connection + if err := db.Ping(); err != nil { + log.Printf("Attempt %d: Failed to ping database: %v", attempt, err) + db.Close() + if attempt == maxAttempts { + return nil, fmt.Errorf("failed to ping database after %d attempts: %w", maxAttempts, err) + } + time.Sleep(1 * time.Second) + continue + } + + // Create items table + _, err = db.Exec(` + CREATE TABLE IF NOT EXISTS items ( + id TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `) + if err != nil { + log.Printf("Attempt %d: Failed to create items table: %v", attempt, err) + db.Close() + if attempt == maxAttempts { + return nil, fmt.Errorf("failed to create items table after %d attempts: %w", maxAttempts, err) + } + time.Sleep(1 * time.Second) + continue + } + + log.Printf("SQLite database initialized successfully at %s on attempt %d", dbPath, attempt) + return db, nil + } + return nil, fmt.Errorf("failed to initialize database after %d attempts", maxAttempts) +} + +// InitializeMemory creates a new PklResourceReader with an initialized SQLite database. +func InitializeMemory(dbPath string) (*PklResourceReader, error) { + db, err := InitializeDatabase(dbPath) + if err != nil { + return nil, fmt.Errorf("error initializing database: %w", err) + } + // Do NOT close db here; caller will manage closing + return &PklResourceReader{DB: db, DBPath: dbPath}, nil +} diff --git a/pkg/resolver/imports.go b/pkg/resolver/imports.go index 8d3bccec..dd93e8ee 100644 --- a/pkg/resolver/imports.go +++ b/pkg/resolver/imports.go @@ -48,6 +48,7 @@ func (dr *DependencyResolver) PrependDynamicImports(pklFile string) error { "pkl:xml": {Alias: "", Check: false}, "pkl:yaml": {Alias: "", Check: false}, fmt.Sprintf("package://schema.kdeps.com/core@%s#/Document.pkl", schema.SchemaVersion(dr.Context)): {Alias: "document", Check: false}, + fmt.Sprintf("package://schema.kdeps.com/core@%s#/Memory.pkl", schema.SchemaVersion(dr.Context)): {Alias: "memory", Check: false}, fmt.Sprintf("package://schema.kdeps.com/core@%s#/Skip.pkl", schema.SchemaVersion(dr.Context)): {Alias: "skip", Check: false}, fmt.Sprintf("package://schema.kdeps.com/core@%s#/Utils.pkl", schema.SchemaVersion(dr.Context)): {Alias: "utils", Check: false}, filepath.Join(dr.ActionDir, "/llm/"+dr.RequestID+"__llm_output.pkl"): {Alias: "llm", Check: true}, diff --git a/pkg/resolver/resolver.go b/pkg/resolver/resolver.go index 12bed6cb..8c5c4a16 100644 --- a/pkg/resolver/resolver.go +++ b/pkg/resolver/resolver.go @@ -15,6 +15,7 @@ import ( "github.com/kdeps/kdeps/pkg/environment" "github.com/kdeps/kdeps/pkg/ktx" "github.com/kdeps/kdeps/pkg/logging" + "github.com/kdeps/kdeps/pkg/memory" "github.com/kdeps/kdeps/pkg/utils" pklRes "github.com/kdeps/schema/gen/resource" pklWf "github.com/kdeps/schema/gen/workflow" @@ -33,6 +34,9 @@ type DependencyResolver struct { Environment *environment.Environment Workflow pklWf.Workflow Request *gin.Context + MemoryReader *memory.PklResourceReader + MemoryDBPath string + AgentName string RequestID string RequestPklFile string ResponsePklFile string @@ -105,6 +109,27 @@ func NewGraphResolver(fs afero.Fs, ctx context.Context, env *environment.Environ responsePklFile := filepath.Join(actionDir, "/api/"+graphID+"__response.pkl") responseTargetFile := filepath.Join(actionDir, "/api/"+graphID+"__response.json") + workflowConfiguration, err := pklWf.LoadFromPath(ctx, pklWfFile) + if err != nil { + return nil, err + } + + var apiServerMode, installAnaconda bool + var agentName, memoryDBPath string + + if workflowConfiguration.GetSettings() != nil { + apiServerMode = workflowConfiguration.GetSettings().APIServerMode + agentSettings := workflowConfiguration.GetSettings().AgentSettings + installAnaconda = agentSettings.InstallAnaconda + agentName = workflowConfiguration.GetName() + } + + memoryDBPath = filepath.Join("/root/.kdeps", agentName+"_memory.db") + reader, err := memory.InitializeMemory(memoryDBPath) + if err != nil { + return nil, fmt.Errorf("failed to initialize DB memory: %w", err) + } + dependencyResolver := &DependencyResolver{ Fs: fs, ResourceDependencies: make(map[string][]string), @@ -123,18 +148,12 @@ func NewGraphResolver(fs afero.Fs, ctx context.Context, env *environment.Environ ResponseTargetFile: responseTargetFile, ProjectDir: projectDir, Request: req, - } - - workflowConfiguration, err := pklWf.LoadFromPath(ctx, pklWfFile) - if err != nil { - return nil, err - } - dependencyResolver.Workflow = workflowConfiguration - if workflowConfiguration.GetSettings() != nil { - dependencyResolver.APIServerMode = workflowConfiguration.GetSettings().APIServerMode - - agentSettings := workflowConfiguration.GetSettings().AgentSettings - dependencyResolver.AnacondaInstalled = agentSettings.InstallAnaconda + Workflow: workflowConfiguration, + APIServerMode: apiServerMode, + AnacondaInstalled: installAnaconda, + AgentName: agentName, + MemoryDBPath: memoryDBPath, + MemoryReader: reader, } dependencyResolver.Graph = graph.NewDependencyGraph(fs, logger.BaseLogger(), dependencyResolver.ResourceDependencies) @@ -263,7 +282,16 @@ func (dr *DependencyResolver) HandleRunAction() (bool, error) { continue } - rsc, err := pklRes.LoadFromPath(dr.Context, res.File) + resPkl, err := dr.LoadResource(dr.Context, res.File, Resource) + if err != nil { + return dr.HandleAPIErrorResponse(500, err.Error(), true) + } + + rsc, ok := resPkl.(*pklRes.Resource) + if !ok { + return dr.HandleAPIErrorResponse(500, err.Error(), true) + } + if err != nil { return dr.HandleAPIErrorResponse(500, err.Error(), true) } diff --git a/pkg/resolver/resource_chat.go b/pkg/resolver/resource_chat.go index 9c6a20fd..0a3d1278 100644 --- a/pkg/resolver/resource_chat.go +++ b/pkg/resolver/resource_chat.go @@ -243,11 +243,16 @@ func processScenarioMessages(scenario *[]*pklLLM.MultiChat, logger *logging.Logg func (dr *DependencyResolver) AppendChatEntry(resourceID string, newChat *pklLLM.ResourceChat) error { pklPath := filepath.Join(dr.ActionDir, "llm/"+dr.RequestID+"__llm_output.pkl") - pklRes, err := pklLLM.LoadFromPath(dr.Context, pklPath) + llmRes, err := dr.LoadResource(dr.Context, pklPath, LLMResource) if err != nil { return fmt.Errorf("failed to load PKL file: %w", err) } + pklRes, ok := llmRes.(*pklLLM.LLMImpl) + if !ok { + return errors.New("failed to cast pklRes to *pklLLM.Resource") + } + resources := pklRes.GetResources() if resources == nil { emptyMap := make(map[string]*pklLLM.ResourceChat) diff --git a/pkg/resolver/resource_exec.go b/pkg/resolver/resource_exec.go index 7884aaf5..0873511b 100644 --- a/pkg/resolver/resource_exec.go +++ b/pkg/resolver/resource_exec.go @@ -1,6 +1,7 @@ package resolver import ( + "errors" "fmt" "path/filepath" "strings" @@ -128,9 +129,14 @@ func (dr *DependencyResolver) WriteStdoutToFile(resourceID string, stdoutEncoded func (dr *DependencyResolver) AppendExecEntry(resourceID string, newExec *pklExec.ResourceExec) error { pklPath := filepath.Join(dr.ActionDir, "exec/"+dr.RequestID+"__exec_output.pkl") - pklRes, err := pklExec.LoadFromPath(dr.Context, pklPath) + res, err := dr.LoadResource(dr.Context, pklPath, ExecResource) if err != nil { - return fmt.Errorf("failed to load PKL file: %w", err) + return fmt.Errorf("failed to load PKL: %w", err) + } + + pklRes, ok := res.(*pklExec.ExecImpl) + if !ok { + return errors.New("failed to cast pklRes to *pklExec.ExecImpl") } resources := pklRes.GetResources() diff --git a/pkg/resolver/resource_http.go b/pkg/resolver/resource_http.go index c6deb5fc..841c623a 100644 --- a/pkg/resolver/resource_http.go +++ b/pkg/resolver/resource_http.go @@ -91,11 +91,16 @@ func (dr *DependencyResolver) WriteResponseBodyToFile(resourceID string, respons func (dr *DependencyResolver) AppendHTTPEntry(resourceID string, client *pklHTTP.ResourceHTTPClient) error { pklPath := filepath.Join(dr.ActionDir, "client/"+dr.RequestID+"__client_output.pkl") - pklRes, err := pklHTTP.LoadFromPath(dr.Context, pklPath) + res, err := dr.LoadResource(dr.Context, pklPath, HTTPResource) if err != nil { return fmt.Errorf("failed to load PKL: %w", err) } + pklRes, ok := res.(*pklHTTP.HTTPImpl) + if !ok { + return errors.New("failed to cast pklRes to *pklHTTP.Resource") + } + resources := pklRes.GetResources() if resources == nil { emptyMap := make(map[string]*pklHTTP.ResourceHTTPClient) diff --git a/pkg/resolver/resource_python.go b/pkg/resolver/resource_python.go index b1c3bb1e..57a5386b 100644 --- a/pkg/resolver/resource_python.go +++ b/pkg/resolver/resource_python.go @@ -2,6 +2,7 @@ package resolver import ( "context" + "errors" "fmt" "path/filepath" "strings" @@ -201,9 +202,14 @@ func (dr *DependencyResolver) WritePythonStdoutToFile(resourceID string, stdoutE func (dr *DependencyResolver) AppendPythonEntry(resourceID string, newPython *pklPython.ResourcePython) error { pklPath := filepath.Join(dr.ActionDir, "python/"+dr.RequestID+"__python_output.pkl") - pklRes, err := pklPython.LoadFromPath(dr.Context, pklPath) + res, err := dr.LoadResource(dr.Context, pklPath, PythonResource) if err != nil { - return fmt.Errorf("failed to load PKL file: %w", err) + return fmt.Errorf("failed to load PKL: %w", err) + } + + pklRes, ok := res.(*pklPython.PythonImpl) + if !ok { + return errors.New("failed to cast pklRes to *pklPython.Resource") } resources := pklRes.GetResources() diff --git a/pkg/resolver/resource_response.go b/pkg/resolver/resource_response.go index 193081d3..64405163 100644 --- a/pkg/resolver/resource_response.go +++ b/pkg/resolver/resource_response.go @@ -52,6 +52,7 @@ func (dr *DependencyResolver) ensureResponsePklFileNotExists() error { func (dr *DependencyResolver) buildResponseSections(requestID string, apiResponseBlock apiserverresponse.APIServerResponse) []string { sections := []string{ fmt.Sprintf(`import "package://schema.kdeps.com/core@%s#/Document.pkl" as document`, schema.SchemaVersion(dr.Context)), + fmt.Sprintf(`import "package://schema.kdeps.com/core@%s#/Memory.pkl" as memory`, schema.SchemaVersion(dr.Context)), fmt.Sprintf("success = %v", apiResponseBlock.GetSuccess()), formatResponseMeta(requestID, apiResponseBlock.GetMeta()), formatResponseData(apiResponseBlock.GetResponse()), diff --git a/pkg/resolver/resources.go b/pkg/resolver/resources.go index 005ebb0d..686a4791 100644 --- a/pkg/resolver/resources.go +++ b/pkg/resolver/resources.go @@ -1,14 +1,32 @@ package resolver import ( + "context" + "errors" "fmt" "os" "path/filepath" - "github.com/kdeps/kdeps/pkg/resource" + "github.com/apple/pkl-go/pkl" + pklExec "github.com/kdeps/schema/gen/exec" + pklHTTP "github.com/kdeps/schema/gen/http" + pklLLM "github.com/kdeps/schema/gen/llm" + pklPython "github.com/kdeps/schema/gen/python" + pklResource "github.com/kdeps/schema/gen/resource" "github.com/spf13/afero" ) +// ResourceType defines the type of resource to load. +type ResourceType string + +const ( + ExecResource ResourceType = "exec" + PythonResource ResourceType = "python" + LLMResource ResourceType = "llm" + HTTPResource ResourceType = "http" + Resource ResourceType = "resource" +) + // LoadResourceEntries loads .pkl resource files from the resources directory. func (dr *DependencyResolver) LoadResourceEntries() error { workflowDir := filepath.Join(dr.WorkflowDir, "resources") @@ -67,9 +85,14 @@ func (dr *DependencyResolver) handleFileImports(path string) error { // processPklFile processes an individual .pkl file and updates dependencies. func (dr *DependencyResolver) processPklFile(file string) error { // Load the resource file - pklRes, err := resource.LoadResource(dr.Context, file, dr.Logger) + res, err := dr.LoadResource(dr.Context, file, Resource) if err != nil { - return fmt.Errorf("failed to load resource from .pkl file %s: %w", file, err) + return fmt.Errorf("failed to load PKL file: %w", err) + } + + pklRes, ok := res.(*pklResource.Resource) + if !ok { + return errors.New("failed to cast pklRes to *pklLLM.Resource") } // Append the resource to the list of resources @@ -87,3 +110,93 @@ func (dr *DependencyResolver) processPklFile(file string) error { return nil } + +// LoadResource reads a resource file and returns the parsed resource object or an error. +func (dr *DependencyResolver) LoadResource(ctx context.Context, resourceFile string, resourceType ResourceType) (interface{}, error) { + // Log additional info before reading the resource + dr.Logger.Debug("reading resource file", "resource-file", resourceFile, "resource-type", resourceType) + + // Define an option function to configure EvaluatorOptions + opts := func(options *pkl.EvaluatorOptions) { + pkl.WithDefaultAllowedResources(options) + pkl.WithOsEnv(options) + pkl.WithDefaultAllowedModules(options) + pkl.WithDefaultCacheDir(options) + options.Logger = pkl.NoopLogger + options.ResourceReaders = []pkl.ResourceReader{ + dr.MemoryReader, + } + options.AllowedResources = []string{ + "memory:/", + "package://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.uri@1.0.3", + "https://pkg.pkl-lang.org/pkl-pantry/pkl.experimental.uri@1.0.3", + "https://github.com/apple/pkl-pantry/releases/download/pkl.experimental.uri@1.0.3/pkl.experimental.uri@1.0.3.zip", + } + } + + // Create evaluator with custom options + evaluator, err := pkl.NewEvaluator(ctx, opts) + if err != nil { + dr.Logger.Error("error creating evaluator", "error", err) + return nil, fmt.Errorf("error creating evaluator: %w", err) + } + defer func() { + if cerr := evaluator.Close(); cerr != nil && err == nil { + err = cerr + dr.Logger.Error("error closing evaluator", "error", err) + } + }() + + // Load the resource based on the resource type + source := pkl.FileSource(resourceFile) + switch resourceType { + case Resource: + res, err := pklResource.Load(ctx, evaluator, source) + if err != nil { + dr.Logger.Error("error reading resource file", "resource-file", resourceFile, "error", err) + return nil, fmt.Errorf("error reading resource file '%s': %w", resourceFile, err) + } + dr.Logger.Debug("successfully loaded resource", "resource-file", resourceFile) + return res, nil + + case ExecResource: + res, err := pklExec.Load(ctx, evaluator, source) + if err != nil { + dr.Logger.Error("error reading exec resource file", "resource-file", resourceFile, "error", err) + return nil, fmt.Errorf("error reading exec resource file '%s': %w", resourceFile, err) + } + dr.Logger.Debug("successfully loaded exec resource", "resource-file", resourceFile) + return res, nil + + case PythonResource: + res, err := pklPython.Load(ctx, evaluator, source) + if err != nil { + dr.Logger.Error("error reading python resource file", "resource-file", resourceFile, "error", err) + return nil, fmt.Errorf("error reading python resource file '%s': %w", resourceFile, err) + } + dr.Logger.Debug("successfully loaded python resource", "resource-file", resourceFile) + return res, nil + + case LLMResource: + res, err := pklLLM.Load(ctx, evaluator, source) + if err != nil { + dr.Logger.Error("error reading llm resource file", "resource-file", resourceFile, "error", err) + return nil, fmt.Errorf("error reading llm resource file '%s': %w", resourceFile, err) + } + dr.Logger.Debug("successfully loaded llm resource", "resource-file", resourceFile) + return res, nil + + case HTTPResource: + res, err := pklHTTP.Load(ctx, evaluator, source) + if err != nil { + dr.Logger.Error("error reading http resource file", "resource-file", resourceFile, "error", err) + return nil, fmt.Errorf("error reading http resource file '%s': %w", resourceFile, err) + } + dr.Logger.Debug("successfully loaded http resource", "resource-file", resourceFile) + return res, nil + + default: + dr.Logger.Error("unknown resource type", "resource-type", resourceType) + return nil, fmt.Errorf("unknown resource type: %s", resourceType) + } +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 0c5e1296..bb45b021 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -12,7 +12,7 @@ import ( var ( cachedVersion string once sync.Once - specifiedVersion string = "0.2.20" // Default specified version + specifiedVersion string = "0.2.23" // Default specified version UseLatest bool = false ) diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index 7ee07b98..40aa0081 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -12,8 +12,8 @@ func TestSchemaVersion(t *testing.T) { t.Parallel() ctx := context.Background() - const mockLockedVersion = "0.2.20" // Define the version once and reuse it - const mockVersion = "0.2.20" // Define the version once and reuse it + const mockLockedVersion = "0.2.23" // Define the version once and reuse it + const mockVersion = "0.2.23" // Define the version once and reuse it // Save the original value of UseLatest to avoid test interference originalUseLatest := UseLatest diff --git a/pkg/template/templates/client.pkl b/pkg/template/templates/client.pkl index ac1a704b..a2ee84b8 100644 --- a/pkg/template/templates/client.pkl +++ b/pkg/template/templates/client.pkl @@ -41,6 +41,7 @@ run { // If any evaluated condition returns true, the resource execution will be bypassed. // "@(request.path)" != "/api/v1/whois" && "@(request.method)" != "GET" } + preflightCheck { validations { // This section expects boolean validations. @@ -54,6 +55,15 @@ run { } } + // The expr block is a dedicated resource for evaluating standard PKL expressions. It is primarily used to execute + // expressions that produce side effects, such as updating resources or triggering actions, but also supports + // general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and + // scripting within a configuration. + expr { + // "@(memory.setItem("foo", "bar"))" + // "@(memory.clear())" + } + // Initiates an HTTP client request for this resource. // // The HTTP resource provides the following helper functions: diff --git a/pkg/template/templates/exec.pkl b/pkg/template/templates/exec.pkl index 0811ee3a..10c562f1 100644 --- a/pkg/template/templates/exec.pkl +++ b/pkg/template/templates/exec.pkl @@ -40,6 +40,7 @@ run { // Conditions under which the execution of this resource should be skipped. // If any evaluated condition returns true, the resource execution will be bypassed. } + preflightCheck { validations { // This section expects boolean validations. @@ -56,6 +57,15 @@ run { } } + // The expr block is a dedicated resource for evaluating standard PKL expressions. It is primarily used to execute + // expressions that produce side effects, such as updating resources or triggering actions, but also supports + // general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and + // scripting within a configuration. + expr { + // "@(memory.setItem("foo", "bar"))" + // "@(memory.clear())" + } + // Initiates a shell session for executing commands within this resource. Any packages // defined in the workflow are accessible here. // diff --git a/pkg/template/templates/llm.pkl b/pkg/template/templates/llm.pkl index 5d3a8380..e88488fc 100644 --- a/pkg/template/templates/llm.pkl +++ b/pkg/template/templates/llm.pkl @@ -41,6 +41,7 @@ run { // Conditions under which the execution of this resource should be skipped. // If any evaluated condition returns true, the resource execution will be bypassed. } + preflightCheck { validations { // This section expects boolean validations. @@ -53,6 +54,15 @@ run { } } + // The expr block is a dedicated resource for evaluating standard PKL expressions. It is primarily used to execute + // expressions that produce side effects, such as updating resources or triggering actions, but also supports + // general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and + // scripting within a configuration. + expr { + // "@(memory.setItem("foo", "bar"))" + // "@(memory.clear())" + } + // Initializes a chat session with the LLM for this resource. // // This resource offers the following helper functions: diff --git a/pkg/template/templates/python.pkl b/pkg/template/templates/python.pkl index 1938e0e7..15ce45ad 100644 --- a/pkg/template/templates/python.pkl +++ b/pkg/template/templates/python.pkl @@ -40,6 +40,7 @@ run { // Conditions under which the execution of this resource should be skipped. // If any evaluated condition returns true, the resource execution will be bypassed. } + preflightCheck { validations { // This section expects boolean validations. @@ -56,6 +57,15 @@ run { } } + // The expr block is a dedicated resource for evaluating standard PKL expressions. It is primarily used to execute + // expressions that produce side effects, such as updating resources or triggering actions, but also supports + // general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and + // scripting within a configuration. + expr { + // "@(memory.setItem("foo", "bar"))" + // "@(memory.clear())" + } + // Initiates a shell session for executing commands within this resource. Any packages // defined in the workflow are accessible here. // diff --git a/pkg/template/templates/response.pkl b/pkg/template/templates/response.pkl index 938b690c..6cce7008 100644 --- a/pkg/template/templates/response.pkl +++ b/pkg/template/templates/response.pkl @@ -47,6 +47,7 @@ run { // Conditions under which the execution of this resource should be skipped. // If any evaluated condition returns true, the resource execution will be bypassed. } + preflightCheck { validations { // This section expects boolean validations. @@ -59,6 +60,15 @@ run { } } + // The expr block is a dedicated resource for evaluating standard PKL expressions. It is primarily used to execute + // expressions that produce side effects, such as updating resources or triggering actions, but also supports + // general-purpose evaluation of any valid PKL expression, making it a versatile tool for inline logic and + // scripting within a configuration. + expr { + // "@(memory.setItem("foo", "bar"))" + // "@(memory.clear())" + } + // Initializes an api response for this agent. // // This resource action is straightforward. It @@ -99,6 +109,7 @@ run { response { data { "@(llm.response("llmResource"))" + // "@(memory.getItem("foo"))" // "@(python.stdout("pythonResource"))" // "@(exec.stdout("shellResource"))" // "@(client.responseBody("httpResource"))"