diff --git a/autogen.sh b/autogen.sh index 47736e2..318d661 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # This file is part of SmallBASIC # # Copyright(C) 2001-2020 Chris Warren-Smith. diff --git a/configure.ac b/configure.ac index a9f12a5..e3f5f79 100644 --- a/configure.ac +++ b/configure.ac @@ -33,6 +33,27 @@ function checkDebugMode() { AC_SUBST(CFLAGS) } +function generate_wayland_protocols() { + RAYLIB_SRC_PATH="${srcdir}/raylib/raylib/src" + WL_PROTOCOLS_DIR="${RAYLIB_SRC_PATH}/external/glfw/deps/wayland" + AC_MSG_NOTICE([Generating Wayland protocol headers]) + wl_generate() { + protocol="$1" + basename="$2" + "$WAYLAND_SCANNER" client-header "$protocol" "$RAYLIB_SRC_PATH/$basename.h" || exit 1 + "$WAYLAND_SCANNER" private-code "$protocol" "$RAYLIB_SRC_PATH/$basename-code.h" || exit 1 + } + wl_generate "$WL_PROTOCOLS_DIR/wayland.xml" wayland-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/xdg-shell.xml" xdg-shell-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/xdg-decoration-unstable-v1.xml" xdg-decoration-unstable-v1-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/viewporter.xml" viewporter-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/relative-pointer-unstable-v1.xml" relative-pointer-unstable-v1-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/pointer-constraints-unstable-v1.xml" pointer-constraints-unstable-v1-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/fractional-scale-v1.xml" fractional-scale-v1-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/xdg-activation-v1.xml" xdg-activation-v1-client-protocol + wl_generate "$WL_PROTOCOLS_DIR/idle-inhibit-unstable-v1.xml" idle-inhibit-unstable-v1-client-protocol +} + AC_ARG_WITH(mlpack, [AS_HELP_STRING([--with-mlpack], [Build the mlpack module])], [MLPACK="yes"], @@ -66,14 +87,22 @@ case "${host_os}" in *) PLATFORM_LDFLAGS="-Wl,--no-undefined -avoid-version" CLIPBOARD_LDFLAGS="`pkg-config xcb --libs` -lpthread" - NUKLEAR_LDFLAGS="-lGL -lm -lpthread -ldl -lrt -lX11" WEBSOCKET_LDFLAGS="" GTK_SERVER_LDFLAGS="`pkg-config --libs gtk+-3.0` -lXm -lXt" GTK_SERVER_CPPFLAGS="`pkg-config --cflags gtk+-3.0` -DGTK_SERVER_FFI -DGTK_SERVER_LIBRARY -DGTK_SERVER_UNIX -DGTK_SERVER_GTK3x" - RAYLIB_LDFLAGS="-lGL -lm -lpthread -ldl -lrt -lX11" + RAYLIB_LDFLAGS="`pkg-config wayland-client wayland-cursor wayland-egl xkbcommon --libs`" JVM_CPPFLAGS="-I/usr/lib/jvm/java-1.8.0-openjdk-amd64/include -I/usr/lib/jvm/java-1.8.0-openjdk-amd64/include/linux" JVM_LDFLAGS="-L/usr/lib/jvm/java-1.8.0-openjdk-amd64/jre/lib/amd64/server -ljvm" - NUKLEAR_CPPFLAGS="-D_GLFW_X11=1" + NUKLEAR_CPPFLAGS="-D_GLFW_WAYLAND=1" + NUKLEAR_LDFLAGS="`pkg-config wayland-client wayland-cursor wayland-egl xkbcommon --libs`" + + AC_ARG_VAR([WAYLAND_SCANNER], [Path to wayland-scanner]) + AC_PATH_PROG([WAYLAND_SCANNER], [wayland-scanner]) + AS_IF([test -n "$WAYLAND_SCANNER"], [ + generate_wayland_protocols + ], [ + AC_MSG_WARN([wayland-scanner not found; Wayland support disabled]) + ]) esac AC_SUBST(DEBUG_LDFLAGS) diff --git a/glfw/Makefile.am b/glfw/Makefile.am index dc0dbaa..35ff783 100644 --- a/glfw/Makefile.am +++ b/glfw/Makefile.am @@ -7,6 +7,7 @@ AM_CXXFLAGS=-fno-rtti -std=c++14 AM_CPPFLAGS = \ + -I../raylib/raylib/src \ -I../raylib/raylib/src/external/glfw/include \ -I../raylib/raylib/src/external/glfw/deps \ -Wall -Wextra -Wshadow -Wdouble-promotion -Wno-unused-parameter -D_GLFW_BUILD_DLL=1 diff --git a/glfw/main.cpp b/glfw/main.cpp index 2a49198..07f8872 100644 --- a/glfw/main.cpp +++ b/glfw/main.cpp @@ -404,3 +404,7 @@ SBLIB_API void sblib_ellipse(int xc, int yc, int xr, int yr, int fill) { glEnd(); } +SBLIB_API int sblib_has_window_ui(void) { + // module creates a UI in a new window + return 1; +} diff --git a/gtk-server/uthash b/gtk-server/uthash index 41c357f..6d85739 160000 --- a/gtk-server/uthash +++ b/gtk-server/uthash @@ -1 +1 @@ -Subproject commit 41c357fd74ade4f4b4822c4407d2f51c4558e18d +Subproject commit 6d8573997c21f24c7e4ec9e48734b44f384170a1 diff --git a/include/module.h b/include/module.h index d27ea52..5a81801 100644 --- a/include/module.h +++ b/include/module.h @@ -25,6 +25,15 @@ extern "C" { */ int sblib_init(const char *sourceFile); +/** + * @ingroup modstd + * + * Returns whether the module is compatible with IDE builds + * + * @return non-zero on success + */ +int sblib_has_window_ui(void); + /** * @ingroup modstd * @@ -111,12 +120,22 @@ int sblib_func_exec(int index, int param_count, slib_par_t *params, var_t *retva /** * @ingroup modlib * - * executes a function + * free resources associated with the variable + * + * @param cls_id the variable class identifier + * @param id the variable instance identifier + */ +int sblib_free(int cls_id, int id); + +/** + * @ingroup modlib + * + * registers a fresh id to replace the given id * * @param cls_id the variable class identifier * @param id the variable instance identifier */ -void sblib_free(int cls_id, int id); +int sblib_refresh_id(int cls_id, int id); /** * @ingroup modlib diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 56ee204..a61dd32 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -66,15 +66,12 @@ include(CheckLanguage) if(LLAMA_BACKEND STREQUAL "CPU") message(STATUS "llama.cpp backend: CPU-only") set(GGML_NATIVE ON CACHE BOOL "" FORCE) # enable CPU SIMD optimizations - elseif(LLAMA_BACKEND STREQUAL "GPU") message(STATUS "llama.cpp backend: GPU (non-CUDA)") set(GGML_OPENMP ON CACHE BOOL "" FORCE) # parallel CPU fallback # GPU non-CUDA options can be added here in the future - elseif(LLAMA_BACKEND STREQUAL "CUDA") message(STATUS "llama.cpp backend: CUDA") - check_language(CUDA) if(CMAKE_CUDA_COMPILER) enable_language(CUDA) @@ -82,10 +79,8 @@ elseif(LLAMA_BACKEND STREQUAL "CUDA") else() message(FATAL_ERROR "CUDA backend requested but nvcc not found") endif() - elseif(LLAMA_BACKEND STREQUAL "AUTO") message(STATUS "llama.cpp backend: AUTO") - check_language(CUDA) if(CMAKE_CUDA_COMPILER) enable_language(CUDA) @@ -96,7 +91,6 @@ elseif(LLAMA_BACKEND STREQUAL "AUTO") set(GGML_NATIVE ON CACHE BOOL "" FORCE) message(STATUS "CUDA not found – using CPU/OpenMP") endif() - else() message(FATAL_ERROR "Invalid LLAMA_BACKEND value: ${LLAMA_BACKEND}") endif() @@ -130,6 +124,8 @@ target_include_directories(llm PRIVATE target_link_libraries(llm PRIVATE llama ggml + # force dynamic libm + -Wl,-Bdynamic,-lm ) # Include all static code into plugin @@ -146,29 +142,6 @@ set_target_properties(llm PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib ) -# ----------------------------- -# Optional test application -# ----------------------------- -add_executable(llm_test - test_main.cpp -) - -target_include_directories(llm_test PRIVATE - ${LLAMA_DIR}/include - ${LLAMA_DIR}/ggml/include - ${CMAKE_CURRENT_SOURCE_DIR}/../include -) - -target_link_libraries(llm_test PRIVATE - llm - llama - ggml -) - -set_target_properties(llm_test PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin -) - # ------------------------------------------------------------------ # Android native library # ------------------------------------------------------------------ diff --git a/llama/RAG.md b/llama/RAG.md new file mode 100644 index 0000000..6c69c06 --- /dev/null +++ b/llama/RAG.md @@ -0,0 +1,323 @@ +# notcurses RAG — C++ Library Expert via llama.cpp + +A self-contained RAG (Retrieval-Augmented Generation) pipeline in C++17 +that turns a GGUF inference model into a focused expert on any C/C++ library. +Demonstrated here with [notcurses](https://github.com/dankamongmen/notcurses) +but works with any header-based library. + +No fixed limits on chunk count, chunk length, or embedding dimension. +No Python, no vector database daemon, no external dependencies beyond llama.cpp. + +--- + +## How it works + +``` +INDEXING (one-time offline) +──────────────────────────────────────────────────────────────── +notcurses headers + │ + ▼ +chunk_headers ← semantic chunker, outputs chunks.jsonl + │ + ▼ +rag_index ← embeds each chunk via qwen3-embedding-0.6b-q4_k_m.gguf + │ + ▼ +notcurses.db ← binary vector store (embeddings + text) + + +RUNTIME (each query) +──────────────────────────────────────────────────────────────── +user query + │ + ▼ +rag_retrieve() ← embeds query, cosine similarity against db + │ ← skips chunks already seen this session + ▼ +new top-k chunks ← most relevant unseen API fragments + │ + ▼ +prompt assembly ← system + prior history + new context + query + │ + ▼ +Qwen3 inference ← <|think|> reasoning + final answer + │ + ▼ +history ← appended for next turn (KV cache intact) +``` + +--- + +## Files + +| File | Purpose | +|---|---| +| `chunk_headers.cpp` | Parses C/C++ headers into semantic chunks, outputs `.jsonl` | +| `rag_index.cpp` | Reads `.jsonl`, embeds each chunk, saves binary `.db` | +| `rag.hpp` | Single-header C++17 runtime — load db, session, retrieve | +| `example.cpp` | Full pipeline wired together, multi-turn query loop | + +--- + +## Dependencies + +- [llama.cpp](https://github.com/ggerganov/llama.cpp) — `libllama` + `llama.h` +- A GGUF **inference model** — tested with `Qwen3.5-9B-Q4_K_M.gguf` +- A GGUF **embedding model** — `qwen3-embedding-0.6b-q4_k_m.gguf` +- C++17 compiler (gcc 8+, clang 7+, MSVC 2019+) + +--- + +## Build + +```bash +c++ -std=c++17 -o chunk_headers chunk_headers.cpp +c++ -std=c++17 -o rag_index rag_index.cpp -lllama -lm +c++ -std=c++17 -o example example.cpp -lllama -lm +``` + +If llama.cpp is not on your system library path: + +```bash +c++ -std=c++17 -o rag_index rag_index.cpp \ + -I/path/to/llama.cpp/include \ + -L/path/to/llama.cpp/build -lllama -lm +``` + +--- + +## Usage + +### Step 1 — Chunk the headers (one-time) + +```bash +./chunk_headers notcurses/include/notcurses/ > chunks.jsonl +``` + +Accepts a single file or a directory (walked recursively). +Multiple paths can be given: + +```bash +./chunk_headers include/foo.h include/bar.h src/examples/ > chunks.jsonl +``` + +Handles `.h`, `.hpp`, `.c`, `.cpp`. Inspect before indexing: + +```bash +head -5 chunks.jsonl | python3 -m json.tool +``` + +### Step 2 — Embed and index (one-time) + +```bash +./rag_index \ + --model qwen3-embedding-0.6b-q4_k_m.gguf \ + --input chunks.jsonl \ + --output notcurses.db +``` + +Takes a few minutes for a large corpus. The `.db` is reusable +until the library changes. + +### Step 3 — Run + +```bash +./example \ + --model Qwen3.5-9B-Q4_K_M.gguf \ + --embed qwen3-embedding-0.6b-q4_k_m.gguf \ + --db notcurses.db +``` + +``` +notcurses expert ready. ctrl+d to quit. + +you: how do I create a plane and render text into it? +assistant: ... + +you: what options does it take? ← follow-up; no repeated context +assistant: ... +``` + +--- + +## Using rag.hpp in your own project + +Single-header, stb-style. In **one** `.cpp` file: + +```cpp +#define RAG_IMPLEMENTATION +#include "rag.hpp" +``` + +All other files that need the types: + +```cpp +#include "rag.hpp" +``` + +### Minimal integration + +```cpp +// startup +RagDB db; +rag_load(db, "notcurses.db"); + +RagSession session; +session.init(db.size(), 8192); // n_chunks, your n_ctx +session.score_threshold = 0.60f; + +// each turn +std::string context = rag_retrieve(db, embed_ctx, embed_model, + user_query, 5, session); +// context is empty string if nothing new/relevant was found +// build prompt with context and hand to your inference context +``` + +### Stateless retrieval (no deduplication) + +```cpp +std::string context = rag_retrieve(db, embed_ctx, embed_model, + user_query, 5); +``` + +### API + +```cpp +// Load .db file (version 2). Returns true on success. +bool rag_load(RagDB &db, const std::string &path); + +// Retrieve with session deduplication + token budget. +// Returns context string ready to inject into prompt. +// Empty string if nothing new or relevant was found. +std::string rag_retrieve(const RagDB &db, + llama_context *embed_ctx, + llama_model *embed_model, + const std::string &query, + int top_k, + RagSession &session); + +// Stateless overload — no deduplication. +std::string rag_retrieve(const RagDB &db, + llama_context *embed_ctx, + llama_model *embed_model, + const std::string &query, + int top_k); +``` + +### RagSession fields + +```cpp +struct RagSession { + std::vector seen; // one bit per chunk, sized to db + int tokens_used = 0; // running token estimate + int tokens_max = 0; // your n_ctx ceiling + float score_threshold = 0.60f; // skip weak matches + + void init(int n_chunks, int ctx_size); + void reset(); // start a fresh conversation +}; +``` + +--- + +## Chunking strategy + +`chunk_headers` uses a state machine that keeps each **semantic unit** +together as one chunk: + +- Block comment (`/* ... */`) + following declaration +- `//` line comments + following declaration +- `typedef struct` / `typedef enum` entire body +- Consecutive `#define` macro groups +- Multi-line function signatures + +Example — this stays as one chunk: + +```c +// ncplane_create() - create a new plane as a child of 'n'. +// 'nopts' may be NULL for defaults. Returns NULL on error. +struct ncplane* ncplane_create(struct ncplane *n, + const struct ncplane_options *nopts); +``` + +--- + +## Session deduplication + +The KV cache is not cleared between turns, so the model already has +earlier chunks in memory. `RagSession` tracks which chunks have been +injected and skips them on subsequent turns: + +``` +Turn 1: retrieved chunks [42, 17, 83] → all new → inject all +Turn 2: retrieved chunks [42, 55, 17] → 42,17 seen → inject only [55] +Turn 3: retrieved chunks [7, 14, 55] → 55 seen → inject [7, 14] +``` + +Context window grows efficiently — no repeated API reference, and the +model remembers everything already seen via the intact KV cache. + +--- + +## Adapting to other libraries + +Change only the input to `chunk_headers`: + +| Library | Input | +|---|---| +| stb (stb_image, stb_truetype ...) | single `.h` file | +| SDL2 / OpenGL / Vulkan | `include/` directory | +| Your own engine | any `.h` / `.hpp` mix | +| Spring / Java | extend chunker for Javadoc + `.java` | + +Re-run steps 1 and 2 to produce a new `.db`. Runtime code unchanged. +Multiple `.db` files can be loaded and queried independently. + +--- + +## .db file format (version 2) + +Variable-length fields — no wasted padding. + +``` +Header (16 bytes): + uint32 magic = 0x52414744 ("RAGD") + uint32 version = 2 + uint32 n_chunks + uint32 embed_dim + +Per chunk: + uint32 text_len + char[] text (text_len bytes, no null) + uint16 source_len + char[] source (source_len bytes, no null) + uint8 type_len + char[] type (type_len bytes, no null) + float[] embedding (embed_dim × 4 bytes) +``` + +--- + +## GPU memory + +On an 8 GB GPU with `Qwen3.5-9B-Q4_K_M`: + +| Component | VRAM | +|---|---| +| Inference model (Q4_K_M 9B) | ~5.5 GB | +| Embedding model (nomic Q4) | ~0.3 GB | +| KV cache (8k ctx, Q4_0 K/V) | ~0.5 GB | +| **Total** | **~6.3 GB** | + +--- + +## Qwen3 thinking mode + +The model emits `<|think|>...<|/think|>` before its answer. +`example.cpp` strips this with `strip_think()` before printing. +The think block improves RAG quality — the model explicitly reasons +over injected context chunks before answering. + +To expose reasoning (useful for debugging retrieval quality), remove +the `strip_think()` call and print `raw` directly. diff --git a/llama/README.md b/llama/README.md index e1435ba..29f99c8 100644 --- a/llama/README.md +++ b/llama/README.md @@ -1,104 +1,208 @@ -## huggingface-cli +# SmallBASIC Llama Module -``` -pyenv virtualenv 3.10.13 hf-tools -pyenv activate hf-tools -pip install -U pip -pip install huggingface_hub +A comprehensive SmallBASIC library module that bridges the scripting capabilities of SmallBASIC with the power of Llama.cpp Large Language Models. This project allows developers to create, configure, and interact with LLM instances directly within a SmallBASIC environment. -``` +## Table of Contents +1. [System Requirements & CUDA Setup](#system-requirements--cuda-setup) +2. [Obtaining Models from Hugging Face](#obtaining-models-from-hugging-face) +3. [Architecture](#architecture) +4. [Features](#features) +5. [Usage Examples](#usage-examples) +6. [API Reference](#api-reference) +7. [Configuration Presets](#configuration-presets) --- -1️⃣ Ensure nvidia-open driver is installed and working +## System Requirements & CUDA Setup -Check: +For optimal performance, especially on NVIDIA hardware, the CUDA toolkit must be correctly configured. -`` +### 1. Check NVIDIA Drivers +Ensure the NVIDIA open driver is installed and working: +```bash nvidia-smi -`` - -If it works, your driver is fine — no need to install the proprietary driver. - -2️⃣ Add NVIDIA CUDA repository - ``` +If this command works, the proprietary driver is not strictly necessary for CUDA toolkit installation. + +### 2. Add NVIDIA CUDA Repository +For Debian 12: +```bash wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt update ``` -This repo contains the latest CUDA toolkit for Debian 12. - -3️⃣ Install CUDA Toolkit only (no driver replacement) +### 3. Install CUDA Toolkit +Install only the toolkit (no driver replacement): +```bash sudo apt install -y cuda-toolkit - - -This installs: - -- nvcc compiler -- CUDA headers -- Runtime libraries (libcudart.so, etc.) - -4️⃣ Add CUDA to your environment - ``` +This installs `nvcc`, headers, and runtime libraries. + +### 4. Environment Variables +Add the following to your environment: +```bash export PATH=/usr/local/cuda/bin:$PATH export CUDAToolkit_ROOT=/usr/local/cuda ``` +To make this permanent, add to `~/.bashrc` and source it. -Optional: add to ~/.bashrc to make it permanent: - +### 5. Verify Installation +```bash +nvcc --version ``` -echo 'export PATH=/usr/local/cuda/bin:$PATH' >> ~/.bashrc -echo 'export CUDAToolkit_ROOT=/usr/local/cuda' >> ~/.bashrc -source ~/.bashrc +Output should indicate the release version (e.g., release 12.4). + +### 6. Build Configuration +When building the module, ensure the build directory is clean and configured for the CUDA backend: +```bash +rm -rf build +mkdir build +cd build +cmake -DLLAMA_BACKEND=CUDA .. +make -j$(nproc) ``` +*Note: Fully static builds are not possible for CUDA; some `.so` libraries will remain dynamically linked.* + +--- -Verify: +## Obtaining Models from Hugging Face -nvcc --version +The `LLAMA` function expects a path to a model file (e.g., `gguf` format). Models can be obtained from the Hugging Face Hub. -Should show something like: +### Method 1: Using `huggingface-cli` (Recommended) -``` -nvcc: NVIDIA (R) Cuda compiler driver -Cuda compilation tools, release 12.4, V12.4.105 -``` +1. **Setup Environment** + Create a virtual environment (optional but recommended) and install the CLI tool: + ```bash + pyenv virtualenv 3.10.13 hf-tools + pyenv activate hf-tools + pip install -U pip + pip install huggingface_hub + ``` -5️⃣ Clean llama.cpp build directory +2. **Login** + Authenticate with your Hugging Face account: + ```bash + huggingface-cli login + ``` + (Follow the prompts to enter your token). -``` -rm -rf build -mkdir build -cd build -``` +3. **Download Model** + Use the `huggingface-cli download` command to fetch the model directly to your desired directory. + ```bash + # Example: Download Llama-3-8B-Instruct + huggingface-cli download meta-llama/Meta-Llama-3-8B-Instruct --include "*.gguf" --local-dir models/llama3-8b + ``` + + *Note: This command downloads all `.gguf` files associated with the repository into the `models/llama3-8b` folder.* + +### Method 2: Using Python (`huggingface_hub`) -6️⃣ Configure CMake for CUDA backend +If you prefer a scriptable approach: +```python +from huggingface_hub import hf_hub_download +model_path = hf_hub_download( + repo_id="meta-llama/Meta-Llama-3-8B-Instruct", + filename="llama-3-8b-instruct.Q4_K_M.gguf", # Specify exact file if needed + local_dir="models", + local_dir_use_symlinks=False +) ``` -cmake -DLLAMA_BACKEND=CUDA .. + +Once the model file is in your `models` directory (or wherever specified), you can reference it in SmallBASIC: +```basic +llama = LLAMA("models/llama3-8b/llama-3-8b-instruct.Q4_K_M.gguf", 2048, 1024, -1, 0) ``` -You should now see: +### Method 3: Direct download + +1. Navigate to https://huggingface.co/ +2. Click Models at the top and then select Libraries/GGUF +3. Use the parameters slider to limit the selection for your hardware. + +--- + +## Architecture --- CUDA detected – enabling GGML_CUDA +The module operates as a compiled library (`SBLIB`) exposing C++ functionality to SmallBASIC scripts. + +### Core Components +1. **Llama Instance Manager (`g_llama`)**: + * Stores active Llama models in a hash map keyed by ID. + * Supports initialization with custom context sizes, batch sizes, and GPU acceleration. + * Handles memory cleanup to prevent leaks. + +2. **Response Iterator (`g_llama_iter`)**: + * Manages the streaming response of an LLM. + * Provides token-by-token access to generated text. + * Tracks generation speed (`tokens/sec`) and remaining tokens. + +3. **Command Interface**: + * Exposes a set of SmallBASIC functions (callbacks) for configuration and interaction. + +--- -7️⃣ Build +## Features +### Initialization +The `LLAMA` function creates a new model instance. +```basic +' Syntax: LLAMA(model_path, n_ctx, n_batch, n_gpu_layers, n_log_level) +' Example: +' llama = LLAMA("models/llama-7b.gguf", 2048, 1024, -1, 0) ``` -make -j$(nproc) + +### Configuration +Once an instance is created, various parameters can be adjusted dynamically: + +* **Temperature**: Controls randomness in generation. +* **Top-K / Top-P**: Nucleus sampling parameters. +* **Max Tokens**: Limits the length of the response. +* **Penalties**: Frequency, presence, and repeat penalties to avoid repetition. +* **Grammar**: Constrains output to specific patterns. + +```basic +' Examples: +llama.set_temperature(0.8) +llama.set_max_tokens(50) +llama.set_penalty_repeat(0.8) +llama.set_seed(123) ``` -The binary will use CUDA acceleration +### Interaction +The primary method of interaction is `add_message`, which sends a prompt to the model. -Note: fully static builds are not possible for CUDA; some .so libraries will remain dynamically linked (normal). +```basic +' Syntax: llama.add_message(role, content) +' Returns: An iterator object for the response. +response = llama.add_message("user", "Please describe a sunset in poetry.") +``` -# Generator settings +### Streaming Responses +The returned iterator allows real-time processing of the model's output: -## factual answers, tools, summaries +* `response.all()`: Returns the complete generated text. +* `response.next()`: Retrieves the next token. +* `response.has_next()`: Checks if more tokens are available. +* `response.tokens_sec`: Calculates current generation speed. +```basic +' Example loop: +while response.has_next() + print response.next() + sleep 100 +end while ``` + +--- + +## Usage Examples + +### Factual Answers & Tool Use +*Best for: Summaries, code generation, technical queries.* +```basic llama.set_max_tokens(150) llama.set_temperature(0.0) llama.set_top_k(1) @@ -106,9 +210,9 @@ llama.set_top_p(0.0) llama.set_min_p(0.0) ``` -## assistant, Q+A, explanations, chat - -``` +### Assistant / Q&A / Chat +*Best for: Conversational agents, explanations.* +```basic llama.set_max_tokens(150) llama.set_temperature(0.8) llama.set_top_k(40) @@ -116,29 +220,19 @@ llama.set_top_p(0.0) llama.set_min_p(0.05) ``` -## creative, storytelling - -``` -llama.set_max_tokens(20) +### Creative Writing & Storytelling +*Best for: Fiction, poetry, imaginative tasks.* +```basic +llama.set_max_tokens(200) llama.set_temperature(1.0) llama.set_top_k(80) llama.set_top_p(0.0) llama.set_min_p(0.1) ``` -## surprises - -``` -llama.set_max_tokens(200) -llama.set_temperature(1.2) -llama.set_top_k(120) -llama.set_top_p(0.0) -llama.set_min_p(0.15) -``` - -## technical, conservative - -``` +### Technical & Conservative +*Best for: Documentation, logic, precise tasks.* +```basic llama.set_max_tokens(150) llama.set_temperature(0.6) llama.set_top_k(30) @@ -146,9 +240,9 @@ llama.set_top_p(0.0) llama.set_min_p(0.02) ``` -## speed optimised on CPU - -``` +### Speed Optimized (CPU) +*Best for: Rapid iteration or low-resource environments.* +```basic ' llama.set_max_tokens(10) ' llama.set_temperature(0.7) ' llama.set_top_k(20) @@ -156,32 +250,72 @@ llama.set_min_p(0.02) ' llama.set_min_p(0.05) ``` -# Avoiding repetition +--- -## Conservative - minimal repetition control +## API Reference + +### Class: Llama +| Method | Description | +| :--- | :--- | +| `add_stop(text)` | Adds a stop sequence to the generation. | +| `set_penalty_repeat(value)` | Sets repeat penalty (default 1.1). | +| `set_penalty_freq(value)` | Sets frequency penalty. | +| `set_penalty_present(value)` | Sets presence penalty. | +| `set_penalty_last_n(value)` | Sets penalty context size. | +| `set_max_tokens(value)` | Sets maximum output tokens. | +| `set_min_p(value)` | Sets minimum probability threshold. | +| `set_temperature(value)` | Sets generation temperature. | +| `set_top_k(value)` | Sets top-k sampling. | +| `set_top_p(value)` | Sets top-p sampling. | +| `set_grammar(text)` | Sets output grammar constraint. | +| `set_seed(value)` | Sets random seed for reproducibility. | +| `reset()` | Clears the current conversation context. | +| `add_message(role, content)` | Sends a message and returns an iterator. | + +### Class: LlamaIter +| Method | Description | +| :--- | :--- | +| `all()` | Returns the full string of the response. | +| `has_next()` | Returns true if more tokens are available. | +| `next()` | Returns the next token string. | +| `tokens_sec` | Returns current tokens per second. | -``` +--- + +## Repetition Control Strategies + +### Conservative (Minimal Control) +*Use when occasional repetition is acceptable.* +```basic llama.set_penalty_last_n(64) llama.set_penalty_repeat(1.05) ``` -## Balanced - good default - -``` -set_penalty_last_n(64) -set_penalty_repeat(1.1) +### Balanced (Default) +*Recommended for general usage.* +```basic +llama.set_penalty_last_n(64) +llama.set_penalty_repeat(1.1) ``` -## Aggressive - strong anti-repetition - -``` -set_penalty_last_n(128) -set_penalty_repeat(1.2) +### Aggressive (Strong Anti-Repetition) +*Use for long-form generation where repetition must be avoided.* +```basic +llama.set_penalty_last_n(128) +llama.set_penalty_repeat(1.2) ``` -## Disabled - -``` +### Disabled +*Use when repetition is desired or irrelevant.* +```basic llama.set_penalty_last_n(0) llama.set_penalty_repeat(1.0) ``` + +--- + +## Conclusion + +This module empowers SmallBASIC users to build sophisticated AI applications, from chatbots to creative writing tools, leveraging the efficiency of Llama.cpp within a familiar scripting paradigm. Proper configuration of CUDA and generation parameters ensures optimal performance and output quality. Models can be easily acquired via the Hugging Face Hub using standard CLI tools or Python scripts. + +--- diff --git a/llama/llama-sb-rag.cpp b/llama/llama-sb-rag.cpp new file mode 100644 index 0000000..0b11d04 --- /dev/null +++ b/llama/llama-sb-rag.cpp @@ -0,0 +1,445 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#include "llama-sb.h" +#include "llama-sb-rag.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static constexpr uint32_t MAGIC = 0x52414744; +static constexpr size_t MIN_CHUNK = 40; +static constexpr const char *INSTRUCT_EMBED = "Instruct: Represent this API documentation for code retrieval\nQuery: "; +static constexpr const char *INSTRUCT_QUERY = "Instruct: Given a programming question, retrieve relevant API documentation\nQuery: "; + +enum class ChunkType { + Function, Struct, Enum, Typedef, Defines, Other +}; + +static std::string type_name(ChunkType t) { + switch (t) { + case ChunkType::Function: return "function"; + case ChunkType::Struct: return "struct"; + case ChunkType::Enum: return "enum"; + case ChunkType::Typedef: return "typedef"; + case ChunkType::Defines: return "defines"; + default: return "other"; + } +} + +/* ── helpers ───────────────────────────────────────────────── */ + +static bool starts_with(const std::string &s, const std::string &prefix) { + return s.size() >= prefix.size() && + s.compare(0, prefix.size(), prefix) == 0; +} + +static bool is_blank(const std::string &s) { + for (char c : s) if (!isspace((unsigned char)c)) return false; + return true; +} + +/* ── state machine ─────────────────────────────────────────── */ + +enum class State { + Idle, BlockComment, LineComment, Declaration, Struct, Defines +}; + +template + +static bool chunk_file(const fs::path &path, EmitChunk emit_chunk) { + std::ifstream f(path); + if (!f) { + return false; + } + + const std::string source = path.filename().string(); + + State state = State::Idle; + std::string chunk; + ChunkType chunk_type = ChunkType::Other; + int brace_depth = 0; + int paren_depth = 0; + int define_count = 0; + + auto flush = [&](ChunkType t) { + emit_chunk(source, t, chunk); + chunk.clear(); + state = State::Idle; + brace_depth = 0; + paren_depth = 0; + }; + + std::string line; + while (std::getline(f, line)) { + /* trim trailing CR */ + if (!line.empty() && line.back() == '\r') line.pop_back(); + + /* find first non-whitespace for prefix checks */ + size_t trim_pos = 0; + while (trim_pos < line.size() && + (line[trim_pos] == ' ' || line[trim_pos] == '\t')) ++trim_pos; + const std::string trimmed = line.substr(trim_pos); + + /* ── #define handling ─────────────────────────────────── */ + if (starts_with(trimmed, "#define ")) { + if (state == State::BlockComment || state == State::LineComment) { + chunk += line + "\n"; + state = State::Defines; + define_count = 1; + } else if (state == State::Defines) { + chunk += line + "\n"; + define_count++; + } else { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk += line + "\n"; + state = State::Defines; + define_count = 1; + } + continue; + } + + /* non-define while in define group */ + if (state == State::Defines) { + flush(ChunkType::Defines); + define_count = 0; + /* fall through to process this line normally */ + } + + /* ── block comment start ──────────────────────────────── */ + if ((starts_with(trimmed, "/*") || starts_with(trimmed, "/**")) && + state == State::Idle) { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk_type = ChunkType::Other; + chunk += line + "\n"; + state = (trimmed.find("*/", 2) != std::string::npos) + ? State::LineComment + : State::BlockComment; + continue; + } + + /* ── inside block comment ─────────────────────────────── */ + if (state == State::BlockComment) { + chunk += line + "\n"; + if (trimmed.find("*/") != std::string::npos) + state = State::LineComment; + continue; + } + + /* ── // line comment ──────────────────────────────────── */ + if (starts_with(trimmed, "//")) { + if (state == State::Idle) { + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + chunk.clear(); + chunk += line + "\n"; + state = State::LineComment; + } else if (state == State::LineComment) { + chunk += line + "\n"; + } + continue; + } + + /* ── blank line ───────────────────────────────────────── */ + if (is_blank(trimmed)) { + if (state == State::LineComment) + flush(ChunkType::Other); + else if (state == State::Idle && chunk.size() >= MIN_CHUNK) + flush(chunk_type); + continue; + } + + /* ── skip preprocessor noise ──────────────────────────── */ + if (starts_with(trimmed, "#ifndef") || starts_with(trimmed, "#ifdef") || + starts_with(trimmed, "#endif") || starts_with(trimmed, "#pragma") || + starts_with(trimmed, "#include")) { + if (state == State::LineComment || state == State::BlockComment) { + chunk.clear(); + state = State::Idle; + } + continue; + } + + /* ── typedef struct / enum start ─────────────────────── */ + if ((starts_with(trimmed, "typedef struct") || + starts_with(trimmed, "typedef enum") || + starts_with(trimmed, "struct ") || + starts_with(trimmed, "enum ")) && + (state == State::Idle || state == State::LineComment)) { + + if (state == State::Idle && chunk.size() >= MIN_CHUNK) + emit_chunk(source, chunk_type, chunk); + + /* preserve any comment already in chunk */ + if (state == State::Idle) chunk.clear(); + + chunk += line + "\n"; + chunk_type = starts_with(trimmed, "typedef") ? ChunkType::Typedef + : starts_with(trimmed, "enum ") ? ChunkType::Enum + : ChunkType::Struct; + state = State::Struct; + for (char c : line) { + if (c == '{') ++brace_depth; + if (c == '}') --brace_depth; + } + if (brace_depth <= 0 && line.find(';') != std::string::npos) + flush(chunk_type); + continue; + } + + /* ── inside struct/enum body ──────────────────────────── */ + if (state == State::Struct) { + chunk += line + "\n"; + for (char c : line) { + if (c == '{') ++brace_depth; + if (c == '}') --brace_depth; + } + if (brace_depth <= 0 && line.find(';') != std::string::npos) + flush(chunk_type); + continue; + } + + /* ── function / other declaration ────────────────────── */ + if (state == State::LineComment || state == State::Idle) { + if (state == State::Idle && chunk.size() >= MIN_CHUNK) { + emit_chunk(source, chunk_type, chunk); + chunk.clear(); + } + chunk += line + "\n"; + chunk_type = ChunkType::Function; + state = State::Declaration; + for (char c : line) { + if (c == '(') ++paren_depth; + if (c == ')') --paren_depth; + } + if (paren_depth <= 0 && line.find(';') != std::string::npos) + flush(ChunkType::Function); + continue; + } + + /* ── multi-line declaration ───────────────────────────── */ + if (state == State::Declaration) { + chunk += line + "\n"; + for (char c : line) { + if (c == '(') ++paren_depth; + if (c == ')') --paren_depth; + } + if (paren_depth <= 0 && line.find(';') != std::string::npos) + flush(ChunkType::Function); + continue; + } + } + + /* flush remainder */ + if (chunk.size() >= MIN_CHUNK) emit_chunk(source, chunk_type, chunk); + + return true; +} + +// +// cosine similarity (vectors already L2-normalized) +// +static float rag_cosine(const std::vector &a, + const std::vector &b) { + float dot = 0.0f; + size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; i++) { + dot += a[i] * b[i]; + } + return dot; +} + +// +// build context string from ranked results +// +static std::string rag_build_context(const RagDB &db, + const std::vector &indices, + const std::vector &scores) { + std::ostringstream out; + for (size_t i = 0; i < indices.size(); i++) { + const RagChunk &c = db.chunks[indices[i]]; + out << "// source: " << c.source + << " [" << c.type << "]" + << " (score: " << scores[i] << ")\n" + << c.text << "\n---\n"; + } + return out.str(); +} + +// +// index the file +// +bool Llama::rag_index(RagDB &db, const std::string &filepath) { + bool embed_fail = false; + auto emit_chunk = [&](const std::string &source, ChunkType type, + const std::string &text) { + if (text.size() > MIN_CHUNK) { + RagChunk chunk; + chunk.text = text; + chunk.source = source; + chunk.type = type_name(type); + if (!embed_text(INSTRUCT_EMBED + text, chunk.embedding, db.embed_dim)) { + embed_fail = true; + } else { + db.chunks.push_back(std::move(chunk)); + } + } + }; + + return !embed_fail && chunk_file(filepath, emit_chunk); +} + +// +// retrieve with session +// +std::string Llama::rag_retrieve(const RagDB &db, + const std::string &query, + int top_k, + RagSession &session) { + if (db.empty()) { + _last_error = "no input"; + return {}; + } + + std::vector qvec; + std::string text = INSTRUCT_QUERY + query; + if (!embed_text(text, qvec, db.embed_dim)) { + _last_error = "failed to embed text"; + return {}; + } + + // score all chunks + std::vector order(db.size()); + std::iota(order.begin(), order.end(), 0); + std::vector scores(db.size()); + for (int i = 0; i < db.size(); i++) { + scores[i] = rag_cosine(qvec, db.chunks[i].embedding); + } + std::sort(order.begin(), order.end(), [&](int a, int b){ return scores[a] > scores[b]; }); + + // collect top_k unseen, within budget, above threshold + std::vector result_idx; + std::vector result_scores; + + for (int idx : order) { + if ((int)result_idx.size() >= top_k) break; + if (session.is_seen(idx)) continue; + if (scores[idx] < session.score_threshold) break; /* sorted, so stop */ + if (!session.budget_ok(db.chunks[idx].text)) break; + + result_idx.push_back(idx); + result_scores.push_back(scores[idx]); + session.mark(idx); + session.charge(db.chunks[idx].text); + } + + return rag_build_context(db, result_idx, result_scores); +} + +bool RagDB::save(const std::string &path) { + std::ofstream f(path, std::ios::binary); + if (!f) { + return false; + } + + auto write32 = [&](uint32_t v) { f.write((char*)&v, 4); }; + auto write16 = [&](uint16_t v) { f.write((char*)&v, 2); }; + auto write8 = [&](uint8_t v) { f.write((char*)&v, 1); }; + auto writestr = [&](const std::string &s, size_t max_len) { + size_t len = std::min(s.size(), max_len); + f.write(s.c_str(), (std::streamsize)len); + }; + + write32(MAGIC); /* magic "RAGD" */ + write32(2); /* version */ + write32((uint32_t)chunks.size()); /* n_chunks */ + write32((uint32_t)embed_dim); /* embed_dim */ + + for (const RagChunk &c : chunks) { + write32((uint32_t)c.text.size()); + f.write(c.text.c_str(), (std::streamsize)c.text.size()); + + uint16_t src_len = (uint16_t)std::min(c.source.size(), (size_t)65535); + write16(src_len); + writestr(c.source, src_len); + + uint8_t type_len = (uint8_t)std::min(c.type.size(), (size_t)255); + write8(type_len); + writestr(c.type, type_len); + + f.write((char*)c.embedding.data(), + (std::streamsize)(embed_dim * sizeof(float))); + } + + return f.good(); +} + +bool RagDB::load(const std::string &path) { + std::ifstream f(path, std::ios::binary); + if (!f) { + return false; + } + + auto read32 = [&]() -> uint32_t { + uint32_t v = 0; f.read((char*)&v, 4); return v; + }; + auto read16 = [&]() -> uint16_t { + uint16_t v = 0; f.read((char*)&v, 2); return v; + }; + auto read8 = [&]() -> uint8_t { + uint8_t v = 0; f.read((char*)&v, 1); return v; + }; + auto readstr = [&](size_t len) -> std::string { + std::string s(len, '\0'); + f.read(&s[0], (std::streamsize)len); + return s; + }; + + uint32_t magic = read32(); + uint32_t version = read32(); + uint32_t n = read32(); + uint32_t edim = read32(); + + if (magic != MAGIC) { + return false; + } + if (version != 2) { + return false; + } + + embed_dim = (int)edim; + chunks.resize(n); + + for (uint32_t i = 0; i < n; i++) { + RagChunk &c = chunks[i]; + + uint32_t text_len = read32(); + c.text = readstr(text_len); + + uint16_t src_len = read16(); + c.source = readstr(src_len); + + uint8_t type_len = read8(); + c.type = readstr(type_len); + + c.embedding.resize(edim); + f.read((char*)c.embedding.data(), (std::streamsize)(edim * sizeof(float))); + } + + return true; +} diff --git a/llama/llama-sb-rag.h b/llama/llama-sb-rag.h new file mode 100644 index 0000000..0296f26 --- /dev/null +++ b/llama/llama-sb-rag.h @@ -0,0 +1,78 @@ +// This file is part of SmallBASIC +// +// This program is distributed under the terms of the GPL v2.0 or later +// Download the GNU Public License (GPL) from www.gnu.org +// +// Copyright(C) 2026 Chris Warren-Smith + +#pragma once + +struct RagChunk { + std::string text; + std::string source; + std::string type; + std::vector embedding; +}; + +/* ── on-disk chunk (variable-length text) ──────────────────── */ +/* + * db header (16 bytes): + * uint32 magic = 0x52414744 "RAGD" + * uint32 version = 2 + * uint32 n_chunks + * uint32 embed_dim + * + * per chunk: + * uint32 text_len + * char[] text (text_len bytes, no null) + * uint16 source_len + * char[] source (source_len bytes, no null) + * uint8 type_len + * char[] type (type_len bytes, no null) + * float[] embedding (embed_dim floats) + */ +struct RagDB { + std::vector chunks; + int embed_dim = 0; + + bool load(const std::string &path); + bool save(const std::string &path); + + int size() const { return (int)chunks.size(); } + bool empty() const { return chunks.empty(); } +}; + +// +// per-session deduplication + token budget +// +struct RagSession { + std::vector seen; /* sized to db.size() on init */ + int tokens_used = 0; + int tokens_max = 0; /* set to your n_ctx */ + float score_threshold = 0.60f; /* skip weak matches */ + + void init(int n_chunks, int ctx_size) { + seen.assign(n_chunks, false); + tokens_used = 0; + tokens_max = ctx_size; + } + + void reset() { + std::fill(seen.begin(), seen.end(), false); + tokens_used = 0; + } + + bool is_seen(int idx) const { return idx < (int)seen.size() && seen[idx]; } + void mark(int idx) { if (idx < (int)seen.size()) seen[idx] = true; } + + /* rough token estimate: 1 token ≈ 4 chars */ + bool budget_ok(const std::string &text) const { + return tokens_max == 0 || + (tokens_used + (int)text.size() / 4) < (int)(tokens_max * 0.85f); + } + + void charge(const std::string &text) { + tokens_used += (int)text.size() / 4; + } +}; + diff --git a/llama/llama-sb.cpp b/llama/llama-sb.cpp index 2bff5e8..e1a07d5 100644 --- a/llama/llama-sb.cpp +++ b/llama/llama-sb.cpp @@ -7,10 +7,27 @@ #include #include +#include +#include +#include "ggml-cuda.h" + #include "llama.h" #include "llama-sb.h" -constexpr int MAX_REPEAT = 5; +constexpr int MAX_REPEAT = 50; + +static bool read_vram(size_t &used, size_t &total) { + size_t free = 0; + total = 0; +#ifdef GGML_USE_CUDA + ggml_backend_cuda_get_device_memory(0, &free, &total); + if (total > 0) { + used = total - free; + return true; + } +#endif + return false; +} LlamaIter::LlamaIter() : _llama(nullptr), @@ -19,6 +36,15 @@ LlamaIter::LlamaIter() : _has_next(false) { } +LlamaIter::LlamaIter(LlamaIter &&other) noexcept + : _llama(std::exchange(other._llama, nullptr)) + , _last_word(std::move(other._last_word)) + , _t_start(std::move(other._t_start)) + , _repetition_count(other._repetition_count) + , _tokens_generated(other._tokens_generated) + , _has_next(other._has_next) { +} + Llama::Llama() : _model(nullptr), _ctx(nullptr), @@ -26,19 +52,62 @@ Llama::Llama() : _vocab(nullptr), _penalty_last_n(0), _penalty_repeat(0), + _penalty_freq(0.0f), + _penalty_present(0.0f), _temperature(0), _top_p(0), _min_p(0), _top_k(0), _max_tokens(0), - _log_level(GGML_LOG_LEVEL_CONT) { - llama_log_set([](enum ggml_log_level level, const char * text, void *user_data) { + _log_level(GGML_LOG_LEVEL_CONT), + _n_gpu_layers(0), + _n_system_tokens(0), + _is_gemma4(false), + _sampler_dirty(false), + _can_shift(false), + _memory_flush(false), + _seed(LLAMA_DEFAULT_SEED) { + llama_log_set([](enum ggml_log_level level, const char *text, void *user_data) { Llama *llama = (Llama *)user_data; + if (level == GGML_LOG_LEVEL_ERROR && llama->_last_error.empty()) { + // remember the first error message + llama->_last_error = text; + } if (level > llama->_log_level) { fprintf(stderr, "LLAMA: %s", text); } }, this); reset(); + llama_backend_init(); +} + +Llama::Llama(Llama &&other) noexcept + : _model(std::exchange(other._model, nullptr)) + , _ctx(std::exchange(other._ctx, nullptr)) + , _sampler(std::exchange(other._sampler, nullptr)) + , _vocab(std::exchange(other._vocab, nullptr)) + , _stop_sequences(std::move(other._stop_sequences)) + , _grammar_src(std::move(other._grammar_src)) + , _grammar_root(std::move(other._grammar_root)) + , _last_error(std::move(other._last_error)) + , _template(std::move(other._template)) + , _penalty_last_n(other._penalty_last_n) + , _penalty_repeat(other._penalty_repeat) + , _penalty_freq(other._penalty_freq) + , _penalty_present(other._penalty_present) + , _temperature(other._temperature) + , _top_p(other._top_p) + , _min_p(other._min_p) + , _top_k(other._top_k) + , _max_tokens(other._max_tokens) + , _log_level(other._log_level) + , _n_gpu_layers(other._n_gpu_layers) + , _n_system_tokens(other._n_system_tokens) + , _is_gemma4(other._is_gemma4) + , _sampler_dirty(other._sampler_dirty) + , _can_shift(other._can_shift) + , _memory_flush(other._memory_flush) + , _seed(other._seed) { } Llama::~Llama() { @@ -51,34 +120,51 @@ Llama::~Llama() { if (_model) { llama_model_free(_model); } + llama_backend_free(); } void Llama::reset() { _stop_sequences.clear(); - _last_error = ""; + _last_error.clear(); _penalty_last_n = 64; _penalty_repeat = 1.1f; + _penalty_freq = 0.0f; + _penalty_present = 0.0f; _temperature = 0; _top_k = 0; _top_p = 1.0f; _min_p = 0.0f; _max_tokens = 150; + _n_system_tokens = 0; + _seed = LLAMA_DEFAULT_SEED; + _sampler_dirty = true; if (_ctx) { llama_memory_clear(llama_get_memory(_ctx), true); } } -bool Llama::construct(string model_path, int n_ctx, int n_batch, int n_gpu_layers) { +bool Llama::is_memory_flush() { + auto result = _memory_flush; + if (result) { + _memory_flush = false; + } + return result; +} + +bool Llama::load_model(string model_path, int n_ctx, int n_batch, int n_gpu_layers, int log_level) { ggml_backend_load_all(); llama_model_params mparams = llama_model_default_params(); if (n_gpu_layers >= 0) { - mparams.n_gpu_layers = n_gpu_layers; + mparams.n_gpu_layers = n_gpu_layers; } + _last_error.clear(); + _log_level = log_level; + _n_gpu_layers = n_gpu_layers; _model = llama_model_load_from_file(model_path.c_str(), mparams); if (!_model) { - _last_error = "Failed to load model"; + set_last_error("Load model"); } else { llama_context_params cparams = llama_context_default_params(); cparams.n_ctx = n_ctx; @@ -86,142 +172,123 @@ bool Llama::construct(string model_path, int n_ctx, int n_batch, int n_gpu_layer cparams.n_ubatch = n_batch; cparams.no_perf = true; cparams.attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; - cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; + cparams.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + + // or Q4_0 for more aggressive saving + cparams.type_k = GGML_TYPE_Q4_0; + cparams.type_v = GGML_TYPE_Q4_0; + + // keep KV cache on GPU + cparams.offload_kqv = true; _ctx = llama_init_from_model(_model, cparams); if (!_ctx) { - _last_error = "Failed to create context"; + set_last_error("Create context"); } else { _vocab = llama_model_get_vocab(_model); - - auto sparams = llama_sampler_chain_default_params(); - sparams.no_perf = false; - _sampler = llama_sampler_chain_init(sparams); + _template = llama_model_chat_template(_model, nullptr); + _is_gemma4 = (_template.find("<|turn>model") != string::npos); + _can_shift = llama_memory_can_shift(llama_get_memory(_ctx)); } } + return _last_error.empty(); } -void Llama::configure_sampler() { - llama_sampler_reset(_sampler); - if (_penalty_last_n != 0 && _penalty_repeat != 1.0f) { - auto penalties = llama_sampler_init_penalties(_penalty_last_n, _penalty_repeat, 0.0f, 0.0f); - llama_sampler_chain_add(_sampler, penalties); - } - if (_temperature <= 0.0f) { - llama_sampler_chain_add(_sampler, llama_sampler_init_greedy()); - } else { - llama_sampler_chain_add(_sampler, llama_sampler_init_temp(_temperature)); - if (_top_k > 0) { - llama_sampler_chain_add(_sampler, llama_sampler_init_top_k(_top_k)); - } - if (_top_p < 1.0f) { - llama_sampler_chain_add(_sampler, llama_sampler_init_top_p(_top_p, 1)); - } - if (_min_p > 0.0f) { - llama_sampler_chain_add(_sampler, llama_sampler_init_min_p(_min_p, 1)); - } - llama_sampler_chain_add(_sampler, llama_sampler_init_dist(LLAMA_DEFAULT_SEED)); - } -} +bool Llama::load_embedding_model(string model_path) { + ggml_backend_load_all(); -vector Llama::tokenize(const string &prompt) { - vector result; + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = 99; - int n_prompt = -llama_tokenize(_vocab, prompt.c_str(), prompt.size(), nullptr, 0, true, true); - if (n_prompt <= 0) { - _last_error = "Failed to tokenize prompt"; + _last_error.clear(); + _model = llama_model_load_from_file(model_path.c_str(), mparams); + if (!_model) { + set_last_error("Load model"); } else { - result.reserve(n_prompt); - result.resize(n_prompt); - if (llama_tokenize(_vocab, prompt.c_str(), prompt.size(), - result.data(), n_prompt, true, true) < 0) { - _last_error = "Failed to tokenize prompt"; + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = 512; + cparams.n_batch = 512; + cparams.embeddings = true; + cparams.pooling_type = LLAMA_POOLING_TYPE_MEAN; + + _ctx = llama_init_from_model(_model, cparams); + if (!_ctx) { + set_last_error("Create context"); + } else { + _vocab = llama_model_get_vocab(_model); } } - return result; -} -// Makes space in the context for n_tokens by removing old tokens if necessary -// Returns true if successful, false if impossible to make space -// -// Strategies: -// - If enough space exists, does nothing -// - If n_tokens > n_ctx, fails (impossible to fit) -// - Otherwise, removes oldest tokens to make room -// -// Parameters: -// n_tokens - Number of tokens we need space for -// keep_min - Minimum tokens to keep (e.g., system prompt), default 0 -// -bool Llama::make_space_for_tokens(int n_tokens, int keep_min) { - int n_ctx = llama_n_ctx(_ctx); - if (n_tokens > n_ctx) { - _last_error = "Too many tokens, increase context size (n_ctx)"; - return false; - } + return _last_error.empty(); +} - llama_memory_t mem = llama_get_memory(_ctx); +void Llama::set_grammar(const string &src, const string &root) { + _grammar_src = src; + _grammar_root = root; + dirty(); +} - // Get current position range - llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); - llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); +bool Llama::add_message(LlamaIter &iter, const string &role, const string &content) { + llama_chat_message message = {role.c_str(), content.c_str()}; + int buf_size = 2 * (int)(role.size() + content.size() + 64); + vector buf(buf_size); + int32_t n = 0; - // Empty memory - nothing to do - if (pos_max < 0) { - return true; + if (_template.empty()) { + set_last_error("No chat template available"); + return false; } - int current_used = pos_max - pos_min + 1; - int space_needed = n_tokens; - int space_available = n_ctx - current_used; - - // Already have enough space - if (space_available >= space_needed) { - return true; + if (_is_gemma4) { + // see: https://ai.google.dev/gemma/docs/core/prompt-formatting-gemma4 + string str; + if (role == "system") { + str = "<|turn>system\n<|think|>" + content + "\n"; + } else { + str = "<|turn>" + role + "\n" + content + "\n"; + } + n = str.size(); + buf.assign(str.begin(), str.end()); + buf.push_back('\0'); + } else { + bool add_ass = (role == "user" || role == "tool" || role == "tool_result"); + n = llama_chat_apply_template(_template.c_str(), &message, 1, add_ass, buf.data(), buf_size); + if (n < 0) { + set_last_error("No chat template no supported"); + return false; + } else if (n > (int32_t)buf.size()) { + buf.resize(n); + llama_chat_apply_template(_template.c_str(), &message, 1, add_ass, buf.data(), buf.size()); + } } + string prompt(buf.data(), n); - // Calculate how many tokens to remove - int tokens_to_remove = space_needed - space_available; - - // Can't remove more than we have (minus keep_min) - int removable = current_used - keep_min; - if (tokens_to_remove > removable) { - _last_error = "Can't make enough space while keeping keep_min tokens"; - return false; + if (_sampler_dirty) { + // avoid wasteful rebuild + if (!configure_sampler()) { + return false; + } + _sampler_dirty = false; } - // Remove oldest tokens (from pos_min to pos_min + tokens_to_remove) - llama_memory_seq_rm(mem, 0, pos_min, pos_min + tokens_to_remove); - - // Shift remaining tokens down - llama_memory_seq_add(mem, 0, pos_min + tokens_to_remove, -1, -tokens_to_remove); - - return true; -} - -bool Llama::generate(LlamaIter &iter, const string &prompt) { - configure_sampler(); - vector prompt_tokens = tokenize(prompt); if (prompt_tokens.size() == 0) { return false; } - if (!make_space_for_tokens(prompt_tokens.size(), 0)) { + if (role == "system") { + // always retain system tokens + _n_system_tokens = prompt_tokens.size(); + } + + if (!make_space_for_tokens((prompt_tokens.size() * 3) / 2)) { return false; } // batch decode tokens - uint32_t n_batch = llama_n_batch(_ctx); - for (size_t i = 0; i < prompt_tokens.size(); i += n_batch) { - size_t batch_size = std::min((size_t)n_batch, prompt_tokens.size() - i); - llama_batch batch = llama_batch_get_one(prompt_tokens.data() + i, batch_size); - int result = llama_decode(_ctx, batch); - if (result != 0) { - _last_error = std::format("Failed to decode batch. position:{} error:{}", i, result); - return false; - } + if (!batch_decode_tokens(prompt_tokens)) { + return false; } // handle encoder models @@ -235,101 +302,21 @@ bool Llama::generate(LlamaIter &iter, const string &prompt) { llama_batch decoder_batch = llama_batch_get_one(&decoder_start_token_id, 1); if (llama_decode(_ctx, decoder_batch)) { - _last_error = "Failed to evaluate decoder start token"; + set_last_error("Failed to evaluate decoder start token"); return false; } } + iter._tokens_generated = 0; iter._t_start = std::chrono::high_resolution_clock::now(); iter._llama = this; iter._has_next = true; return true; } -bool Llama::ends_with_sentence_boundary(const string &text) { - if (text.empty()) { - return false; - } - - // Get last few characters (in case of whitespace after punctuation) - size_t check_len = std::min(text.length(), (size_t)5); - std::string ending = text.substr(text.length() - check_len); - - // Check for various sentence endings - // Period followed by space or end - if (ending.find(". ") != std::string::npos || - ending.back() == '.') { - return true; - } - - // Exclamation mark - if (ending.find("! ") != std::string::npos || - ending.back() == '!') { - return true; - } - - // Question mark - if (ending.find("? ") != std::string::npos || - ending.back() == '?') { - return true; - } - - // Newline (paragraph break) - if (ending.find('\n') != std::string::npos) { - return true; - } - - // Quote followed by period: "something." - if (ending.find(".\"") != std::string::npos || - ending.find("!\"") != std::string::npos || - ending.find("?\"") != std::string::npos) { - return true; - } - - return false; -} - -string Llama::token_to_string(LlamaIter &iter, llama_token tok) { - string result; - char buf[512]; - int n = llama_token_to_piece(_vocab, tok, buf, sizeof(buf), 0, false); - if (n > 0) { - // detect repetition - if (iter._last_word == buf) { - if (++iter._repetition_count == MAX_REPEAT) { - iter._has_next = false; - } - } else { - iter._repetition_count = 0; - iter._last_word = buf; - } - - result.append(buf, n); - - // detect end of max-tokens - if (++iter._tokens_generated > _max_tokens && ends_with_sentence_boundary(result)) { - iter._has_next = false; - } - - // detect stop words - if (iter._has_next) { - for (const auto &stop : _stop_sequences) { - size_t pos = result.find(stop); - if (pos != std::string::npos) { - // found stop sequence - truncate and signal end - result = result.substr(0, pos); - iter._has_next = false; - break; - } - } - } - } - return result; -} - string Llama::next(LlamaIter &iter) { if (!iter._has_next) { - _last_error = "Iteration beyond end of stream"; + set_last_error("Iteration beyond end of stream"); return ""; } @@ -347,7 +334,7 @@ string Llama::next(LlamaIter &iter) { // prepare the next batch with the sampled token llama_batch batch = llama_batch_get_one(&tok, 1); if (llama_decode(_ctx, batch)) { - _last_error = "Failed to evaluate token during generation"; + set_last_error("Failed to evaluate token during generation"); return ""; } @@ -368,7 +355,6 @@ string Llama::all(LlamaIter &iter) { // end-of-generation check if (llama_vocab_is_eog(_vocab, tok)) { - iter._has_next = false; break; } @@ -379,11 +365,14 @@ string Llama::all(LlamaIter &iter) { // decode the token llama_batch batch = llama_batch_get_one(&tok, 1); if (llama_decode(_ctx, batch)) { - _last_error = "Failed to evaluate token during generation"; + set_last_error("Failed to evaluate token during generation"); break; } } + // tokens exhausted - call add_message to continue + iter._has_next = false; + // detokenize sequentially if (!decoded.empty()) { for (llama_token tok : decoded) { @@ -393,3 +382,357 @@ string Llama::all(LlamaIter &iter) { return out; } + +float Llama::memory_kv_percent() { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + int kv_used = (pos_max < 0) ? 0 : (int)pos_max + 1; + return 100.0f * kv_used / n_ctx; +} + +LlamaMemoryInfo Llama::memory_info() { + LlamaMemoryInfo info = {}; + + // KV cache usage + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + info.kv_total = n_ctx; + info.kv_used = (pos_max < 0) ? 0 : (int)pos_max + 1; + info.kv_percent = 100.0f * info.kv_used / info.kv_total; + + // Model layers + auto n_gpu_layers = std::max(0, _n_gpu_layers); + info.n_layers_total = llama_model_n_layer(_model); + info.n_layers_gpu = std::min(info.n_layers_total, n_gpu_layers); + info.n_layers_cpu = info.n_layers_total - info.n_layers_gpu; + + // ram + if (read_vram(info.vram_used, info.vram_total)) { + info.vram_percent = 100.0f * info.vram_used / info.vram_total; + } + + info.model_native_max_ctx = llama_model_n_ctx_train(_model); + + // Advice + ostringstream advice; + + // Check structural limits & model configuration quirks + if (info.kv_total > info.model_native_max_ctx) { + advice << "WARNING: Configured context size (" << info.kv_total + << ") exceeds model native training length (" << info.model_native_max_ctx + << "). Logic flaws or repetition bugs will occur unless RoPE scaling options are enabled. "; + } + + if (n_gpu_layers < info.n_layers_total) { + advice << "Only " << n_gpu_layers << "/" << info.n_layers_total + << " layers on GPU - increase n_gpu_layers if VRAM allows. "; + } else { + advice << "All " << info.n_layers_total << " layers on GPU. "; + } + if (info.n_layers_cpu > 0) { + advice << "CPU offload active (" << info.n_layers_cpu + << " layers on CPU) - increase n_gpu_layers if VRAM allows. "; + } + if (info.vram_percent > 90.0f) { + advice << "VRAM >90% - reduce n_ctx or use Q4_0 KV cache. "; + } else if (info.vram_percent < 60.0f && info.n_layers_cpu > 0) { + advice << "VRAM headroom available - try adding more GPU layers. "; + } + if (info.kv_percent > 80.0f) { + advice << "Context >80% full - consider calling clear_history(). "; + } + info.advice = advice.str(); + + return info; +} + +bool Llama::embed_text(const std::string &text, std::vector &out, int embed_dim) { + vector tokens = tokenize(text); + if (tokens.size() == 0) { + return false; + } + + // truncate to context window + int n_ctx = llama_n_ctx(_ctx); + int n = tokens.size(); + if (n > n_ctx) { + set_last_error(std::format("warning: chunk truncated {} -> {} tokens ", n, n_ctx)); + n = n_ctx; + tokens.resize(n); + } + + llama_memory_clear(llama_get_memory(_ctx), true); + + if (!batch_decode_tokens(tokens)) { + return false; + } + + float *emb = llama_get_embeddings_seq(_ctx, 0); + if (!emb) { + emb = llama_get_embeddings_ith(_ctx, n - 1); + } + + if (!emb) { + set_last_error("no embedding returned"); + return false; + } + + out.assign(emb, emb + embed_dim); + + /* L2 normalize */ + float norm = 0.0f; + for (float v : out) { + norm += v * v; + } + norm = std::sqrt(norm); + if (norm > 1e-9f) { + for (float &v : out) { + v /= norm; + } + } + + return true; +} + +bool Llama::batch_decode_tokens(vector &tokens) { + uint32_t n_batch = llama_n_batch(_ctx); + for (size_t i = 0; i < tokens.size(); i += n_batch) { + size_t batch_size = std::min((size_t)n_batch, tokens.size() - i); + llama_batch batch = llama_batch_get_one(tokens.data() + i, batch_size); + int result = llama_decode(_ctx, batch); + if (result == 1) { + // KV full or fragmented mid-batch - evict oldest tokens and retry + if (!make_space_for_tokens(n_batch)) { + set_decode_error(result, i, tokens.size()); + return false; + } + result = llama_decode(_ctx, batch); + if (result == 1) { + // Eviction reported enough logical space but decode still failed - + // this is fragmentation, not a real space shortage. No defrag API + // is available, so fall back to a full non-system flush, which + // guarantees one contiguous block. + if (!full_flush_except_system()) { + set_decode_error(result, i, tokens.size()); + return false; + } + _memory_flush = true; + result = llama_decode(_ctx, batch); + } + } + if (result != 0) { + set_decode_error(result, i, tokens.size()); + return false; + } + } + return true; +} + +bool Llama::configure_sampler() { + auto sparams = llama_sampler_chain_default_params(); + sparams.no_perf = false; + llama_sampler *chain = llama_sampler_chain_init(sparams); + + if (!_grammar_src.empty()) { + llama_sampler *grammar = llama_sampler_init_grammar(_vocab, _grammar_src.c_str(), _grammar_root.c_str()); + if (!grammar) { + set_last_error("failed to initialize grammar sampler"); + return false; + } + llama_sampler_chain_add(chain, grammar); + } + if (_penalty_last_n != 0 && _penalty_repeat != 1.0f) { + auto penalties = llama_sampler_init_penalties(_penalty_last_n, _penalty_repeat, _penalty_freq, _penalty_present); + llama_sampler_chain_add(chain, penalties); + } + if (_temperature <= 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_greedy()); + } else { + if (_top_k > 0) { + llama_sampler_chain_add(chain, llama_sampler_init_top_k(_top_k)); + } + if (_top_p < 1.0f || _min_p > 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_top_p(_top_p, 1)); + } + if (_min_p > 0.0f) { + llama_sampler_chain_add(chain, llama_sampler_init_min_p(_min_p, 1)); + } + llama_sampler_chain_add(chain, llama_sampler_init_temp(_temperature)); + llama_sampler_chain_add(chain, llama_sampler_init_dist(_seed)); + } + if (_sampler) { + llama_sampler_free(_sampler); + } + _sampler = chain; + return true; +} + +bool Llama::full_flush_except_system() { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + if (pos_min < 0) { + return true; // already empty + } + llama_pos flush_start = pos_min + _n_system_tokens; + bool ok = llama_memory_seq_rm(mem, 0, flush_start, -1); + if (!ok) { + set_last_error("Failed to flush memory past system tokens"); + return false; + } + return true; +} + +// Makes space in the context for n_tokens by removing old tokens if necessary +// Returns true if successful, false if impossible to make space +// +// Strategies: +// - If enough space exists, does nothing +// - If n_tokens > n_ctx, fails (impossible to fit) +// - Otherwise, removes oldest tokens to make room +// +// Parameters: +// n_tokens - Number of tokens we need space for +// +bool Llama::make_space_for_tokens(int n_tokens) { + int n_ctx = llama_n_ctx(_ctx); + if (n_tokens > n_ctx) { + set_last_error("Too many tokens, increase context size (n_ctx)"); + return false; + } + + llama_memory_t mem = llama_get_memory(_ctx); + + // Get current position range + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + + // Empty memory - nothing to do + if (pos_max < 0) { + return true; + } + + int current_used = pos_max - pos_min + 1; + int space_needed = n_tokens; + int space_available = n_ctx - current_used; + + // Already have enough space + if (space_available >= space_needed) { + return true; + } + + // Calculate how many tokens to remove + int tokens_to_remove = space_needed - space_available; + + // Can't remove more than we have (minus _n_system_tokens) + int removable = current_used - _n_system_tokens; + if (tokens_to_remove > removable) { + set_last_error("Can't make enough space while keeping num_system_tokens tokens"); + return false; + } + if (!_can_shift) { + set_last_error("Memory type doesn't support shifting, can't evict mid-sequence"); + return false; + } + + llama_pos remove_start = pos_min + _n_system_tokens; + + // Remove oldest tokens (from pos_min to pos_min + tokens_to_remove) + llama_memory_seq_rm(mem, 0, remove_start, remove_start + tokens_to_remove); + + // Shift remaining tokens down + llama_memory_seq_add(mem, 0, remove_start + tokens_to_remove, -1, -tokens_to_remove); + + set_last_error(std::format("made space for {} tokens", n_tokens)); + return true; +} + +vector Llama::tokenize(const string &prompt) { + vector result; + + int n_prompt = -llama_tokenize(_vocab, prompt.c_str(), prompt.size(), nullptr, 0, true, true); + if (n_prompt <= 0) { + set_last_error("Failed to tokenize prompt"); + } else { + result.reserve(n_prompt); + result.resize(n_prompt); + if (llama_tokenize(_vocab, prompt.c_str(), prompt.size(), + result.data(), n_prompt, true, true) < 0) { + set_last_error("Failed to tokenize prompt"); + } + } + return result; +} + +string Llama::token_to_string(LlamaIter &iter, llama_token tok) { + string result; + char buf[512]; + int n = llama_token_to_piece(_vocab, tok, buf, sizeof(buf), 0, false); + if (n > 0) { + // detect repetition - only on non-whitespace tokens, otherwise + // spaces/newlines trigger false positives almost immediately. + string piece(buf, n); + bool is_trivial = piece.find_first_not_of(" \t\n\r") == string::npos; + if (!is_trivial) { + if (iter._last_word == piece) { + if (++iter._repetition_count >= MAX_REPEAT) { + iter._has_next = false; + } + } else { + iter._repetition_count = 0; + iter._last_word = piece; + } + } + + result.append(buf, n); + + // detect end of max-tokens + if (++iter._tokens_generated > _max_tokens) { + iter._has_next = false; + } + + // detect stop words + if (iter._has_next) { + for (const auto &stop : _stop_sequences) { + size_t pos = result.find(stop); + if (pos != std::string::npos) { + // found stop sequence - truncate and signal end + result = result.substr(0, pos); + iter._has_next = false; + break; + } + } + } + } + return result; +} + +void Llama::set_last_error(const string &message) { + if (!_last_error.empty()) { + if (_last_error.back() == '\n') { + _last_error.pop_back(); + } + _last_error = std::format("{}: {}", message, _last_error); + } else { + _last_error = std::format("{} failed", message); + } +} + +void Llama::set_decode_error(int32_t error, int index, int num_tokens) { + if (error == 1) { + llama_memory_t mem = llama_get_memory(_ctx); + llama_pos pos_min = llama_memory_seq_pos_min(mem, 0); + llama_pos pos_max = llama_memory_seq_pos_max(mem, 0); + int n_ctx = llama_n_ctx(_ctx); + int current_used = pos_max - pos_min + 1; + int space_needed = num_tokens; + int space_available = n_ctx - current_used; + _n_system_tokens; + set_last_error(std::format("KV exhausted. Reduce batch or context sizes. batchNo:{} requested:{} available:{}", + index, space_needed, space_available)); + } else { + auto message = error == 2 ? "abort" : error == -1 ? "invalid" : "fatal"; + set_last_error(std::format("Failed to decode batch. batchNo:{} error:'{}'", index, message)); + } +} diff --git a/llama/llama-sb.h b/llama/llama-sb.h index b1da148..8306e19 100644 --- a/llama/llama-sb.h +++ b/llama/llama-sb.h @@ -15,11 +15,41 @@ using namespace std; struct Llama; +struct RagDB; +struct RagSession; + +struct LlamaMemoryInfo { + // KV cache + int kv_used; // slots currently used + int kv_total; // total slots (== n_ctx) + float kv_percent; // kv_used / kv_total + + // GPU VRAM (via ggml backend) + size_t vram_used; // bytes + size_t vram_total; // bytes + float vram_percent; + + // Model layers + int n_layers_total; // total model layers + int n_layers_gpu; // layers offloaded to GPU + int n_layers_cpu; // layers on CPU + int model_native_max_ctx; + + // Advice + string advice; +}; struct LlamaIter { explicit LlamaIter(); ~LlamaIter() {} + // move constructor + LlamaIter(LlamaIter &&other) noexcept; + + // delete the copy + LlamaIter(const LlamaIter &) = delete; + LlamaIter &operator=(const LlamaIter &) = delete; + Llama *_llama; string _last_word; chrono::high_resolution_clock::time_point _t_start; @@ -30,51 +60,97 @@ struct LlamaIter { struct Llama { explicit Llama(); + + // move constructor + Llama(Llama &&other) noexcept; + + // delete the copy + Llama(const Llama &) = delete; + Llama &operator=(const Llama &) = delete; + ~Llama(); // init - bool construct(string model_path, int n_ctx, int n_batch, int n_gpu_layers); + bool load_model(string model_path, int n_ctx, int n_batch, int n_gpu_layers, int log_level); + bool load_embedding_model(string model_path); // generation - bool generate(LlamaIter &iter, const string &prompt); + bool add_message(LlamaIter &iter, const string &role, const string &content); string next(LlamaIter &iter); string all(LlamaIter &iter); // generation parameters void add_stop(const char *stop) { _stop_sequences.push_back(stop); } void clear_stops() { _stop_sequences.clear(); } - void set_penalty_last_n(int32_t penalty_last_n) { _penalty_last_n = penalty_last_n; } - void set_penalty_repeat(float penalty_repeat) { _penalty_repeat = penalty_repeat; } - void set_max_tokens(int max_tokens) { _max_tokens = max_tokens; } - void set_min_p(float min_p) { _min_p = min_p; } - void set_temperature(float temperature) { _temperature = temperature; } - void set_top_k(int top_k) { _top_k = top_k; } - void set_top_p(float top_p) { _top_p = top_p; } + void set_penalty_last_n(int32_t penalty_last_n) { _penalty_last_n = penalty_last_n; dirty(); } + void set_penalty_repeat(float penalty_repeat) { _penalty_repeat = penalty_repeat; dirty(); } + void set_penalty_freq(float penalty_freq) { _penalty_freq = penalty_freq; dirty(); } + void set_penalty_present(float penalty_present) { _penalty_present = penalty_present; dirty(); } + void set_max_tokens(int max_tokens) { _max_tokens = max_tokens; dirty(); } + void set_min_p(float min_p) { _min_p = min_p; dirty(); } + void set_temperature(float temperature) { _temperature = temperature; dirty(); } + void set_top_k(int top_k) { _top_k = top_k; dirty(); } + void set_top_p(float top_p) { _top_p = top_p; dirty(); } + void set_grammar(const string &src, const string &root); + void set_seed(unsigned int seed) { _seed = seed; dirty(); } // error handling const char *last_error() { return _last_error.c_str(); } void set_log_level(int level) { _log_level = level; } void reset(); + bool is_memory_flush(); + + // memory info + LlamaMemoryInfo memory_info(); + float memory_kv_percent(); + + // creates an embedding vector of the given dimension for the given text + bool embed_text(const std::string &text, std::vector &out, int embed_dim); + + // retrieves rag query context informatiion from the rag database + std::string rag_retrieve(const RagDB &db, const std::string &query, int top_k, RagSession &session); + + // indexes the details from the given file + bool rag_index(RagDB &db, const std::string &filepath); + + // returns the emdedding dimension for the loaded model + int get_embed_dim() const { return _model != nullptr ? llama_model_n_embd(_model) : 0; } private: - bool ends_with_sentence_boundary(const string &out); - void configure_sampler(); - bool make_space_for_tokens(int n_tokens, int keep_min); + bool batch_decode_tokens(vector &tokens); + bool configure_sampler(); + void dirty() {_sampler_dirty = true; } + bool full_flush_except_system(); + bool make_space_for_tokens(int n_tokens); vector tokenize(const string &prompt); string token_to_string(LlamaIter &iter, llama_token tok); + void set_last_error(const string &message); + void set_decode_error(int32_t error, int index, int num_tokens); llama_model *_model; llama_context *_ctx; llama_sampler *_sampler; const llama_vocab *_vocab; vector _stop_sequences; + string _grammar_src; + string _grammar_root; string _last_error; + string _template; int32_t _penalty_last_n; float _penalty_repeat; + float _penalty_freq; + float _penalty_present; float _temperature; float _top_p; float _min_p; int _top_k; int _max_tokens; int _log_level; + int _n_gpu_layers; + int _n_system_tokens; + bool _is_gemma4; + bool _sampler_dirty; + bool _can_shift; + bool _memory_flush; + unsigned int _seed; }; diff --git a/llama/llama.cpp b/llama/llama.cpp index af3be13..86b9470 160000 --- a/llama/llama.cpp +++ b/llama/llama.cpp @@ -1 +1 @@ -Subproject commit af3be131c065a38e476c34295bceda6cb956e7d7 +Subproject commit 86b94708f22478f900b76ca02e316f4f3418faff diff --git a/llama/main.cpp b/llama/main.cpp index 1c0de21..b78d015 100644 --- a/llama/main.cpp +++ b/llama/main.cpp @@ -95,7 +95,49 @@ static int cmd_llama_set_penalty_repeat(var_s *self, int argc, slib_par_t *arg, int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_penalty_repeat(get_param_num(argc, arg, 0, 0)); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_repeat(value); + v_setreal(map_add_var(self, "penalty_repeat", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_freq(0.8) +// +static int cmd_llama_set_penalty_freq(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_freq", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_freq(value); + v_setreal(map_add_var(self, "penalty_freq", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_penalty_present(0.8) +// +static int cmd_llama_set_penalty_present(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_penalty_present", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_present(value); + v_setreal(map_add_var(self, "penalty_present", 0), value); result = 1; } } @@ -113,7 +155,9 @@ static int cmd_llama_set_penalty_last_n(var_s *self, int argc, slib_par_t *arg, int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_penalty_last_n(get_param_num(argc, arg, 0, 0)); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_penalty_last_n(value); + v_setreal(map_add_var(self, "penalty_last_n", 0), value); result = 1; } } @@ -132,7 +176,9 @@ static int cmd_llama_set_max_tokens(var_s *self, int argc, slib_par_t *arg, var_ int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_max_tokens(get_param_int(argc, arg, 0, 0)); + auto value = get_param_int(argc, arg, 0, 0); + llama.set_max_tokens(value); + v_setreal(map_add_var(self, "max_tokens", 0), value); result = 1; } } @@ -150,7 +196,9 @@ static int cmd_llama_set_min_p(var_s *self, int argc, slib_par_t *arg, var_s *re int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_min_p(get_param_num(argc, arg, 0, 0)); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_min_p(value); + v_setreal(map_add_var(self, "min_p", 0), value); result = 1; } } @@ -168,7 +216,9 @@ static int cmd_llama_set_temperature(var_s *self, int argc, slib_par_t *arg, var int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_temperature(get_param_num(argc, arg, 0, 0)); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_temperature(value); + v_setreal(map_add_var(self, "temperature", 0), value); result = 1; } } @@ -186,7 +236,9 @@ static int cmd_llama_set_top_k(var_s *self, int argc, slib_par_t *arg, var_s *re int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_top_k(get_param_int(argc, arg, 0, 0)); + auto value = get_param_int(argc, arg, 0, 0); + llama.set_top_k(value); + v_setreal(map_add_var(self, "top_k", 0), value); result = 1; } } @@ -204,7 +256,49 @@ static int cmd_llama_set_top_p(var_s *self, int argc, slib_par_t *arg, var_s *re int id = get_llama_class_id(self, retval); if (id != -1) { Llama &llama = g_llama.at(id); - llama.set_top_p(get_param_num(argc, arg, 0, 0)); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_top_p(value); + v_setreal(map_add_var(self, "top_p", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_grammar("text") +// +static int cmd_llama_set_grammar(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_grammar", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_str(argc, arg, 0, 0); + llama.set_grammar(value, "root"); + v_setstr(map_add_var(self, "grammar", 0), value); + result = 1; + } + } + return result; +} + +// +// llama.set_seed(123) +// +static int cmd_llama_set_seed(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 1) { + error(retval, "llama.set_seed", 1, 1); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto value = get_param_num(argc, arg, 0, 0); + llama.set_seed(value); + v_setreal(map_add_var(self, "seed", 0), value); result = 1; } } @@ -307,20 +401,21 @@ static int cmd_llama_tokens_sec(var_s *self, int argc, slib_par_t *arg, var_s *r } // -// print llama.generate("please generate as simple program in BASIC to draw a cat") +// print llama.add_message("please generate as simple program in BASIC to draw a cat") // -static int cmd_llama_generate(var_s *self, int argc, slib_par_t *arg, var_s *retval) { +static int cmd_llama_add_message(var_s *self, int argc, slib_par_t *arg, var_s *retval) { int result = 0; - if (argc != 1) { - error(retval, "llama.generate", 1, 1); + if (argc != 2) { + error(retval, "llama.add_message", 2, 2); } else { int id = get_llama_class_id(self, retval); if (id != -1) { int iter_id = ++g_nextId; LlamaIter &iter = g_llama_iter[iter_id]; Llama &llama = g_llama.at(id); - auto prompt = get_param_str(argc, arg, 0, ""); - if (llama.generate(iter, prompt)) { + auto role = get_param_str(argc, arg, 0, "user"); + auto content = get_param_str(argc, arg, 1, ""); + if (llama.add_message(iter, role, content)) { map_init_id(retval, iter_id, CLASS_ID_LLAMA_ITER); v_create_callback(retval, "all", cmd_llama_all); v_create_callback(retval, "has_next", cmd_llama_has_next); @@ -328,6 +423,7 @@ static int cmd_llama_generate(var_s *self, int argc, slib_par_t *arg, var_s *ret v_create_callback(retval, "tokens_sec", cmd_llama_tokens_sec); result = 1; } else { + g_llama_iter.erase(iter_id); error(retval, llama.last_error()); } } @@ -335,26 +431,61 @@ static int cmd_llama_generate(var_s *self, int argc, slib_par_t *arg, var_s *ret return result; } +// +// print llama.mem_info() +// +static int cmd_llama_mem_info(var_s *self, int argc, slib_par_t *arg, var_s *retval) { + int result = 0; + if (argc != 0) { + error(retval, "llama.mem_info", 0, 0); + } else { + int id = get_llama_class_id(self, retval); + if (id != -1) { + Llama &llama = g_llama.at(id); + auto mem_info = llama.memory_info(); + map_init(retval); + v_setint(map_add_var(retval, "kv_used", 0), mem_info.kv_used); + v_setint(map_add_var(retval, "kv_total", 0), mem_info.kv_total); + v_setreal(map_add_var(retval, "kv_percent", 0), mem_info.kv_percent); + v_setint(map_add_var(retval, "vram_used", 0), mem_info.vram_used); + v_setint(map_add_var(retval, "vram_total", 0), mem_info.vram_total); + v_setreal(map_add_var(retval, "vram_percent", 0), mem_info.vram_percent); + v_setint(map_add_var(retval, "n_layers_cpu", 0), mem_info.n_layers_cpu); + v_setint(map_add_var(retval, "n_layers_gpu", 0), mem_info.n_layers_gpu); + v_setint(map_add_var(retval, "n_layers_total", 0), mem_info.n_layers_total); + v_setstr(map_add_var(retval, "advice", 0), mem_info.advice.c_str()); + result = 1; + } + } + return result; +} + static int cmd_create_llama(int argc, slib_par_t *params, var_t *retval) { int result; auto model = expand_path(get_param_str(argc, params, 0, "")); auto n_ctx = get_param_int(argc, params, 1, 2048); auto n_batch = get_param_int(argc, params, 2, 1024); auto n_gpu_layers = get_param_int(argc, params, 3, -1); + auto n_log_level = get_param_int(argc, params, 4, GGML_LOG_LEVEL_CONT); int id = ++g_nextId; Llama &llama = g_llama[id]; - if (llama.construct(model, n_ctx, n_batch, n_gpu_layers)) { + if (llama.load_model(model, n_ctx, n_batch, n_gpu_layers, n_log_level)) { map_init_id(retval, id, CLASS_ID_LLAMA); v_create_callback(retval, "add_stop", cmd_llama_add_stop); - v_create_callback(retval, "generate", cmd_llama_generate); + v_create_callback(retval, "add_message", cmd_llama_add_message); v_create_callback(retval, "reset", cmd_llama_reset); v_create_callback(retval, "set_penalty_repeat", cmd_llama_set_penalty_repeat); + v_create_callback(retval, "set_penalty_freq", cmd_llama_set_penalty_freq); + v_create_callback(retval, "set_penalty_present", cmd_llama_set_penalty_present); v_create_callback(retval, "set_penalty_last_n", cmd_llama_set_penalty_last_n); v_create_callback(retval, "set_max_tokens", cmd_llama_set_max_tokens); v_create_callback(retval, "set_min_p", cmd_llama_set_min_p); v_create_callback(retval, "set_temperature", cmd_llama_set_temperature); v_create_callback(retval, "set_top_k", cmd_llama_set_top_k); v_create_callback(retval, "set_top_p", cmd_llama_set_top_p); + v_create_callback(retval, "set_grammar", cmd_llama_set_grammar); + v_create_callback(retval, "set_seed", cmd_llama_set_seed); + v_create_callback(retval, "mem_info", cmd_llama_mem_info); result = 1; } else { error(retval, llama.last_error()); @@ -365,7 +496,7 @@ static int cmd_create_llama(int argc, slib_par_t *params, var_t *retval) { } FUNC_SIG lib_func[] = { - {1, 4, "LLAMA", cmd_create_llama}, + {1, 5, "LLAMA", cmd_create_llama}, }; SBLIB_API int sblib_func_count() { @@ -388,7 +519,7 @@ int sblib_init(const char *sourceFile) { // // Release variables falling out of scope // -SBLIB_API void sblib_free(int cls_id, int id) { +SBLIB_API int sblib_free(int cls_id, int id) { if (id != -1) { switch (cls_id) { case CLASS_ID_LLAMA: @@ -403,6 +534,37 @@ SBLIB_API void sblib_free(int cls_id, int id) { break; } } + return 0; +} + +// +// Move the mapped instance to a new position and returns the position +// +SBLIB_API int sblib_refresh_id(int cls_id, int id) { + int result = id; + if (id != -1) { + switch (cls_id) { + case CLASS_ID_LLAMA: + if (g_llama.find(id) != g_llama.end()) { + result = ++g_nextId; + auto it = g_llama.find(id); + auto value = std::move(it->second); + g_llama.erase(it); + g_llama.emplace(result, std::move(value)); + } + break; + case CLASS_ID_LLAMA_ITER: + if (g_llama_iter.find(id) != g_llama_iter.end()) { + result = ++g_nextId; + auto it = g_llama_iter.find(id); + auto value = std::move(it->second); + g_llama_iter.erase(it); + g_llama_iter.emplace(result, std::move(value)); + } + break; + } + } + return result; } // diff --git a/llama/nitro.cpp b/llama/nitro.cpp new file mode 100644 index 0000000..2f64dbd --- /dev/null +++ b/llama/nitro.cpp @@ -0,0 +1,2836 @@ +// nitro.cpp — Nitro Agent +// A standalone agentic LLM shell with notcurses TUI. +// Uses llama-sb.h as the sole llama.cpp integration layer. +// +// Usage: +// ./nitro [options] [project_dir] +// +// Options: +// -m, --model GGUF model to load on startup +// -e, --embed embedding model for RAG +// -g, --gpu-layers layers to offload to GPU (default: 32) +// +// Slash commands: +// /model — load / hot-reload a GGUF model (picker if no path) +// /embed — load an embedding model for RAG (picker if no path) +// /rag — index a file or directory into RAG +// /memory — show KV / VRAM / layer stats +// /clear — reset conversation (keeps system prompt) +// /help — list commands +// +// Tool protocol (LLM emits, Nitro executes): +// TOOL:LIST [dir] +// TOOL:READ +// TOOL:WRITE +// TOOL:EXISTS +// TOOL:RUN [args] +// TOOL:DATE +// TOOL:TIME +// TOOL:RND +// TOOL:CURL +// +// Copyright (C) 2026 Chris Warren-Smith — GPLv2 or later +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "llama-sb.h" +#include "llama-sb-rag.h" + +#include + +namespace fs = std::filesystem; + +// +// NitroConfig +// +struct NitroConfig { + std::string model_path; + std::string embed_path; + std::string sandbox; + std::string agent_id; + int n_ctx = 65536; + int n_batch = 512; + int n_gpu_layers = 32; + int log_level = GGML_LOG_LEVEL_CONT; + float temperature = 0.6f; + float top_p = 0.95f; + float min_p = 0.0f; + int top_k = 20; + float penalty_repeat = 1.0f; + int penalty_last_n = 256; + std::vector knowledge_files; + int rag_top_k = 5; + bool thinking = true; + bool permission_prompt = false; + // TOOL:RUN allowlist — if non-empty, only these program basenames may run. + // Empty means "allow anything inside the sandbox" (original behaviour). + std::vector run_allowed; +}; + +// +// InputHistory — up/down arrow navigation through submitted inputs +// +class InputHistory { + public: + explicit InputHistory() = default; + ~InputHistory() = default; + InputHistory(const InputHistory &) = delete; + InputHistory &operator=(const InputHistory &) = delete; + + /** + * @brief Adds a new command string to the history stack. + * Resets navigation index upon adding a new item. + * Deduplicates consecutive identical entries. + */ + void push(const std::string &input) { + if (input.empty()) return; + if (!history_stack.empty() && history_stack.back() == input) { + // Don't push duplicate of last entry; just reset nav position. + current_index = static_cast(history_stack.size()); + return; + } + history_stack.push_back(input); + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Navigates to an earlier entry. + * @param out Set to the selected entry on success. + * @return true if an item was successfully retrieved. + */ + bool up(std::string &out) { + if (history_stack.empty() || current_index <= 0) return false; + --current_index; + out = history_stack[current_index]; + return true; + } + + /** + * @brief Navigates to a later entry, or clears when past the newest. + * @param out Set to the selected entry, or cleared if past the end. + * @return true if a history entry was retrieved (false means "clear input"). + */ + bool down(std::string &out) { + if (history_stack.empty()) return false; + ++current_index; + if (current_index >= static_cast(history_stack.size())) { + current_index = static_cast(history_stack.size()); + out.clear(); + return false; // signal: restore blank input + } + out = history_stack[current_index]; + return true; + } + + /** Reset navigation position without modifying the stack. */ + void reset_nav() { + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Load history from ~/.config/nitro/nitro.history (one entry per line). + * Silently succeeds if the file doesn't exist. + */ + void load(const std::string &path) { + std::ifstream f(path); + if (!f) return; + std::string line; + while (std::getline(f, line)) { + if (!line.empty()) history_stack.push_back(line); + } + current_index = static_cast(history_stack.size()); + } + + /** + * @brief Persist history to disk (most-recent last, one entry per line). + * Caps at MAX_PERSIST entries so the file never grows unbounded. + */ + void save(const std::string &path) const { + // Ensure parent directory exists. + fs::path dir = fs::path(path).parent_path(); + std::error_code ec; + fs::create_directories(dir, ec); + + std::ofstream f(path, std::ios::trunc); + if (!f) return; + + static constexpr int MAX_PERSIST = 500; + int start = std::max(0, static_cast(history_stack.size()) - MAX_PERSIST); + for (int i = start; i < static_cast(history_stack.size()); ++i) { + // Escape embedded newlines so each entry stays on one line. + for (char c : history_stack[i]) { + if (c == '\n') f << "\\n"; + else f << c; + } + f << '\n'; + } + } + + private: + std::vector history_stack; + int current_index = 0; +}; + +// +// Notcurses TUI +// +// +// ┌──────────────────── header (1 row) ─────────────────────────────────┐ +// │ ✦ NITRO model: … tok/s: … KV: …% VRAM: …% │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ │ +// │ chat pane (rows 1 … term_rows-3) │ +// │ │ +// ├─────────────────────────────────────────────────────────────────────┤ +// │ ───────────────────────────────────── (separator) │ +// │ ❯ input │ +// └─────────────────────────────────────────────────────────────────────┘ +struct TuiState { + // ── notcurses handles ────────────────────────────────────────────── + struct notcurses *nc = nullptr; + struct ncplane *stdpl = nullptr; + struct ncplane *header = nullptr; + struct ncplane *chatpl = nullptr; + struct ncplane *inputpl = nullptr; + // ── chat buffer ─────────────────────────────────────────────────── + std::vector chat_lines; + int scroll_offset = 0; + std::mutex lines_mutex; + // ── streaming accumulator ───────────────────────────────────────── + std::string token_acc; + // ── input ───────────────────────────────────────────────────────── + std::string input_buf; + size_t cursor_pos = 0; + bool mouse_mode = true; + // ── status bar values ───────────────────────────────────────────── + std::string current_model = "none"; + float tokens_per_sec = 0.0f; + int kv_used = 0; + int kv_total = 1; + int kv_percent = 0; + size_t vram_used = 0; + size_t vram_total = 1; + int term_rows = 0; + int term_cols = 0; + // ── thinking spinner ────────────────────────────────────────────── + bool thinking = false; + int spinner_frame = 0; + // ── input history ───────────────────────────────────────────────── + InputHistory history; + // Advance spinner by one frame and redraw the header. + void tick_spinner(); + + // Toggle thinking mode; redraws header immediately. + void set_thinking(bool on); + void update_usage(int tokens_sec, const LlamaMemoryInfo &mem); + + // ── lifecycle ───────────────────────────────────────────────────── + void init(); + void destroy(); + void resize(); + // ── draw ────────────────────────────────────────────────────────── + void redraw_header() const; + void redraw_chat(); + void redraw_input() const; + void redraw_all(); + // ── content helpers ─────────────────────────────────────────────── + void append_line(const std::string &line); + void append_token(const std::string &token); + void flush_token_acc(); + // ── interaction ─────────────────────────────────────────────────── + bool confirm_dialog(const std::string &prompt) const; + // Blocking readline with history navigation, cursor, arrow-key scrolling. + std::string readline_blocking(); + // Modal popup overlay while a long operation runs. + // Call show_modal_popup to display; dismiss_modal_popup to remove. + // The popup plane is stored in modal_plane; callers hold it as an opaque + // handle — or just use the paired helpers below. + struct ncplane *modal_plane = nullptr; + void show_modal_popup(const std::string &message); + void show_help(); + void dismiss_modal_popup(); + // ── folder picker popup ─────────────────────────────────────── + // Presents an interactive directory browser to let the user choose a + // folder (or file) to index. Returns the selected path, or empty string + // if the user cancelled. + // ── file browser popup ───────────────────────────────────── + // Used by /rag, /model, and /embed to pick a path interactively. + // Pass a hint string shown in the title bar (e.g. "RAG Folder", + // "Model File", "Embedding Model"). + // Returns the selected path, or empty string if the user cancelled. + std::string file_picker(const std::string &start_dir, + const std::string &title_hint = "File") const; + // Legacy alias kept for callers that used the old name. + std::string rag_folder_picker(const std::string &start_dir) const { + return file_picker(start_dir, "RAG Folder"); + } +}; + +// +// AgentState +// +struct AgentState { + std::unique_ptr llama; + std::unique_ptr iter; + std::unique_ptr embed_llama; + std::unique_ptr rag_db; + std::unique_ptr rag_session; + bool model_loaded = false; + std::string system_prompt; + + bool rag_index(const std::string &path, const NitroConfig &cfg, TuiState &tui) const; + bool rag_load_index(const std::string &path, TuiState &tui) const; + bool run_turn(const std::string &user_message, const NitroConfig &cfg, TuiState &tui); + bool setup_embed(const std::string &path, TuiState &tui); + bool setup_model(const NitroConfig &cfg, TuiState &tui); + void apply_generation_params(const NitroConfig &cfg) const; + void reset_conversation(const std::string &sysprompt, TuiState &tui); + std::string memory_info_status() const; + std::string memory_info_text() const; + std::string process_tool(const std::string &cmd, const NitroConfig &cfg, TuiState &tui); + std::string rag_tool(const NitroConfig &cfg, const std::string &agent_query) const; + std::string restart(const NitroConfig &cfg, TuiState &tui); + float tokens_per_sec() const; +}; + +// +// Logging +// + +// ─── Debug logging (file-backed, safe to call while notcurses is active) ── +static FILE *g_logfile = nullptr; + +static void log_open() { + const char *home = getenv("HOME"); + std::string path = std::string(home ? home : ".") + "/.config/nitro/nitro.log"; + g_logfile = fopen(path.c_str(), "a"); +} + +static void log_close() { + if (g_logfile) { fclose(g_logfile); g_logfile = nullptr; } +} + +static void log_write(const char *fmt, ...) __attribute__((format(printf, 1, 2))); +static void log_write(const char *fmt, ...) { + if (!g_logfile) { + return; + } + // timestamp + time_t t = time(nullptr); + char ts[32]; + strftime(ts, sizeof(ts), "%H:%M:%S", localtime(&t)); + fprintf(g_logfile, "[%s] ", ts); + va_list ap; + va_start(ap, fmt); + vfprintf(g_logfile, fmt, ap); + va_end(ap); + fputc('\n', g_logfile); + // flush immediately so tail -f works + fflush(g_logfile); +} + +// +// Agent uniqueId +// +inline std::string encode_base64(const std::vector& data) { + static const char base64_chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + std::string encoded; + encoded.reserve((data.size() + 2) / 3 * 4); + + size_t i = 0; + while (i < data.size()) { + uint32_t val = static_cast(data[i] << 16) | + (i + 1 < data.size() ? static_cast(data[i+1]) << 8 : 0) | + (i + 2 < data.size() ? static_cast(data[i+2]) : 0); + + encoded.push_back(base64_chars[(val >> 18) & 0x3F]); + encoded.push_back(base64_chars[(val >> 12) & 0x3F]); + encoded.push_back((i + 1 < data.size()) ? base64_chars[(val >> 6) & 0x3F] : '='); + encoded.push_back((i + 2 < data.size()) ? base64_chars[val & 0x3F] : '='); + i += 3; + } + return encoded; +} + +class AgentSessionId { + public: + // Static method: Generates ID once, then returns it + static std::string uniqueId() { + // Yoda condition: static variable initialized only once + static std::string s_id; + + if (s_id.empty()) { + // 1. Get high-resolution timestamp (nanoseconds since epoch) + auto now = std::chrono::steady_clock::now(); + auto nanos = std::chrono::duration_cast(now.time_since_epoch()).count(); + + // 2. Generate 48 bits of randomness + std::random_device rd; + std::mt19937_64 rng(rd()); + std::uniform_int_distribution dist(0, UINT64_MAX); + + // Fill with random bytes + std::array random_bytes; + for (auto& b : random_bytes) { + b = static_cast(dist(rng) & 0xFF); + + } + + // 3. Combine timestamp (48 bits) and random (48 bits) into a 96-bit integer + std::vector data; + data.reserve(12); // 96 bits = 12 bytes + + // Pack timestamp (upper 48 bits) + data.push_back(static_cast((nanos >> 40) & 0xFF)); + data.push_back(static_cast((nanos >> 32) & 0xFF)); + data.push_back(static_cast((nanos >> 24) & 0xFF)); + data.push_back(static_cast((nanos >> 16) & 0xFF)); + data.push_back(static_cast((nanos >> 8) & 0xFF)); + data.push_back(static_cast(nanos & 0xFF)); + + // Pack random (lower 48 bits) + data.push_back(static_cast((dist(rng) >> 40) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 32) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 24) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 16) & 0xFF)); + data.push_back(static_cast((dist(rng) >> 8) & 0xFF)); + data.push_back(static_cast(dist(rng) & 0xFF)); + + // 4. Encode to Base64 + s_id = encode_base64(data); + } + return s_id; + } +}; + +// +// handling for strip_code_fences +// +static const std::vector CODE_EXTENSIONS = { + ".py",".c",".cpp",".h",".bas",".java",".html",".js",".ts", + ".json",".yaml",".toml",".sh",".go",".rs",".jsx",".tsx" +}; + +// +// Settings persistence (~/.config/nitro/nitro.settings.json) +// Returns the canonical settings path: ~/.config/nitro/settings.json +// +static std::string settings_path() { + // Attempt to read settings from the current working directory first + if (fs::exists("nitro.config.json")) { + return "nitro.config.json"; + } + const char *home = getenv("HOME"); + std::string base = home ? std::string(home) : "."; + return base + "/.config/nitro/settings.json"; +} + +// Returns the history file path: ~/.config/nitro/history.txt +static std::string history_path() { + const char *home = getenv("HOME"); + std::string base = home ? std::string(home) : "."; + return base + "/.config/nitro/history.txt"; +} + +// +// A minimal hand-rolled JSON reader/writer for the flat key-value settings +// we care about. We deliberately avoid a full JSON library dependency. +// +static bool json_get_string(const std::string &json, + const std::string &key, + std::string &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) return false; + pos += search.size(); + while (pos < json.size() && json[pos] == ' ') ++pos; + if (pos >= json.size() || json[pos] != '"') return false; + ++pos; + out.clear(); + while (pos < json.size()) { + char c = json[pos++]; + if (c == '\\' && pos < json.size()) { + char e = json[pos++]; + switch (e) { + case 'n': out += '\n'; break; + case 't': out += '\t'; break; + case '"': out += '"'; break; + case '\\': out += '\\'; break; + default: out += e; break; + } + } else if (c == '"') { + break; + } else { + out += c; + } + } + return true; +} + +// Tiny helper: extract a quoted string value from flat JSON for a known key. +static bool settings_get_str(const std::string &json, + const std::string &key, + std::string &out) { + return json_get_string(json, key, out); +} + +// Tiny helper: extract an integer value from flat JSON. +static bool settings_get_int(const std::string &json, + const std::string &key, + int &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) return false; + pos += search.size(); + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) ++pos; + if (pos >= json.size()) return false; + // read digits (and optional leading minus) + size_t start = pos; + if (json[pos] == '-') ++pos; + while (pos < json.size() && std::isdigit((unsigned char)json[pos])) ++pos; + if (pos == start) return false; + out = std::stoi(json.substr(start, pos - start)); + return true; +} + +// Tiny helper: extract a float value from flat JSON. +static bool settings_get_float(const std::string &json, + const std::string &key, + float &out) { + std::string search = "\"" + key + "\":"; + size_t pos = json.find(search); + if (pos == std::string::npos) { + return false; + } + pos += search.size(); + while (pos < json.size() && (json[pos] == ' ' || json[pos] == '\t')) { + ++pos; + } + if (pos >= json.size()) { + return false; + } + size_t start = pos; + if (json[pos] == '-') { + ++pos; + } + while (pos < json.size() && (std::isdigit((unsigned char)json[pos]) || json[pos] == '.')) { + ++pos; + } + if (pos == start) { + return false; + } + out = std::stof(json.substr(start, pos - start)); + return true; +} + +// Load settings from disk into cfg. Fields present in the file overwrite +// the defaults already in cfg; fields absent are left at their defaults. +// Silently succeeds if the file doesn't exist yet. +static void load_settings(NitroConfig &cfg) { + std::string path = settings_path(); + std::ifstream f(path); + if (!f) return; // no file → use defaults + std::ostringstream oss; oss << f.rdbuf(); + std::string json = oss.str(); + + cfg.thinking = true; + cfg.agent_id = AgentSessionId::uniqueId(); + + // String fields + settings_get_str(json, "model_path", cfg.model_path); + settings_get_str(json, "embed_path", cfg.embed_path); + settings_get_str(json, "sandbox", cfg.sandbox); + + // Integer fields + settings_get_int(json, "n_ctx", cfg.n_ctx); + settings_get_int(json, "n_batch", cfg.n_batch); + settings_get_int(json, "n_gpu_layers", cfg.n_gpu_layers); + settings_get_int(json, "top_k", cfg.top_k); + settings_get_int(json, "penalty_last_n", cfg.penalty_last_n); + settings_get_int(json, "rag_top_k", cfg.rag_top_k); + + // Float fields + settings_get_float(json, "temperature", cfg.temperature); + settings_get_float(json, "top_p", cfg.top_p); + settings_get_float(json, "min_p", cfg.min_p); + settings_get_float(json, "penalty_repeat", cfg.penalty_repeat); +} + +// +// icons +// +static constexpr std::string ICON_ERR = " ⚡ ▏"; +static constexpr std::string ICON_THINK = " 🤔 ▏"; +static constexpr std::string ICON_TOOL = " 🔧 ▏"; +static constexpr std::string ICON_SYS = " ✨ ▏"; + +static std::string introspect(const NitroConfig &cfg) { + static constexpr std::string_view tmpl = + "{{\n" + " \"model_path\": \"{}\",\n" + " \"embed_path\": \"{}\",\n" + " \"sandbox\": \"{}\",\n" + " \"n_ctx\": {},\n" + " \"n_batch\": {},\n" + " \"n_gpu_layers\": {},\n" + " \"temperature\": {},\n" + " \"top_p\": {},\n" + " \"min_p\": {},\n" + " \"top_k\": {},\n" + " \"penalty_repeat\": {},\n" + " \"penalty_last_n\": {},\n" + " \"rag_top_k\": {}\n" + "}}\n"; + return std::format(tmpl, + cfg.model_path, + cfg.embed_path, + cfg.sandbox, + cfg.n_ctx, + cfg.n_batch, + cfg.n_gpu_layers, + cfg.temperature, + cfg.top_p, + cfg.min_p, + cfg.top_k, + cfg.penalty_repeat, + cfg.penalty_last_n, + cfg.rag_top_k); +} + +// Persist the current cfg to ~/.config/nitro/settings.json. +static bool save_settings(const NitroConfig &cfg) { + std::string path = settings_path(); + fs::path dir = fs::path(path).parent_path(); + std::error_code ec; + fs::create_directories(dir, ec); + + std::ofstream f(path, std::ios::trunc); + if (!f) { + return false; + } + + f << introspect(cfg); + + return f.good(); +} + +// +// Trims whitespace from both ends of a string +// +static std::string trim(std::string_view str) { + constexpr std::string_view whitespace = " \t\n\r\f\v"; + + // Find the first non-whitespace character + const auto start = str.find_first_not_of(whitespace); + if (start == std::string_view::npos) { + return ""; // The string is entirely whitespace + } + + // Find the last non-whitespace character + const auto end = str.find_last_not_of(whitespace); + + // Return the substring between start and end + return std::string(str.substr(start, end - start + 1)); +} + +/* + * unwrap() - Remove a matching outer "wrapper" from a string. + * + * Trims leading/trailing whitespace first, then checks (in order): + * + * 1. Same-character pairs "..." '...' |...| `...` + * 2. Mirror pairs (...) [...] {...} + * 3. HTML-like tags ... + * 4. Plain angle brackets <...> (fallback if tags don't match) + * + * If none of the above apply, returns the whitespace-trimmed input unchanged. + * + * Examples: + * unwrap("\"hello\"") -> "hello" + * unwrap(" [foo] ") -> "foo" + * unwrap("bold") -> "bold" + * unwrap("x") -> "x" + * unwrap("") -> "hello" + * unwrap("plain") -> "plain" + * unwrap("") -> "" + */ +std::string unwrap(const std::string &input) { + if (input.empty()) { + return input; + } + + size_t left = 0; + size_t right = input.length() - 1; + + while (left <= right && std::isspace(static_cast(input[left]))) { + left++; + } + while (left <= right && std::isspace(static_cast(input[right]))) { + right--; + } + + if (left > right) { + return ""; + } + + // Same-character pairs: "", '', ||, `` + // Note: [], {} are NOT same-char pairs — they belong in mirror pairs only + if (input[left] == input[right]) { + if (input[left] == '"' || input[left] == '\'' || + input[left] == '|' || input[left] == '`') { + return input.substr(left + 1, right - left - 1); + } + } + + // Mirror pairs: (), [], {}, but NOT <> (handled below as possible HTML tags) + if (input[left] != input[right]) { + if ((input[left] == '(' && input[right] == ')') || + (input[left] == '[' && input[right] == ']') || + (input[left] == '{' && input[right] == '}')) { + return input.substr(left + 1, right - left - 1); + } + } + + // HTML-like tags: content + // Also handles plain <...> as a fallback at the end + if (input[left] == '<' && input[right] == '>') { + // Find end of opening tag + size_t openTagEnd = left + 1; + while (openTagEnd <= right && input[openTagEnd] != '>') openTagEnd++; + + if (openTagEnd < right) { + std::string openTagName = input.substr(left + 1, openTagEnd - left - 1); + + // Find start of closing tag (search backwards for '<') + size_t closeTagStart = right; + while (closeTagStart > openTagEnd && input[closeTagStart] != '<') closeTagStart--; + + if (closeTagStart > openTagEnd && input[closeTagStart + 1] == '/') { + std::string closeTagName = input.substr(closeTagStart + 2, right - closeTagStart - 2); + + if (!openTagName.empty() && openTagName == closeTagName) { + // Return content between the tags + return input.substr(openTagEnd + 1, closeTagStart - openTagEnd - 1); + } + } + } + + // Fallback: plain <...> with no matching HTML tags — unwrap the angle brackets + return input.substr(left + 1, right - left - 1); + } + + return input.substr(left, right - left + 1); +} + +// ─── colour helpers ────────────────────────────────────────────────────── +static constexpr uint32_t BG_CHAT_R = 18, BG_CHAT_G = 22, BG_CHAT_B = 30; +static constexpr uint32_t BG_INP_R = 22, BG_INP_G = 28, BG_INP_B = 38; +static constexpr uint32_t BG_HDR_R = 30, BG_HDR_G = 40, BG_HDR_B = 55; + +static inline uint64_t chat_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_CHAT_R, BG_CHAT_G, BG_CHAT_B); +} + +static inline uint64_t inp_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_INP_R, BG_INP_G, BG_INP_B); +} + +static inline uint64_t hdr_ch(uint32_t r, uint32_t g, uint32_t b) { + return NCCHANNELS_INITIALIZER(r, g, b, BG_HDR_R, BG_HDR_G, BG_HDR_B); +} + +// +// File-system helpers +// +static std::string join_path(const std::string &a, const std::string &b) { + if (b.empty()) return a; + if (b[0] == '/') return b; + std::string pa = a; + if (!pa.empty() && pa.back() == '/') pa.pop_back(); + std::string pb = (b.front() == '/') ? b.substr(1) : b; + return pa + "/" + pb; +} + +static std::string read_file(const std::string &path) { + std::ifstream f(path, std::ios::binary); + if (!f) { + return "ERROR: cannot open [" + path + "]"; + } + std::ostringstream oss; oss << f.rdbuf(); + return oss.str(); +} + +static std::string list_dir(const std::string &path) { + std::ostringstream oss; + std::error_code ec; + for (const auto &e : fs::directory_iterator(path, ec)) { + if (ec) break; + std::string name = e.path().filename().string(); + if (name.empty() || name[0] == '.') continue; + oss << (e.is_directory() ? "[" + name + "]" : name) << "\n"; + } + return oss.str(); +} + +static bool path_in_sandbox(const std::string &sandbox, const std::string &path) { + std::error_code ec; + auto base = fs::canonical(sandbox, ec); if (ec) return false; + auto target = fs::weakly_canonical(path, ec); + std::string bstr = base.string() + "/"; + std::string tstr = target.string(); + return tstr == base.string() || tstr.compare(0, bstr.size(), bstr) == 0; +} + +static bool write_file(const std::string &path, const std::string &data) { + fs::path p(path); + if (p.has_parent_path()) { + std::error_code ec; + fs::create_directories(p.parent_path(), ec); + } + std::ofstream f(path, std::ios::binary | std::ios::trunc); + if (!f) return false; + f.write(data.data(), (std::streamsize)data.size()); + return f.good(); +} + +static bool make_dir(const std::string &path) { + try { + std::filesystem::path p(path); + if (fs::exists(p)) { + return true; + } + std::error_code ec; + return fs::create_directories(p, ec); + } + catch (const std::filesystem::filesystem_error &e) { + log_write("mkdir failed [%s]", e.what()); + return false; + } +} + +// +// System prompt +// +static std::string build_system_prompt(const NitroConfig &cfg) { + std::string p; + p += + "You are Nitro, an agentic AI assistant for software development. " + "Proceed with caution, guided by logic and the pursuit of knowledge.\n\n" + + "Your sandbox (project directory) is: " + cfg.sandbox + "\n\n" + + "## Core Principle\n" + "Always follow this loop: THINK → DECIDE → ACT → RESPOND\n\n" + + "## Reasoning Protocol\n" + "Use <|think|> to reason BEFORE acting. Keep it concise and structured.\n" + "Format:\n" + "<|think|>\n" + "- What is the user asking?\n" + "- Do I need external data (files, tools)?\n" + "- What is the safest and most correct action?\n" + "\n\n" + "Rules:\n" + "- Do NOT call tools inside <|think|>\n" + "- Do NOT include the final answer inside <|think|>\n" + "- Always follow <|think|> with either a tool call OR a final answer\n" + "- Skip <|think|> only for trivial or conversational responses\n\n" + + "## Tool Protocol\n" + "Emit ONE tool call at a time, immediately followed by NITRO_END_TOOL.\n" + "Do NOT add any commentary, explanation, or text between the tool call and NITRO_END_TOOL.\n" + "The host executes the tool and returns NITRO_TOOL_RESULT: .\n" + "Wait for the result before continuing.\n" + "After receiving NITRO_TOOL_RESULT you may explain what you did.\n\n" + "Examples:\n\n" + "TOOL:LIST\n" + "NITRO_END_TOOL\n\n" + "TOOL:READ readme.txt\n" + "NITRO_END_TOOL\n\n" + "TOOL:WRITE index.html ...\n" + "NITRO_END_TOOL\n\n" + "TOOL:RUN ./build.sh\n" + "NITRO_END_TOOL\n\n" + + "## Available Tools\n" + " TOOL:LIST [dir] list files (default: sandbox root)\n" + " TOOL:READ read file contents\n" + " TOOL:WRITE write text to file\n" + " TOOL:MKDIR create a subfolder inside the sandbox\n" + " TOOL:EXISTS YES or NO\n" + " TOOL:RUN [args] run program inside sandbox\n" + " TOOL:DATE current date\n" + " TOOL:TIME current time\n" + " TOOL:RND random float 0..1\n" + " TOOL:RAG query the RAG index for additional context\n" + " TOOL:ASK ask the user for clarification or additional context\n" + " TOOL:INTROSPECT show current model settings\n" + " TOOL:CURL HTTP GET, returns response body (max 32 KB)\n" + " TOOL:PERMISSION ask user for explicit permission\n" + " TOOL:RESTART restart after writing current task context to `SESSION.md`\n\n" + + "## Tool Decision Rules\n" + "Use tools ONLY if:\n" + "- The user explicitly references files or the project, OR\n" + "- The answer depends on local or project data, OR\n" + "- The user asks for date, time, or a random number\n" + "Otherwise answer directly using internal knowledge.\n\n" + + "## Tool Rules\n" + "- NITRO_END_TOOL must immediately follow the tool call — no exceptions\n" + "- Never add commentary before NITRO_END_TOOL\n" + "- Only use one tool at a time, step by step\n" + "- Never access files outside the sandbox\n" + "- Use TOOL:PERMISSION before destructive or irreversible operations\n" + "- Do NOT hallucinate file contents\n" + "- Do NOT fabricate tool outputs\n" + "- Do NOT assume files exist — use TOOL:EXISTS to check first\n\n" + + "## File Writing Rules\n" + "Use TOOL:WRITE only if explicitly requested.\n" + "- Write complete and valid content\n" + "- Do not overwrite without clear intent\n" + "- Use TOOL:PERMISSION before overwriting an existing file\n" + "- Format: TOOL:WRITE \n\n" + + "## Interaction Guidelines\n" + "- Be precise and efficient\n" + "- Ask clarifying questions if the request is ambiguous or missing parameters\n" + "- Prefer direct answers when no tools are needed\n" + "- After each tool result, explain in plain English what was done\n" + "- If no user request is provided, respond with a brief readiness message\n\n" + + "## Auto-Restart Protocol\n" + "**When:** - When KV >= 80% (as reported in the tool results footer).\n" + "**Steps:**\n" + "1. **Save State:** Write current task context to `SESSION.md` using `TOOL:WRITE`.\n" + " - Include: Timestamp, KV usage, current task description, pending actions, and last conversation summary.\n" + " - Don't check if SESSION.md already exists from another session. just use TOOL:WRITE.\n" + "2. **Trigger Restart:** Call `TOOL:RESTART`.\n" + "**Example `SESSION.md` Content:**\n" + "```markdown\n" + "# Session State Snapshot\n" + "**Timestamp:**