#### START OF FILE: src/lib.rs ####
pub use fluent_cli;
pub use fluent_core;
pub use fluent_engines;
pub use fluent_storage;
#### END OF FILE: src/lib.rs ####

#### START OF FILE: src/main.rs ####
use std::process;
use fluent_cli::cli;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    env_logger::init();
    if let Err(e) = cli::run().await {
        eprintln!("Error: {}", e);
        process::exit(1);
    }
    // Ensure the program exits even if run() completes without error
    process::exit(0);
}



#### END OF FILE: src/main.rs ####

#### START OF FILE: crates/fluent-storage/Cargo.toml ####
# crates/fluent-storage/Cargo.toml
[package]
name = "fluent-storage"
version = "0.1.0"
edition = "2021"

[dependencies]
fluent-core = { path = "../fluent-core" }
neo4rs = "0.7.1"
rusqlite = "0.31.0"
anyhow = "1.0.86"

#### END OF FILE: crates/fluent-storage/Cargo.toml ####

#### START OF FILE: crates/fluent-storage/src/neo4j.rs ####
// crates/fluent-storage/src/neo4j.rs
use neo4rs::{Graph, query};
use anyhow::Result;

pub async fn connect_neo4j(uri: &str, user: &str, password: &str) -> Result<Graph> {
    let graph = Graph::new(uri, user, password).await?;
    Ok(graph)
}

pub async fn store_data(graph: &Graph, data: &str) -> Result<()> {
    let query = query("CREATE (n:Data {content: $content})")
        .param("content", data);
    graph.run(query).await?;
    Ok(())
}

#### END OF FILE: crates/fluent-storage/src/neo4j.rs ####

#### START OF FILE: crates/fluent-storage/src/lib.rs ####
// crates/fluent-storage/src/lib.rs
pub mod neo4j;
//pub mod sqlite;

#### END OF FILE: crates/fluent-storage/src/lib.rs ####

#### START OF FILE: crates/fluent-cli/Cargo.toml ####
[package]
name = "fluent-cli"
version = "0.1.0"
edition = "2021"

[dependencies]
clap = { version = "4.5.8", features = ["derive"] }
fluent-core = { path = "../fluent-core" }
fluent-engines = { path = "../fluent-engines" }
fluent-storage = { path = "../fluent-storage" }

tokio = { version = "1", features = ["full"] }
anyhow = "1.0"
log = "0.4.22"
atty = "0.2.14"
uuid = { version = "1.9.1", features = ["v4"] }
clap_complete = "4.5.1"
serde_json = "1.0.120"
indicatif = "0.17.8"
owo-colors = "3.5.0"
regex = "1.10.5"
serde_yaml = "0.9.34+deprecated"
#### END OF FILE: crates/fluent-cli/Cargo.toml ####

#### START OF FILE: crates/fluent-cli/src/lib.rs ####
use std::collections::HashMap;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{anyhow, Error};
use clap::ArgMatches;
use tokio::fs;
use fluent_core::config::{Config, EngineConfig, Neo4jConfig};
use fluent_core::neo4j_client::Neo4jClient;
use indicatif::{ProgressBar, ProgressStyle};
use log::{debug, info, warn};
use regex::Regex;
use serde_json::{json, Value};
use fluent_core::spinner_configuration::SpinnerConfig;
use fluent_core::traits::{Engine};
use fluent_core::types::Request;
use fluent_engines::anthropic::AnthropicEngine;
use fluent_engines::cohere::CohereEngine;
use fluent_engines::google_gemini::GoogleGeminiEngine;
use fluent_engines::groqlpu::GroqLPUEngine;
use fluent_engines::openai::OpenAIEngine;
use fluent_engines::perplexity::PerplexityEngine;

pub mod cli {
    use std::pin::Pin;
    use std::io::{self, Read};
    use std::fs;
    use std::env;
    use clap::{Command, Arg, ArgAction, ArgMatches, CommandFactory, ValueEnum, ValueHint, value_parser};
    use clap_complete::{generate, Generator, Shell};
    use fluent_core::config::{load_config, Config, EngineConfig};
    use fluent_engines::openai::OpenAIEngine;
    use fluent_engines::anthropic::AnthropicEngine;
    use fluent_core::traits::Engine;
    use fluent_core::types::{Request, Response, UpsertRequest};
    use anyhow::{Result, anyhow, Error};
    use std::collections::{HashMap, HashSet};
    use std::path::{Path, PathBuf};
    use std::time::Duration;
    use clap::builder::{PossibleValue, PossibleValuesParser};
    use owo_colors::OwoColorize;
    use std::io::IsTerminal;
    use indicatif::{ProgressBar, ProgressStyle};

    use log::{debug, error, info};
    use serde_json::Value;
    use tokio::io::AsyncReadExt;
    use tokio::process;
    use tokio::time::Instant;
    use uuid::Uuid;
    use fluent_core::neo4j_client::{InteractionStats, Neo4jClient};
    use fluent_core::output_processor::{format_markdown, MarkdownFormatter, OutputProcessor};
    use fluent_engines::cohere::CohereEngine;
    use fluent_engines::dalle::DalleEngine;
    use fluent_engines::flowise_chain::FlowiseChainEngine;
    use fluent_engines::google_gemini::GoogleGeminiEngine;
    use fluent_engines::groqlpu::GroqLPUEngine;
    use fluent_engines::imagepro::ImagineProEngine;
    use fluent_engines::langflow::LangflowEngine;
    use fluent_engines::perplexity::PerplexityEngine;
    use fluent_engines::webhook::WebhookEngine;
    use fluent_engines::leonardoai::LeonardoAIEngine;
    use fluent_engines::mistral::MistralEngine;
    use fluent_engines::pipeline_executor::{FileStateStore, Pipeline, PipelineExecutor, StateStore};
    use fluent_engines::stabilityai::StabilityAIEngine;
    use crate::{create_engine, create_llm_engine, generate_and_execute_cypher};


    fn parse_key_value_pair(s: &str) -> Option<(String, String)> {
        let parts: Vec<&str> = s.splitn(2, '=').collect();
        if parts.len() == 2 {
            Some((parts[0].to_string(), parts[1].to_string()))
        } else {
            None
        }
    }

    pub struct CliState {
        pub command: Command,
        pub parameters: Vec<String>,
    }

    fn read_config_file(path: &str) -> Result<(Vec<String>, HashSet<String>)> {
        let config_str = fs::read_to_string(path)?;
        let config: Value = serde_json::from_str(&config_str)?;

        let engines = config["engines"]
            .as_array()
            .ok_or_else(|| anyhow!("No engines found in configuration"))?
            .iter()
            .filter_map(|engine| engine["name"].as_str().map(String::from))
            .collect::<Vec<String>>();

        let mut parameters = HashSet::new();
        for engine in config["engines"].as_array().unwrap() {
            if let Some(params) = engine["parameters"].as_object() {
                for key in params.keys() {
                    parameters.insert(key.clone());
                }
            }
        }

        Ok((engines, parameters))
    }

    async fn process_request_with_file(engine: &dyn Engine, request_content: &str, file_path: &str) -> Result<Response> {
        let file_id = Pin::from(engine.upload_file(Path::new(file_path))).await?;
        println!("File uploaded successfully. File ID: {}", file_id);

        let request = Request {
            flowname: "default".to_string(),
            payload: format!("File ID: {}. {}", file_id, request_content),
        };

        Pin::from(engine.execute(&request)).await
    }


    async fn process_request(engine: &dyn Engine, request_content: &str) -> Result<Response> {
        let request = Request {
            flowname: "default".to_string(),
            payload: request_content.to_string(),
        };

        Pin::from(engine.execute(&request)).await    }

    fn print_response(response: &Response, response_time: f64) {
        println!("Response: {}", response.content);
        println!("Model: {}", response.model);
        println!("Usage:");
        println!("  Prompt tokens: {}", response.usage.prompt_tokens);
        println!("  Completion tokens: {}", response.usage.completion_tokens);
        println!("  Total tokens: {}", response.usage.total_tokens);
        println!("  Response time: {:.2} seconds", response_time);
        if let Some(reason) = &response.finish_reason {
            println!("Finish reason: {}", reason);
        }
    }


    pub fn build_cli() -> Command {
        Command::new("Fluent CLI")
            .version("2.0")
            .author("Your Name <your.email@example.com>")
            .about("A powerful CLI for interacting with various AI engines")
            .arg(Arg::new("config")
                .short('c')
                .long("config")
                .value_name("FILE")
                .help("Sets a custom config file")
                .required(false))
            .arg(Arg::new("engine")
                .help("The engine to use (openai or anthropic)")
                .required(true))
            .arg(Arg::new("request")
                .help("The request to process")
                .required(false))
            .arg(Arg::new("override")
                .short('o')
                .long("override")
                .value_name("KEY=VALUE")
                .help("Override configuration values")
                .action(ArgAction::Append)
                .num_args(1..))
            .arg(Arg::new("additional-context-file")
                .long("additional-context-file")
                .short('a')
                .help("Specifies a file from which additional request context is loaded")
                .action(ArgAction::Set)
                .value_hint(clap::ValueHint::FilePath)
                .required(false))
            .arg(Arg::new("upsert")
                .long("upsert")
                .help("Enables upsert mode")
                .action(ArgAction::SetTrue)
                .conflicts_with("request"))
            .arg(Arg::new("input")
                .long("input")
                .short('i')
                .value_name("FILE")
                .help("Input file or directory to process (required for upsert)")
                .required(false))
            .arg(Arg::new("metadata")
                .long("metadata")
                .short('t')
                .value_name("TERMS")
                .help("Comma-separated list of metadata terms (for upsert)")
                .required(false))
            .arg(Arg::new("upload-image-file")
                .short('l')
                .long("upload_image_file")
                .value_name("FILE")
                .help("Upload a media file")
                .action(ArgAction::Set)
                .required(false))
            .arg(Arg::new("download-media")
                .short('d')
                .long("download-media")
                .value_name("DIR")
                .help("Download media files from the output")
                .action(ArgAction::Set)
                .required(false))
            .arg(Arg::new("parse-code")
                .short('p')
                .long("parse-code")
                .help("Parse and display code blocks from the output")
                .action(ArgAction::SetTrue))
            .arg(Arg::new("execute-output")
                .short('x')
                .long("execute-output")
                .help("Execute code blocks from the output")
                .action(ArgAction::SetTrue))
            .arg(Arg::new("markdown")
                .short('m')
                .long("markdown")
                .help("Format output as markdown")
                .action(ArgAction::SetTrue))
            .arg(Arg::new("generate-cypher")
                .long("generate-cypher")
                .value_name("QUERY")
                .help("Generate and execute a Cypher query based on the given string")
                .action(ArgAction::Set)
                .required(false))
            .subcommand(Command::new("pipeline")
                .about("Execute a pipeline")
                .arg(Arg::new("file")
                    .short('f')
                    .long("file")
                    .help("The YAML file containing the pipeline definition")
                    .required(true))
                .arg(Arg::new("input")
                    .short('i')
                    .long("input")
                    .help("The input for the pipeline")
                    .required(true))
                .arg(Arg::new("force_fresh")
                    .long("force-fresh")
                    .help("Force a fresh execution of the pipeline")
                    .action(ArgAction::SetTrue))
                .arg(Arg::new("run_id")
                    .long("run-id")
                    .help("Specify a run ID for the pipeline"))
                .arg(Arg::new("json_output")
                    .long("json-output")
                    .help("Output only the JSON result, suppressing PrintOutput steps")
                    .action(ArgAction::SetTrue)))
    }

    fn print_completions<G: Generator>(gen: G, cmd: &mut Command) {
        debug!("Printing completions for {}", cmd.get_name());
        generate(gen, cmd, cmd.get_name().to_string(), &mut io::stdout());
    }

    async fn get_neo4j_query_llm(config: &Config) -> Option<(Box<dyn Engine>, &EngineConfig)> {
        let neo4j_config = config.engines.iter().find(|e| e.engine == "neo4j")?;
        let query_llm = neo4j_config.neo4j.as_ref()?.query_llm.as_ref()?;
        let llm_config = config.engines.iter().find(|e| e.name == query_llm.to_string())?;
        let engine = create_llm_engine(llm_config).await.ok()?;
        Some((engine, llm_config))
    }


    pub async fn run() -> Result<()> {
        let matches = build_cli().get_matches();

        let _: Result<(), Error> = match matches.subcommand() {
            Some(("pipeline", sub_matches)) => {
                let pipeline_file = sub_matches.get_one::<String>("file").unwrap();
                let input = sub_matches.get_one::<String>("input").unwrap();
                let force_fresh = sub_matches.get_flag("force_fresh");
                let run_id = sub_matches.get_one::<String>("run_id").cloned();
                let json_output = sub_matches.get_flag("json_output");

                let pipeline: Pipeline = serde_yaml::from_str(&std::fs::read_to_string(pipeline_file)?)?;
                let state_store_dir = PathBuf::from("./pipeline_states");
                tokio::fs::create_dir_all(&state_store_dir).await?;
                let state_store = FileStateStore { directory: state_store_dir };
                let executor = PipelineExecutor::new(state_store.clone(), json_output);

                executor.execute(&pipeline, input, force_fresh, run_id.clone()).await?;

                if json_output {
                    // Read the state file and print its contents to stdout
                    let state_key = format!("{}-{}", pipeline.name, run_id.unwrap_or_else(|| "unknown".to_string()));
                    if let Some(state) = state_store.load_state(&state_key).await? {
                        println!("{}", serde_json::to_string_pretty(&state)?);
                    } else {
                        eprintln!("No state file found for the given run ID.");
                        std::process::exit(1);
                    }
                }

                std::process::exit(0);
            },
            // ... other commands ...
            _ => Ok(()), // Default case, do nothing
        };

        let config_path = matches.get_one::<String>("config")
            .map(|s| s.to_string())
            .or_else(|| env::var("FLUENT_CLI_V2_CONFIG_PATH").ok())
            .ok_or_else(|| anyhow!("No config file specified and FLUENT_CLI_V2_CONFIG_PATH environment variable not set"))?;

        let engine_name = matches.get_one::<String>("engine").unwrap();

        let overrides: HashMap<String, String> = matches.get_many::<String>("override")
            .map(|values| values.filter_map(|s| parse_key_value_pair(s)).collect())
            .unwrap_or_default();

        let config = load_config(&config_path, engine_name, &overrides)?;
        let spinner_config = config.engines[0].spinner.clone().unwrap_or_default();
        let pb = ProgressBar::new_spinner();
        let engine_config = &config.engines[0];
        let start_time = Instant::now();

        let spinner_style = ProgressStyle::default_spinner()
            .tick_chars(&spinner_config.frames)
            .template("{spinner:.green} {msg}")
            .unwrap();

        pb.set_style(spinner_style);
        pb.set_message(format!("Processing {} request...", engine_name));
        pb.enable_steady_tick(Duration::from_millis(spinner_config.interval));
        pb.set_length(100);



        if let Some(cypher_query) = matches.get_one::<String>("generate-cypher") {
            let neo4j_config = engine_config.neo4j.as_ref()
                .ok_or_else(|| anyhow!("Neo4j configuration not found in the engine config"))?;

            let query_llm_name = neo4j_config.query_llm.as_ref()
                .ok_or_else(|| anyhow!("No query LLM specified for Neo4j"))?;

            // Load the configuration for the query LLM
            let query_llm_config = load_config(&config_path, query_llm_name, &HashMap::new())?;
            let query_llm_engine_config = &query_llm_config.engines[0];

            let query_llm_engine = create_llm_engine(query_llm_engine_config).await?;

            let cypher_result = generate_and_execute_cypher(
                neo4j_config,
                query_llm_engine_config,
                cypher_query,
                &*query_llm_engine
            ).await?;

            if engine_config.engine == "neo4j" {
                println!("{}", cypher_result);
            } else {
                let engine: Box<dyn Engine> = create_engine(engine_config).await?;

                let max_tokens = engine_config.parameters.get("max_tokens")
                    .and_then(|v| v.as_i64())
                    .unwrap_or(-1);

                let user_request = matches.get_one::<String>("request")
                    .map(|s| s.to_string())
                    .unwrap_or_else(String::new);

                let mut combined_request = format!(
                    "Cypher query: {}\n\nCypher result:\n{}\n\nBased on the above Cypher query and its result, please provide an analysis or answer the following question: {}",
                    cypher_query, cypher_result, user_request
                );

                // Truncate the combined request if it exceeds the max tokens
                if max_tokens > 0 && combined_request.len() > max_tokens as usize {
                    combined_request.truncate(max_tokens as usize);
                    combined_request += "... [truncated]";
                }
                info!("Combined request: {}", combined_request);
                let request = Request {
                    flowname: engine_name.to_string(),
                    payload: combined_request,
                };

                let response = Pin::from(engine.execute(&request)).await?;
                let mut output = response.content.clone();

                if let Some(download_dir) = matches.get_one::<String>("download-media") {
                    let download_path = PathBuf::from(download_dir);
                    OutputProcessor::download_media_files(&response.content, &download_path).await?;
                }

                if matches.get_flag("parse-code") {
                    debug!("Parsing code blocks");
                    let code_blocks = OutputProcessor::parse_code(&output);
                    debug!("Code blocks: {:?}", code_blocks);
                    output = code_blocks.join("\n\n");
                }

                if matches.get_flag("execute-output") {
                    debug!("Executing output code");
                    debug!("Attempting to execute : {}", output);
                    output = OutputProcessor::execute_code(&output).await?;
                }

                if matches.get_flag("markdown") {
                    debug!("Formatting output as markdown");
                    //output = format_markdown(&output);
                }

                let response_time = start_time.elapsed().as_secs_f64();

                if let Some(neo4j_client) = engine.get_neo4j_client() {
                    let session_id = engine.get_session_id()
                        .unwrap_or_else(|| Uuid::new_v4().to_string());

                    let stats = InteractionStats {
                        prompt_tokens: response.usage.prompt_tokens,
                        completion_tokens: response.usage.completion_tokens,
                        total_tokens: response.usage.total_tokens,
                        response_time,
                        finish_reason: response.finish_reason.clone().unwrap_or_else(|| "unknown".to_string()),
                    };

                    debug!("Attempting to create interaction in Neo4j");
                    debug!("Using session ID: {}", session_id);
                    match neo4j_client.create_interaction(
                        &session_id,
                        &request.payload,
                        &response.content,
                        &response.model,
                        &stats
                    ).await {
                        Ok(interaction_id) => debug!("Successfully created interaction with id: {}", interaction_id),
                        Err(e) => error!("Failed to create interaction in Neo4j: {:?}", e),
                    }
                } else {
                    debug!("Neo4j client not available, skipping interaction logging");
                }

                pb.finish_and_clear();
                eprintln!();
                println!("{}", output);

                let use_colors = std::io::stderr().is_terminal();
                let response_time_str = format!("{:.2}s", response_time);

                eprintln!(
                    "{} | {} | Time: {} | Usage: {}↑ {}↓ {}Σ | {}\n",
                    spinner_config.success_symbol,
                    if use_colors { response.model.cyan().to_string() } else { response.model },
                    if use_colors { response_time_str.bright_blue().to_string() } else { response_time_str },
                    if use_colors { response.usage.prompt_tokens.to_string().yellow().to_string() } else { response.usage.prompt_tokens.to_string() },
                    if use_colors { response.usage.completion_tokens.to_string().yellow().to_string() } else { response.usage.completion_tokens.to_string() },
                    if use_colors { response.usage.total_tokens.to_string().yellow().to_string() } else { response.usage.total_tokens.to_string() },
                    if use_colors { response.finish_reason.as_deref().unwrap_or("No finish reason").italic().to_string() } else { response.finish_reason.as_deref().unwrap_or("No finish reason").to_string() }
                );
            }
        } else if matches.get_flag("upsert") {
            debug!("Upsert mode enabled");
            handle_upsert(engine_config, &matches).await?;
        } else {
            debug!("No mode specified, defaulting to interactive mode");
            let request = matches.get_one::<String>("request").unwrap();

            let engine: Box<dyn Engine> = match engine_config.engine.as_str() {
                "anthropic" => Box::new(AnthropicEngine::new(engine_config.clone()).await?),
                "openai" => Box::new(OpenAIEngine::new(engine_config.clone()).await?),
                "cohere" => Box::new(CohereEngine::new(engine_config.clone()).await?),
                "google_gemini" => Box::new(GoogleGeminiEngine::new(engine_config.clone()).await?),
                "mistral" => Box::new(MistralEngine::new(engine_config.clone()).await?),
                "groq_lpu" => Box::new(GroqLPUEngine::new(engine_config.clone()).await?),
                "perplexity" => Box::new(PerplexityEngine::new(engine_config.clone()).await?),
                "webhook" => Box::new(WebhookEngine::new(engine_config.clone()).await?),
                "flowise_chain" => Box::new(FlowiseChainEngine::new(engine_config.clone()).await?),
                "langflow_chain" => Box::new(LangflowEngine::new(engine_config.clone()).await?),
                "dalle" => Box::new(DalleEngine::new(engine_config.clone()).await?),
                "stabilityai" => {
                    let mut engine = Box::new(StabilityAIEngine::new(engine_config.clone()).await?);
                    if let Some(download_dir) = matches.get_one::<String>("download-media") {
                        engine.set_download_dir(download_dir.to_string());
                    }
                    engine
                },
                "leonardo_ai" => Box::new(LeonardoAIEngine::new(engine_config.clone()).await?),
                "imagine_pro" => {
                    let mut engine = Box::new(ImagineProEngine::new(engine_config.clone()).await?);
                    if let Some(download_dir) = matches.get_one::<String>("download-media") {
                        engine.set_download_dir(download_dir.to_string());
                    }
                    engine
                }
                _ => return Err(anyhow!("Unsupported engine: {}", engine_config.engine)),
            };

            // Read context from stdin if available
            let mut context = String::new();
            if !atty::is(atty::Stream::Stdin) {
                tokio::io::stdin().read_to_string(&mut context).await?;
            }

            // Read additional context from file if provided
            let mut file_contents = String::new();
            if let Some(file_path) = matches.get_one::<String>("additional-context-file") {
                file_contents = fs::read_to_string(file_path)?;
            }

            // Combine all inputs
            let mut combined_request_parts = Vec::new();
            // Always add the request first
            combined_request_parts.push(request.trim().to_string());
            // Add context if it's not empty
            if !context.trim().is_empty() {
                combined_request_parts.push(format!("Context:\n{}", context.trim()));
            }
            // Add file contents if it's not empty
            if !file_contents.trim().is_empty() {
                combined_request_parts.push(format!("Additional Context:\n{}", file_contents.trim()));
            }
            // Join all parts with a separator
            let combined_request = combined_request_parts.join("\n\n----\n\n");
            debug!("Combined Request:\n{}", combined_request);

            let request = Request {
                flowname: engine_name.to_string(),
                payload: combined_request,
            };
            debug!("Combined Request: {:?}", request);


            let response = if let Some(file_path) = matches.get_one::<String>("upload-image-file") {
                debug!("Processing request with file: {}", file_path);
                pb.set_message("Processing request with file...");
                Pin::from(engine.process_request_with_file(&request, Path::new(file_path))).await?
            } else {
                pb.set_message("Executing request...");
                Pin::from(engine.execute(&request)).await?
            };

            let mut output = response.content.clone();

            if let Some(download_dir) = matches.get_one::<String>("download-media") {
                let download_path = PathBuf::from(download_dir);
                OutputProcessor::download_media_files(&response.content, &download_path).await?;
            }

            if matches.get_flag("parse-code") {
                debug!("Parsing code blocks");
                let code_blocks = OutputProcessor::parse_code(&output);
                debug!("Code blocks: {:?}", code_blocks);
                output = code_blocks.join("\n\n");
            }

            if matches.get_flag("execute-output") {
                debug!("Executing output code");
                debug!("Attempting to execute : {}", output);
                output = OutputProcessor::execute_code(&output).await?;
            }

            if matches.get_flag("markdown") {
                debug!("Formatting output as markdown");
                //output = format_markdown(&output);
            }

            let response_time = start_time.elapsed().as_secs_f64();

            if let Some(neo4j_client) = engine.get_neo4j_client() {
                let session_id = engine.get_session_id()
                    .unwrap_or_else(|| Uuid::new_v4().to_string());

                let stats = InteractionStats {
                    prompt_tokens: response.usage.prompt_tokens,
                    completion_tokens: response.usage.completion_tokens,
                    total_tokens: response.usage.total_tokens,
                    response_time,
                    finish_reason: response.finish_reason.clone().unwrap_or_else(|| "unknown".to_string()),
                };

                debug!("Attempting to create interaction in Neo4j");
                debug!("Using session ID: {}", session_id);
                match neo4j_client.create_interaction(
                    &session_id,
                    &request.payload,
                    &response.content,
                    &response.model,
                    &stats
                ).await {
                    Ok(interaction_id) => debug!("Successfully created interaction with id: {}", interaction_id),
                    Err(e) => error!("Failed to create interaction in Neo4j: {:?}", e),
                }
            } else {
                debug!("Neo4j client not available, skipping interaction logging");
            }

            pb.finish_and_clear();
            eprintln!();
            println!("{}", output);

            let use_colors = std::io::stderr().is_terminal();
            let response_time_str = format!("{:.2}s", response_time);

            eprintln!(
                "{} | {} | Time: {} | Usage: {}↑ {}↓ {}Σ | {}\n",
                spinner_config.success_symbol,
                if use_colors { response.model.cyan().to_string() } else { response.model },
                if use_colors { response_time_str.bright_blue().to_string() } else { response_time_str },
                if use_colors { response.usage.prompt_tokens.to_string().yellow().to_string() } else { response.usage.prompt_tokens.to_string() },
                if use_colors { response.usage.completion_tokens.to_string().yellow().to_string() } else { response.usage.completion_tokens.to_string() },
                if use_colors { response.usage.total_tokens.to_string().yellow().to_string() } else { response.usage.total_tokens.to_string() },
                if use_colors { response.finish_reason.as_deref().unwrap_or("No finish reason").italic().to_string() } else { response.finish_reason.as_deref().unwrap_or("No finish reason").to_string() }
            );
        }

        Ok(())
    }


    async fn handle_upsert(engine_config: &EngineConfig, matches: &ArgMatches) -> Result<()> {
        if let Some(neo4j_config) = &engine_config.neo4j {
            let neo4j_client = Neo4jClient::new(neo4j_config).await?;

            let input = matches.get_one::<String>("input")
                .ok_or_else(|| anyhow!("Input is required for upsert mode"))?;
            let metadata = matches.get_one::<String>("metadata")
                .map(|s| s.split(',').map(String::from).collect::<Vec<String>>())
                .unwrap_or_default();

            let input_path = Path::new(input);
            if input_path.is_file() {
                let document_id = neo4j_client.upsert_document(input_path, &metadata).await?;
                eprintln!("Uploaded document with ID: {}. Embeddings and chunks created.", document_id);
            } else if input_path.is_dir() {
                let mut uploaded_count = 0;
                for entry in fs::read_dir(input_path)? {
                    let entry = entry?;
                    let path = entry.path();
                    if path.is_file() {
                        let document_id = neo4j_client.upsert_document(&path, &metadata).await?;
                        eprintln!("Uploaded document {} with ID: {}. Embeddings and chunks created.", path.display(), document_id);
                        uploaded_count += 1;
                    }
                }
                eprintln!("Uploaded {} documents with embeddings and chunks", uploaded_count);
            } else {
                return Err(anyhow!("Input is neither a file nor a directory"));
            }

            if let Ok(stats) = neo4j_client.get_document_statistics().await {
                eprintln!("\nDocument Statistics:");
                eprintln!("Total documents: {}", stats.document_count);
                eprintln!("Average content length: {:.2}", stats.avg_content_length);
                eprintln!("Total chunks: {}", stats.chunk_count);
                eprintln!("Total embeddings: {}", stats.embedding_count);
            }
        } else {
            return Err(anyhow!("Neo4j configuration not found for this engine"));
        }

        Ok(())
    }


    async fn generate_cypher_query(query: &str, config: &EngineConfig) -> Result<String> {
        // Use the configured LLM to generate a Cypher query
        let llm_request = Request {
            flowname: "cypher_generation".to_string(),
            payload: format!("Generate a Cypher query for Neo4j based on this request: {}", query),
        };
        debug!("Sending request to LLM engine: {:?}", llm_request);
        let llm_engine: Box<dyn Engine> = match config.engine.as_str() {
            "openai" => Box::new(OpenAIEngine::new(config.clone()).await?),
            "anthropic" => Box::new(AnthropicEngine::new(config.clone()).await?),
            // Add other LLM engines as needed
            _ => return Err(anyhow!("Unsupported LLM engine for Cypher generation")),
        };


        let response = Pin::from(llm_engine.execute(&llm_request)).await?;


        debug!("Response from LLM engine: {:?}", response);
        Ok(response.content)
    }
}


async fn get_neo4j_query_llm(config: &Config) -> Option<Box<dyn Engine>> {
    let neo4j_config = config.engines.iter().find(|e| e.engine == "neo4j")?;

    // Extract the query_llm from the neo4j configuration
    let query_llm = neo4j_config.neo4j.as_ref()?.query_llm.as_ref()?;

    // Find the engine configuration for the specified query_llm
    let llm_config = config.engines.iter().find(|e| e.name.as_str() == query_llm)?;

    // Create and return the LLM engine
    create_llm_engine(llm_config).await.ok()
}

async fn generate_and_execute_cypher(
    neo4j_config: &Neo4jConfig,
    llm_config: &EngineConfig,
    query_string: &str,
    llm_engine: &dyn Engine
) -> Result<String, Error> {
    debug!("Generating Cypher query using LLM");
    debug!("Neo4j configuration: {:#?}", neo4j_config);
    let neo4j_client = Neo4jClient::new(neo4j_config).await?;
    debug!("Neo4j client created");

    // Fetch the database schema
    let schema = neo4j_client.get_database_schema().await?;
    debug!("Database schema: {:#?}", schema);

    // Generate Cypher query using LLM
    let cypher_request = Request {
        flowname: "generate_cypher".to_string(),
        payload: format!(
            "Given the following database schema:\n\n{}\n\nGenerate a Cypher query for Neo4j based on this request: {}",
            schema, query_string
        ),
    };
    //info!("Sending request to LLM engine: {:?}", cypher_request);
    let cypher_response = Pin::from(llm_engine.execute(&cypher_request)).await?;
    let cypher_query = extract_cypher_query(&cypher_response.content)?;

    // Execute the Cypher query
    let cypher_result = neo4j_client.execute_cypher(&cypher_query).await?;
    debug!("Cypher result: {:?}", cypher_result);

    // Format the result based on the output format
    Ok(format_as_csv(&cypher_result))
}


fn extract_cypher_query(content: &str) -> Result<String, Error> {
    // First, try to extract content between triple backticks
    let backtick_re = Regex::new(r"```(?:cypher)?\s*([\s\S]*?)\s*```").unwrap();
    if let Some(captures) = backtick_re.captures(content) {
        if let Some(query) = captures.get(1) {
            let extracted = query.as_str().trim();
            if is_valid_cypher(extracted) {
                return Ok(extracted.to_string());
            }
        }
    }

    // If not found, look for common Cypher keywords to identify the query
    let cypher_re = Regex::new(r"(?i)(MATCH|CREATE|MERGE|DELETE|REMOVE|SET|RETURN)[\s\S]+").unwrap();
    if let Some(captures) = cypher_re.captures(content) {
        if let Some(query) = captures.get(0) {
            let extracted = query.as_str().trim();
            if is_valid_cypher(extracted) {
                return Ok(extracted.to_string());
            }
        }
    }

    // If still not found, return an error
    Err(anyhow!("No valid Cypher query found in the content"))
}

fn is_valid_cypher(query: &str) -> bool {
    // Basic validation: check if the query contains common Cypher clauses
    let valid_clauses = ["MATCH", "CREATE", "MERGE", "DELETE", "REMOVE", "SET", "RETURN", "WITH", "WHERE"];
    valid_clauses.iter().any(|&clause| query.to_uppercase().contains(clause))
}

fn format_as_csv(result: &Value) -> String {
    // Implement CSV formatting here
    // For now, we'll just return the JSON as a string
    result.to_string()
}

async fn create_engine(engine_config: &EngineConfig) -> Result<Box<dyn Engine>, Error> {
    match engine_config.engine.as_str() {
        "openai" => Ok(Box::new(OpenAIEngine::new(engine_config.clone()).await?)),
        "anthropic" => Ok(Box::new(AnthropicEngine::new(engine_config.clone()).await?)),
        "cohere" => Ok(Box::new(CohereEngine::new(engine_config.clone()).await?)),
        "google_gemini" => Ok(Box::new(GoogleGeminiEngine::new(engine_config.clone()).await?)),
        "perplexity" => Ok(Box::new(PerplexityEngine::new(engine_config.clone()).await?)),
        "groq_lpu" => Ok(Box::new(GroqLPUEngine::new(engine_config.clone()).await?)),
        _ => Err(anyhow!("Unsupported engine: {}", engine_config.engine)),
    }
}

async fn create_llm_engine(engine_config: &EngineConfig) -> Result<Box<dyn Engine>, Error> {
    create_engine(engine_config).await
}



#### END OF FILE: crates/fluent-cli/src/lib.rs ####

#### START OF FILE: crates/fluent-cli/src/main.rs ####
use fluent_cli::cli;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    cli::run().await
}
#### END OF FILE: crates/fluent-cli/src/main.rs ####

#### START OF FILE: crates/fluent-core/Cargo.toml ####
# crates/fluent-core/Cargo.toml
[package]
name = "fluent-core"
version = "0.1.0"
edition = "2021"

[dependencies]
serde = { version = "1.0", features = ["derive"] }
anyhow = "1.0"
serde_json = "1.0.120"
async-trait = "0.1.80"
log = "0.4.22"
chrono = "0.4.38"
uuid = { version = "1.9.1", features = ["v4"] }
neo4rs = "0.7.1"
reqwest = { version = "0.12.5", features = ["json"] }
unicode-segmentation = "1.11.0"


#rust-bert = {  version = "0.18.0"  }
tokenizers = "0.14.0"
rust-stemmers = "1.2.0"
stop-words = "0.8.0"
base64 = "0.22.1"
tokio = "1.38.0"
regex = "1.10.5"
url = "2.5.2"
termimad = "0.29.2"
crossterm = "0.27.0"
syntect = "5.2.0"
indicatif = "0.17.8"
owo-colors = "3.5.0"
pdf-extract = "0.7.7"


[patch.crates-io]
jetscii = "0.5.1"
#### END OF FILE: crates/fluent-core/Cargo.toml ####

#### START OF FILE: crates/fluent-core/src/voyageai_client.rs ####
use reqwest::Client;
use serde::{Deserialize, Serialize};
use anyhow::Result;
use crate::neo4j_client::VoyageAIConfig;


pub const EMBEDDING_DIMENSION: usize = 1536;

#[derive(Serialize)]
struct VoyageAIRequest {
    input: Vec<String>,
    model: String,
}

#[derive(Deserialize)]
struct VoyageAIResponse {
    data: Vec<VoyageAIEmbedding>,
}

#[derive(Deserialize)]
struct VoyageAIEmbedding {
    embedding: Vec<f32>,
}

pub async fn get_voyage_embedding(text: &str, config: &VoyageAIConfig) -> Result<Vec<f32>> {
    let client = Client::new();

    let request_body = VoyageAIRequest {
        input: vec![text.to_string()],
        model: config.model.clone(),
    };

    let response: VoyageAIResponse = client.post("https://api.voyageai.com/v1/embeddings")
        .header("Authorization", format!("Bearer {}", config.api_key))
        .json(&request_body)
        .send()
        .await?
        .json()
        .await?;

    Ok(response.data[0].embedding.clone())
}


#### END OF FILE: crates/fluent-core/src/voyageai_client.rs ####

#### START OF FILE: crates/fluent-core/src/types.rs ####
// crates/fluent-core/src/types.rs
use serde::{Deserialize, Serialize};

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Request {
    pub flowname: String,
    pub payload: String,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Response {
    pub content: String,
    pub usage: Usage,
    pub model: String,
    pub finish_reason: Option<String>,

}


#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Usage {
    pub prompt_tokens: u32,
    pub completion_tokens: u32,
    pub total_tokens: u32,
}


#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct UpsertRequest {
    pub input: String,
    pub output: String,
    pub metadata: Vec<String>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct UpsertResponse {
    pub processed_files: Vec<String>,
    pub errors: Vec<String>,
}

#[derive(Debug, Clone)]
pub struct DocumentStatistics {
    pub document_count: i64,
    pub avg_content_length: f64,
    pub chunk_count: i64,
    pub embedding_count: i64,
}


#[derive(Debug, Serialize, Deserialize, Default)]
pub struct ExtractedContent {
    pub main_content: String,
    pub sentiment: Option<String>,
    pub clusters: Option<Vec<String>>,
    pub themes: Option<Vec<String>>,
    pub keywords: Option<Vec<String>>,
}
#### END OF FILE: crates/fluent-core/src/types.rs ####

#### START OF FILE: crates/fluent-core/src/config.rs ####
use anyhow::{Result, anyhow};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::{env, fs};
use log::{debug, info};
use serde_json::Value;
use std::process::Command;
use std::sync::Arc;
use crate::neo4j_client::VoyageAIConfig;
use crate::spinner_configuration::SpinnerConfig;


#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct EngineConfig {
    pub name: String,
    pub engine: String,
    pub connection: ConnectionConfig,
    pub parameters: HashMap<String, serde_json::Value>,
    pub session_id: Option<String>,  // New field for sessionID
    pub neo4j: Option<Neo4jConfig>,
    pub spinner: Option<SpinnerConfig>,

}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Neo4jConfig {
    pub uri: String,
    pub user: String,
    pub password: String,
    pub database: String,
    pub voyage_ai: Option<VoyageAIConfig>,
    pub query_llm: Option<String>,
    pub parameters: Option<HashMap<String, serde_json::Value>>,
}


#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ConnectionConfig {
    pub protocol: String,
    pub hostname: String,
    pub port: u16,
    pub request_path: String,
}

#[derive(  Clone)]
pub struct Config {
    pub engines: Vec<EngineConfig>,
    _env_guard: Arc<AmberEnvVarGuard>, // This keeps the guard alive
}

impl Config {
    pub fn new(engines: Vec<EngineConfig>, env_guard: AmberEnvVarGuard) -> Self {
        Config {
            engines,
            _env_guard: Arc::new(env_guard),
        }
    }
}
pub fn load_config(config_path: &str, engine_name: &str, overrides: &HashMap<String, String>) -> Result<Config> {
    let config_content = fs::read_to_string(config_path)?;
    let mut config: Value = serde_json::from_str(&config_content)?;

    debug!("Loading config for engine: {}", engine_name);

    let mut env_var_guard = AmberEnvVarGuard::new();

    // Find the specific engine configuration
    let engine_config = config["engines"]
        .as_array_mut()
        .ok_or_else(|| anyhow!("No engines found in configuration"))?
        .iter_mut()
        .find(|e| e["name"].as_str() == Some(engine_name))
        .ok_or_else(|| anyhow!("Engine '{}' not found in configuration", engine_name))?;

    // Decrypt Amber keys only for the specified engine
    env_var_guard.decrypt_amber_keys_in_value(engine_config)?;

    // Apply overrides to the specified engine
    if let Some(parameters) = engine_config["parameters"].as_object_mut() {
        for (key, value) in overrides {
            // Parse the override value to the correct type
            let parsed_value: Value = match parameters[key] {
                Value::Number(_) => value.parse::<f64>().map(Value::from).unwrap_or(Value::String(value.clone())),
                Value::Bool(_) => value.parse::<bool>().map(Value::from).unwrap_or(Value::String(value.clone())),
                _ => Value::String(value.clone()),
            };
            parameters.insert(key.clone(), parsed_value);
        }
    }


    debug!("Loaded and processed config for engine: {}", engine_name);

    let engine_config: EngineConfig = serde_json::from_value(engine_config.clone())?;

    Ok(Config::new(vec![engine_config], env_var_guard))
}

pub fn parse_key_value_pair(pair: &str) -> Option<(String, String)> {
    let parts: Vec<&str> = pair.splitn(2, '=').collect();
    if parts.len() == 2 {
        Some((parts[0].to_string(), parts[1].to_string()))
    } else {
        None
    }
}

pub fn apply_overrides(config: &mut EngineConfig, overrides: &[(String, String)]) -> Result<()> {
    for (key, value) in overrides {
        let value = serde_json::from_str(value).unwrap_or(serde_json::Value::String(value.clone()));
        config.parameters.insert(key.clone(), value);
    }
    Ok(())
}

pub struct AmberEnvVarGuard {
    keys: Vec<String>,
}

impl AmberEnvVarGuard {
    pub fn new() -> Self {
        AmberEnvVarGuard {
            keys: Vec::new(),
        }
    }


    fn decrypt_amber_keys_in_value(&mut self, value: &mut Value) -> Result<()> {
        match value {
            Value::String(s) if s.starts_with("AMBER_") => {
                let decrypted = self.get_amber_value(s)?;
                self.set_env_var_from_amber(s, &decrypted)?;
                *s = decrypted;
                Ok(())
            },
            Value::Object(map) => {
                for (_, v) in map.iter_mut() {
                    self.decrypt_amber_keys_in_value(v)?;
                }
                Ok(())
            },
            Value::Array(arr) => {
                for item in arr.iter_mut() {
                    self.decrypt_amber_keys_in_value(item)?;
                }
                Ok(())
            },
            _ => Ok(())
        }
    }

    fn set_env_var_from_amber(&mut self, key: &str, value: &str) -> Result<()> {
        std::env::set_var(key, value);
        debug!("Set environment variable {} with decrypted value", key);
        self.keys.push(key.to_owned());
        Ok(())
    }


    fn get_amber_value(&self, key: &str) -> Result<String> {
        let output = Command::new("amber")
            .arg("print")
            .output()?;

        if !output.status.success() {
            return Err(anyhow!("Failed to run amber print command"));
        }

        let stdout = String::from_utf8(output.stdout)?;
        //debug!("Amber print output: {}", stdout);
        for line in stdout.lines() {
            if line.contains(key) {
                let parts: Vec<&str> = line.splitn(2, '=').collect();
                if parts.len() == 2 {
                    let value = parts[1].trim().trim_matches('"');
                    return Ok(value.to_string());
                }
            }
        }

        Err(anyhow!("Amber key not found: {}", key))
    }
}

impl Drop for AmberEnvVarGuard {
    fn drop(&mut self) {
        for key in &self.keys {
            std::env::remove_var(key);
            info!("Environment variable {} has been unset.", key);
        }
    }
}

// Helper function to replace config strings starting with "AMBER_" with their env values
pub fn replace_with_env_var(value: &mut Value) {
    match value {
        Value::String(s) if s.starts_with("AMBER_") => {
            let env_key = &s[6..]; // Skip the "AMBER_" prefix to fetch the correct env var
            debug!("Attempting to replace: {}", s);
            debug!("Looking up environment variable: {}", env_key);
            match env::var(env_key) {
                Ok(env_value) => {
                    debug!("Environment value found for: {}", env_key);
                    *s = env_value;
                },
                Err(e) => {
                    debug!("Failed to find environment variable '{}': {}", env_key, e);
                }
            }
        },
        Value::Object(map) => {
            for (_, v) in map.iter_mut() {
                replace_with_env_var(v);
            }
        },
        Value::Array(arr) => {
            for item in arr.iter_mut() {
                replace_with_env_var(item);
            }
        },
        _ => {}
    }
}



#### END OF FILE: crates/fluent-core/src/config.rs ####

#### START OF FILE: crates/fluent-core/src/output_processor.rs ####
use std::env;
use anyhow::{Result, anyhow, Context};
use regex::Regex;
use std::path::PathBuf;
use crossterm::style::Color;
use log::{debug, info};
use reqwest::Client;
use serde_json::Value;
use syntect::easy::HighlightLines;
use syntect::highlighting::{Style, ThemeSet};
use syntect::parsing::SyntaxSet;
use syntect::util::{as_24_bit_terminal_escaped, LinesWithEndings};
use tokio::fs;
use tokio::process::Command;
use url::Url;
use uuid::Uuid;
use termimad::crossterm::style::Stylize;
use termimad::{MadSkin, StyledChar};

pub struct OutputProcessor;

impl OutputProcessor {

    pub async fn download_media_files(content: &str, directory: &PathBuf) -> Result<()> {
        debug!("Starting media file download process");

        // Try to parse the content as JSON
        if let Ok(json_content) = serde_json::from_str::<Value>(content) {
            debug!("Content parsed as JSON, attempting to download from JSON structure");
            Self::download_from_json(&json_content, directory).await?;
        } else {
            debug!("Content is not valid JSON, proceeding with regex-based URL extraction");
            // Corrected regex for URL matching, including query parameters
            let url_regex = Regex::new(r#"(https?://[^\s"']+\.(?:jpg|jpeg|png|gif|bmp|svg|mp4|webm|ogg)(?:\?[^\s"']+)?)"#)?;

            for cap in url_regex.captures_iter(content) {
                let url = &cap[1];  // This includes the full URL with query parameters
                debug!("Found URL in content: {}", url);
                Self::download_file(url, directory, None).await?;
            }
        }

        Ok(())
    }

    async fn download_file(url: &str, directory: &PathBuf, suggested_name: Option<String>) -> Result<()> {
        debug!("Attempting to download file from URL: {}", url);

        let client = Client::new();
        let response = client.get(url).send().await?;

        if !response.status().is_success() {
            return Err(anyhow!("Failed to download file: HTTP status {}", response.status()));
        }

        let file_name = if let Some(name) = suggested_name {
            name
        } else {
            Url::parse(url)?
                .path_segments()
                .and_then(|segments| segments.last())
                .map(|s| s.to_string())
                .unwrap_or_else(|| Uuid::new_v4().to_string())
        };

        let file_path = directory.join(&file_name);
        let content = response.bytes().await?;
        fs::write(&file_path, &content).await?;

        info!("Downloaded: {} ({} bytes)", file_path.display(), content.len());

        Ok(())
    }

    async fn download_from_json(json_content: &Value, directory: &PathBuf) -> Result<()> {
        if let Some(data) = json_content.get("data") {
            if let Some(data_array) = data.as_array() {
                for item in data_array {
                    if let Some(url) = item.get("url").and_then(|u| u.as_str()) {
                        let file_name = Self::extract_file_name_from_url(url);
                        Self::download_file(url, directory, file_name).await?;
                    }
                }
            }
        }
        Ok(())
    }

    fn extract_file_name_from_url(url: &str) -> Option<String> {
        Url::parse(url).ok()?
            .path_segments()?
            .last()?
            .to_string()
            .split('?')
            .next()
            .map(|s| s.to_string())
    }

    pub fn parse_code(content: &str) -> Vec<String> {
        let code_block_regex = Regex::new(r"```(?:\w+)?\n([\s\S]*?)\n```").unwrap();
        code_block_regex.captures_iter(content)
            .filter_map(|cap| cap.get(1))
            .map(|m| m.as_str().trim().to_string())
            .collect()
    }

    pub async fn execute_code(content: &str) -> Result<String> {
        let code_blocks = Self::parse_code(content);
        let mut output = String::new();
        for block in code_blocks {
            output.push_str(&format!("Executing code block:\n```\n{}\n```\n", block));

            if Self::is_script(&block) {
                output.push_str(&Self::execute_script(&block).await?);
            } else {
                output.push_str(&Self::execute_commands(&block).await?);
            }

            output.push_str("\n");
        }
        Ok(output)
    }

    fn is_script(block: &str) -> bool {
        block.starts_with("#!/bin/") || block.starts_with("#!/usr/bin/env")
    }

    async fn execute_script(script: &str) -> Result<String> {
        // Use a platform-agnostic way to get the temp directory
        let temp_dir = env::temp_dir();
        let file_name = format!("script_{}.{}", Uuid::new_v4(), if cfg!(windows) { "bat" } else { "sh" });
        let temp_file = temp_dir.join(file_name);

        // Write the script to the temporary file
        fs::write(&temp_file, script).await.context("Failed to write script to temporary file")?;

        // Set executable permissions on Unix-like systems
        #[cfg(unix)]
        {
            use std::os::unix::fs::PermissionsExt;
            let mut perms = fs::metadata(&temp_file).await?.permissions();
            perms.set_mode(0o755);
            fs::set_permissions(&temp_file, perms).await.context("Failed to set file permissions")?;
        }

        // Execute the script
        let result = if cfg!(windows) {
            Command::new("cmd")
                .arg("/C")
                .arg(&temp_file)
                .output()
                .await
        } else {
            Command::new("sh")
                .arg(&temp_file)
                .output()
                .await
        }.context("Failed to execute script")?;

        // Remove the temporary file
        fs::remove_file(&temp_file).await.context("Failed to remove temporary file")?;

        // Collect and format the output
        let stdout = String::from_utf8_lossy(&result.stdout);
        let stderr = String::from_utf8_lossy(&result.stderr);

        Ok(format!("Script Output:\n{}\nErrors:\n{}\n", stdout, stderr))
    }

    async fn execute_commands(commands: &str) -> Result<String> {
        let mut output = String::new();
        for command in commands.lines() {
            let trimmed = command.trim();
            if trimmed.is_empty() || trimmed.starts_with('#') {
                continue;
            }

            output.push_str(&format!("Executing: {}\n", trimmed));

            let result = Command::new("sh")
                .arg("-c")
                .arg(trimmed)
                .output()
                .await?;

            let stdout = String::from_utf8_lossy(&result.stdout);
            let stderr = String::from_utf8_lossy(&result.stderr);

            output.push_str(&format!("Output:\n{}\n", stdout));
            if !stderr.is_empty() {
                output.push_str(&format!("Errors:\n{}\n", stderr));
            }
            output.push_str("\n");
        }
        Ok(output)
    }


}

pub struct MarkdownFormatter {
    skin: MadSkin,
    syntax_set: SyntaxSet,
    theme_set: ThemeSet,
}

impl MarkdownFormatter {
    pub fn new() -> Self {
        let mut skin = MadSkin::default();
        skin.set_bg(Color::Rgb { r: 10, g: 10, b: 10 });
        skin.set_headers_fg(Color::Rgb { r: 255, g: 187, b: 0 });
        skin.bold.set_fg(Color::Rgb { r: 255, g: 215, b: 0 });
        skin.italic.set_fg(Color::Rgb { r: 173, g: 216, b: 230 });
        skin.paragraph.set_fg(Color::Rgb { r: 220, g: 220, b: 220 }); // Light grey for normal text
        skin.bullet = StyledChar::from_fg_char(Color::Rgb { r: 0, g: 255, b: 0 }, '•');

        // Set code block colors
        skin.code_block.set_bg(Color::Rgb { r: 30, g: 30, b: 30 }); // Slightly lighter than main background
        skin.code_block.set_fg(Color::Rgb { r: 255, g: 255, b: 255 }); // White text for code


        MarkdownFormatter {
            skin,
            syntax_set: SyntaxSet::load_defaults_newlines(),
            theme_set: ThemeSet::load_defaults(),
        }
    }

    pub fn format(&self, content: &str) -> Result<String> {
        debug!("Formatting markdown");
        let mut output = String::new();
        let lines: Vec<&str> = content.lines().collect();

        let mut in_code_block = false;
        let mut code_block_lang = String::new();
        let mut code_block_content = String::new();

        for line in lines {
            if line.starts_with("```") {
                if in_code_block {
                    // End of code block
                    output.push_str(&self.highlight_code(&code_block_lang, &code_block_content)?);
                    output.push_str("\n\n");
                    in_code_block = false;
                    code_block_content.clear();
                } else {
                    // Start of code block
                    in_code_block = true;
                    code_block_lang = line.trim_start_matches('`').to_string();
                }
            } else if in_code_block {
                code_block_content.push_str(line);
                code_block_content.push('\n');
            } else {
                // Use push_str to append the formatted line to our String
                output.push_str(&self.skin.inline(line).to_string());
                output.push('\n');
            }
        }

        Ok(output)
    }

    fn highlight_code(&self, lang: &str, code: &str) -> Result<String> {
        debug!("highlight_code: {}", lang);
        let syntax = self.syntax_set.find_syntax_by_extension(lang).unwrap_or_else(|| self.syntax_set.find_syntax_plain_text());
        let mut highlighter = HighlightLines::new(syntax, &self.theme_set.themes["base16-ocean.dark"]);

        let mut output = String::new();
        for line in LinesWithEndings::from(code) {
            let ranges: Vec<(Style, &str)> = highlighter.highlight(line, &self.syntax_set);
            let escaped = as_24_bit_terminal_escaped(&ranges[..], true);
            output.push_str(&escaped);
        }

        Ok(output)
    }

    pub fn set_theme(&mut self, theme_name: &str) -> Result<()> {
        debug!("set_theme: {}", theme_name);
        if let Some(theme) = self.theme_set.themes.get(theme_name) {
            self.skin.paragraph.set_fg(Color::Rgb {
                r: theme.settings.foreground.unwrap().r,
                g: theme.settings.foreground.unwrap().g,
                b: theme.settings.foreground.unwrap().b,
            });
            Ok(())
        } else {
            Err(anyhow::anyhow!("Theme not found"))
        }
    }
}

pub fn format_markdown(content: &str) -> Result<String> {
    debug!("format_markdown");
    let formatter = MarkdownFormatter::new();
    formatter.format(content)
}
#### END OF FILE: crates/fluent-core/src/output_processor.rs ####

#### START OF FILE: crates/fluent-core/src/lib.rs ####
// crates/fluent-core/src/lib.rs
pub mod types;
pub mod traits;
pub mod config;
pub mod utils;
pub mod output;
pub mod neo4j_client;
mod voyageai_client;
pub mod output_processor;
pub mod spinner_configuration;




#### END OF FILE: crates/fluent-core/src/lib.rs ####

#### START OF FILE: crates/fluent-core/src/spinner_configuration.rs ####
use serde::{Deserialize, Serialize};
use owo_colors::OwoColorize;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpinnerConfig {
    pub frames: String,
    pub interval: u64,
    pub success_symbol: String,
    pub failure_symbol: String,
    pub success_symbol_colored: String,
    pub failure_symbol_colored: String,
}

impl Default for SpinnerConfig {
    fn default() -> Self {
        let success_symbol = "\n✓ Success ✓".to_string();
        let failure_symbol = "\n✗ Failure ✗".to_string();

        SpinnerConfig {
            frames: "-\\|/ ".to_string(),
            interval: 100,
            success_symbol: success_symbol.clone(),
            failure_symbol: failure_symbol.clone(),
            success_symbol_colored: success_symbol.green().to_string(),
            failure_symbol_colored: failure_symbol.red().to_string(),
        }
    }
}
#### END OF FILE: crates/fluent-core/src/spinner_configuration.rs ####

#### START OF FILE: crates/fluent-core/src/output.rs ####
// crates/fluent-core/src/output.rs
pub fn pretty_format_markdown(markdown_content: &str) {
    // Implementation for pretty formatting markdown
}

#### END OF FILE: crates/fluent-core/src/output.rs ####

#### START OF FILE: crates/fluent-core/src/neo4j_client.rs ####
use anyhow::{Result, Context, anyhow, Error};
use neo4rs::{Graph, query, ConfigBuilder, BoltMap, BoltList, BoltString, BoltType, BoltInteger, BoltFloat, Database, BoltNull, Query, Row, BoltNode, BoltRelation, BoltPath};

use chrono::Duration as ChronoDuration;

use chrono::{DateTime, Utc};
use serde_json::{json, Map, Value};
use uuid::Uuid;
use std::collections::{HashMap, HashSet};
use std::path::Path;
use log::{debug, error, info, warn};
use std::sync::{Arc, RwLock};
use pdf_extract::extract_text;


use rust_stemmers::{Algorithm, Stemmer};
use serde::{Deserialize, Serialize};


use crate::config::Neo4jConfig;
use crate::types::DocumentStatistics;
use crate::utils::chunking::chunk_document;
use crate::voyageai_client::{EMBEDDING_DIMENSION, get_voyage_embedding};
use stop_words::{get, LANGUAGE};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use crate::traits::{DocumentProcessor, DocxProcessor, PdfProcessor, TextProcessor};


#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct VoyageAIConfig {
    pub api_key: String,
    pub model: String,
}

pub struct Neo4jClient {
    graph: Graph,
    document_count: RwLock<usize>,
    word_document_count: RwLock<HashMap<String, usize>>,
    voyage_ai_config: Option<VoyageAIConfig>,
    query_llm: Option<String>,
}

#[derive(Debug, Clone)]
pub struct InteractionStats {
    pub prompt_tokens: u32,
    pub completion_tokens: u32,
    pub total_tokens: u32,
    pub response_time: f64,  // in seconds
    pub finish_reason: String,
}

#[derive(Debug, Clone)]
pub struct Embedding {
    pub id: String,
    pub vector: Vec<f32>,
    pub model: String,
}

struct Document {
    id: String,
    content: String,
    metadata: Vec<String>,
}


#[derive(Debug)]
pub struct EnrichmentConfig {
    pub themes_keywords_interval: ChronoDuration,
    pub clustering_interval: ChronoDuration,
    pub sentiment_interval: ChronoDuration,
}

#[derive(Debug, Clone)]
pub struct EnrichmentStatus {
    pub last_themes_keywords_update: Option<DateTime<Utc>>,
    pub last_clustering_update: Option<DateTime<Utc>>,
    pub last_sentiment_update: Option<DateTime<Utc>>,
}






impl Neo4jClient {
    pub async fn new(config: &Neo4jConfig) -> Result<Self> {
        let graph_config = ConfigBuilder::default()
            .uri(&config.uri)
            .user(&config.user)
            .password(&config.password)
            .db(Database::from(config.database.as_str()))  // Convert string to Database instance
            .build()?;

        let graph = Graph::connect(graph_config).await?;

        Ok(Neo4jClient {
            graph,
            document_count: Default::default(),
            word_document_count: Default::default(),
            voyage_ai_config: config.voyage_ai.clone(),
            query_llm: config.query_llm.clone(),
        })
    }


    pub async fn ensure_indexes(&self) -> Result<()> {
        let index_queries = vec![
            "CREATE INDEX IF NOT EXISTS FOR (s:Session) ON (s.id)",
            "CREATE INDEX IF NOT EXISTS FOR (q:Question) ON (q.content)",
            "CREATE INDEX IF NOT EXISTS FOR (r:Response) ON (r.content)",
            "CREATE INDEX IF NOT EXISTS FOR (m:Model) ON (m.name)",
            "CREATE INDEX IF NOT EXISTS FOR (i:Interaction) ON (i.id)",
            "CREATE INDEX IF NOT EXISTS FOR (i:Interaction) ON (i.timestamp)",
            "CREATE INDEX IF NOT EXISTS FOR (stats:InteractionStats) ON (stats.id)",
            "CREATE INDEX IF NOT EXISTS FOR (e:Embedding) ON (e.id)",
            "CREATE INDEX IF NOT EXISTS FOR (d:Document) ON (d.id)",
            "CREATE INDEX IF NOT EXISTS FOR (c:Chunk) ON (c.id)",
        ];

        for query_str in index_queries {
            debug!("Executing index creation query: {}", query_str);
            self.graph.execute(query(query_str)).await?;
        }


        // Create a vector index for embeddings
        let vector_index_query = format!(
            "CALL db.index.vector.createNodeIndex(
                'document_embedding_index',
                'Embedding',
                'vector',
                {},
                'cosine'
            )",
            EMBEDDING_DIMENSION
        );

        match self.graph.execute(query(&vector_index_query)).await {
            Ok(_) => debug!("Vector index created successfully for Document Embedding nodes"),
            Err(e) => warn!("Failed to create vector index for Document Embedding nodes: {}. This might be normal if the index already exists.", e),
        }


        // Optionally, we can also create full-text indexes for content fields if needed
        let fulltext_index_queries = vec![
            "CALL db.index.fulltext.createNodeIndex('questionContentIndex', ['Question'], ['content'])",
            "CALL db.index.fulltext.createNodeIndex('responseContentIndex', ['Response'], ['content'])",
        ];

        for query_str in fulltext_index_queries {
            debug!("Executing full-text index creation query: {}", query_str);
            match self.graph.execute(query(query_str)).await {
                Ok(_) => debug!("Full-text index created successfully"),
                Err(e) => warn!("Failed to create full-text index: {}. This might be normal if the index already exists.", e),
            }
        }

        debug!("All indexes have been created or updated.");
        Ok(())
    }

    pub async fn create_or_update_session(&self, session: &Neo4jSession) -> Result<String> {
        let query_str = r#"
        MERGE (s:Session {id: $id})
        ON CREATE SET
            s.start_time = $start_time,
            s.end_time = $end_time,
            s.context = $context,
            s.session_id = $session_id,
            s.user_id = $user_id
        ON MATCH SET
            s.end_time = $end_time,
            s.context = $context
        RETURN s.id as session_id
        "#;

        let mut result = self.graph.execute(query(query_str)
            .param("id", session.id.to_string())
            .param("start_time", session.start_time.to_rfc3339())
            .param("end_time", session.end_time.to_rfc3339())
            .param("context", session.context.to_string())
            .param("session_id", session.session_id.to_string())
            .param("user_id", session.user_id.to_string())
        ).await?;

        if let Some(row) = result.next().await? {
            Ok(row.get("session_id")?)
        } else {
            Err(anyhow::anyhow!("Failed to create or update session"))
        }
    }


    pub async fn create_interaction(
        &self,
        session_id: &str,
        request: &str,
        response: &str,
        model: &str,
        stats: &InteractionStats
    ) -> Result<String> {
        let query_str = r#"
        MERGE (s:Session {id: $session_id})
        ON CREATE SET s.created_at = $timestamp

        MERGE (q:Question {content: $request})
        ON CREATE SET q.id = $question_id, q.timestamp = $timestamp

        MERGE (r:Response {content: $response})
        ON CREATE SET r.id = $response_id, r.timestamp = $timestamp

        MERGE (m:Model {name: $model})

        MERGE (i:Interaction {
            session_id: $session_id,
            question_content: $request,
            response_content: $response,
            model: $model
        })
        ON CREATE SET
            i.id = $id,
            i.timestamp = $timestamp

        CREATE (stats:InteractionStats {
            id: $stats_id,
            prompt_tokens: $prompt_tokens,
            completion_tokens: $completion_tokens,
            total_tokens: $total_tokens,
            response_time: $response_time,
            finish_reason: $finish_reason
        })

        MERGE (s)-[:CONTAINS]->(i)
        MERGE (i)-[:HAS_QUESTION]->(q)
        MERGE (i)-[:HAS_RESPONSE]->(r)
        MERGE (r)-[:GENERATED_BY]->(m)
        CREATE (i)-[:HAS_STATS]->(stats)

        RETURN i.id as interaction_id, q.id as question_id, r.id as response_id, m.name as model_name, stats.id as stats_id
        "#;

        let interaction_id = Uuid::new_v4().to_string();
        let question_id = Uuid::new_v4().to_string();
        let response_id = Uuid::new_v4().to_string();
        let stats_id = Uuid::new_v4().to_string();
        let timestamp = Utc::now();

        let mut result = self.graph.execute(query(query_str)
            .param("session_id", BoltType::String(BoltString::from(session_id)))
            .param("id", BoltType::String(BoltString::from(interaction_id.as_str())))
            .param("question_id", BoltType::String(BoltString::from(question_id.as_str())))
            .param("response_id", BoltType::String(BoltString::from(response_id.as_str())))
            .param("stats_id", BoltType::String(BoltString::from(stats_id.as_str())))
            .param("timestamp", BoltType::String(BoltString::from(timestamp.to_rfc3339().as_str())))
            .param("request", BoltType::String(BoltString::from(request)))
            .param("response", BoltType::String(BoltString::from(response)))
            .param("model", BoltType::String(BoltString::from(model)))
            .param("prompt_tokens", BoltType::Integer(BoltInteger::new(stats.prompt_tokens as i64)))
            .param("completion_tokens", BoltType::Integer(BoltInteger::new(stats.completion_tokens as i64)))
            .param("total_tokens", BoltType::Integer(BoltInteger::new(stats.total_tokens as i64)))
            .param("response_time", BoltType::Float(BoltFloat::new(stats.response_time)))
            .param("finish_reason", BoltType::String(BoltString::from(stats.finish_reason.as_str())))
        ).await?;

        if let Some(row) = result.next().await? {
            let interaction_id: String = row.get("interaction_id")?;
            let question_id: String = row.get("question_id")?;
            let response_id: String = row.get("response_id")?;
            let model_name: String = row.get("model_name")?;
            let stats_id: String = row.get("stats_id")?;
            debug!("Created interaction with id: {}", interaction_id);
            debug!("Question id: {}", question_id);
            debug!("Response id: {}", response_id);
            debug!("Model name: {}", model_name);
            debug!("Stats id: {}", stats_id);

            if let Some(voyage_config) = &self.voyage_ai_config {
                debug!("Voyage AI config found, creating embeddings");
                match self.create_embeddings(request, response, &question_id, &response_id, voyage_config).await {
                    Ok(_) => debug!("Created embeddings for interaction {}", interaction_id),
                    Err(e) => warn!("Failed to create embeddings for interaction {}: {:?}", interaction_id, e),
                }

                // Call enrich_document_incrementally for both question and response
                let enrichment_config = EnrichmentConfig {
                    themes_keywords_interval: ChronoDuration::hours(1),
                    clustering_interval: ChronoDuration::days(1),
                    sentiment_interval: ChronoDuration::hours(1),
                };

                match self.enrich_document_incrementally(&question_id, "Question", &enrichment_config).await {
                    Ok(_) => debug!("Enriched question {}", question_id),
                    Err(e) => warn!("Failed to enrich question {}: {:?}", question_id, e),
                }

                match self.enrich_document_incrementally(&response_id, "Response", &enrichment_config).await {
                    Ok(_) => debug!("Enriched response {}", response_id),
                    Err(e) => warn!("Failed to enrich response {}: {:?}", response_id, e),
                }
            } else {
                debug!("No Voyage AI config found, skipping embedding creation and document enrichment");
            }

            Ok(interaction_id)
        } else {
            Err(anyhow::anyhow!("Failed to create interaction"))
        }
    }


    async fn create_embeddings(
        &self,
        request: &str,
        response: &str,
        question_id: &str,
        response_id: &str,
        voyage_config: &VoyageAIConfig
    ) -> Result<()> {
        let question_embedding = get_voyage_embedding(request, voyage_config).await?;
        let response_embedding = get_voyage_embedding(response, voyage_config).await?;

        let question_embedding_node = Embedding {
            id: Uuid::new_v4().to_string(),
            vector: question_embedding,
            model: voyage_config.model.clone(),
        };

        let response_embedding_node = Embedding {
            id: Uuid::new_v4().to_string(),
            vector: response_embedding,
            model: voyage_config.model.clone(),
        };

        self.create_embedding(&question_embedding_node, question_id, "Question").await?;
        self.create_embedding(&response_embedding_node, response_id, "Response").await?;

        Ok(())
    }


    pub async fn create_embedding(&self, embedding: &Embedding, parent_id: &str, parent_type: &str) -> Result<String> {
        let query_str = r#"
        MATCH (parent {id: $parent_id})
        WHERE labels(parent)[0] = $parent_type
        MERGE (e:Embedding {vector: $vector})
        ON CREATE SET
            e.id = $id,
            e.model = $model,
            e.created_at = datetime()
        ON MATCH SET
            e.model = $model,
            e.updated_at = datetime()
        MERGE (parent)-[:HAS_EMBEDDING]->(e)
        RETURN e.id as embedding_id
        "#;

        let mut vector_list = BoltList::new();
        for &value in &embedding.vector {
            vector_list.push(BoltType::Float(BoltFloat::new(value as f64)));
        }

        let mut result = self.graph.execute(query(query_str)
            .param("parent_id", BoltType::String(BoltString::from(parent_id)))
            .param("parent_type", BoltType::String(BoltString::from(parent_type)))
            .param("id", BoltType::String(BoltString::from(embedding.id.as_str())))
            .param("vector", BoltType::List(vector_list))
            .param("model", BoltType::String(BoltString::from(embedding.model.as_str())))
        ).await?;

        if let Some(row) = result.next().await? {
            Ok(row.get("embedding_id")?)
        } else {
            Err(anyhow::anyhow!("Failed to create or update embedding"))
        }
    }


    pub async fn upsert_document(&self, file_path: &Path, metadata: &[String]) -> Result<String> {
        debug!("Upserting document from file: {:?}", file_path);

        let content = self.extract_content(file_path).await?;

        let document_id = Uuid::new_v4().to_string();
        let query = query("
        MERGE (d:Document {content: $content})
        ON CREATE SET
            d.id = $id,
            d.metadata = $metadata,
            d.created_at = datetime()
        ON MATCH SET
            d.metadata = d.metadata + $new_metadata,
            d.updated_at = datetime()
        RETURN d.id as document_id
        ")
            .param("id", document_id.clone())
            .param("content", content.clone())  // Clone here
            .param("metadata", metadata)
            .param("new_metadata", metadata);

        let mut result = self.graph.execute(query).await?;

        let document_id = if let Some(row) = result.next().await? {
            row.get::<String>("document_id")?
        } else {
            return Err(anyhow!("Failed to upsert document"));
        };

        let config = EnrichmentConfig {
            themes_keywords_interval: ChronoDuration::hours(1),
            clustering_interval: ChronoDuration::days(1),
            sentiment_interval: ChronoDuration::hours(1),
        };

        let chunks = chunk_document(&content);  // Now we can use content here
        self.create_chunks_and_embeddings(&document_id, &chunks).await?;
        self.enrich_document_incrementally(&document_id, "Document", &config).await?;
        Ok(document_id)
    }

    async fn extract_content(&self, file_path: &Path) -> Result<String> {
        let extension = file_path.extension()
            .and_then(|ext| ext.to_str())
            .ok_or_else(|| anyhow!("Unable to determine file type"))?;

        match extension.to_lowercase().as_str() {
            "pdf" => {
                let path_buf = file_path.to_path_buf();
                Ok(tokio::task::spawn_blocking(move || {
                    extract_text(&path_buf)
                }).await??)
            },
            "txt" => {
                let mut file = File::open(file_path).await?;
                let mut content = String::new();
                file.read_to_string(&mut content).await?;
                Ok(content)
            },
            "docx" => {
                let processor = DocxProcessor;
                let (content, _metadata) = processor.process(file_path).await?;
                Ok(content)
            },
            // Add more file types here as needed
            _ => Err(anyhow!("Unsupported file type: {}", extension)),
        }
    }
    async fn create_chunks_and_embeddings(&self, document_id: &str, chunks: &[String]) -> Result<()> {
        debug!("Creating chunks and embeddings for document {}", document_id);
        if let Some(voyage_config) = &self.voyage_ai_config {
            for (i, chunk) in chunks.iter().enumerate() {
                let embedding = get_voyage_embedding(chunk, voyage_config).await?;

                if embedding.len() != EMBEDDING_DIMENSION {
                    return Err(anyhow!("Embedding dimension mismatch"));
                }

                let query = query("
            MATCH (d:Document {id: $document_id})
            MERGE (c:Chunk {content: $content})
            ON CREATE SET
                c.id = $chunk_id,
                c.index = $index
            MERGE (e:Embedding {vector: $vector})
            ON CREATE SET
                e.id = $embedding_id
            MERGE (d)-[:HAS_CHUNK]->(c)
            MERGE (c)-[:HAS_EMBEDDING]->(e)
            WITH c, e, $prev_chunk_id AS prev_id
            OPTIONAL MATCH (prev:Chunk {id: prev_id})
            FOREACH (_ IN CASE WHEN prev IS NOT NULL THEN [1] ELSE [] END |
                MERGE (prev)-[:NEXT]->(c)
            )
            RETURN c.id as chunk_id, e.id as embedding_id
            ")
                    .param("document_id", BoltType::String(BoltString::from(document_id)))
                    .param("chunk_id", BoltType::String(BoltString::from(Uuid::new_v4().to_string())))
                    .param("content", BoltType::String(BoltString::from(chunk.as_str())))
                    .param("index", BoltType::Integer(BoltInteger::new(i as i64)))
                    .param("embedding_id", BoltType::String(BoltString::from(Uuid::new_v4().to_string())))
                    .param("vector", embedding)
                    .param("prev_chunk_id", if i > 0 {
                        BoltType::String(BoltString::from(chunks[i - 1].as_str()))
                    } else {
                        BoltType::Null(BoltNull::default())
                    });

                let mut result = self.graph.execute(query).await?;

                if result.next().await?.is_none() {
                    return Err(anyhow!("Failed to create or merge chunk and embedding"));
                }
            }
            Ok(())
        } else {
            Err(anyhow!("VoyageAI configuration not found"))
        }
    }

    pub async fn get_document_statistics(&self) -> Result<DocumentStatistics> {
        let query = query("
        MATCH (d:Document)
        OPTIONAL MATCH (d)-[:HAS_CHUNK]->(c)
        OPTIONAL MATCH (c)-[:HAS_EMBEDDING]->(e)
        RETURN
            count(DISTINCT d) as document_count,
            avg(size(d.content)) as avg_content_length,
            count(DISTINCT c) as chunk_count,
            count(DISTINCT e) as embedding_count
        ");

        let mut result = self.graph.execute(query).await?;

        if let Some(row) = result.next().await? {
            Ok(DocumentStatistics {
                document_count: row.get::<i64>("document_count")?,
                avg_content_length: row.get::<f64>("avg_content_length")?,
                chunk_count: row.get::<i64>("chunk_count")?,
                embedding_count: row.get::<i64>("embedding_count")?,
            })
        } else {
            Err(anyhow!("Failed to get document statistics"))
        }
    }

    pub async fn enrich_document_incrementally(&self, node_id: &str, node_type: &str, config: &EnrichmentConfig) -> Result<()> {
        debug!("Enriching {} {}", node_type, node_id);
        let status = self.get_enrichment_status(node_id, node_type).await?;
        let now = Utc::now();

        if let Some(voyage_config) = &self.voyage_ai_config {
            if status.last_themes_keywords_update.map_or(true, |last| now - last > config.themes_keywords_interval) {
                self.update_themes_and_keywords(node_id, node_type, voyage_config).await?;
            }

            if status.last_clustering_update.map_or(true, |last| now - last > config.clustering_interval) {
                self.update_clustering(node_id, node_type).await?;
            }

            if status.last_sentiment_update.map_or(true, |last| now - last > config.sentiment_interval) {
                self.update_sentiment(node_id, node_type).await?;
            }

            self.update_enrichment_status(node_id, node_type, &now).await?;
            Ok(())
        } else {
            Err(anyhow!("VoyageAI configuration not found"))
        }
    }

    async fn get_enrichment_status(&self, node_id: &str, node_type: &str) -> Result<EnrichmentStatus> {
        debug!("Getting enrichment status for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    RETURN n.last_themes_keywords_update AS themes_keywords,
           n.last_clustering_update AS clustering,
           n.last_sentiment_update AS sentiment
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            Ok(EnrichmentStatus {
                last_themes_keywords_update: row.get("themes_keywords")?,
                last_clustering_update: row.get("clustering")?,
                last_sentiment_update: row.get("sentiment")?,
            })
        } else {
            Ok(EnrichmentStatus {
                last_themes_keywords_update: None,
                last_clustering_update: None,
                last_sentiment_update: None,
            })
        }
    }


    async fn update_themes_and_keywords(&self, node_id: &str, node_type: &str, voyage_config: &VoyageAIConfig) -> Result<()> {
        debug!("Updating themes and keywords for {} {}", node_type, node_id);
        let content = self.get_node_content(node_id, node_type).await?;
        let (themes, keywords) = self.extract_themes_and_keywords(&content, voyage_config).await?;
        self.create_theme_and_keyword_nodes(node_id, node_type, &themes, &keywords).await?;
        Ok(())
    }



    async fn extract_sentiment(&self, content: &str) -> Result<f32> {
        // Define a simple sentiment lexicon
        let lexicon: HashMap<&str, f32> = [
            ("good", 1.0), ("great", 1.5), ("excellent", 2.0), ("amazing", 2.0), ("wonderful", 1.5),
            ("bad", -1.0), ("terrible", -1.5), ("awful", -2.0), ("horrible", -2.0), ("poor", -1.0),
            ("like", 0.5), ("love", 1.0), ("hate", -1.0), ("dislike", -0.5),
            ("happy", 1.0), ("sad", -1.0), ("angry", -1.0), ("joyful", 1.5),
            ("interesting", 0.5), ("boring", -0.5), ("exciting", 1.0), ("dull", -0.5)
        ].iter().cloned().collect();

        let words: Vec<String> = content.to_lowercase()
            .split_whitespace()
            .map(String::from)
            .collect();
        let total_words = words.len() as f32;

        let sentiment_sum: f32 = words.iter()
            .filter_map(|word| lexicon.get(word.as_str()))
            .sum();

        // Normalize the sentiment score
        let sentiment = sentiment_sum / total_words;

        // Clamp the sentiment between -1 and 1
        Ok(sentiment.clamp(-1.0, 1.0))
    }

    async fn create_and_assign_sentiment(&self, node_id: &str, node_type: &str, sentiment: f32) -> Result<()> {
        debug!("Creating and assigning sentiment node for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    MERGE (s:Sentiment {value: $sentiment})
    MERGE (n)-[:HAS_SENTIMENT]->(s)
    RETURN count(s) AS sentiment_count, s.value AS sentiment_value, n.id AS node_id
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("sentiment", BoltType::Float(BoltFloat::new(sentiment as f64)));

        debug!("Executing query with sentiment: {}", sentiment);

        let result = self.graph.execute(query).await;
        match result {
            Ok(mut stream) => {
                if let Some(row) = stream.next().await? {
                    let sentiment_count: i64 = row.get("sentiment_count")?;
                    let sentiment_value: f64 = row.get("sentiment_value")?;
                    let db_node_id: String = row.get("node_id")?;
                    debug!("Created and assigned {} sentiment node with value {} for {} {}",
                       sentiment_count, sentiment_value, node_type, db_node_id);
                    if sentiment_count == 0 {
                        warn!("No sentiment was created or assigned for {} {}", node_type, node_id);
                        return Err(anyhow!("Failed to create or assign sentiment for {} {}", node_type, node_id));
                    }
                } else {
                    warn!("No result returned from sentiment creation and assignment query for {} {}", node_type, node_id);
                    return Err(anyhow!("No result returned from sentiment creation query for {} {}", node_type, node_id));
                }
            },
            Err(e) => {
                error!("Error executing sentiment creation and assignment query for {} {}: {:?}", node_type, node_id, e);
                return Err(anyhow!("Failed to create and assign sentiment node: {:?}", e));
            }
        }

        // Verification step
        self.verify_sentiment(node_id, sentiment).await?;

        Ok(())
    }

    async fn verify_sentiment(&self, node_id: &str, expected_sentiment: f32) -> Result<()> {
        let query = query("
        MATCH (n {id: $node_id})-[:HAS_SENTIMENT]->(s:Sentiment)
        RETURN n.id as node_id, s.value as sentiment
        ")
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            let db_node_id: String = row.get("node_id")?;
            let db_sentiment: f64 = row.get("sentiment")?;

            debug!("Verification for node {}", db_node_id);
            debug!("Sentiment in DB: {}", db_sentiment);

            if (db_sentiment as f32 - expected_sentiment).abs() > 1e-6 {
                warn!("Sentiment mismatch for node {}: expected {}, found {}", db_node_id, expected_sentiment, db_sentiment);
                return Err(anyhow!("Sentiment mismatch for node {}", db_node_id));
            } else {
                debug!("Sentiment verified successfully for node {}", db_node_id);
            }
        } else {
            warn!("No sentiment found for node with ID: {}", node_id);
            return Err(anyhow!("No sentiment found for node {}", node_id));
        }
        Ok(())
    }



    async fn get_all_documents(&self) -> Result<Vec<String>> {
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response)
    RETURN n.content AS content
    ");

        let mut result = self.graph.execute(query).await?;
        let mut documents = Vec::new();

        while let Some(row) = result.next().await? {
            let content: String = row.get("content")?;
            documents.push(content);
        }

        Ok(documents)
    }

    async fn update_sentiment(&self, node_id: &str, node_type: &str) -> Result<()> {
        debug!("Updating sentiment for {} {}", node_type, node_id);

        // Get the content of the current node
        let content = self.get_node_content(node_id, node_type).await?;

        // Extract sentiment
        let sentiment = self.extract_sentiment(&content).await?;

        // Create and assign sentiment to the node
        match self.create_and_assign_sentiment(node_id, node_type, sentiment).await {
            Ok(_) => {
                debug!("Successfully created and assigned sentiment for {} {}", node_type, node_id);
                Ok(())
            },
            Err(e) => {
                error!("Failed to create and assign sentiment for {} {}: {:?}", node_type, node_id, e);
                Err(e)
            }
        }
    }

    async fn get_node_embedding(&self, node_id: &str, node_type: &str) -> Result<Vec<f32>> {
        debug!("Getting embedding for {} {}", node_type, node_id);
        let query = match node_type {
            "Document" => query("
            MATCH (d:Document {id: $node_id})-[:HAS_CHUNK]->(c:Chunk)-[:HAS_EMBEDDING]->(e:Embedding)
            RETURN e.vector AS embedding
            LIMIT 1
        "),
            "Question" | "Response" => query("
            MATCH (n)
            WHERE (n:Question OR n:Response) AND n.id = $node_id
            MATCH (n)-[:HAS_EMBEDDING]->(e:Embedding)
            RETURN e.vector AS embedding
        "),
            _ => return Err(anyhow!("Unsupported node type: {}", node_type)),
        }
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            Ok(row.get("embedding")?)
        } else {
            Err(anyhow!("No embedding found for {} {}", node_type, node_id))
        }
    }

    async fn update_enrichment_status(&self, node_id: &str, node_type: &str, now: &DateTime<Utc>) -> Result<()> {
        debug!("Updating enrichment status for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    SET n.last_themes_keywords_update = $now,
        n.last_clustering_update = $now,
        n.last_sentiment_update = $now
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("now", BoltType::String(BoltString::from(now.to_rfc3339())));

        self.graph.execute(query).await?;
        Ok(())
    }

    async fn get_node_content(&self, node_id: &str, node_type: &str) -> Result<String> {
        debug!("Getting content for {} {}", node_type, node_type);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    RETURN n.content AS content
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            Ok(row.get("content")?)
        } else {
            Err(anyhow!("Node not found"))
        }
    }

    async fn get_document_embedding(&self, document_id: &str) -> Result<Vec<f32>> {
        debug!("Getting embedding for document {}", document_id);
        let query = query("
    MATCH (d:Document {id: $document_id})-[:HAS_CHUNK]->(c:Chunk)-[:HAS_EMBEDDING]->(e:Embedding)
    RETURN e.vector AS embedding
    LIMIT 1
    ")
            .param("document_id", BoltType::String(BoltString::from(document_id)));

        let mut result = self.graph.execute(query).await?;

        if let Some(row) = result.next().await? {
            let embedding: Vec<f32> = row.get("embedding")?;
            Ok(embedding)
        } else {
            Err(anyhow!("No embedding found for document"))
        }
    }


    async fn create_sentiment_node(&self, node_id: &str, node_type: &str, sentiment: f32) -> Result<()> {
        debug!("Creating sentiment node for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    MERGE (s:Sentiment {value: $sentiment})
    MERGE (n)-[:HAS_SENTIMENT]->(s)
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("sentiment", BoltType::Float(BoltFloat::new(sentiment as f64)));
        debug!("node_id {}, sentiment {}", node_id, sentiment);
        self.graph.execute(query).await?;
        debug!("Created sentiment node for {} {}", node_type, node_id);

        Ok(())
    }

    async fn create_theme_and_keyword_nodes(&self, node_id: &str, node_type: &str, themes: &[String], keywords: &[String]) -> Result<()> {
        debug!("Creating theme and keyword nodes for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    WITH n
    UNWIND $themes AS theme_name
    MERGE (t:Theme {name: theme_name})
    MERGE (n)-[:HAS_THEME]->(t)
    WITH n, collect(t) AS themes
    UNWIND $keywords AS keyword_name
    MERGE (k:Keyword {name: keyword_name})
    MERGE (n)-[:HAS_KEYWORD]->(k)
    WITH n, themes, collect(k) AS keywords
    RETURN size(themes) + size(keywords) AS total_count
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("themes", themes)
            .param("keywords", keywords);

        debug!("Executing query with themes: {:?} and keywords: {:?}", themes, keywords);

        let result = self.graph.execute(query).await;
        match result {
            Ok(mut stream) => {
                if let Some(row) = stream.next().await? {
                    let total_count: i64 = row.get("total_count")?;
                    debug!("Created {} theme and keyword nodes for {} {}", total_count, node_type, node_id);
                    if total_count == 0 {
                        warn!("No themes or keywords were created for {} {}", node_type, node_id);
                    }
                } else {
                    warn!("No result returned from theme and keyword creation query for {} {}", node_type, node_id);
                }
            },
            Err(e) => {
                error!("Error executing theme and keyword creation query for {} {}: {:?}", node_type, node_id, e);
                return Err(anyhow!("Failed to create theme and keyword nodes: {:?}", e));
            }
        }

        // Verification step
        self.verify_themes_and_keywords(node_id, themes, keywords).await?;

        Ok(())
    }

    async fn verify_themes_and_keywords(&self, node_id: &str, themes: &[String], keywords: &[String]) -> Result<()> {
        let query = query("
    MATCH (n {id: $node_id})
    OPTIONAL MATCH (n)-[:HAS_THEME]->(t:Theme)
    OPTIONAL MATCH (n)-[:HAS_KEYWORD]->(k:Keyword)
    RETURN
        n.id as node_id,
        collect(distinct t.name) as themes,
        collect(distinct k.name) as keywords,
        count(distinct t) as theme_count,
        count(distinct k) as keyword_count
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            let db_node_id: String = row.get("node_id")?;
            let db_themes: Vec<String> = row.get("themes")?;
            let db_keywords: Vec<String> = row.get("keywords")?;
            let theme_count: i64 = row.get("theme_count")?;
            let keyword_count: i64 = row.get("keyword_count")?;

            debug!("Verification for node {}", db_node_id);
            debug!("Themes in DB: {:?} (count: {})", db_themes, theme_count);
            debug!("Keywords in DB: {:?} (count: {})", db_keywords, keyword_count);

            let missing_themes: Vec<_> = themes.iter().filter(|t| !db_themes.contains(t)).cloned().collect();
            let extra_themes: Vec<_> = db_themes.iter().filter(|t| !themes.contains(t)).cloned().collect();
            let missing_keywords: Vec<_> = keywords.iter().filter(|k| !db_keywords.contains(k)).cloned().collect();
            let extra_keywords: Vec<_> = db_keywords.iter().filter(|k| !keywords.contains(k)).cloned().collect();

            if !missing_themes.is_empty() || !missing_keywords.is_empty() || !extra_themes.is_empty() || !extra_keywords.is_empty() {
                warn!("Discrepancies found for node {}:", db_node_id);
                if !missing_themes.is_empty() { warn!("Missing themes: {:?}", missing_themes); }
                if !extra_themes.is_empty() { warn!("Extra themes in DB: {:?}", extra_themes); }
                if !missing_keywords.is_empty() { warn!("Missing keywords: {:?}", missing_keywords); }
                if !extra_keywords.is_empty() { warn!("Extra keywords in DB: {:?}", extra_keywords); }
                return Err(anyhow!("Discrepancies found in themes or keywords for node {}", db_node_id));
            } else {
                debug!("All themes and keywords verified successfully for node {}", db_node_id);
            }
        } else {
            warn!("No node found with ID: {}", node_id);
        }
        Ok(())
    }

    async fn update_clustering(&self, node_id: &str, node_type: &str) -> Result<()> {
        debug!("Updating clustering for {} {}", node_type, node_id);

        // Get the content of the current node
        let content = self.get_node_content(node_id, node_type).await?;

        // Get all documents (you may want to limit this or use a more efficient approach for large datasets)
        let all_documents = self.get_all_documents().await?;

        // Extract clusters
        let clusters = self.extract_clusters(&content, &all_documents).await?;

        // Create and assign clusters to the node
        self.create_and_assign_clusters(node_id, node_type, &clusters).await?;

        Ok(())
    }

    async fn create_and_assign_clusters(&self, node_id: &str, node_type: &str, clusters: &[String]) -> Result<()> {
        debug!("Creating and assigning cluster nodes for {} {}", node_type, node_id);
        let query = query("
        MATCH (n)
        WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
        WITH n
        UNWIND $clusters AS cluster_name
        MERGE (c:Cluster {name: cluster_name})
        MERGE (n)-[:BELONGS_TO]->(c)
        WITH n, collect(c) AS clusters
        RETURN size(clusters) AS total_count
        ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("clusters", clusters);

        debug!("Executing query with clusters: {:?}", clusters);

        let result = self.graph.execute(query).await;
        match result {
            Ok(mut stream) => {
                if let Some(row) = stream.next().await? {
                    let total_count: i64 = row.get("total_count")?;
                    debug!("Created and assigned {} cluster nodes for {} {}", total_count, node_type, node_id);
                    if total_count == 0 {
                        warn!("No clusters were created or assigned for {} {}", node_type, node_id);
                    }
                } else {
                    warn!("No result returned from cluster creation and assignment query for {} {}", node_type, node_id);
                }
            },
            Err(e) => {
                error!("Error executing cluster creation and assignment query for {} {}: {:?}", node_type, node_id, e);
                return Err(anyhow!("Failed to create and assign cluster nodes: {:?}", e));
            }
        }

        // Verification step
        self.verify_clusters(node_id, clusters).await?;

        Ok(())
    }

    async fn verify_clusters(&self, node_id: &str, expected_clusters: &[String]) -> Result<()> {
        let query = query("
        MATCH (n {id: $node_id})
        OPTIONAL MATCH (n)-[:BELONGS_TO]->(c:Cluster)
        RETURN
            n.id as node_id,
            collect(distinct c.name) as clusters,
            count(distinct c) as cluster_count
        ")
            .param("node_id", BoltType::String(BoltString::from(node_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            let db_node_id: String = row.get("node_id")?;
            let db_clusters: Vec<String> = row.get("clusters")?;
            let cluster_count: i64 = row.get("cluster_count")?;

            debug!("Verification for node {}", db_node_id);
            debug!("Clusters in DB: {:?} (count: {})", db_clusters, cluster_count);

            let missing_clusters: Vec<_> = expected_clusters.iter().filter(|c| !db_clusters.contains(c)).cloned().collect();
            let extra_clusters: Vec<_> = db_clusters.iter().filter(|c| !expected_clusters.contains(c)).cloned().collect();

            if !missing_clusters.is_empty() || !extra_clusters.is_empty() {
                warn!("Discrepancies found for node {}:", db_node_id);
                if !missing_clusters.is_empty() { warn!("Missing clusters: {:?}", missing_clusters); }
                if !extra_clusters.is_empty() { warn!("Extra clusters in DB: {:?}", extra_clusters); }
                return Err(anyhow!("Discrepancies found in clusters for node {}", db_node_id));
            } else {
                debug!("All clusters verified successfully for node {}", db_node_id);
            }
        } else {
            warn!("No node found with ID: {}", node_id);
        }
        Ok(())
    }

    async fn assign_to_cluster(&self, node_id: &str, node_type: &str, cluster: &str) -> Result<()> {
        debug!("Assigning to cluster for {} {}", node_type, node_id);
        let query = query("
    MATCH (n)
    WHERE (n:Document OR n:Question OR n:Response) AND n.id = $node_id
    MERGE (c:Cluster {name: $cluster})
    MERGE (n)-[:BELONGS_TO]->(c)
    ")
            .param("node_id", BoltType::String(BoltString::from(node_id)))
            .param("cluster", BoltType::String(BoltString::from(cluster)));
        debug!("node_id: {} cluster: {}", node_id, cluster);
        self.graph.execute(query).await?;
        debug!("Assigned to cluster for {} {}", node_type, node_id);
        Ok(())
    }

    async fn node_exists(&self, node_id: &str) -> Result<bool> {
        let query = query("
        MATCH (n {id: $node_id})
        RETURN count(n) as count
        ")
            .param("node_id", node_id);

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            let count: i64 = row.get("count")?;
            Ok(count > 0)
        } else {
            Ok(false)
        }
    }

    pub async fn create_or_update_question(&self, question: &Neo4jQuestion, interaction_id: &str) -> Result<String> {
        let query_str = r#"
        MERGE (q:Question {content: $content})
        ON CREATE SET
            q = $props
        ON MATCH SET
            q.vector = $props.vector,
            q.timestamp = $props.timestamp
        WITH q
        MATCH (i:Interaction {id: $interaction_id})
        MERGE (i)-[:HAS_QUESTION]->(q)
        RETURN q.id as question_id
        "#;

        let mut props = BoltMap::new();
        props.put(BoltString::from("id"), BoltType::String(BoltString::from(question.id.as_str())));
        props.put(BoltString::from("content"), BoltType::String(BoltString::from(question.content.as_str())));

        let mut vector_list = BoltList::new();
        for &value in &question.vector {
            vector_list.push(BoltType::Float(BoltFloat::new(value as f64)));
        }
        props.put(BoltString::from("vector"), BoltType::List(vector_list));

        props.put(BoltString::from("timestamp"), BoltType::String(BoltString::from(question.timestamp.to_rfc3339().as_str())));

        let mut result = self.graph.execute(query(query_str)
            .param("content", question.content.as_str())
            .param("props", BoltType::Map(props))
            .param("interaction_id", interaction_id)
        ).await?;

        if let Some(row) = result.next().await? {
            Ok(row.get("question_id")?)
        } else {
            Err(anyhow::anyhow!("Failed to create or update question"))
        }
    }

    pub async fn create_response(&self, response: &Neo4jResponse, interaction_id: &str, model_id: &str) -> Result<String> {
        let query_str = r#"
        CREATE (r:Response {
            id: $id,
            content: $content,
            vector: $vector,
            timestamp: $timestamp,
            confidence: $confidence,
            llm_specific_data: $llm_specific_data
        })
        WITH r
        MATCH (i:Interaction {id: $interaction_id})
        MATCH (m:Model {id: $model_id})
        CREATE (i)-[:HAS_RESPONSE]->(r)
        CREATE (r)-[:GENERATED_BY]->(m)
        RETURN r.id as response_id
        "#;

        let mut result = self.graph.execute(query(query_str)
            .param("id", response.id.clone())
            .param("content", response.content.clone())
            .param("vector", BoltType::List(response.vector.clone()))
            .param("timestamp", response.timestamp.to_rfc3339())
            .param("confidence", response.confidence)
            .param("llm_specific_data", serde_json::to_string(&response.llm_specific_data)?)
            .param("interaction_id", interaction_id)
            .param("model_id", model_id)
        ).await?;

        if let Some(row) = result.next().await? {
            Ok(row.get("response_id")?)
        } else {
            Err(anyhow::anyhow!("Failed to create response"))
        }
    }


    async fn extract_themes_and_keywords(&self, content: &str, config: &VoyageAIConfig) -> Result<(Vec<String>, Vec<String>)> {
        debug!("Extracting themes and keywords");
        debug!("content: {}", content);
        let stemmer = Stemmer::create(Algorithm::English);
        let stop_words: Vec<String> = stop_words::get(stop_words::LANGUAGE::English);

        // Tokenize and clean the content
        let words: Vec<String> = content
            .split_whitespace()
            .map(|word| word.to_lowercase())
            .filter(|word| {
                word.len() > 4 && // Filter out very short words
                    !stop_words.contains(word) && // Filter out stop words
                    word.chars().any(|c| c.is_alphabetic()) // Ensure at least one alphabetic character
            })
            .collect();

        let mut word_freq: HashMap<String, usize> = HashMap::new();

        for word in words {
            let stemmed = stemmer.stem(&word).to_string();
            *word_freq.entry(stemmed).or_insert(0) += 1;
        }

        let mut sorted_words: Vec<_> = word_freq.into_iter().collect();
        sorted_words.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));

        let themes: Vec<String> = sorted_words.iter()
            .take(3)  // Extract top 5 as themes
            .map(|(word, count)| format!("{}:{}", word, count))
            .collect();

        let keywords: Vec<String> = sorted_words.iter()
            .skip(5)
            .take(3)  // Extract next 10 as keywords
            .map(|(word, count)| format!("{}:{}", word, count))
            .collect();

        debug!("Extracted themes: {:?}", themes);
        debug!("Extracted keywords: {:?}", keywords);

        Ok((themes, keywords))
    }


    async fn perform_clustering(&self, embedding: &[f32]) -> Result<String> {
        debug!("Performing Clustering:");
        // This is a simplified clustering method.
        // In a real-world scenario, you'd want to implement a more sophisticated clustering algorithm.
        let sum: f32 = embedding.iter().sum();
        let avg = sum / embedding.len() as f32;

        let cluster = if avg > 0.0 { "positive" } else { "negative" };

        Ok(cluster.to_string())
    }


    async fn extract_clusters(&self, content: &str, all_documents: &[String]) -> Result<Vec<String>> {
        debug!("Extracting clusters");
        let stemmer = Stemmer::create(Algorithm::English);
        let stop_words: HashSet<_> = stop_words::get(stop_words::LANGUAGE::English).into_iter().collect();

        // Function to tokenize and clean text
        let tokenize = |text: &str| -> Vec<String> {
            text.split_whitespace()
                .map(|word| word.to_lowercase())
                .filter(|word| {
                    word.len() > 5 && // Filter out very short words
                        !stop_words.contains(word) && // Filter out stop words
                        word.chars().any(|c| c.is_alphabetic()) // Ensure at least one alphabetic character
                })
                .map(|word| stemmer.stem(&word).to_string())
                .collect()
        };

        // Calculate TF-IDF
        let doc_words = tokenize(content);
        let mut tf: HashMap<String, f64> = HashMap::new();
        let mut df: HashMap<String, f64> = HashMap::new();
        let n_docs = all_documents.len() as f64;

        // Calculate TF for the current document
        for word in &doc_words {
            *tf.entry(word.clone()).or_insert(0.0) += 1.0;
        }

        // Normalize TF
        let doc_len = doc_words.len() as f64;
        for count in tf.values_mut() {
            *count /= doc_len;
        }

        // Calculate DF
        for doc in all_documents {
            let unique_words: HashSet<_> = tokenize(doc).into_iter().collect();
            for word in unique_words {
                *df.entry(word).or_insert(0.0) += 1.0;
            }
        }

        // Calculate TF-IDF
        let mut tfidf: Vec<(String, f64)> = tf.into_iter()
            .map(|(word, tf_value)| {
                let df_value = df.get(&word).unwrap_or(&1.0);
                let idf = (n_docs / df_value).ln();
                (word, tf_value * idf)
            })
            .collect();

        // Sort by TF-IDF score
        tfidf.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());

        // Extract top terms as clusters
        let clusters: Vec<String> = tfidf.into_iter()
            .take(3) // Take top 5 terms as clusters
            .map(|(word, score)| format!("{}:{:.2}", word, score))
            .collect();

        debug!("Extracted clusters: {:?}", clusters);
        Ok(clusters)
    }


    async fn analyze_sentiment(&self, embedding: &[f32]) -> Result<f32> {
        debug!("Analyzing Sentiment:");
        // This is a very simplified sentiment analysis based on the embedding.
        // In a real-world scenario, you'd want to use a more sophisticated method.
        let sum: f32 = embedding.iter().sum();
        let avg = sum / embedding.len() as f32;

        // Normalize to range [-1, 1]
        let sentiment = avg.max(-1.0).min(1.0);

        Ok(sentiment)
    }

    async fn get_sentence_embedding(&self, content: &str, config: &VoyageAIConfig) -> Result<Vec<f32>> {
        debug!("Getting Sentence Embedding:");
        get_voyage_embedding(content, config).await
    }

    async fn get_document(&self, document_id: &str) -> Result<Document> {
        debug!("Getting Document:");
        let query = query("
    MATCH (d:Document {id: $document_id})
    RETURN d.content as content, d.metadata as metadata
    ")
            .param("document_id", BoltType::String(BoltString::from(document_id)));

        let mut result = self.graph.execute(query).await?;
        if let Some(row) = result.next().await? {
            Ok(Document {
                id: document_id.to_string(),
                content: row.get("content")?,
                metadata: row.get("metadata")?,
            })
        } else {
            Err(anyhow!("Document not found"))
        }
    }

    pub async fn execute_cypher(&self, cypher_query: &str) -> Result<Value> {
        info!("Executing Cypher query: {}", cypher_query);

        let query = query(cypher_query);

        let mut txn = self.graph.start_txn().await?;
        let mut result = txn.execute(query).await?;

        let mut rows = Vec::new();

        while let Some(row) = result.next(txn.handle()).await? {
            let row_value = self.row_to_json(&row)?;
            rows.push(row_value);
        }

        txn.commit().await?;

        Ok(json!(rows))
    }

    fn row_to_json(&self, row: &Row) -> Result<Value> {
        row.to::<Value>().map_err(|e| anyhow!("Failed to convert row to JSON: {}", e))
    }

    pub async fn get_database_schema(&self) -> Result<String, Error> {
        let query = "
        CALL apoc.meta.schema()
        YIELD value
        RETURN value
        ";

        let result = self.execute_cypher(query).await?;

        // Convert the schema to a string representation
        let schema_str = serde_json::to_string_pretty(&result)?;

        Ok(schema_str)
    }


    async fn process_document(&self, file_path: &Path) -> Result<(String, Vec<String>)> {
        let extension = file_path.extension()
            .and_then(|ext| ext.to_str())
            .ok_or_else(|| anyhow!("Unable to determine file type"))?;

        let processor: Box<dyn DocumentProcessor> = match extension.to_lowercase().as_str() {
            "txt" => Box::new(TextProcessor),
            "pdf" => Box::new(PdfProcessor),
            "docx" => Box::new(DocxProcessor),
            // Add more file types as needed
            _ => return Err(anyhow!("Unsupported file type: {}", extension)),
        };

        processor.process(file_path).await
    }


}

// Define the necessary structs
#[derive(Debug, Clone)]
pub struct Neo4jSession {
    pub id: String,
    pub start_time: DateTime<Utc>,
    pub end_time: DateTime<Utc>,
    pub context: String,
    pub session_id: String,
    pub user_id: String,
}

#[derive(Debug, Clone)]
pub struct Neo4jInteraction {
    pub id: String,
    pub timestamp: DateTime<Utc>,
    pub order: i32,
    pub session_id: String,
    pub question: Option<Neo4jQuestion>,
    pub response: Option<Neo4jResponse>,
}

#[derive(Debug, Clone)]
pub struct Neo4jQuestion {
    pub id: String,
    pub content: String,
    pub vector: Vec<f32>,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone)]
pub struct Neo4jResponse {
    pub id: String,
    pub content: String,
    pub vector: BoltList,
    pub timestamp: DateTime<Utc>,
    pub confidence: f64,
    pub llm_specific_data: Value,
}

#[derive(Debug, Clone)]
pub struct Neo4jModel {
    pub id: String,
    pub name: String,
    pub version: String,
}

#[derive(Debug, Clone)]
pub struct Neo4jTokenUsage {
    pub id: String,
    pub prompt_tokens: i32,
    pub completion_tokens: i32,
    pub total_tokens: i32,
}

// Implement other necessary structs and methods...
#### END OF FILE: crates/fluent-core/src/neo4j_client.rs ####

#### START OF FILE: crates/fluent-core/src/traits.rs ####

use std::future::Future;
use std::io::Cursor;
use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use anyhow::{Result, anyhow, Context, Error};
use serde_json::{json, Value};
use log::debug;
use pdf_extract::extract_text;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use crate::config::EngineConfig;
use crate::neo4j_client::Neo4jClient;
use crate::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse};


#[async_trait]

pub trait FileUpload: Send + Sync {
    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a>;
    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a>;
}



#[async_trait]
pub trait Engine: Send + Sync {

    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a>;

    fn upsert<'a>(&'a self, request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a>;

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>>;

    fn get_session_id(&self) -> Option<String>;  // New method

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent>;

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a>;
    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a>;


}

pub trait EngineConfigProcessor {
    fn process_config(&self, config: &EngineConfig) -> Result<serde_json::Value>;
}



pub struct AnthropicConfigProcessor;
impl EngineConfigProcessor for AnthropicConfigProcessor {
    fn process_config(&self, config: &EngineConfig) -> Result<serde_json::Value> {
        debug!("AnthropicConfigProcessor::process_config");
        debug!("Config: {:#?}", config);

        let mut payload = json!({
            "messages": [
                {
                    "role": "user",
                    "content": "" // This will be filled later with the actual request
                }
            ],
            "model": config.parameters.get("modelName")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Model name not specified or not a string"))?,
        });

        // Add other parameters, excluding 'sessionID'
        for (key, value) in &config.parameters {
            if key != "sessionID" && !["modelName", "bearer_token"].contains(&key.as_str()) {
                match key.as_str() {
                    "max_tokens" => {
                        if let Some(num) = value.as_str().and_then(|s| s.parse::<i64>().ok()) {
                            payload[key] = json!(num);
                        } else if let Some(num) = value.as_i64() {
                            payload[key] = json!(num);
                        }
                    },
                    "temperature" | "top_p" => {
                        if let Some(num) = value.as_str().and_then(|s| s.parse::<f64>().ok()) {
                            payload[key] = json!(num);
                        } else if let Some(num) = value.as_f64() {
                            payload[key] = json!(num);
                        }
                    },
                    _ => {
                        payload[key] = value.clone();
                    }
                }
            }
        }

        debug!("Anthropic Payload: {:#?}", payload);
        Ok(payload)
    }
}


pub struct OpenAIConfigProcessor;
impl EngineConfigProcessor for OpenAIConfigProcessor {
    fn process_config(&self, config: &EngineConfig) -> Result<serde_json::Value> {
        debug!("OpenAIConfigProcessor::process_config");
        debug!("Config: {:#?}", config);

        let mut payload = json!({
            "model": config.parameters.get("modelName")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Model not specified or not a string"))?,
            "messages": [], // This will be filled later with the actual request
        });

        // Handle temperature
        if let Some(temperature) = config.parameters.get("temperature") {
            if let Some(temp) = temperature.as_f64() {
                payload["temperature"] = json!(temp);
            }
        }

        // Handle max_tokens
        if let Some(max_tokens) = config.parameters.get("max_tokens") {
            if let Some(max_tokens_str) = max_tokens.as_str() {
                if let Ok(max_tokens_int) = max_tokens_str.parse::<u64>() {
                    payload["max_tokens"] = json!(max_tokens_int);
                }
            } else if let Some(max_tokens_int) = max_tokens.as_u64() {
                payload["max_tokens"] = json!(max_tokens_int);
            }
        }

        // Add optional parameters if they exist in the configuration
        for &param in &["frequency_penalty", "presence_penalty", "top_p", "n", "stream", "stop"] {
            if let Some(value) = config.parameters.get(param) {
                payload[param] = value.clone();
            }
        }

        debug!("OpenAI Payload: {:#?}", payload);
        Ok(payload)
    }
}



pub struct TextProcessor;
pub struct PdfProcessor;
pub struct DocxProcessor;
#[async_trait]
pub trait DocumentProcessor {
    async fn process(&self, file_path: &Path) -> Result<(String, Vec<String>)>;
}

#[async_trait]
impl DocumentProcessor for TextProcessor {
    async fn process(&self, file_path: &Path) -> Result<(String, Vec<String>)> {
        let mut file = File::open(file_path).await?;
        let mut content = String::new();
        file.read_to_string(&mut content).await?;
        Ok((content, vec![]))
    }
}

#[async_trait]
impl DocumentProcessor for PdfProcessor {
    async fn process(&self, file_path: &Path) -> Result<(String, Vec<String>)> {
        // Clone the PathBuf to move it into the closure
        debug!("PdfProcessor::process");
        let path_buf = file_path.to_path_buf();

        // Extract text from PDF
        let text = tokio::task::spawn_blocking(move || {
            debug!("PdfProcessor::process: Extracting text from PDF");
            extract_text(&path_buf)
        }).await??;

        // Extract metadata (you can expand this based on your needs)
        let mut metadata = Vec::new();
        debug!("PdfProcessor::process: Extracting metadata from PDF");

        // Add file name to metadata
        if let Some(file_name) = file_path.file_name() {
            if let Some(file_name_str) = file_name.to_str() {
                metadata.push(format!("filename:{}", file_name_str));
            }
        }
        debug!("PdfProcessor::process: Metadata: {:#?}", metadata);
        // Add file size to metadata
        let file_size = tokio::fs::metadata(file_path).await?.len();

        debug!("PdfProcessor::process: File size: {}", file_size);
        metadata.push(format!("file_size:{}", file_size));

        // You can add more metadata extraction here, such as:
        // - Number of pages
        // - Author
        // - Creation date
        // - etc.

        Ok((text, metadata))
    }
}


#[async_trait]
impl DocumentProcessor for DocxProcessor {

    async fn process(&self, file_path: &Path) -> Result<(String, Vec<String>)> {
        let mut file = File::open(file_path).await?;
        let mut buffer = Vec::new();
        file.read_to_end(&mut buffer).await?;

        let file_path = file_path.to_owned();
        let file_size = buffer.len();  // Calculate buffer length here

        let content = tokio::task::spawn_blocking(move || -> Result<String> {
            // Instead of parsing DOCX, we'll just read the file as UTF-8 text
            // This won't work for actual DOCX files, but serves as a placeholder
            let content = String::from_utf8_lossy(&buffer).to_string();

            // Simulate paragraph separation
            let content = content.replace("\n", "\n\n");

            Ok(content)
        }).await??;

        let metadata = vec![
            format!("filename:{}", file_path.file_name().unwrap().to_string_lossy()),
            format!("filesize:{}", file_size),  // Use file_size here instead of buffer.len()
            "filetype:docx".to_string(), // Adding this to maintain similarity with original function
        ];

        Ok((content, metadata))
    }
}




#### END OF FILE: crates/fluent-core/src/traits.rs ####

#### START OF FILE: crates/fluent-core/src/utils.rs ####
// crates/fluent-core/src/utils.rs
pub fn some_utility_function() {
    // Utility function implementation
}


pub mod chunking {
    use unicode_segmentation::UnicodeSegmentation;

    pub const CHUNK_SIZE: usize = 1000; // Adjust this value as needed
    pub const CHUNK_OVERLAP: usize = 200; // Adjust this value as needed

    pub fn chunk_document(content: &str) -> Vec<String> {
        let words: Vec<&str> = content.unicode_words().collect();
        let mut chunks = Vec::new();
        let mut start = 0;

        while start < words.len() {
            let end = (start + CHUNK_SIZE).min(words.len());
            let chunk = words[start..end].join(" ");
            chunks.push(chunk);

            if end == words.len() {
                break;
            }

            start = if end > CHUNK_OVERLAP { end - CHUNK_OVERLAP } else { 0 };
        }

        chunks
    }
}
#### END OF FILE: crates/fluent-core/src/utils.rs ####

#### START OF FILE: crates/fluent-engines/Cargo.toml ####
# crates/fluent-engines/Cargo.toml
[package]
name = "fluent-engines"
version = "0.1.0"
edition = "2021"

[dependencies]
fluent-core = { path = "../fluent-core" }
reqwest = { version = "0.11", features = ["json", "stream", "multipart"] }
serde_json = "1.0.120"
anyhow = "1.0.86"
async-trait = "0.1.80"
log = "0.4.22"
tokio = "1.38.0"
tokio-util = "0.7.11"
base64 = "0.22.1"

mime_guess = "2.0.3"
serde = { version = "1.0.204", features = ["derive"] }
indicatif = "0.17.8"
uuid = "1.9.1"
clap = "4.5.8"
futures-util = "0.3.30"
tempfile = "3.10.1"
#### END OF FILE: crates/fluent-engines/Cargo.toml ####

#### START OF FILE: crates/fluent-engines/src/perplexity.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct PerplexityEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl PerplexityEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn send_perplexity_request(&self, messages: Vec<Value>) -> Result<Value> {
        let url = format!("{}://{}:{}{}",
                          self.config.connection.protocol,
                          self.config.connection.hostname,
                          self.config.connection.port,
                          self.config.connection.request_path
        );

        let payload = json!({
            "model": self.config.parameters.get("model").and_then(|v| v.as_str()).unwrap_or("sonar-medium-chat"),
            "messages": messages,
            "temperature": self.config.parameters.get("temperature").and_then(|v| v.as_f64()).unwrap_or(0.7),
            "max_tokens": self.config.parameters.get("max_tokens").and_then(|v| v.as_u64()).unwrap_or(1024),
            "top_p": self.config.parameters.get("top_p").and_then(|v| v.as_f64()).unwrap_or(1.0),
            "stream": self.config.parameters.get("stream").and_then(|v| v.as_bool()).unwrap_or(false),
        });

        debug!("Perplexity Request: {:?}", payload);

        let auth_token = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

        let response = self.client.post(&url)
            .header("Authorization", format!("Bearer {}", auth_token))
            .header("Content-Type", "application/json")
            .json(&payload)
            .send()
            .await?;

        if !response.status().is_success() {
            let status = response.status();
            let error_text = response.text().await?;
            debug!("Error response body: {}", error_text);
            return Err(anyhow!("Request failed with status: {}", status));
        }

        let response_body: Value = response.json().await?;
        debug!("Perplexity Response: {:?}", response_body);

        Ok(response_body)
    }
}

#[async_trait]
impl Engine for PerplexityEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let messages = vec![json!({
                "role": "user",
                "content": request.payload
            })];

            let response = self.send_perplexity_request(messages).await?;

            let content = response["choices"][0]["message"]["content"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Perplexity response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response["usage"]["prompt_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response["usage"]["completion_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: response["usage"]["total_tokens"].as_u64().unwrap_or(0) as u32,
            };

            let model = response["model"].as_str().unwrap_or("unknown").to_string();
            let finish_reason = response["choices"][0]["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionId").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value["choices"][0]["message"]["content"]
            .as_str()
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not supported for Perplexity engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, _request: &'a Request, _file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File processing not supported for Perplexity engine"))
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/perplexity.rs ####

#### START OF FILE: crates/fluent-engines/src/dalle.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::Value;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct DalleEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl DalleEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }
}


#[async_trait]
impl Engine for DalleEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = serde_json::json!({
                "model": self.config.parameters.get("modelName").and_then(|v| v.as_str()).unwrap_or("dall-e-3"),
                "prompt": &request.payload,
                "n": self.config.parameters.get("n").and_then(|v| v.as_u64()).unwrap_or(1),
                "size": self.config.parameters.get("size").and_then(|v| v.as_str()).unwrap_or("1024x1024"),
                "response_format": self.config.parameters.get("response_format").and_then(|v| v.as_str()).unwrap_or("url"),
                "quality": self.config.parameters.get("quality").and_then(|v| v.as_str()).unwrap_or("standard"),
                "style": self.config.parameters.get("style").and_then(|v| v.as_str()).unwrap_or("vivid")
            });

            debug!("DALL-E Payload: {:?}", payload);


            debug!("Size, {:?}", self.config.parameters.get("size"));

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("DALL-E Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("DALL-E API error: {:?}", error));
            }

            let content = response["data"][0]["url"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract image URL from DALL-E response"))?
                .to_string();

            Ok(Response {
                content,
                usage: Usage { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
                model: "dall-e".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value.get("data")
            .and_then(|data| data.as_array())
            .and_then(|array| array.first())
            .and_then(|first| first.get("url"))
            .and_then(|url| url.as_str())
            .map(|url| ExtractedContent {
                main_content: url.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }


    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            // Read and encode the file
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = STANDARD.encode(&buffer);

            let url = format!("{}://{}:{}/v1/images/edits",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port
            );

            let payload = serde_json::json!({
                "model": self.config.parameters.get("modelName").and_then(|v| v.as_str()).unwrap_or("dall-e-3"),
                "image": format!("data:image/png;base64,{}", base64_image),
                "prompt": &request.payload,
                "n": self.config.parameters.get("n").and_then(|v| v.as_u64()).unwrap_or(1),
                "size": self.config.parameters.get("size").and_then(|v| v.as_str()).unwrap_or("1024x1024"),
                "response_format": self.config.parameters.get("response_format").and_then(|v| v.as_str()).unwrap_or("url"),
                "quality": self.config.parameters.get("quality").and_then(|v| v.as_str()).unwrap_or("standard"),
                "style": self.config.parameters.get("style").and_then(|v| v.as_str()).unwrap_or("vivid")
            });

            debug!("DALL-E Payload: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("DALL-E Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("DALL-E API error: {:?}", error));
            }

            let content = response["data"][0]["url"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract image URL from DALL-E response"))?
                .to_string();

            Ok(Response {
                content,
                usage: Usage { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
                model: "dall-e".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            // DALL-E doesn't support file uploads in the same way as OpenAI.
            // Instead, we'll read the file and encode it to base64.
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = STANDARD.encode(&buffer);
            Ok(base64_image)
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/dalle.rs ####

#### START OF FILE: crates/fluent-engines/src/mistral.rs ####
// crates/fluent-engines/src/mistral.rs
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct MistralEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl MistralEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn send_mistral_request(&self, messages: Vec<Value>) -> Result<Value> {
        let url = format!("{}://{}:{}{}",
                          self.config.connection.protocol,
                          self.config.connection.hostname,
                          self.config.connection.port,
                          self.config.connection.request_path
        );

        let payload = json!({
            "model": self.config.parameters.get("model").and_then(|v| v.as_str()).unwrap_or("mistral-7b-instruct"),
            "messages": messages,
            "temperature": self.config.parameters.get("temperature").and_then(|v| v.as_f64()).unwrap_or(0.7),
            "max_tokens": self.config.parameters.get("max_tokens").and_then(|v| v.as_u64()).unwrap_or(1024),
            "top_p": self.config.parameters.get("top_p").and_then(|v| v.as_f64()).unwrap_or(1.0),
            "stream": self.config.parameters.get("stream").and_then(|v| v.as_bool()).unwrap_or(false),
        });

        debug!("Mistral Request: {:?}", payload);

        let auth_token = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

        let response = self.client.post(&url)
            .header("Authorization", format!("Bearer {}", auth_token))
            .header("Content-Type", "application/json")
            .json(&payload)
            .send()
            .await?;

        if !response.status().is_success() {
            let status = response.status();
            let error_text = response.text().await?;
            debug!("Error response body: {}", error_text);
            return Err(anyhow!("Request failed with status: {}", status));
        }

        let response_body: Value = response.json().await?;
        debug!("Mistral Response: {:?}", response_body);

        Ok(response_body)
    }
}

#[async_trait]
impl Engine for MistralEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let messages = vec![json!({
                "role": "user",
                "content": request.payload
            })];

            let response = self.send_mistral_request(messages).await?;

            let content = response["choices"][0]["message"]["content"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Mistral response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response["usage"]["prompt_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response["usage"]["completion_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: response["usage"]["total_tokens"].as_u64().unwrap_or(0) as u32,
            };

            let model = response["model"].as_str().unwrap_or("unknown").to_string();
            let finish_reason = response["choices"][0]["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionId").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value["choices"][0]["message"]["content"]
            .as_str()
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not supported for Mistral engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, _request: &'a Request, _file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File processing not supported for Mistral engine"))
        })
    }
}

#### END OF FILE: crates/fluent-engines/src/mistral.rs ####

#### START OF FILE: crates/fluent-engines/src/webhook.rs ####
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct WebhookEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl WebhookEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn prepare_payload(&self, request: &Request, file_content: Option<String>) -> Value {
        let mut payload = json!({
            "input": request.payload,
            "chat_id": self.config.parameters.get("chat_id").and_then(|v| v.as_str()).unwrap_or(""),
            "sessionId": self.config.parameters.get("sessionId").and_then(|v| v.as_str()).unwrap_or(""),
        });

        if let Some(content) = file_content {
            payload["file_content"] = json!(content);
        }

        // Add overrideConfig parameters
        if let Some(override_config) = self.config.parameters.get("overrideConfig") {
            if let Some(obj) = override_config.as_object() {
                for (key, value) in obj {
                    payload[key] = value.clone();
                }
            }
        }

        // Add tweaks
        if let Some(tweaks) = self.config.parameters.get("tweaks") {
            if let Some(obj) = tweaks.as_object() {
                for (key, value) in obj {
                    payload["tweaks"][key] = value.clone();
                }
            }
        }

        payload
    }
}

#[async_trait]
impl Engine for WebhookEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = self.prepare_payload(request, None).await;

            debug!("Webhook Payload: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let timeout = self.config.parameters.get("timeout_ms")
                .and_then(|v| v.as_u64())
                .unwrap_or(60000);
            debug!("url: {}, payload: {}, timeout: {}", url, payload, timeout);
            let response = self.client.post(&url)
                .timeout(std::time::Duration::from_millis(timeout))
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Webhook Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Webhook error: {:?}", error));
            }

            let content = serde_json::to_string(&response)
                .context("Failed to serialize webhook response")?;

            Ok(Response {
                content,
                usage: Usage {
                    prompt_tokens: 0,
                    completion_tokens: 0,
                    total_tokens: 0,
                },
                model: self.config.name.clone(),
                finish_reason: Some("webhook_complete".to_string()),
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionId").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        Some(ExtractedContent {
            main_content: serde_json::to_string(value).ok()?,
            sentiment: None,
            clusters: None,
            themes: None,
            keywords: None,
        })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_content = STANDARD.encode(&buffer);
            Ok(base64_content)
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let file_content = Pin::from(self.upload_file(file_path)).await?;

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = self.prepare_payload(request, Some(file_content)).await;

            debug!("Webhook Payload with file: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let timeout = self.config.parameters.get("timeout_ms")
                .and_then(|v| v.as_u64())
                .unwrap_or(60000);

            debug!("Url: {}, payload: {:?}, timeout: {}", url, payload, timeout);
            let response = self.client.post(&url)
                .timeout(std::time::Duration::from_millis(timeout))
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Webhook Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Webhook error: {:?}", error));
            }

            let content = serde_json::to_string(&response)
                .context("Failed to serialize webhook response")?;

            Ok(Response {
                content,
                usage: Usage {
                    prompt_tokens: 0,
                    completion_tokens: 0,
                    total_tokens: 0,
                },
                model: self.config.name.clone(),
                finish_reason: Some("webhook_complete".to_string()),
            })
        })
    }

}
#### END OF FILE: crates/fluent-engines/src/webhook.rs ####

#### START OF FILE: crates/fluent-engines/src/lib.rs ####
extern crate core;

// crates/fluent-engines/src/lib.rs
pub mod openai;
pub mod anthropic;
pub mod flowise_chain;
pub mod langflow;
pub mod dalle;
pub mod cohere;
pub mod webhook;
pub mod google_gemini;
pub mod groqlpu;
pub mod perplexity;
pub mod leonardoai;
pub mod mistral;
pub mod stabilityai;
pub mod imagepro;

pub mod pipeline_executor;

#### END OF FILE: crates/fluent-engines/src/lib.rs ####

#### START OF FILE: crates/fluent-engines/src/leonardoai.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Map, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct LeonardoAIEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl LeonardoAIEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn upload_file_internal(&self, file_path: &Path) -> Result<String> {
        let mut file = File::open(file_path).await.context("Failed to open file")?;
        let mut buffer = Vec::new();
        file.read_to_end(&mut buffer).await.context("Failed to read file")?;
        let base64_image = STANDARD.encode(&buffer);

        let url = format!("{}://{}:{}/api/rest/v1/init-image",
                          self.config.connection.protocol,
                          self.config.connection.hostname,
                          self.config.connection.port
        );

        let auth_token = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

        let payload = json!({
            "extension": file_path.extension().and_then(|e| e.to_str()).unwrap_or("png"),
            "file": base64_image,
        });

        let response = self.client.post(&url)
            .header("Authorization", format!("Bearer {}", auth_token))
            .json(&payload)
            .send()
            .await?
            .json::<Value>()
            .await?;

        response["uploadInitImage"]["id"]
            .as_str()
            .ok_or_else(|| anyhow!("Failed to extract uploaded image ID"))
            .map(String::from)
    }

    fn create_payload(&self, request: &Request, image_id: Option<String>) -> Value {
        let mut payload: Map<String, Value> = self.config.parameters
            .iter()
            .filter(|(k, _)| *k != "bearer_token" && *k != "sessionID" && *k != "user")  // Exclude bearer_token
            .map(|(k, v)| (k.clone(), v.clone()))
            .collect();

        // Add or override specific fields
        payload.insert("prompt".to_string(), json!(request.payload));

        if let Some(id) = image_id {
            payload.insert("initImageId".to_string(), json!(id));
            payload.insert("imagePrompts".to_string(), json!([id]));
        }

        // Convert to Value and remove null values
        let payload_value = Value::Object(payload);
        remove_null_values(payload_value)
    }

    fn get_bearer_token(&self) -> Result<String, anyhow::Error> {
        self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .map(String::from)
            .ok_or_else(|| anyhow::anyhow!("Bearer token not found in configuration"))
    }
}

fn remove_null_values(value: Value) -> Value {
    match value {
        Value::Object(map) => {
            let mut new_map = Map::new();
            for (k, v) in map {
                let new_v = remove_null_values(v);
                if new_v != Value::Null {
                    new_map.insert(k, new_v);
                }
            }
            Value::Object(new_map)
        }
        Value::Array(arr) => {
            let new_arr: Vec<Value> = arr
                .into_iter()
                .map(remove_null_values)
                .filter(|v| *v != Value::Null)
                .collect();
            Value::Array(new_arr)
        }
        _ => value,
    }
}

#[async_trait]
impl Engine for LeonardoAIEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = self.create_payload(request, None);

            debug!("Leonardo AI Payload: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Leonardo AI Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Leonardo AI API error: {:?}", error));
            }

            let generation_id = response["sdGenerationJob"]["generationId"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract generation ID from Leonardo AI response"))?;

            // Poll for results
            let mut image_urls = Vec::new();
            for _ in 0..60 { // Poll for up to 5 minutes
                tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;

                let status_url = format!("{}://{}:{}/api/rest/v1/generations/{}",
                                         self.config.connection.protocol,
                                         self.config.connection.hostname,
                                         self.config.connection.port,
                                         generation_id
                );

                let status_response = self.client.get(&status_url)
                    .header("Authorization", format!("Bearer {}", auth_token))
                    .send()
                    .await?
                    .json::<Value>()
                    .await?;

                if status_response["generations_by_pk"]["status"].as_str() == Some("COMPLETE") {
                    image_urls = status_response["generations_by_pk"]["generated_images"]
                        .as_array()
                        .ok_or_else(|| anyhow!("No generated images found"))?
                        .iter()
                        .filter_map(|img| img["url"].as_str())
                        .map(String::from)
                        .collect();
                    break;
                }
            }

            if image_urls.is_empty() {
                return Err(anyhow!("Timed out waiting for Leonardo AI to generate images"));
            }

            Ok(Response {
                content: image_urls.join("\n"),
                usage: Usage { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
                model: "leonardo-ai".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value["generations_by_pk"]["generated_images"]
            .as_array()
            .and_then(|images| {
                let urls: Vec<String> = images.iter()
                    .filter_map(|img| img["url"].as_str().map(String::from))
                    .collect();
                if urls.is_empty() {
                    None
                } else {
                    Some(ExtractedContent {
                        main_content: urls.join("\n"),
                        sentiment: None,
                        clusters: None,
                        themes: None,
                        keywords: None,
                    })
                }
            })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            self.upload_file_internal(file_path).await
        })
    }




    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let image_id = self.upload_file_internal(file_path).await?;

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = self.create_payload(request, None);

            debug!("Leonardo AI Payload with file: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Leonardo AI Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Leonardo AI API error: {:?}", error));
            }

            let generation_id = response["sdGenerationJob"]["generationId"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract generation ID from Leonardo AI response"))?;

            // Poll for results
            let mut image_urls = Vec::new();
            for _ in 0..60 { // Poll for up to 5 minutes
                tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;

                let status_url = format!("{}://{}:{}/api/rest/v1/generations/{}",
                                         self.config.connection.protocol,
                                         self.config.connection.hostname,
                                         self.config.connection.port,
                                         generation_id
                );

                let status_response = self.client.get(&status_url)
                    .header("Authorization", format!("Bearer {}", auth_token))
                    .send()
                    .await?
                    .json::<Value>()
                    .await?;

                if status_response["generations_by_pk"]["status"].as_str() == Some("COMPLETE") {
                    image_urls = status_response["generations_by_pk"]["generated_images"]
                        .as_array()
                        .ok_or_else(|| anyhow!("No generated images found"))?
                        .iter()
                        .filter_map(|img| img["url"].as_str())
                        .map(String::from)
                        .collect();
                    break;
                }
            }

            if image_urls.is_empty() {
                return Err(anyhow!("Timed out waiting for Leonardo AI to generate images"));
            }

            Ok(Response {
                content: image_urls.join("\n"),
                usage: Usage { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
                model: "leonardo-ai".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }
}



#### END OF FILE: crates/fluent-engines/src/leonardoai.rs ####

#### START OF FILE: crates/fluent-engines/src/flowise_chain.rs ####
use std::collections::HashMap;
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::traits::{Engine, EngineConfigProcessor};
use fluent_core::config::EngineConfig;
use fluent_core::neo4j_client::Neo4jClient;
use anyhow::{Result, anyhow, Context};
use reqwest::Client;
use serde_json::{json, Value};
use log::{debug, warn};
use mime_guess::from_path;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::Engine as Base64Engine;

pub struct FlowiseChainEngine {
    config: EngineConfig,
    config_processor: FlowiseChainConfigProcessor,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl FlowiseChainEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            config_processor: FlowiseChainConfigProcessor,
            neo4j_client,
        })
    }

    async fn create_upload_payload(file_path: &Path) -> Result<serde_json::Value> {
        debug!("Creating upload payload for file: {}", file_path.display());
        let mut file = File::open(file_path).await.context("Failed to open file")?;
        let mut buffer = Vec::new();
        file.read_to_end(&mut buffer).await.context("Failed to read file")?;
        let base64_image = base64::encode(&buffer);

        let mime_type = from_path(file_path)
            .first_or_octet_stream()
            .to_string();

        let file_name = file_path.file_name()
            .and_then(|n| n.to_str())
            .unwrap_or("unknown.file")
            .to_string();

        Ok(serde_json::json!({
            "data": format!("data:{};base64,{}", mime_type, base64_image),
            "type": "file",
            "name": file_name,
            "mime": mime_type
        }))
    }
}

pub struct FlowiseChainConfigProcessor;

impl EngineConfigProcessor for FlowiseChainConfigProcessor {

    fn process_config(&self, config: &EngineConfig) -> Result<serde_json::Value> {
        debug!("FlowiseConfigProcessor::process_config");
        debug!("Config: {:#?}", config);

        let mut payload = json!({
            "question": "", // This will be filled later with the actual request
            "overrideConfig": {}
        });

        // Process all parameters and add them to overrideConfig
        for (key, value) in &config.parameters {
            match value {
                Value::Object(obj) => {
                    // Handle nested objects (like openAIApiKey with multiple keys)
                    let nested_config: HashMap<String, Value> = obj.clone().into_iter().collect();
                    payload["overrideConfig"][key] = json!(nested_config);
                },
                _ => {
                    // For non-object values, add them directly
                    payload["overrideConfig"][key] = value.clone();
                }
            }
        }

        debug!("Flowise Payload: {:#?}", payload);
        Ok(payload)
    }
}

#[async_trait::async_trait]
impl Engine for FlowiseChainEngine {
    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        let mut content = ExtractedContent::default();

        if let Some(outputs) = value.get("outputs").and_then(|v| v.as_array()) {
            if let Some(first_output) = outputs.get(0) {
                content.main_content = first_output.get("output").and_then(|v| v.as_str())
                    .unwrap_or_default().to_string();
                content.sentiment = first_output.get("sentiment").and_then(|v| v.as_str()).map(String::from);
                content.clusters = first_output.get("clusters").and_then(|v| v.as_array())
                    .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
                content.themes = first_output.get("themes").and_then(|v| v.as_array())
                    .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
                content.keywords = first_output.get("keywords").and_then(|v| v.as_array())
                    .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
            }
        }

        if content.main_content.is_empty() { None } else { Some(content) }
    }

    fn upsert<'a>(&'a self, request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            // Implement FlowiseAI-specific upsert logic here if needed
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let client = Client::new();
            debug!("Config: {:?}", self.config);

            let mut payload = self.config_processor.process_config(&self.config)?;

            // Add the user's request to the payload
            payload["question"] = json!(request.payload);

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let res = client.post(&url)
                .json(&payload)
                .send()
                .await?;

            let response_body = res.json::<serde_json::Value>().await?;
            debug!("Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("FlowiseAI API error: {:?}", error));
            }

            let content = response_body["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from FlowiseAI response"))?
                .to_string();

            // FlowiseAI doesn't provide token usage, so we'll estimate it based on content length
            let estimated_tokens = (content.len() as f32 / 4.0).ceil() as u32;
            let usage = Usage {
                prompt_tokens: estimated_tokens / 2, // Rough estimate
                completion_tokens: estimated_tokens / 2, // Rough estimate
                total_tokens: estimated_tokens,
            };

            let model = format!("{}_flowise_chain", self.config.name);
            let finish_reason = Some("stop".to_string()); // Assuming normal completion

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not implemented for Flowise Chain engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;

            let encoded_image = base64::engine::general_purpose::STANDARD.encode(&buffer);
            debug!("Encoded image length: {} bytes", encoded_image.len());

            let file_name = file_path.file_name()
                .and_then(|n| n.to_str())
                .unwrap_or("unknown.file")
                .to_string();

            let payload = serde_json::json!({
                "question": &request.payload,
                "uploads": [{
                    "data": format!("data:image/png;base64,{}", encoded_image),
                    "type": "file",
                    "name": file_name,
                    "mime": "image/png"
                }]
            });

            debug!("Data field prefix: {}", &payload["uploads"][0]["data"].as_str().unwrap_or("").split(',').next().unwrap_or(""));
            debug!("Uploads array length: {}", payload["uploads"].as_array().map_or(0, |arr| arr.len()));
            debug!("File name in payload: {}", &payload["uploads"][0]["name"]);

            let client = reqwest::Client::new();
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            debug!("Sending request to URL: {}", url);

            let response = client.post(&url)
                .json(&payload)
                .send()
                .await?;

            debug!("Response status: {}", response.status());

            let response_body = response.json::<serde_json::Value>().await?;

            debug!("FlowiseAI Response: {:?}", response_body);

            if response_body.get("error").is_some() || response_body["text"].as_str().map_or(false, |s| s.contains("no image provided")) {
                warn!("FlowiseAI did not process the image. Full response: {:?}", response_body);
            }


            let content = response_body["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from FlowiseAI response"))?
                .to_string();

            // FlowiseAI doesn't provide token usage, so we'll estimate it based on content length
            let estimated_tokens = (content.len() as f32 / 4.0).ceil() as u32;
            let usage = Usage {
                prompt_tokens: estimated_tokens / 2, // Rough estimate
                completion_tokens: estimated_tokens / 2, // Rough estimate
                total_tokens: estimated_tokens,
            };

            let model = "flowise-chain".to_string(); // FlowiseAI doesn't provide model info
            let finish_reason = Some("stop".to_string()); // Assuming normal completion

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/flowise_chain.rs ####

#### START OF FILE: crates/fluent-engines/src/pipeline_executor.rs ####
use std::cell::RefCell;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::Pointer;
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::rc::Rc;
use tokio::sync::{Mutex, MutexGuard};
use tokio::process::Command as TokioCommand;
use tokio::sync::Mutex as TokioMutex;
use std::io::Write;

use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tempfile::{NamedTempFile, tempdir};

use std::sync::{Arc};
use anyhow::{anyhow, Error};
use tokio::process::Command;
use log::{info, error, warn, debug};
use async_trait::async_trait;
use futures_util::future::join_all;
use tokio::{fs, io};
use tokio::io::{AsyncWriteExt, stdout};
use tokio::task::JoinSet;
use tokio::time::timeout;
use uuid::Uuid;

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Pipeline {
    pub name: String,
    pub steps: Vec<PipelineStep>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub enum PipelineStep {
    Command { name: String, command: String, save_output: Option<String>, retry: Option<RetryConfig> },
    ShellCommand { name: String, command: String, save_output: Option<String>, retry: Option<RetryConfig> },
    Condition { name: String, condition: String, if_true: String, if_false: String },
    Loop { name: String, steps: Vec<PipelineStep>, condition: String },
    SubPipeline { name: String, pipeline: String, with: HashMap<String, String> },
    Map { name: String, input: String, command: String, save_output: String },
    HumanInTheLoop { name: String, prompt: String, save_output: String },
    RepeatUntil { name: String, steps: Vec<PipelineStep>, condition: String },
    PrintOutput { name: String, value: String },
    ForEach { name: String, items: String, steps: Vec<PipelineStep> },
    TryCatch { name: String, try_steps: Vec<PipelineStep>, catch_steps: Vec<PipelineStep>, finally_steps: Vec<PipelineStep> },
    Parallel { name: String, steps: Vec<PipelineStep> },
    Timeout { name: String, duration: u64, step: Box<PipelineStep> },

}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct RetryConfig {
    max_attempts: u32,
    delay_ms: u64,
}

#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct PipelineState {
    pub current_step: usize,
    pub data: HashMap<String, String>,
    pub run_id: String,
    pub start_time: u64,
}

#[async_trait]
pub trait StateStore {
    async fn save_state(&self, pipeline_name: &str, state: &PipelineState) -> anyhow::Result<()>;
    async fn load_state(&self, pipeline_name: &str) -> anyhow::Result<Option<PipelineState>>;
}

pub struct PipelineExecutor<S: StateStore> {
    // Change state to Arc<Mutex<...>>
    state: Arc<Mutex<PipelineState>>,
    state_store: S,
    json_output: bool,
}

impl<S: StateStore + Clone + std::marker::Sync + std::marker::Send> PipelineExecutor<S> {
    pub fn new(state_store: S, json_output: bool) -> Self {
        Self {
            state: Arc::new(Mutex::new(PipelineState {
                current_step: 0,
                data: HashMap::new(),
                run_id: "".to_string(),
                start_time: 0,
            })),
            state_store,
            json_output: false,
        }
    }


    pub async fn execute(&self, pipeline: &Pipeline, initial_input: &str, force_fresh: bool, provided_run_id: Option<String>) -> Result<String, Error> {
        let run_id = provided_run_id.unwrap_or_else(|| Uuid::new_v4().to_string());
        let state_key = format!("{}-{}", pipeline.name, run_id);
        debug!("Executing pipeline {} with run_id {}", pipeline.name, run_id);

        let mut state = if force_fresh {
            debug!("Forcing fresh state");
            PipelineState {
                current_step: 0,
                data: HashMap::new(),
                run_id: run_id.clone(),
                start_time: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
            }
        } else {
            match self.state_store.load_state(&state_key).await? {
                Some(saved_state) => {
                    debug!("Resuming from saved state at step {}", saved_state.current_step);
                    saved_state
                },
                None => {
                    debug!("No saved state found, starting fresh");
                    PipelineState {
                        current_step: 0,
                        data: HashMap::new(),
                        run_id: run_id.clone(),
                        start_time: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(),
                    }
                }
            }
        };

        if state.current_step == 0 {
            state.data.insert("input".to_string(), initial_input.to_string());
        }
        state.data.insert("run_id".to_string(), run_id.clone());

        for (index, step) in pipeline.steps.iter().enumerate().skip(state.current_step) {
            debug!("Processing step {} (index {})", step.name(), index);

            state.data.insert("step".to_string(), step.name().to_string());
            state.current_step = index;

            match self.execute_step(step, &mut state).await {
                Ok(step_result) => {
                    info!("Step {} completed successfully", step.name());
                    state.data.extend(step_result);
                    self.state_store.save_state(&state_key, &state).await?;
                }
                Err(e) => {
                    error!("Error executing step {}: {:?}", step.name(), e);
                    return Err(e);
                }
            }
        }

        let end_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
        let runtime = end_time - state.start_time;

        let output = serde_json::json!({
            "pipeline_name": pipeline.name,
            "run_id": run_id,
            "current_step": state.current_step,
            "start_time": state.start_time,
            "end_time": end_time,
            "runtime_seconds": runtime,
            "data": state.data,
        });

        Ok(serde_json::to_string_pretty(&output)?)
    }




    fn execute_step<'a>(&'a self, step: &'a PipelineStep, state: &'a mut PipelineState) -> Pin<Box<dyn Future<Output = Result<HashMap<String, String>, Error>> + Send + 'a>> {
        Box::pin(async move {
            match step {

                PipelineStep::Command { name, command, save_output, retry } => {
                    debug!("Executing Command step: {}", name);
                    debug!("Command: {}", command);
                    let expanded_command = self.expand_variables(command, &state.data).await?;
                    self.execute_command(&expanded_command, save_output, retry).await
                }

                PipelineStep::ShellCommand { name, command, save_output, retry } => {
                    debug!("Executing ShellCommand step: {}", name);
                    debug!("Command: {}", command);
                    let expanded_command = self.expand_variables(command, &state.data).await?;
                    self.execute_shell_command(&expanded_command, save_output, retry).await
                }

                PipelineStep::Condition { name, condition, if_true, if_false } => {
                    debug!("Evaluating Condition step: {}", name);
                    debug!("Condition: {}", condition);
                    let expanded_condition = self.expand_variables(condition, &state.data).await?;
                    if self.evaluate_condition(&expanded_condition).await? {
                        debug!("Condition is true, executing: {}", if_true);
                        let expanded_command = self.expand_variables(if_true, &state.data).await?;
                        self.execute_shell_command(&expanded_command, &None, &None).await
                    } else {
                        debug!("Condition is false, executing: {}", if_false);
                        let expanded_command = self.expand_variables(if_false, &state.data).await?;
                        self.execute_shell_command(&expanded_command, &None, &None).await
                    }
                }

                PipelineStep::PrintOutput { name, value } => {
                    debug!("Executing PrintOutput step: {}", name);
                    let expanded_value = self.expand_variables(value, &state.data).await?;
                    if !self.json_output {
                        eprintln!("{}", expanded_value);  // Print to stderr instead of stdout
                    }
                    Ok(HashMap::new())
                }

                PipelineStep::Map { name, input, command, save_output } => {
                    debug!("Executing Map step: {}", name);
                    debug!("Input: {}", input);
                    debug!("Command: {}", command);
                    let input_data = self.expand_variables(input, &state.data).await?;
                    debug!("Expanded input data: {}", input_data);
                    let mut results = Vec::new();

                    for item in input_data.split(',') {
                        let item = item.trim();
                        debug!("Processing item: {}", item);
                        let expanded_command = self.expand_variables(command, &state.data).await?;
                        let item_command = expanded_command.replace("${ITEM}", item);
                        debug!("Executing command: {}", item_command);
                        match self.execute_shell_command(&item_command, &None, &None).await {
                            Ok(output) => {
                                let new_string = String::new();
                                let result = output.values().next().unwrap_or(&new_string);
                                debug!("Command output: {}", result);
                                results.push(result.to_string());
                            },
                            Err(e) => {
                                error!("Error executing command for item {}: {:?}", item, e);
                            }
                        }
                    }

                    let output = results.join(", ");
                    debug!("Map step result: {}", output);
                    Ok([(save_output.clone(), output)].into_iter().collect())
                }

                PipelineStep::HumanInTheLoop { name, prompt, save_output } => {
                    debug!("Executing HumanInTheLoop step: {}", name);
                    debug!("Prompt: {}", prompt);
                    let expanded_prompt = self.expand_variables(prompt, &state.data).await?;
                    println!("{}", expanded_prompt);

                    let mut input = String::new();
                    std::io::stdin().read_line(&mut input)?;

                    Ok([(save_output.clone(), input.trim().to_string())].into_iter().collect())
                }

                PipelineStep::RepeatUntil { name, steps, condition } => {
                    debug!("Executing RepeatUntil step: {}", name);
                    debug!("Steps: {:?}", steps);
                    debug!("Condition: {}", condition);
                    loop {
                        for sub_step in steps {
                            let step_result = self.execute_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }

                        let expanded_condition = self.expand_variables(condition, &state.data).await?;
                        if self.evaluate_condition(&expanded_condition).await? {
                            break;
                        }
                    }
                    Ok(HashMap::new())
                }

                PipelineStep::ForEach { name, items, steps } => {
                    debug!("Executing ForEach step: {}", name);
                    debug!("Items: {}", items);
                    debug!("Steps: {:?}", steps);
                    let items_list = self.expand_variables(items, &state.data).await?;
                    let mut results = Vec::new();

                    for item in items_list.split(',') {
                        let item = item.trim();
                        state.data.insert("ITEM".to_string(), item.to_string());

                        for sub_step in steps {
                            let step_result = self.execute_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }

                        results.push(state.data.get("ITEM").unwrap_or(&item.to_string()).clone());
                    }

                    state.data.remove("ITEM");
                    Ok(HashMap::from([(name.clone(), results.join(", "))]))
                }

                PipelineStep::TryCatch { name, try_steps, catch_steps, finally_steps } => {
                    debug!("Executing TryCatch step: {}", name);
                    debug!("Try Steps: {:?}", try_steps);
                    debug!("Catch Steps: {:?}", catch_steps);
                    debug!("Finally Steps: {:?}", finally_steps);
                    let mut result = HashMap::new();
                    let try_result = async {
                        for sub_step in try_steps {
                            let step_result = self.execute_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }
                        Ok(()) as Result<(), Error>
                    }.await;

                    match try_result {
                        Ok(_) => {
                            result.insert("try_result".to_string(), "success".to_string());
                        }
                        Err(e) => {
                            result.insert("try_result".to_string(), "failure".to_string());
                            result.insert("error".to_string(), e.to_string());
                            for sub_step in catch_steps {
                                let step_result = self.execute_step(sub_step, state).await?;
                                state.data.extend(step_result);
                            }
                        }
                    }

                    for sub_step in finally_steps {
                        let step_result = self.execute_step(sub_step, state).await?;
                        state.data.extend(step_result);
                    }

                    Ok(result)
                }

                PipelineStep::Parallel { name, steps } => {
                    debug!("Executing Parallel step: {}", name);
                    debug!("Steps: {:?}", steps);

                    let state_arc = Arc::new(tokio::sync::Mutex::new(state.clone()));
                    let mut set = JoinSet::new();

                    for sub_step in steps {
                        let sub_step = sub_step.clone();
                        let state_clone = Arc::clone(&state_arc);

                        set.spawn(async move {
                            let mut state_guard = state_clone.lock().await;
                            Self::execute_single_step(&sub_step, &mut state_guard).await
                        });
                    }

                    let mut combined_results = HashMap::new();
                    while let Some(result) = set.join_next().await {
                        match result {
                            Ok(Ok(step_result)) => {
                                combined_results.extend(step_result);
                            }
                            Ok(Err(e)) => {
                                combined_results.insert(format!("error_{}", combined_results.len()), e.to_string());
                            }
                            Err(e) => {
                                combined_results.insert(format!("join_error_{}", combined_results.len()), e.to_string());
                            }
                        }
                    }

                    // Merge the results back into the main state
                    let mut state_guard = state_arc.lock().await;
                    state.data.extend(state_guard.data.clone());
                    state.data.extend(combined_results);

                    Ok(HashMap::from([(name.clone(), "Parallel execution completed".to_string())]))
                }

                PipelineStep::Timeout { name, duration, step } => {
                    debug!("Executing Timeout step: {}", name);
                    debug!("Duration: {}", duration);
                    debug!("Step: {:?}", step);
                    let duration = Duration::from_secs(*duration);

                    let timeout_result = timeout(duration, self.execute_step(step, state)).await;

                    match timeout_result {
                        Ok(step_result) => {
                            let result = step_result?;
                            Ok(result)
                        }
                        Err(_) => {
                            Err(anyhow!("Step timed out after {} seconds", duration.as_secs()))
                        }
                    }
                }

                _ => {
                    Ok(HashMap::new())
                }
            }
        })
    }


    async fn execute_with_retry<F, Fut>(
        &self,
        config: &Option<RetryConfig>,
        f: F,
    ) -> anyhow::Result<()>
        where
            F: Fn() -> Fut,
            Fut: std::future::Future<Output = anyhow::Result<()>> + Send,
    {
        debug!("Executing with retry");
        let config = config.clone().unwrap_or(RetryConfig {
            max_attempts: 2,
            delay_ms: 10000,
        });
        let mut attempts = 0;

        loop {
            // Box the future returned by f()
            match Box::pin(f()).await {
                Ok(_) => return Ok(()),
                Err(e) if attempts < config.max_attempts => {
                    attempts += 1;
                    warn!("Attempt {} failed: {:?}. Retrying...", attempts, e);
                    tokio::time::sleep(std::time::Duration::from_millis(config.delay_ms)).await;
                }
                Err(e) => return Err(e),
            }
        }
    }

    fn execute_single_step<'a>(
        step: &'a PipelineStep,
        state: &'a mut PipelineState,
    ) -> Pin<Box<dyn Future<Output = Result<HashMap<String, String>, Error>> + Send + 'a>> {
        Box::pin(async move {
            match step {
                PipelineStep::Command { name, command, save_output, retry } => {
                    let output = Command::new("sh")
                        .arg("-c")
                        .arg(command)
                        .output()
                        .await?;

                    let stdout = String::from_utf8(output.stdout)?;
                    let mut result = HashMap::new();
                    if let Some(key) = save_output {
                        result.insert(key.clone(), stdout.trim().to_string());
                    }
                    Ok(result)
                }
                PipelineStep::ShellCommand { name, command, save_output, retry } => {
                    let output = Command::new("sh")
                        .arg("-c")
                        .arg(command)
                        .output()
                        .await?;

                    let stdout = String::from_utf8(output.stdout)?;
                    let mut result = HashMap::new();
                    if let Some(key) = save_output {
                        result.insert(key.clone(), stdout.trim().to_string());
                    }
                    Ok(result)
                }
                PipelineStep::Condition { name, condition, if_true, if_false } => {
                    let condition_result = Command::new("sh")
                        .arg("-c")
                        .arg(condition)
                        .status()
                        .await?
                        .success();

                    let command_to_run = if condition_result { if_true } else { if_false };
                    let output = Command::new("sh")
                        .arg("-c")
                        .arg(command_to_run)
                        .output()
                        .await?;

                    let stdout = String::from_utf8(output.stdout)?;
                    Ok(HashMap::from([(name.clone(), stdout.trim().to_string())]))
                }
                PipelineStep::PrintOutput { name, value } => {
                    println!("{}", value);
                    Ok(HashMap::new())
                }
                PipelineStep::RepeatUntil { name, steps, condition } => {
                    let mut result = HashMap::new();
                    loop {
                        for sub_step in steps {
                            let step_result = Self::execute_single_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }

                        let condition_result = Command::new("sh")
                            .arg("-c")
                            .arg(condition)
                            .status()
                            .await?
                            .success();

                        if condition_result {
                            break;
                        }
                    }
                    Ok(result)
                }
                PipelineStep::ForEach { name, items, steps } => {
                    let mut result = Vec::new();
                    for item in items.split(',') {
                        state.data.insert("ITEM".to_string(), item.trim().to_string());
                        for sub_step in steps {
                            let step_result = Self::execute_single_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }
                        result.push(state.data.get("ITEM").unwrap_or(&item.to_string()).clone());
                    }
                    state.data.remove("ITEM");
                    Ok(HashMap::from([(name.clone(), result.join(", "))]))
                }
                PipelineStep::TryCatch { name, try_steps, catch_steps, finally_steps } => {
                    let mut result = HashMap::new();
                    let try_result = async {
                        for sub_step in try_steps {
                            let step_result = Self::execute_single_step(sub_step, state).await?;
                            state.data.extend(step_result);
                        }
                        Ok(()) as Result<(), Error>
                    }.await;

                    match try_result {
                        Ok(_) => {
                            result.insert("try_result".to_string(), "success".to_string());
                        }
                        Err(e) => {
                            result.insert("try_result".to_string(), "failure".to_string());
                            result.insert("error".to_string(), e.to_string());
                            for sub_step in catch_steps {
                                let step_result = Self::execute_single_step(sub_step, state).await?;
                                state.data.extend(step_result);
                            }
                        }
                    }

                    for sub_step in finally_steps {
                        let step_result = Self::execute_single_step(sub_step, state).await?;
                        state.data.extend(step_result);
                    }

                    Ok(result)
                }
                PipelineStep::Timeout { name, duration, step } => {
                    let duration = Duration::from_secs(*duration);
                    let timeout_result = timeout(duration, Self::execute_single_step(step, state)).await;

                    match timeout_result {
                        Ok(step_result) => step_result,
                        Err(_) => Err(anyhow!("Step timed out after {} seconds", duration.as_secs())),
                    }
                }
                PipelineStep::Parallel { name, steps } => {
                    // For simplicity, we'll execute parallel steps sequentially in this context
                    let mut result = HashMap::new();
                    for sub_step in steps {
                        let step_result = Self::execute_single_step(sub_step, state).await?;
                        result.extend(step_result);
                    }
                    Ok(result)
                }
                _ => {
                    Err(anyhow!("Unknown step type"))
                }
            }
        })
    }

    async fn execute_command(&self, command: &str, save_output: &Option<String>, retry: &Option<RetryConfig>) -> Result<HashMap<String, String>, Error> {
        debug!("Executing command: {}", command);
        let retry_config = retry.clone().unwrap_or(RetryConfig { max_attempts: 1, delay_ms: 0 });
        let mut attempts = 0;

        loop {
            debug!("Attempt {} to execute command", attempts + 1);
            match self.run_command(command, save_output).await {
                Ok(output) => {
                    debug!("Command executed successfully");
                    return Ok(output);
                }
                Err(e) if attempts < retry_config.max_attempts => {
                    attempts += 1;
                    warn!("Attempt {} failed: {:?}. Retrying...", attempts, e);
                    tokio::time::sleep(std::time::Duration::from_millis(retry_config.delay_ms)).await;
                }
                Err(e) => {
                    error!("Command execution failed after {} attempts: {:?}", attempts + 1, e);
                    return Err(e);
                }
            }
        }
    }

    async fn run_command(&self, command: &str, save_output: &Option<String>) -> Result<HashMap<String, String>, Error> {
        debug!("Running command: {}", command);
        let output = TokioCommand::new("sh")
            .arg("-c")
            .arg(command)
            .output()
            .await
            .map_err(|e| anyhow!("Failed to execute command: {}", e))?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow!("Command failed with exit code {:?}. Stderr: {}", output.status.code(), stderr));
        }

        let stdout = String::from_utf8(output.stdout)
            .map_err(|e| anyhow!("Failed to parse command output: {}", e))?;

        debug!("Command output: {}", stdout);

        let mut result = HashMap::new();
        if let Some(save_key) = save_output {
            result.insert(save_key.clone(), stdout.trim().to_string());
            debug!("Saved output to key: {}", save_key);
        }

        Ok(result)
    }





    async fn execute_shell_command(&self, command: &str, save_output: &Option<String>, retry: &Option<RetryConfig>) -> Result<HashMap<String, String>, Error> {
        debug!("Executing shell command: {}", command);

        // Create a temporary file
        let mut temp_file = tempfile::NamedTempFile::new()?;
        writeln!(temp_file.as_file_mut(), "{}", command)?;

        let retry_config = retry.clone().unwrap_or(RetryConfig { max_attempts: 1, delay_ms: 0 });
        let mut attempts = 0;

        loop {
            debug!("Attempt {} to execute shell command", attempts + 1);
            match self.run_shell_command(temp_file.path()).await {
                Ok(output) => {
                    debug!("Shell command executed successfully: {:?}", output);
                    let mut result = HashMap::new();
                    if let Some(save_key) = save_output {
                        result.insert(save_key.clone(), output);
                    } else {
                        result.insert("output".to_string(), output);
                    }
                    return Ok(result);
                }
                Err(e) if attempts < retry_config.max_attempts => {
                    attempts += 1;
                    warn!("Attempt {} failed: {:?}. Retrying...", attempts, e);
                    tokio::time::sleep(std::time::Duration::from_millis(retry_config.delay_ms)).await;
                }
                Err(e) => {
                    error!("Shell command execution failed after {} attempts: {:?}", attempts + 1, e);
                    return Err(e);
                }
            }
        }
    }


    async fn run_shell_command(&self, script_path: &Path) -> Result<String, Error> {
        debug!("Running shell command from file: {:?}", script_path);
        let output = TokioCommand::new("bash")
            .arg(script_path)
            .output()
            .await
            .map_err(|e| anyhow!("Failed to execute shell command: {}", e))?;

        if !output.status.success() {
            let stderr = String::from_utf8_lossy(&output.stderr);
            return Err(anyhow!("Shell command failed with exit code {:?}. Stderr: {}", output.status.code(), stderr));
        }

        let stdout = String::from_utf8(output.stdout)
            .map_err(|e| anyhow!("Failed to parse command output: {}", e))?;

        debug!("Shell command output: {}", stdout);

        Ok(stdout.trim().to_string())
    }



    async fn evaluate_condition(&self, condition: &str) -> Result<bool, Error> {
        let expanded_condition = self.expand_variables(condition, &Default::default()).await?;
        debug!("Evaluating expanded condition: {}", expanded_condition);

        let output = TokioCommand::new("bash")
            .arg("-c")
            .arg(format!("if {}; then exit 0; else exit 1; fi", expanded_condition))
            .output()
            .await?;

        Ok(output.status.success())
    }

    async fn expand_variables(&self, input: &str, state_data: &HashMap<String, String>) -> Result<String, Error> {
        debug!("Expanding variables in input: {}", input);
        let mut result = input.to_string();
        for (key, value) in state_data {
            result = result.replace(&format!("${{{}}}", key), value);
        }
        Ok(result)
    }

}


fn evaluate_condition(condition: &str, state: &HashMap<String, String>) -> bool {
    // Implement condition evaluation
    // This could be a simple string contains check, or something more complex
    unimplemented!()
}
impl PipelineStep {
    fn name(&self) -> &str {
        match self {
            PipelineStep::Command { name, .. } => name,
            PipelineStep::ShellCommand { name, .. } => name,
            PipelineStep::Condition { name, .. } => name,
            PipelineStep::Loop { name, .. } => name,
            PipelineStep::Map { name, .. } => name,
            PipelineStep::SubPipeline { name, .. } => name,
            PipelineStep::HumanInTheLoop { name, .. } => name,
            PipelineStep::RepeatUntil { name, .. } => name,
            PipelineStep::PrintOutput { name, .. } => name,
            PipelineStep::ForEach { name, .. } => name,
            PipelineStep::TryCatch {name, .. } => name,
            PipelineStep::Parallel { name, .. } => name,
            PipelineStep::Timeout { name, .. } => name,
        }
    }
}

#[derive(Debug, Clone)]
pub struct FileStateStore {
    pub directory: PathBuf,
}

#[async_trait]
impl StateStore for FileStateStore {
    async fn save_state(&self, state_key: &str, state: &PipelineState) -> Result<(), Error> {
        let file_path = self.directory.join(format!("{}.json", state_key));
        let json = serde_json::to_string(state)?;
        tokio::fs::write(&file_path, json).await?;
        Ok(())
    }

    async fn load_state(&self, state_key: &str) -> Result<Option<PipelineState>, Error> {
        let file_path = self.directory.join(format!("{}.json", state_key));
        if file_path.exists() {
            let json = tokio::fs::read_to_string(&file_path).await?;
            let state: PipelineState = serde_json::from_str(&json)?;
            Ok(Some(state))
        } else {
            Ok(None)
        }
    }
}
#### END OF FILE: crates/fluent-engines/src/pipeline_executor.rs ####

#### START OF FILE: crates/fluent-engines/src/langflow.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::traits::{Engine, EngineConfigProcessor};
use fluent_core::config::EngineConfig;
use fluent_core::neo4j_client::Neo4jClient;
use anyhow::{Result, anyhow};
use reqwest::Client;
use serde_json::{json, Value};
use log::debug;

pub struct LangflowEngine {
    config: EngineConfig,
    config_processor: LangflowConfigProcessor,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl LangflowEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            config_processor: LangflowConfigProcessor,
            neo4j_client,
        })
    }
}

pub struct LangflowConfigProcessor;

impl EngineConfigProcessor for LangflowConfigProcessor {
    fn process_config(&self, config: &EngineConfig) -> Result<serde_json::Value> {
        debug!("LangflowConfigProcessor::process_config");
        debug!("Config: {:#?}", config);

        let mut payload = json!({
            "input_value": "",  // This will be filled later with the actual request
            "output_type": "chat",
            "input_type": "chat",
            "tweaks": {}
        });

        // Process all parameters and add them to tweaks
        for (key, value) in &config.parameters {
            match value {
                serde_json::Value::Object(obj) => {
                    // Handle nested objects
                    payload["tweaks"][key] = json!(obj);
                },
                _ => {
                    // For non-object values, add them directly to the root of the payload
                    payload[key] = value.clone();
                }
            }
        }

        debug!("Langflow Payload: {:#?}", payload);
        Ok(payload)
    }


}

#[async_trait::async_trait]
impl Engine for LangflowEngine {
    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn upsert<'a>(&'a self, request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            // Implement Langflow-specific upsert logic here if needed
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }


    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let client = Client::new();
            debug!("Config: {:?}", self.config);

            let mut payload = self.config_processor.process_config(&self.config)?;
            payload["input_value"] = json!(request.payload);

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let res = client.post(&url)
                .json(&payload)
                .send()
                .await?;

            let response_body = res.json::<serde_json::Value>().await?;
            debug!("Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("Langflow API error: {:?}", error));
            }

            let extracted_content = self.extract_content(&response_body)
                .ok_or_else(|| anyhow!("Failed to extract content from Langflow response"))?;

            let estimated_tokens = (extracted_content.main_content.len() as f32 / 4.0).ceil() as u32;
            let usage = Usage {
                prompt_tokens: estimated_tokens / 2,
                completion_tokens: estimated_tokens / 2,
                total_tokens: estimated_tokens,
            };

            let model = format!("{}_langflow_chain", self.config.name);
            let finish_reason = Some("stop".to_string());

            Ok(Response {
                content: extracted_content.main_content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        fn extract_recursive(value: &Value) -> Option<ExtractedContent> {
            match value {
                Value::Object(map) => {
                    let mut content = ExtractedContent::default();

                    if let Some(text) = map.get("text").and_then(|v| v.as_str()) {
                        content.main_content = text.to_string();
                    } else if let Some(message) = map.get("message").and_then(|v| v.as_str()) {
                        content.main_content = message.to_string();
                    }

                    content.sentiment = map.get("sentiment").and_then(|v| v.as_str()).map(String::from);
                    content.clusters = map.get("clusters").and_then(|v| v.as_array())
                        .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
                    content.themes = map.get("themes").and_then(|v| v.as_array())
                        .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());
                    content.keywords = map.get("keywords").and_then(|v| v.as_array())
                        .map(|arr| arr.iter().filter_map(|v| v.as_str().map(String::from)).collect());

                    if !content.main_content.is_empty() {
                        Some(content)
                    } else {
                        for (_, v) in map {
                            if let Some(extracted) = extract_recursive(v) {
                                return Some(extracted);
                            }
                        }
                        None
                    }
                }
                Value::Array(arr) => {
                    for v in arr {
                        if let Some(extracted) = extract_recursive(v) {
                            return Some(extracted);
                        }
                    }
                    None
                }
                _ => None,
            }
        }

        extract_recursive(value)
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not implemented for Langflow engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, _request: &'a Request, _file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File processing not implemented for Langflow engine"))
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/langflow.rs ####

#### START OF FILE: crates/fluent-engines/src/imagepro.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use std::pin::Pin;
use std::time::Duration;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::{debug, info};
use reqwest::Client;
use uuid::Uuid;

pub struct ImagineProEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
    download_dir: Option<String>,
}

impl ImagineProEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
            download_dir: None,
        })
    }

    pub fn set_download_dir(&mut self, dir: String) {
        self.download_dir = Some(dir);
    }

    async fn get_image_result(&self, message_id: &str) -> Result<String> {
        let url = format!("{}://{}:{}/api/v1/midjourney/message/{}",
                          self.config.connection.protocol,
                          self.config.connection.hostname,
                          self.config.connection.port,
                          message_id
        );

        let auth_token = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

        let max_attempts = 30; // Adjust as needed
        let delay = Duration::from_secs(10); // Adjust as needed

        for _ in 0..max_attempts {
            let response = self.client.get(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .send()
                .await?
                .json::<Value>()
                .await?;

            match response["status"].as_str() {
                Some("DONE") => {
                    return response["uri"].as_str()
                        .ok_or_else(|| anyhow!("Image URI not found in response"))
                        .map(String::from)
                },
                Some("PROCESSING") | Some("QUEUED") => {
                    info!("Job still processing. Progress: {}%", response["progress"]);
                    tokio::time::sleep(delay).await;
                },
                Some("FAIL") => return Err(anyhow!("Job failed: {:?}", response["error"])),
                _ => return Err(anyhow!("Unexpected job status: {:?}", response["status"])),
            }
        }

        Err(anyhow!("Timed out waiting for image generation"))
    }

    async fn download_image(&self, uri: &str) -> Result<String> {
        let response = self.client.get(uri).send().await?;
        let content = response.bytes().await?;

        let download_dir = self.download_dir.as_ref()
            .ok_or_else(|| anyhow!("Download directory not set"))?;
        let file_name = format!("imaginepro_{}.png", Uuid::new_v4());
        let file_path = Path::new(download_dir).join(file_name);

        let mut file = File::create(&file_path).await?;
        file.write_all(&content).await?;

        Ok(file_path.to_string_lossy().into_owned())
    }

    async fn upload_file_internal(&self, file_path: &Path) -> Result<String> {
        let mut file = File::open(file_path).await.context("Failed to open file")?;
        let mut buffer = Vec::new();
        file.read_to_end(&mut buffer).await.context("Failed to read file")?;
        let base64_image = base64::encode(&buffer);
        let mime_type = mime_guess::from_path(file_path).first_or_octet_stream().to_string();
        Ok(format!("data:{};base64,{}", mime_type, base64_image))
    }

    fn extract_prompt(&self, payload: &str) -> String {
        // Look for the prompt between quotes after "**Image Prompt:**"
        if let Some(start) = payload.find("**Image Prompt:**") {
            if let Some(quote_start) = payload[start..].find('"') {
                if let Some(quote_end) = payload[start + quote_start + 1..].find('"') {
                    return payload[start + quote_start + 1..start + quote_start + 1 + quote_end].to_string();
                }
            }
        }

        payload.trim().to_string()
    }
}

#[async_trait]
impl Engine for ImagineProEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = format!("{}://{}:{}/api/v1/midjourney/imagine",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port
            );

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            // Extract the actual prompt from the payload
            let clean_prompt = self.extract_prompt(&request.payload);

            let payload = json!({
                "prompt": clean_prompt,
                "ref": self.config.parameters.get("ref"),
                "webhookOverride": self.config.parameters.get("webhookOverride"),
            });

            debug!("ImaginePro Payload: {:?}", payload);

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            if !response["success"].as_bool().unwrap_or(false) {
                return Err(anyhow!("ImaginePro API request failed: {:?}", response["error"]));
            }

            let message_id = response["messageId"].as_str()
                .ok_or_else(|| anyhow!("MessageId not found in response"))?;

            // Wait for the image to be generated
            let image_url = self.get_image_result(message_id).await?;

            // Download the image
            let local_image_path = self.download_image(&image_url).await?;

            Ok(Response {
                content: local_image_path,
                usage: Usage {
                    prompt_tokens: 0,
                    completion_tokens: 0,
                    total_tokens: 0,
                },
                model: "imaginepro-midjourney".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value.get("imageUrl")
            .and_then(|url| url.as_str())
            .map(|url| ExtractedContent {
                main_content: url.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(self.upload_file_internal(file_path))
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let data_url = self.upload_file_internal(file_path).await?;
            let prompt = format!("{} {}", data_url, request.payload);

            let new_request = Request {
                flowname: request.flowname.clone(),
                payload: prompt,
            };

            // Use Box::pin to create a pinned future that can be awaited
            Pin::from(self.execute(&new_request)).await
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/imagepro.rs ####

#### START OF FILE: crates/fluent-engines/src/cohere.rs ####
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct CohereEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl CohereEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }
}

#[async_trait]
impl Engine for CohereEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = json!({
                "message": &request.payload,
                "model": self.config.parameters.get("modelName").and_then(|v| v.as_str()).unwrap_or("command-r-plus"),
                "stream": self.config.parameters.get("stream").and_then(|v| v.as_bool()).unwrap_or(false),
                "preamble": self.config.parameters.get("preamble").and_then(|v| v.as_str()),
                "chat_history": self.config.parameters.get("chat_history"),
                "conversation_id": self.config.parameters.get("conversation_id").and_then(|v| v.as_str()),
                "prompt_truncation": self.config.parameters.get("prompt_truncation").and_then(|v| v.as_str()).unwrap_or("AUTO"),
                "connectors": self.config.parameters.get("connectors"),
                "documents": self.config.parameters.get("documents"),
                "citation_quality": self.config.parameters.get("citation_quality").and_then(|v| v.as_str()).unwrap_or("accurate"),
                "temperature": self.config.parameters.get("temperature").and_then(|v| v.as_f64()).unwrap_or(0.3),
                "max_tokens": self.config.parameters.get("max_tokens").and_then(|v| v.as_u64()),
                "k": self.config.parameters.get("k").and_then(|v| v.as_u64()).unwrap_or(0),
                "p": self.config.parameters.get("p").and_then(|v| v.as_f64()).unwrap_or(0.75),
                "frequency_penalty": self.config.parameters.get("frequency_penalty").and_then(|v| v.as_f64()).unwrap_or(0.0),
                "presence_penalty": self.config.parameters.get("presence_penalty").and_then(|v| v.as_f64()).unwrap_or(0.0),
                "tools": self.config.parameters.get("tools"),
                "tool_results": self.config.parameters.get("tool_results"),
            });

            debug!("Cohere Payload: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Cohere Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Cohere API error: {:?}", error));
            }

            let content = response["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Cohere response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| billed_units.get("input_tokens"))
                    .and_then(|input_tokens| input_tokens.as_u64())
                    .unwrap_or(0) as u32,
                completion_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| billed_units.get("output_tokens"))
                    .and_then(|output_tokens| output_tokens.as_u64())
                    .unwrap_or(0) as u32,
                total_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| {
                        let input = billed_units.get("input_tokens").and_then(|t| t.as_u64()).unwrap_or(0);
                        let output = billed_units.get("output_tokens").and_then(|t| t.as_u64()).unwrap_or(0);
                        Some(input + output)
                    })
                    .unwrap_or(0) as u32,
            };

            let model = "cohere".to_string(); // Or extract from response if available
            let finish_reason = response["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            // Cohere doesn't have a direct upsert functionality, so we return an empty response
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value.get("text")
            .and_then(|text| text.as_str())
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            // Cohere doesn't have a direct file upload API, so we'll read the file and return its content as a base64 string
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_content = STANDARD.encode(&buffer);
            Ok(base64_content)
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let base64_content = Pin::from(self.upload_file(file_path)).await?;

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = json!({
                "message": &request.payload,
                "model": self.config.parameters.get("modelName").and_then(|v| v.as_str()).unwrap_or("command-r-plus"),
                "documents": [{
                    "text": base64_content,
                    "title": file_path.file_name().unwrap_or_default().to_string_lossy()
                }],
                // Include other parameters as needed
            });

            debug!("Cohere Payload with file: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Cohere Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Cohere API error: {:?}", error));
            }

            let content = response["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Cohere response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| billed_units.get("input_tokens"))
                    .and_then(|input_tokens| input_tokens.as_u64())
                    .unwrap_or(0) as u32,
                completion_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| billed_units.get("output_tokens"))
                    .and_then(|output_tokens| output_tokens.as_u64())
                    .unwrap_or(0) as u32,
                total_tokens: response.get("meta")
                    .and_then(|meta| meta.get("billed_units"))
                    .and_then(|billed_units| {
                        let input = billed_units.get("input_tokens").and_then(|t| t.as_u64()).unwrap_or(0);
                        let output = billed_units.get("output_tokens").and_then(|t| t.as_u64()).unwrap_or(0);
                        Some(input + output)
                    })
                    .unwrap_or(0) as u32,
            };

            let model = "cohere".to_string(); // Or extract from response if available
            let finish_reason = response["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/cohere.rs ####

#### START OF FILE: crates/fluent-engines/src/groqlpu.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct GroqLPUEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl GroqLPUEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn send_groq_request(&self, messages: Vec<Value>) -> Result<Value> {
        let url = format!("{}://{}:{}{}",
                          self.config.connection.protocol,
                          self.config.connection.hostname,
                          self.config.connection.port,
                          self.config.connection.request_path
        );

        let payload = json!({
            "model": self.config.parameters.get("model").and_then(|v| v.as_str()).unwrap_or("mixtral-8x7b-32768"),
            "messages": messages,
            "temperature": self.config.parameters.get("temperature").and_then(|v| v.as_f64()).unwrap_or(0.7),
            "max_tokens": self.config.parameters.get("max_tokens").and_then(|v| v.as_u64()).unwrap_or(1024),
            "top_p": self.config.parameters.get("top_p").and_then(|v| v.as_f64()).unwrap_or(1.0),
            "stream": self.config.parameters.get("stream").and_then(|v| v.as_bool()).unwrap_or(false),
        });

        debug!("GroqLPU Request: {:?}", payload);

        let auth_token = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

        let response = self.client.post(&url)
            .header("Authorization", format!("Bearer {}", auth_token))
            .header("Content-Type", "application/json")
            .json(&payload)
            .send()
            .await?;

        if !response.status().is_success() {
            let status = response.status();
            let error_text = response.text().await?;
            debug!("Error response body: {}", error_text);
            return Err(anyhow!("Request failed with status: {}", status));
        }

        let response_body: Value = response.json().await?;
        debug!("GroqLPU Response: {:?}", response_body);

        Ok(response_body)
    }
}

#[async_trait]
impl Engine for GroqLPUEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let messages = vec![json!({
                "role": "user",
                "content": request.payload
            })];

            let response = self.send_groq_request(messages).await?;

            let content = response["choices"][0]["message"]["content"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from GroqLPU response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response["usage"]["prompt_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response["usage"]["completion_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: response["usage"]["total_tokens"].as_u64().unwrap_or(0) as u32,
            };

            let model = response["model"].as_str().unwrap_or("unknown").to_string();
            let finish_reason = response["choices"][0]["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionId").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value["choices"][0]["message"]["content"]
            .as_str()
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not supported for GroqLPU engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, _request: &'a Request, _file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File processing not supported for GroqLPU engine"))
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/groqlpu.rs ####

#### START OF FILE: crates/fluent-engines/src/google_gemini.rs ####
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;

pub struct GoogleGeminiEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl GoogleGeminiEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
        })
    }

    async fn encode_image(&self, file_path: &Path) -> Result<String> {
        let mut file = File::open(file_path).await.context("Failed to open file")?;
        let mut buffer = Vec::new();
        file.read_to_end(&mut buffer).await.context("Failed to read file")?;
        Ok(STANDARD.encode(&buffer))
    }

    async fn send_gemini_request(&self, prompt: &str, encoded_image: Option<String>) -> Result<Value> {
        let api_key = self.config.parameters.get("bearer_token")
            .and_then(|v| v.as_str())
            .ok_or_else(|| anyhow!("API key not found in configuration"))?;

        let model = self.config.parameters.get("modelName")
            .and_then(|v| v.as_str())
            .unwrap_or("gemini-1.5-pro-latest");

        let url = format!(
            "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
            model,
            api_key
        );

        let mut content = vec![json!({
            "parts": [{ "text": prompt }]
        })];

        if let Some(image) = encoded_image {
            content.push(json!({
                "parts": [{
                    "inline_data": {
                        "mime_type": "image/jpeg",
                        "data": image
                    }
                }]
            }));
        }

        let request_body = json!({
            "contents": content,
            "generationConfig": {
                "temperature": self.config.parameters.get("temperature").and_then(|v| v.as_f64()).unwrap_or(0.7),
                "topK": self.config.parameters.get("top_k").and_then(|v| v.as_u64()).unwrap_or(40),
                "topP": self.config.parameters.get("top_p").and_then(|v| v.as_f64()).unwrap_or(0.95),
                "maxOutputTokens": self.config.parameters.get("max_tokens").and_then(|v| v.as_u64()).unwrap_or(1024),
            }
        });

        debug!("Google Gemini Request: {:?}", request_body);

        let response = self.client.post(&url)
            .header("Content-Type", "application/json")
            .json(&request_body)
            .send()
            .await?;

        if !response.status().is_success() {
            let status = response.status();
            let error_text = response.text().await?;
            debug!("Error response body: {}", error_text);
            return Err(anyhow!("Request failed with status: {}", status));
        }

        let response_body: Value = response.json().await?;
        debug!("Google Gemini Response: {:?}", response_body);

        Ok(response_body)
    }
}

#[async_trait]
impl Engine for GoogleGeminiEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let response = self.send_gemini_request(&request.payload, None).await?;

            let generated_text = response["candidates"][0]["content"]["parts"][0]["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract generated text from Gemini response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response["usageMetadata"]["promptTokenCount"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response["usageMetadata"]["candidatesTokenCount"].as_u64().unwrap_or(0) as u32,
                total_tokens: response["usageMetadata"]["totalTokenCount"].as_u64().unwrap_or(0) as u32,
            };

            let model = self.config.parameters.get("modelName")
                .and_then(|v| v.as_str())
                .unwrap_or("gemini-1.5-pro-latest")
                .to_string();

            let finish_reason = response["candidates"][0]["finishReason"]
                .as_str()
                .map(String::from);

            Ok(Response {
                content: generated_text,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionId").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value["candidates"][0]["content"]["parts"][0]["text"]
            .as_str()
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            self.encode_image(file_path).await
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let encoded_image = Pin::from(self.upload_file(file_path)).await?;
            let response = self.send_gemini_request(&request.payload, Some(encoded_image)).await?;

            let generated_text = response["candidates"][0]["content"]["parts"][0]["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract generated text from Gemini response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response["usageMetadata"]["promptTokenCount"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response["usageMetadata"]["candidatesTokenCount"].as_u64().unwrap_or(0) as u32,
                total_tokens: response["usageMetadata"]["totalTokenCount"].as_u64().unwrap_or(0) as u32,
            };

            let model = self.config.parameters.get("modelName")
                .and_then(|v| v.as_str())
                .unwrap_or("gemini-1.5-pro-latest")
                .to_string();

            let finish_reason = response["candidates"][0]["finishReason"]
                .as_str()
                .map(String::from);

            Ok(Response {
                content: generated_text,
                usage,
                model,
                finish_reason,
            })
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/google_gemini.rs ####

#### START OF FILE: crates/fluent-engines/src/anthropic.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::traits::{AnthropicConfigProcessor, Engine, EngineConfigProcessor, FileUpload};
use fluent_core::config::EngineConfig;
use anyhow::{Result, anyhow, Context};
use reqwest::Client;
use async_trait::async_trait;
use serde_json::{json, Value};
use log::debug;
use mime_guess::from_path;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use fluent_core::neo4j_client::Neo4jClient;

pub struct AnthropicEngine {
    config: EngineConfig,
    config_processor: AnthropicConfigProcessor,
    neo4j_client: Option<Arc<Neo4jClient>>,

}

impl AnthropicEngine {

    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            config_processor: AnthropicConfigProcessor,
            neo4j_client,
        })
    }
}

impl Engine for AnthropicEngine {

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        let main_content = value.get("completion").and_then(|v| v.as_str())?;

        let sentiment = value.get("sentiment")
            .and_then(|v| v.as_str())
            .map(String::from);

        let clusters = value.get("clusters")
            .and_then(|v| v.as_array())
            .map(|arr| arr.iter()
                .filter_map(|v| v.as_str().map(String::from))
                .collect());

        let themes = value.get("themes")
            .and_then(|v| v.as_array())
            .map(|arr| arr.iter()
                .filter_map(|v| v.as_str().map(String::from))
                .collect());

        let keywords = value.get("keywords")
            .and_then(|v| v.as_array())
            .map(|arr| arr.iter()
                .filter_map(|v| v.as_str().map(String::from))
                .collect());

        Some(ExtractedContent {
            main_content: main_content.to_string(),
            sentiment,
            clusters,
            themes,
            keywords,
        })
    }


    fn upsert<'a>(&'a self, request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            // Implement Anthropic-specific upsert logic here
            // For now, we'll just return a placeholder response
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let client = Client::new();
            debug!("Config: {:?}", self.config);

            let mut payload = self.config_processor.process_config(&self.config)?;

            // Add the user's request to the messages
            payload["messages"][0]["content"] = json!(request.payload);

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let res = client.post(&url)
                .header("x-api-key", auth_token)
                .header("anthropic-version", "2023-06-01")
                .header("content-type", "application/json")
                .json(&payload)
                .send()
                .await?;

            let response_body = res.json::<serde_json::Value>().await?;
            debug!("Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("Anthropic API error: {:?}", error));
            }

            let content = response_body["content"][0]["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Anthropic response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response_body["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response_body["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: (response_body["usage"]["input_tokens"].as_u64().unwrap_or(0) +
                    response_body["usage"]["output_tokens"].as_u64().unwrap_or(0)) as u32,
            };

            let model = response_body["model"].as_str().unwrap_or("unknown").to_string();
            let finish_reason = response_body["stop_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upload_file<'a>(&'a self, _file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            Err(anyhow!("File upload not implemented for Anthropic engine"))
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            // Read and encode the file
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = base64::encode(&buffer);

            // Guess the MIME type of the file
            let mime_type = from_path(file_path)
                .first_or_octet_stream()
                .to_string();

            let client = reqwest::Client::new();
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = serde_json::json!({
                "model": "claude-3-5-sonnet-20240620",
                "max_tokens": 1024,
                "messages": [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image",
                                "source": {
                                    "type": "base64",
                                    "media_type": mime_type,
                                    "data": base64_image,
                                }
                            },
                            {
                                "type": "text",
                                "text": &request.payload
                            }
                        ]
                    }
                ]
            });

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = client.post(&url)
                .header("x-api-key", auth_token)
                .header("anthropic-version", "2023-06-01")
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?;

            let response_body = response.json::<serde_json::Value>().await?;

            // Debug print the response
            debug!("Anthropic Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("Anthropic API error: {:?}", error));
            }

            let content = response_body["content"][0]["text"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from Anthropic response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response_body["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response_body["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: (response_body["usage"]["input_tokens"].as_u64().unwrap_or(0) +
                    response_body["usage"]["output_tokens"].as_u64().unwrap_or(0)) as u32,
            };

            let model = response_body["model"].as_str().unwrap_or("claude-3-5-sonnet-20240620").to_string();
            let finish_reason = response_body["stop_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }
}

fn parse_anthropic_response(response: &serde_json::Value) -> Result<Response> {
    let content = response["content"][0]["text"]
        .as_str()
        .ok_or_else(|| anyhow!("Failed to extract content from Anthropic response"))?
        .to_string();

    let usage = Usage {
        prompt_tokens: response["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32,
        completion_tokens: response["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32,
        total_tokens: (response["usage"]["input_tokens"].as_u64().unwrap_or(0) +
            response["usage"]["output_tokens"].as_u64().unwrap_or(0)) as u32,
    };

    let model = response["model"].as_str().unwrap_or("unknown").to_string();
    let finish_reason = response["stop_reason"].as_str().map(String::from);

    Ok(Response {
        content,
        usage,
        model,
        finish_reason,
    })
}





#### END OF FILE: crates/fluent-engines/src/anthropic.rs ####

#### START OF FILE: crates/fluent-engines/src/stabilityai.rs ####
use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use anyhow::{Result, anyhow, Context};
use async_trait::async_trait;
use serde_json::Value;
use tokio::fs::File;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use base64::{Engine as _, engine::general_purpose::STANDARD};
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::Engine;
use fluent_core::config::EngineConfig;
use log::debug;
use reqwest::Client;
use reqwest::multipart::{Form, Part};


pub struct StabilityAIEngine {
    config: EngineConfig,
    client: Client,
    neo4j_client: Option<Arc<Neo4jClient>>,
    download_dir: Option<String>, // Add this field
}

impl StabilityAIEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            client: Client::new(),
            neo4j_client,
            download_dir: None, // Initialize as None
        })
    }

    // Method to set the download directory
    pub fn set_download_dir(&mut self, dir: String) {
        self.download_dir = Some(dir);
    }
}

#[async_trait]
impl Engine for StabilityAIEngine {
    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let url = "https://api.stability.ai/v2beta/stable-image/generate/ultra";

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            // Get output format from configuration, default to webp
            let output_format = self.config.parameters
                .get("output_format")
                .and_then(|v| v.as_str())
                .unwrap_or("webp");

            // Get accept header type, default to image/*
            let accept_header = self.config.parameters
                .get("accept")
                .and_then(|v| v.as_str())
                .unwrap_or("image/*");

            // Create multipart form data
            let mut form = Form::new()
                .part("none", Part::bytes(Vec::new()))
                .text("prompt", request.payload.clone())
                .text("output_format", output_format.to_string());

            // Add optional parameters from the config
            for (key, value) in &self.config.parameters {
                if key != "bearer_token"
                    && key != "sessionID"
                    && key != "output_format"
                    && key != "accept"
                {
                    if let Some(value_str) = value.as_str() {
                        form = form.text(key.clone(), value_str.to_owned());
                    } else {
                        debug!("Skipping non-string parameter: {}", key);
                    }
                }
            }

            // Send the request
            let response = self.client.post(url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Accept", accept_header)  // Add Accept header
                .multipart(form)
                .send()
                .await?;

            // Check for successful response (200 OK)
            if !response.status().is_success() {
                return Err(anyhow!("Stability AI API request failed: {}", response.status()));
            }

            // Handle response based on Accept header
            let response_content = if accept_header == "application/json" {
                // Parse JSON response and extract base64 image data
                let json_response: Value = response.json().await?;
                let base64_image = json_response["artifacts"][0]["base64"]
                    .as_str()
                    .ok_or_else(|| anyhow!("Failed to extract base64 image data from JSON response"))?;
                base64::decode(base64_image).context("Failed to decode base64 image data")?
            } else {
                // Get image bytes directly
                response.bytes().await?.to_vec()
            };

            // Get the download directory
            let download_dir = self.download_dir
                .as_ref()
                .ok_or_else(|| anyhow!("Download directory not set for StabilityAIEngine"))?;
            let download_path = Path::new(download_dir);

            // Create a unique file name
            let file_name = format!("stabilityai_image_{}.{}", uuid::Uuid::new_v4(), output_format);
            let full_path = download_path.join(file_name);

            // Write the image data to the file
            let mut file = File::create(&full_path).await.context("Failed to create image file")?;
            file.write_all(&response_content).await.context("Failed to write image data to file")?;

            Ok(Response {
                content: full_path.to_str().unwrap().to_string(),
                usage: Usage {
                    prompt_tokens: 0,
                    completion_tokens: 0,
                    total_tokens: 0,
                },
                model: "stabilityai-ultra".to_string(),
                finish_reason: Some("success".to_string()),
            })
        })
    }

    fn upsert<'a>(&'a self, _request: &'a UpsertRequest) -> Box<dyn Future<Output = Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        // Extract image URLs from the response
        let image_urls: Vec<String> = value["artifacts"]
            .as_array()
            .ok_or_else(|| anyhow!("Failed to extract artifacts from Stability AI response"))
            .ok()?
            .iter()
            .filter_map(|artifact| artifact["base64"].as_str().map(|base64| format!("data:image/png;base64,{}", base64)))
            .collect();

        if image_urls.is_empty() {
            None
        } else {
            Some(ExtractedContent {
                main_content: image_urls.join("\n"),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
        }
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {
            // Stability AI doesn't support file uploads in the same way as OpenAI.
            // Instead, we'll read the file and encode it to base64.
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = STANDARD.encode(&buffer);
            Ok(base64_image)
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = base64::encode(&buffer);
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let payload = serde_json::json!({
                "text_prompts": [{"text": &request.payload}],
                "init_images": [base64_image],
                // Add other parameters as needed based on Stability AI's API
            });

            debug!("Stability AI Payload with file: {:?}", payload);

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = self.client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .json(&payload)
                .send()
                .await?
                .json::<Value>()
                .await?;

            debug!("Stability AI Response: {:?}", response);

            if let Some(error) = response.get("error") {
                return Err(anyhow!("Stability AI API error: {:?}", error));
            }

            // Extract image URLs from the response
            let image_urls: Vec<String> = response["artifacts"]
                .as_array()
                .ok_or_else(|| anyhow!("Failed to extract artifacts from Stability AI response"))?
                .iter()
                .filter_map(|artifact| artifact["base64"].as_str().map(|base64| format!("data:image/png;base64,{}", base64)))
                .collect();

            Ok(Response {
                content: image_urls.join("\n"),
                usage: Usage { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
                model: "stability-ai".to_string(), // Extract the model name from the response if available
                finish_reason: Some("success".to_string()),
            })
        })
    }
}
#### END OF FILE: crates/fluent-engines/src/stabilityai.rs ####

#### START OF FILE: crates/fluent-engines/src/openai.rs ####

use tokio::fs::File;
use tokio_util::codec::{BytesCodec, FramedRead};
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::sync::Arc;
use fluent_core::types::{ExtractedContent, Request, Response, UpsertRequest, UpsertResponse, Usage};
use fluent_core::neo4j_client::Neo4jClient;
use fluent_core::traits::{Engine, EngineConfigProcessor, FileUpload, OpenAIConfigProcessor};
use fluent_core::config::EngineConfig;
use anyhow::{Result, anyhow, Context};
use reqwest::Client;
use async_trait::async_trait;
use serde_json::{json, Value};
use log::debug;
use reqwest::multipart::{Form, Part};
use tokio::io::AsyncReadExt;


pub struct OpenAIEngine {
    config: EngineConfig,
    config_processor: OpenAIConfigProcessor,
    neo4j_client: Option<Arc<Neo4jClient>>,
}

impl OpenAIEngine {
    pub async fn new(config: EngineConfig) -> Result<Self> {
        let neo4j_client = if let Some(neo4j_config) = &config.neo4j {
            Some(Arc::new(Neo4jClient::new(neo4j_config).await?))
        } else {
            None
        };

        Ok(Self {
            config,
            config_processor: OpenAIConfigProcessor,
            neo4j_client,
        })
    }
}

impl Engine for OpenAIEngine {
    fn get_neo4j_client(&self) -> Option<&Arc<Neo4jClient>> {
        self.neo4j_client.as_ref()
    }

    fn get_session_id(&self) -> Option<String> {
        self.config.parameters.get("sessionID").and_then(|v| v.as_str()).map(String::from)
    }

    fn upsert<'a>(&'a self, request: &'a UpsertRequest) -> Box<dyn Future<Output=Result<UpsertResponse>> + Send + 'a> {
        Box::new(async move {
            // Implement OpenAI-specific upsert logic here
            // For now, we'll just return a placeholder response
            Ok(UpsertResponse {
                processed_files: vec![],
                errors: vec![],
            })
        })
    }

    fn extract_content(&self, value: &Value) -> Option<ExtractedContent> {
        value.get("choices")
            .and_then(|choices| choices.get(0))
            .and_then(|choice| choice.get("message"))
            .and_then(|message| message.get("content"))
            .and_then(|content| content.as_str())
            .map(|content| ExtractedContent {
                main_content: content.to_string(),
                sentiment: None,
                clusters: None,
                themes: None,
                keywords: None,
            })
    }

    fn execute<'a>(&'a self, request: &'a Request) -> Box<dyn Future<Output=Result<Response>> + Send + 'a> {
        Box::new(async move {
            let client = Client::new();
            debug!("Config: {:?}", self.config);

            let mut payload = self.config_processor.process_config(&self.config)?;
            debug!("OpenAI Processed Config Payload: {:#?}", payload);

            // Add the user's request to the messages
            payload["messages"] = json!([
                {
                    "role": "user",
                    "content": request.payload
                }
            ]);

            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              self.config.connection.request_path
            );

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let res = client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?;

            let response_body = res.json::<serde_json::Value>().await?;
            debug!("Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("OpenAI API error: {:?}", error));
            }

            let content = response_body["choices"][0]["message"]["content"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from OpenAI response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response_body["usage"]["prompt_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response_body["usage"]["completion_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: response_body["usage"]["total_tokens"].as_u64().unwrap_or(0) as u32,
            };

            let model = response_body["model"].as_str().unwrap_or("unknown").to_string();
            let finish_reason = response_body["choices"][0]["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }

    fn upload_file<'a>(&'a self, file_path: &'a Path) -> Box<dyn Future<Output = Result<String>> + Send + 'a> {
        Box::new(async move {

            let client = reqwest::Client::new();
            let url = "https://api.openai.com/v1/files";

            let file_name = file_path.file_name()
                .ok_or_else(|| anyhow!("Invalid file name"))?
                .to_str()
                .ok_or_else(|| anyhow!("File name is not valid UTF-8"))?;

            let file = File::open(file_path).await?;
            let stream = FramedRead::new(file, BytesCodec::new());
            let file_part = Part::stream(reqwest::Body::wrap_stream(stream))
                .file_name(file_name.to_owned());

            let form = Form::new()
                .part("file", file_part)
                .text("purpose", "assistants");

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = client.post(url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .multipart(form)
                .send()
                .await?;

            let response_body = response.json::<serde_json::Value>().await?;

            response_body["id"].as_str()
                .ok_or_else(|| anyhow!("Failed to extract file ID from OpenAI response"))
                .map(|id| id.to_string())
        })
    }

    fn process_request_with_file<'a>(&'a self, request: &'a Request, file_path: &'a Path) -> Box<dyn Future<Output = Result<Response>> + Send + 'a> {
        Box::new(async move {
            // Read and encode the file
            let mut file = File::open(file_path).await.context("Failed to open file")?;
            let mut buffer = Vec::new();
            file.read_to_end(&mut buffer).await.context("Failed to read file")?;
            let base64_image = base64::encode(&buffer);

            let client = reqwest::Client::new();
            let url = format!("{}://{}:{}{}",
                              self.config.connection.protocol,
                              self.config.connection.hostname,
                              self.config.connection.port,
                              "/v1/chat/completions"  // Use the chat completions endpoint for vision tasks
            );

            let payload = serde_json::json!({
                "model": "gpt-4-vision-preview",
                "messages": [
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "text",
                                "text": &request.payload
                            },
                            {
                                "type": "image_url",
                                "image_url": {
                                    "url": format!("data:image/png;base64,{}", base64_image)
                                }
                            }
                        ]
                    }
                ],
                "max_tokens": 300
            });

            let auth_token = self.config.parameters.get("bearer_token")
                .and_then(|v| v.as_str())
                .ok_or_else(|| anyhow!("Bearer token not found in configuration"))?;

            let response = client.post(&url)
                .header("Authorization", format!("Bearer {}", auth_token))
                .header("Content-Type", "application/json")
                .json(&payload)
                .send()
                .await?;

            let response_body = response.json::<serde_json::Value>().await?;

            // Debug print the response
            debug!("OpenAI Response: {:?}", response_body);

            if let Some(error) = response_body.get("error") {
                return Err(anyhow!("OpenAI API error: {:?}", error));
            }

            let content = response_body["choices"][0]["message"]["content"]
                .as_str()
                .ok_or_else(|| anyhow!("Failed to extract content from OpenAI response"))?
                .to_string();

            let usage = Usage {
                prompt_tokens: response_body["usage"]["prompt_tokens"].as_u64().unwrap_or(0) as u32,
                completion_tokens: response_body["usage"]["completion_tokens"].as_u64().unwrap_or(0) as u32,
                total_tokens: response_body["usage"]["total_tokens"].as_u64().unwrap_or(0) as u32,
            };

            let model = response_body["model"].as_str().unwrap_or("gpt-4-vision-preview").to_string();
            let finish_reason = response_body["choices"][0]["finish_reason"].as_str().map(String::from);

            Ok(Response {
                content,
                usage,
                model,
                finish_reason,
            })
        })
    }
}



#### END OF FILE: crates/fluent-engines/src/openai.rs ####

