harness_bench.h

Go to the documentation of this file.

 
#pragma once
 
#include <neograph/api.h>
#include <neograph/graph/types.h>
#include <neograph/graph/compiler.h>
#include <neograph/graph/validator.h>
 
#include <string>
#include <vector>
 
namespace neograph::graph {
 
// ── Feedback mode ────────────────────────────────────────────────────
 
enum class NEOGRAPH_API FeedbackMode : uint8_t {
    FULL_DIAGNOSTIC = 0,  
    FAIL_ONLY       = 1,  
    RUNTIME_SYMPTOMS= 2,  
};
 
NEOGRAPH_API const char* feedback_mode_name(FeedbackMode mode);
 
// ── Harness task ─────────────────────────────────────────────────────
 
struct NEOGRAPH_API HarnessTask {
    std::string name;
    std::string description;
    std::string natural_spec;
 
    int min_nodes = 0;
    int max_nodes = 100;
    bool has_conditional_edges = false;
    bool has_barrier = false;
    bool has_interrupt = false;
    bool has_tool_calls = false;
};
 
NEOGRAPH_API HarnessTask parse_task(const json& j);
 
// ── Feedback ─────────────────────────────────────────────────────────
 
struct NEOGRAPH_API Feedback {
    std::string prompt;
    int estimated_tokens = 0;
    bool converged = false;
};
 
NEOGRAPH_API Feedback format_feedback(
    const json& core,
    const std::string& compile_err,
    const ValidationReport& validation,
    FeedbackMode mode,
    int attempted_routes = -1);
 
// ── Convergence metrics ──────────────────────────────────────────────
 
struct NEOGRAPH_API ConvergenceMetrics {
    std::string task_name;
    FeedbackMode mode = FeedbackMode::FULL_DIAGNOSTIC;
    int turns = 0;
    int total_estimated_tokens = 0;
    int total_errors = 0;
    bool converged = false;
    int final_warnings = 0;
    bool meets_spec = false;
 
    json to_json() const;
};
 
// ── Benchmark runner (simulation) ────────────────────────────────────
 
NEOGRAPH_API ConvergenceMetrics run_simulation(
    const HarnessTask& task,
    const json& seed_harness,
    FeedbackMode mode,
    int max_turns = 20,
    unsigned int seed = 42);
 
NEOGRAPH_API json generate_report(
    const std::vector<ConvergenceMetrics>& results);
 
} // namespace neograph::graph