ai_gui_controller.h

Go to the documentation of this file.

#ifndef YAZE_CLI_SERVICE_AI_AI_GUI_CONTROLLER_H_
#define YAZE_CLI_SERVICE_AI_AI_GUI_CONTROLLER_H_
 
#include <filesystem>
#include <memory>
#include <string>
#include <vector>
 
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "cli/service/ai/ai_action_parser.h"
#include "cli/service/ai/vision_action_refiner.h"
#include "cli/service/gui/gui_action_generator.h"
 
namespace yaze {
namespace cli {
 
// Forward declares
class GeminiAIService;
class GuiAutomationClient;  // In cli namespace, not cli::gui
 
namespace ai {
 
struct ControlLoopConfig {
  int max_iterations = 10;                  // Max attempts before giving up
  int screenshot_delay_ms = 500;            // Delay before taking screenshots
  bool enable_vision_verification = true;   // Use vision to verify actions
  bool enable_iterative_refinement = true;  // Retry with refined actions
  int max_retries_per_action = 3;           // Max retries for a single action
  std::string screenshots_dir = "/tmp/yaze/ai_gui_control";
};
 
struct ControlResult {
  bool success = false;
  int iterations_performed = 0;
  std::vector<ai::AIAction> actions_executed;
  std::vector<VisionAnalysisResult> vision_analyses;
  std::vector<std::filesystem::path> screenshots_taken;
  std::string error_message;
  std::string final_state_description;
};
 
class AIGUIController {
 public:
  AIGUIController(GeminiAIService* gemini_service,
                  GuiAutomationClient* gui_client);
 
  ~AIGUIController() = default;
 
  absl::Status Initialize(const ControlLoopConfig& config);
 
  absl::StatusOr<ControlResult> ExecuteCommand(const std::string& command);
 
  absl::StatusOr<ControlResult> ExecuteActions(
      const std::vector<ai::AIAction>& actions);
 
  absl::StatusOr<VisionAnalysisResult> ExecuteSingleAction(
      const AIAction& action, bool verify_with_vision = true);
 
  absl::StatusOr<VisionAnalysisResult> AnalyzeCurrentGUIState(
      const std::string& context = "");
 
  const ControlLoopConfig& config() const { return config_; }
 
  void SetConfig(const ControlLoopConfig& config) { config_ = config; }
 
 private:
  GeminiAIService* gemini_service_;  // Not owned
  GuiAutomationClient* gui_client_;  // Not owned
  std::unique_ptr<VisionActionRefiner> vision_refiner_;
  gui::GuiActionGenerator action_generator_;
  ControlLoopConfig config_;
  std::filesystem::path screenshots_dir_;
 
  // Helper methods
  absl::StatusOr<std::filesystem::path> CaptureCurrentState(
      const std::string& description);
 
  absl::Status ExecuteGRPCAction(const AIAction& action);
 
  absl::StatusOr<VisionAnalysisResult> VerifyActionSuccess(
      const AIAction& action, const std::filesystem::path& before_screenshot,
      const std::filesystem::path& after_screenshot);
 
  absl::StatusOr<AIAction> RefineActionWithVision(
      const AIAction& original_action, const VisionAnalysisResult& analysis);
 
  void EnsureScreenshotsDirectory();
  std::filesystem::path GenerateScreenshotPath(const std::string& suffix);
};
 
}  // namespace ai
}  // namespace cli
}  // namespace yaze
 
#endif  // YAZE_CLI_SERVICE_AI_AI_GUI_CONTROLLER_H_