vision_action_refiner.h

Go to the documentation of this file.

#ifndef YAZE_CLI_SERVICE_AI_VISION_ACTION_REFINER_H_
#define YAZE_CLI_SERVICE_AI_VISION_ACTION_REFINER_H_
 
#include <filesystem>
#include <string>
#include <vector>
 
#include "absl/status/statusor.h"
#include "cli/service/ai/ai_action_parser.h"
 
namespace yaze {
namespace cli {
 
// Forward declare
class GeminiAIService;
 
namespace ai {
 
struct VisionAnalysisResult {
  std::string description;               // What Gemini sees in the image
  std::vector<std::string> widgets;      // Detected UI widgets
  std::vector<std::string> suggestions;  // Action suggestions
  bool action_successful = false;        // Whether the last action succeeded
  std::string error_message;             // Error description if action failed
};
 
struct ActionRefinement {
  bool needs_retry = false;
  bool needs_different_approach = false;
  std::map<std::string, std::string> adjusted_parameters;
  std::string reasoning;
};
 
class VisionActionRefiner {
 public:
  explicit VisionActionRefiner(GeminiAIService* gemini_service);
 
  absl::StatusOr<VisionAnalysisResult> AnalyzeScreenshot(
      const std::filesystem::path& screenshot_path,
      const std::string& context = "");
 
  absl::StatusOr<VisionAnalysisResult> VerifyAction(
      const AIAction& action, const std::filesystem::path& before_screenshot,
      const std::filesystem::path& after_screenshot);
 
  absl::StatusOr<ActionRefinement> RefineAction(
      const AIAction& original_action, const VisionAnalysisResult& analysis);
 
  absl::StatusOr<std::map<std::string, std::string>> LocateUIElement(
      const std::filesystem::path& screenshot_path,
      const std::string& element_name);
 
  absl::StatusOr<std::vector<std::string>> ExtractVisibleWidgets(
      const std::filesystem::path& screenshot_path);
 
 private:
  GeminiAIService* gemini_service_;  // Not owned
 
  // Build prompts for different vision analysis tasks
  std::string BuildAnalysisPrompt(const std::string& context);
  std::string BuildVerificationPrompt(const AIAction& action);
  std::string BuildElementLocationPrompt(const std::string& element_name);
  std::string BuildWidgetExtractionPrompt();
 
  // Parse Gemini vision responses
  VisionAnalysisResult ParseAnalysisResponse(const std::string& response);
  VisionAnalysisResult ParseVerificationResponse(const std::string& response,
                                                 const AIAction& action);
};
 
}  // namespace ai
}  // namespace cli
}  // namespace yaze
 
#endif  // YAZE_CLI_SERVICE_AI_VISION_ACTION_REFINER_H_