#ifndef COMPONENTS_CONTEXTUAL_SEARCH_INTERNAL_COMPOSEBOX_QUERY_CONTROLLER_H_
#define COMPONENTS_CONTEXTUAL_SEARCH_INTERNAL_COMPOSEBOX_QUERY_CONTROLLER_H_
#include <map>
#include <memory>
#include <optional>
#include <string>
#include <vector>
#include "base/functional/callback_forward.h"
#include "base/memory/scoped_refptr.h"
#include "base/memory/weak_ptr.h"
#include "base/observer_list.h"
#include "components/contextual_search/contextual_search_context_controller.h"
#include "components/endpoint_fetcher/endpoint_fetcher.h"
#include "components/lens/lens_overlay_request_id_generator.h"
#include "components/lens/proto/server/lens_overlay_response.pb.h"
#include "services/network/public/cpp/shared_url_loader_factory.h"
#include "third_party/lens_server_proto/aim_communication.pb.h"
#include "third_party/lens_server_proto/lens_overlay_cluster_info.pb.h"
#include "third_party/lens_server_proto/lens_overlay_request_id.pb.h"
#include "third_party/lens_server_proto/lens_overlay_server.pb.h"
namespace base {
class TaskRunner;
}
class TemplateURLService;
namespace lens {
enum class RequestIdUpdateMode;
class ImageData;
class LensOverlayClientContext;
}
namespace signin {
class IdentityManager;
class PrimaryAccountAccessTokenFetcher;
}
namespace variations {
class VariationsClient;
}
namespace version_info {
enum class Channel;
}
class SkBitmap;
using RequestBodyProtoCreatedCallback = base::OnceCallback<void(
lens::LensOverlayServerRequest,
std::optional<contextual_search::FileUploadErrorType>)>;
using InteractionRequestBodyProtoCreatedCallback =
base::OnceCallback<void(lens::LensOverlayServerRequest)>;
class ComposeboxQueryController
: public contextual_search::ContextualSearchContextController {
public:
ComposeboxQueryController(
signin::IdentityManager* identity_manager,
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
version_info::Channel channel,
std::string locale,
TemplateURLService* template_url_service,
variations::VariationsClient* variations_client,
std::unique_ptr<
contextual_search::ContextualSearchContextController::ConfigParams>
config_params);
~ComposeboxQueryController() override;
void InitializeIfNeeded() override;
void CreateSearchUrl(
std::unique_ptr<CreateSearchUrlRequestInfo> search_url_request_info,
base::OnceCallback<void(GURL)> callback) override;
lens::ClientToAimMessage CreateClientToAimRequest(
std::unique_ptr<CreateClientToAimRequestInfo>
create_client_to_aim_request_info) override;
void AddObserver(FileUploadStatusObserver* obs) override;
void RemoveObserver(FileUploadStatusObserver* obs) override;
void StartFileUploadFlow(
const base::UnguessableToken& file_token,
std::unique_ptr<lens::ContextualInputData> contextual_input_data,
std::optional<lens::ImageEncodingOptions> image_options) override;
bool DeleteFile(const base::UnguessableToken& file_token) override;
void ClearFiles() override;
std::unique_ptr<lens::proto::LensOverlaySuggestInputs> CreateSuggestInputs(
const std::vector<base::UnguessableToken>& attached_context_tokens)
override;
const contextual_search::FileInfo* GetFileInfo(
const base::UnguessableToken& file_token) override;
std::vector<const contextual_search::FileInfo*> GetFileInfoList() override;
base::WeakPtr<ContextualSearchContextController> AsWeakPtr() override;
virtual lens::LensOverlayRequestId GetRequestIdForViewportImage(
const base::UnguessableToken& file_token);
enum class QueryControllerState {
kOff = 0,
kAwaitingClusterInfoResponse = 1,
kClusterInfoReceived = 2,
kClusterInfoInvalid = 3,
};
protected:
struct UploadRequest {
public:
UploadRequest();
~UploadRequest();
base::Time start_time;
base::Time response_time;
int response_code = 0;
std::unique_ptr<lens::LensOverlayServerRequest> request_body;
std::unique_ptr<endpoint_fetcher::EndpointFetcher> endpoint_fetcher_;
};
struct FileInfo : public contextual_search::FileInfo {
public:
FileInfo();
~FileInfo() override;
lens::LensOverlayRequestId GetRequestIdForTesting() const {
return request_id;
}
lens::LensOverlayRequestId* GetViewportRequestIdForTesting() const {
return viewport_request_id_.get();
}
private:
friend class ComposeboxQueryController;
friend class ComposeboxQueryControllerIOS;
std::unique_ptr<lens::LensOverlayRequestId> viewport_request_id_;
std::unique_ptr<std::vector<std::string>> request_headers_;
std::vector<uint8_t> file_content;
std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
file_upload_access_token_fetcher_;
std::vector<std::unique_ptr<UploadRequest>> upload_requests_;
size_t num_outstanding_network_requests_ = 0;
};
static void CreateFileUploadRequestProtoWithImageDataAndContinue(
lens::LensOverlayRequestId request_id,
lens::LensOverlayClientContext client_context,
scoped_refptr<lens::RefCountedLensOverlayClientLogs> client_logs,
RequestBodyProtoCreatedCallback callback,
lens::ImageData image_data);
virtual void CreateImageUploadRequest(
lens::LensOverlayRequestId request_id,
const std::vector<uint8_t>& image_data,
std::optional<lens::ImageEncodingOptions> options,
RequestBodyProtoCreatedCallback callback);
using UploadProgressCallback =
base::RepeatingCallback<void(uint64_t position, uint64_t total)>;
virtual std::unique_ptr<endpoint_fetcher::EndpointFetcher>
CreateEndpointFetcher(std::string request_string,
const GURL& fetch_url,
endpoint_fetcher::HttpMethod http_method,
base::TimeDelta timeout,
const std::vector<std::string>& request_headers,
const std::vector<std::string>& cors_exempt_headers,
UploadProgressCallback upload_progress_callback);
lens::LensOverlayClientContext CreateClientContext() const;
void ClearClusterInfo();
virtual void ResetRequestClusterInfoState();
virtual void SendInteractionRequest(
std::unique_ptr<lens::LensOverlayRequestId> request_id,
std::string query_text,
std::optional<lens::ImageCrop> image_crop,
std::optional<lens::LensOverlayClientLogs> client_logs,
std::optional<lens::LensOverlaySelectionType> lens_overlay_selection_type,
base::OnceCallback<void(lens::LensOverlayInteractionResponse)>
interaction_response_callback);
QueryControllerState query_controller_state_ = QueryControllerState::kOff;
base::RepeatingCallback<void(QueryControllerState state)>
on_query_controller_state_changed_callback_;
std::map<base::UnguessableToken, std::unique_ptr<FileInfo>> active_files_;
scoped_refptr<base::TaskRunner> create_request_task_runner_;
private:
struct LensServerInteractionRequest {
public:
explicit LensServerInteractionRequest(
std::unique_ptr<lens::LensOverlayRequestId> request_id);
~LensServerInteractionRequest();
int sequence_id() const { return request_id_->sequence_id(); }
const std::unique_ptr<lens::LensOverlayRequestId> request_id_;
std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
interaction_access_token_fetcher_;
std::unique_ptr<endpoint_fetcher::EndpointFetcher>
interaction_endpoint_fetcher_;
std::unique_ptr<lens::LensOverlayServerRequest> request_;
std::unique_ptr<std::vector<std::string>> request_headers_;
std::optional<base::OnceClosure> request_sent_callback_;
base::OnceCallback<void(lens::LensOverlayInteractionResponse)>
interaction_response_callback_;
bool request_sent_ = false;
};
FileInfo* GetMutableFileInfo(const base::UnguessableToken& file_token);
using OAuthHeadersCreatedCallback =
base::OnceCallback<void(std::vector<std::string>)>;
std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
CreateOAuthHeadersAndContinue(OAuthHeadersCreatedCallback callback);
void FetchClusterInfo();
void SendClusterInfoNetworkRequest(std::vector<std::string> request_headers);
void HandleClusterInfoResponse(
std::unique_ptr<endpoint_fetcher::EndpointResponse> response);
void SetQueryControllerState(QueryControllerState new_state);
void UpdateFileUploadStatus(
const base::UnguessableToken& file_token,
contextual_search::FileUploadStatus status,
std::optional<contextual_search::FileUploadErrorType> error_type);
void ProcessDecodedImageAndContinue(lens::LensOverlayRequestId request_id,
const lens::ImageEncodingOptions& options,
RequestBodyProtoCreatedCallback callback,
const SkBitmap& bitmap);
void CreateUploadRequestBodiesAndContinue(
const base::UnguessableToken& file_token,
std::unique_ptr<lens::ContextualInputData> contextual_input_data,
std::optional<lens::ImageEncodingOptions> options);
void AddPageIndexToImageUploadRequestAndContinue(
std::optional<size_t> pdf_page_index,
RequestBodyProtoCreatedCallback callback,
lens::LensOverlayServerRequest request,
std::optional<contextual_search::FileUploadErrorType> error_type);
void OnUploadRequestBodyReady(
const base::UnguessableToken& file_token,
size_t request_index,
lens::LensOverlayServerRequest request,
std::optional<contextual_search::FileUploadErrorType> error_type);
void OnUploadRequestHeadersReady(const base::UnguessableToken& file_token,
std::vector<std::string> headers);
void MaybeSendUploadNetworkRequest(const base::UnguessableToken& file_token,
size_t request_index);
void SendUploadNetworkRequest(FileInfo* file_info, size_t request_index);
void OnInteractionRequestHeadersReady(std::vector<std::string> headers);
void TrySendInteractionRequest();
void OnInteractionEndpointFetcherCreated(
std::unique_ptr<endpoint_fetcher::EndpointFetcher> endpoint_fetcher);
void HandleInteractionResponse(
std::unique_ptr<endpoint_fetcher::EndpointResponse> response);
void OnUploadEndpointFetcherCreated(
const base::UnguessableToken& file_token,
size_t request_index,
std::unique_ptr<endpoint_fetcher::EndpointFetcher> endpoint_fetcher);
void HandleUploadResponse(
const base::UnguessableToken& file_token,
size_t request_index,
std::unique_ptr<endpoint_fetcher::EndpointResponse> response);
void PerformFetchRequest(
lens::LensOverlayServerRequest* request,
std::vector<std::string>* request_headers,
base::TimeDelta timeout,
base::OnceCallback<
void(std::unique_ptr<endpoint_fetcher::EndpointFetcher>)>
fetcher_created_callback,
endpoint_fetcher::EndpointFetcherCallback response_received_callback,
UploadProgressCallback upload_progress_callback = base::NullCallback());
void AddEncodedVisualSearchInteractionLogDataParam(
const FileInfo* file_info,
const std::optional<std::string>& query_text,
std::optional<lens::LensOverlaySelectionType> lens_overlay_selection_type,
std::map<std::string, std::string>& url_params_map);
std::optional<lens::LensOverlayVisualSearchInteractionData>
ConstructVisualSearchInteractionData(
const FileInfo* file_info,
const std::optional<std::string>& query_text,
std::optional<lens::LensOverlaySelectionType>
lens_overlay_selection_type);
std::optional<lens::LensOverlayClusterInfo> cluster_info_ = std::nullopt;
std::unique_ptr<endpoint_fetcher::EndpointFetcher>
cluster_info_endpoint_fetcher_;
std::unique_ptr<signin::PrimaryAccountAccessTokenFetcher>
cluster_info_access_token_fetcher_;
const raw_ptr<signin::IdentityManager> identity_manager_;
scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
version_info::Channel channel_;
std::string locale_;
lens::LensOverlayRequestIdGenerator request_id_generator_;
base::ObserverList<FileUploadStatusObserver> observers_;
const raw_ptr<TemplateURLService> template_url_service_;
const raw_ptr<variations::VariationsClient> variations_client_;
bool send_lns_surface_;
bool suppress_lns_surface_param_if_no_image_;
bool enable_multi_context_input_flow_;
bool enable_viewport_images_;
bool use_separate_request_ids_for_multi_context_viewport_images_;
bool prioritize_suggestions_for_the_first_attached_document_;
bool attach_page_title_and_url_to_suggest_requests_;
std::unique_ptr<LensServerInteractionRequest>
latest_interaction_request_data_;
int num_files_in_request_ = 0;
base::OnceClosure pending_search_url_request_;
base::WeakPtrFactory<ComposeboxQueryController> weak_ptr_factory_{this};
};
#endif