// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.discoveryengine.v1alpha;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/discoveryengine/v1alpha/search_service.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option csharp_namespace = "Google.Cloud.DiscoveryEngine.V1Alpha";
option go_package = "cloud.google.com/go/discoveryengine/apiv1alpha/discoveryenginepb;discoveryenginepb";
option java_multiple_files = true;
option java_outer_classname = "EvaluationProto";
option java_package = "com.google.cloud.discoveryengine.v1alpha";
option objc_class_prefix = "DISCOVERYENGINE";
option php_namespace = "Google\\Cloud\\DiscoveryEngine\\V1alpha";
option ruby_package = "Google::Cloud::DiscoveryEngine::V1alpha";

// An evaluation is a single execution (or run) of an evaluation process. It
// encapsulates the state of the evaluation and the resulting data.
message Evaluation {
  option (google.api.resource) = {
    type: "discoveryengine.googleapis.com/Evaluation"
    pattern: "projects/{project}/locations/{location}/evaluations/{evaluation}"
  };

  // Describes the specification of the evaluation.
  message EvaluationSpec {
    // Describes the specification of the query set.
    message QuerySetSpec {
      // Required. The full resource name of the
      // [SampleQuerySet][google.cloud.discoveryengine.v1alpha.SampleQuerySet]
      // used for the evaluation, in the format of
      // `projects/{project}/locations/{location}/sampleQuerySets/{sampleQuerySet}`.
      string sample_query_set = 1 [
        (google.api.field_behavior) = REQUIRED,
        (google.api.resource_reference) = {
          type: "discoveryengine.googleapis.com/SampleQuerySet"
        }
      ];
    }

    // The search specification.
    oneof search_spec {
      // Required. The search request that is used to perform the evaluation.
      //
      // Only the following fields within SearchRequest are supported; if any
      // other fields are provided, an UNSUPPORTED error will be returned:
      //
      // * [SearchRequest.serving_config][google.cloud.discoveryengine.v1alpha.SearchRequest.serving_config]
      // * [SearchRequest.branch][google.cloud.discoveryengine.v1alpha.SearchRequest.branch]
      // * [SearchRequest.canonical_filter][google.cloud.discoveryengine.v1alpha.SearchRequest.canonical_filter]
      // * [SearchRequest.query_expansion_spec][google.cloud.discoveryengine.v1alpha.SearchRequest.query_expansion_spec]
      // * [SearchRequest.spell_correction_spec][google.cloud.discoveryengine.v1alpha.SearchRequest.spell_correction_spec]
      // * [SearchRequest.content_search_spec][google.cloud.discoveryengine.v1alpha.SearchRequest.content_search_spec]
      // * [SearchRequest.user_pseudo_id][google.cloud.discoveryengine.v1alpha.SearchRequest.user_pseudo_id]
      SearchRequest search_request = 2 [(google.api.field_behavior) = REQUIRED];
    }

    // Required. The specification of the query set.
    QuerySetSpec query_set_spec = 1 [(google.api.field_behavior) = REQUIRED];
  }

  // Describes the state of an evaluation.
  enum State {
    // The evaluation is unspecified.
    STATE_UNSPECIFIED = 0;

    // The service is preparing to run the evaluation.
    PENDING = 1;

    // The evaluation is in progress.
    RUNNING = 2;

    // The evaluation completed successfully.
    SUCCEEDED = 3;

    // The evaluation failed.
    FAILED = 4;
  }

  // Identifier. The full resource name of the
  // [Evaluation][google.cloud.discoveryengine.v1alpha.Evaluation], in the
  // format of
  // `projects/{project}/locations/{location}/evaluations/{evaluation}`.
  //
  // This field must be a UTF-8 encoded string with a length limit of 1024
  // characters.
  string name = 1 [(google.api.field_behavior) = IDENTIFIER];

  // Required. The specification of the evaluation.
  EvaluationSpec evaluation_spec = 2 [(google.api.field_behavior) = REQUIRED];

  // Output only. The metrics produced by the evaluation, averaged across all
  // [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery]s in the
  // [SampleQuerySet][google.cloud.discoveryengine.v1alpha.SampleQuerySet].
  //
  // Only populated when the evaluation's state is SUCCEEDED.
  QualityMetrics quality_metrics = 3
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The state of the evaluation.
  State state = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The error that occurred during evaluation. Only populated when
  // the evaluation's state is FAILED.
  google.rpc.Status error = 5 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp the
  // [Evaluation][google.cloud.discoveryengine.v1alpha.Evaluation] was created
  // at.
  google.protobuf.Timestamp create_time = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp the
  // [Evaluation][google.cloud.discoveryengine.v1alpha.Evaluation] was completed
  // at.
  google.protobuf.Timestamp end_time = 7
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. A sample of errors encountered while processing the request.
  repeated google.rpc.Status error_samples = 8
      [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Describes the metrics produced by the evaluation.
message QualityMetrics {
  // Stores the metric values at specific top-k levels.
  message TopkMetrics {
    // The top-1 value.
    double top_1 = 1;

    // The top-3 value.
    double top_3 = 2;

    // The top-5 value.
    double top_5 = 3;

    // The top-10 value.
    double top_10 = 4;
  }

  // Recall per document, at various top-k cutoff levels.
  //
  // Recall is the fraction of relevant documents retrieved out of all
  // relevant documents.
  //
  // Example (top-5):
  //  * For a single
  //  [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery], If 3 out
  //  of 5 relevant documents are retrieved in the top-5, recall@5 = 3/5 = 0.6
  TopkMetrics doc_recall = 1;

  // Precision per document, at various top-k cutoff levels.
  //
  // Precision is the fraction of retrieved documents that are relevant.
  //
  // Example (top-5):
  //  * For a single
  //  [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery], If 4 out
  //  of 5 retrieved documents in the top-5 are relevant, precision@5 = 4/5 =
  //  0.8
  TopkMetrics doc_precision = 2;

  // Normalized discounted cumulative gain (NDCG) per document, at various top-k
  // cutoff levels.
  //
  // NDCG measures the ranking quality, giving higher relevance to top
  // results.
  //
  // Example (top-3):
  //  Suppose [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery]
  //  with three retrieved documents (D1, D2, D3) and binary relevance
  //  judgements (1 for relevant, 0 for not relevant):
  //
  //   Retrieved:  [D3 (0), D1 (1), D2 (1)]
  //   Ideal:      [D1 (1), D2 (1), D3 (0)]
  //
  //   Calculate NDCG@3 for each
  //   [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery]:
  //    * DCG@3: 0/log2(1+1) + 1/log2(2+1) + 1/log2(3+1) = 1.13
  //    * Ideal DCG@3: 1/log2(1+1) + 1/log2(2+1) + 0/log2(3+1) = 1.63
  //    * NDCG@3: 1.13/1.63 = 0.693
  TopkMetrics doc_ndcg = 3;

  // Recall per page, at various top-k cutoff levels.
  //
  // Recall is the fraction of relevant pages retrieved out of all relevant
  // pages.
  //
  // Example (top-5):
  //  * For a single
  //  [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery], if 3 out
  //  of 5 relevant pages are retrieved in the top-5, recall@5 = 3/5 = 0.6
  TopkMetrics page_recall = 4;

  // Normalized discounted cumulative gain (NDCG) per page, at various top-k
  // cutoff levels.
  //
  // NDCG measures the ranking quality, giving higher relevance to top
  // results.
  //
  // Example (top-3):
  //  Suppose [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery]
  //  with three retrieved pages (P1, P2, P3) and binary relevance judgements (1
  //  for relevant, 0 for not relevant):
  //
  //   Retrieved:  [P3 (0), P1 (1), P2 (1)]
  //   Ideal:      [P1 (1), P2 (1), P3 (0)]
  //
  //   Calculate NDCG@3 for
  //   [SampleQuery][google.cloud.discoveryengine.v1alpha.SampleQuery]:
  //    * DCG@3: 0/log2(1+1) + 1/log2(2+1) + 1/log2(3+1) = 1.13
  //    * Ideal DCG@3: 1/log2(1+1) + 1/log2(2+1) + 0/log2(3+1) = 1.63
  //    * NDCG@3: 1.13/1.63 = 0.693
  TopkMetrics page_ndcg = 5;
}