// Copyright 2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.cloud.orchestration.airflow.service.v1; import "google/api/annotations.proto"; import "google/api/client.proto"; import "google/api/field_behavior.proto"; import "google/api/resource.proto"; import "google/cloud/orchestration/airflow/service/v1/operations.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/empty.proto"; import "google/protobuf/field_mask.proto"; import "google/protobuf/timestamp.proto"; option go_package = "cloud.google.com/go/orchestration/airflow/service/apiv1/servicepb;servicepb"; option java_multiple_files = true; option java_package = "com.google.cloud.orchestration.airflow.service.v1"; // Managed Apache Airflow Environments. service Environments { option (google.api.default_host) = "composer.googleapis.com"; option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; // Create a new environment. rpc CreateEnvironment(CreateEnvironmentRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*}/environments" body: "environment" }; option (google.api.method_signature) = "parent,environment"; option (google.longrunning.operation_info) = { response_type: "Environment" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Get an existing environment. rpc GetEnvironment(GetEnvironmentRequest) returns (Environment) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/environments/*}" }; option (google.api.method_signature) = "name"; } // List environments. rpc ListEnvironments(ListEnvironmentsRequest) returns (ListEnvironmentsResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*}/environments" }; option (google.api.method_signature) = "parent"; } // Update an environment. rpc UpdateEnvironment(UpdateEnvironmentRequest) returns (google.longrunning.Operation) { option (google.api.http) = { patch: "/v1/{name=projects/*/locations/*/environments/*}" body: "environment" }; option (google.api.method_signature) = "name,environment,update_mask"; option (google.longrunning.operation_info) = { response_type: "Environment" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Delete an environment. rpc DeleteEnvironment(DeleteEnvironmentRequest) returns (google.longrunning.Operation) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/environments/*}" }; option (google.api.method_signature) = "name"; option (google.longrunning.operation_info) = { response_type: "google.protobuf.Empty" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Executes Airflow CLI command. rpc ExecuteAirflowCommand(ExecuteAirflowCommandRequest) returns (ExecuteAirflowCommandResponse) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:executeAirflowCommand" body: "*" }; } // Stops Airflow CLI command execution. rpc StopAirflowCommand(StopAirflowCommandRequest) returns (StopAirflowCommandResponse) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:stopAirflowCommand" body: "*" }; } // Polls Airflow CLI command execution and fetches logs. rpc PollAirflowCommand(PollAirflowCommandRequest) returns (PollAirflowCommandResponse) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:pollAirflowCommand" body: "*" }; } // Lists workloads in a Cloud Composer environment. Workload is a unit that // runs a single Composer component. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc ListWorkloads(ListWorkloadsRequest) returns (ListWorkloadsResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*/environments/*}/workloads" }; option (google.api.method_signature) = "parent"; } // Check if an upgrade operation on the environment will succeed. // // In case of problems detailed info can be found in the returned Operation. rpc CheckUpgrade(CheckUpgradeRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:checkUpgrade" body: "*" }; option (google.longrunning.operation_info) = { response_type: "google.cloud.orchestration.airflow.service.v1.CheckUpgradeResponse" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Creates a user workloads Secret. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc CreateUserWorkloadsSecret(CreateUserWorkloadsSecretRequest) returns (UserWorkloadsSecret) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsSecrets" body: "user_workloads_secret" }; option (google.api.method_signature) = "parent,user_workloads_secret"; } // Gets an existing user workloads Secret. // Values of the "data" field in the response are cleared. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc GetUserWorkloadsSecret(GetUserWorkloadsSecretRequest) returns (UserWorkloadsSecret) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" }; option (google.api.method_signature) = "name"; } // Lists user workloads Secrets. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc ListUserWorkloadsSecrets(ListUserWorkloadsSecretsRequest) returns (ListUserWorkloadsSecretsResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsSecrets" }; option (google.api.method_signature) = "parent"; } // Updates a user workloads Secret. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc UpdateUserWorkloadsSecret(UpdateUserWorkloadsSecretRequest) returns (UserWorkloadsSecret) { option (google.api.http) = { put: "/v1/{user_workloads_secret.name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" body: "user_workloads_secret" }; option (google.api.method_signature) = "user_workloads_secret"; } // Deletes a user workloads Secret. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc DeleteUserWorkloadsSecret(DeleteUserWorkloadsSecretRequest) returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsSecrets/*}" }; option (google.api.method_signature) = "name"; } // Creates a user workloads ConfigMap. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc CreateUserWorkloadsConfigMap(CreateUserWorkloadsConfigMapRequest) returns (UserWorkloadsConfigMap) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsConfigMaps" body: "user_workloads_config_map" }; option (google.api.method_signature) = "parent,user_workloads_config_map"; } // Gets an existing user workloads ConfigMap. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc GetUserWorkloadsConfigMap(GetUserWorkloadsConfigMapRequest) returns (UserWorkloadsConfigMap) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" }; option (google.api.method_signature) = "name"; } // Lists user workloads ConfigMaps. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc ListUserWorkloadsConfigMaps(ListUserWorkloadsConfigMapsRequest) returns (ListUserWorkloadsConfigMapsResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*/environments/*}/userWorkloadsConfigMaps" }; option (google.api.method_signature) = "parent"; } // Updates a user workloads ConfigMap. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc UpdateUserWorkloadsConfigMap(UpdateUserWorkloadsConfigMapRequest) returns (UserWorkloadsConfigMap) { option (google.api.http) = { put: "/v1/{user_workloads_config_map.name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" body: "user_workloads_config_map" }; option (google.api.method_signature) = "user_workloads_config_map"; } // Deletes a user workloads ConfigMap. // // This method is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. rpc DeleteUserWorkloadsConfigMap(DeleteUserWorkloadsConfigMapRequest) returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/environments/*/userWorkloadsConfigMaps/*}" }; option (google.api.method_signature) = "name"; } // Creates a snapshots of a Cloud Composer environment. // // As a result of this operation, snapshot of environment's state is stored // in a location specified in the SaveSnapshotRequest. rpc SaveSnapshot(SaveSnapshotRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:saveSnapshot" body: "*" }; option (google.longrunning.operation_info) = { response_type: "google.cloud.orchestration.airflow.service.v1.SaveSnapshotResponse" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Loads a snapshot of a Cloud Composer environment. // // As a result of this operation, a snapshot of environment's specified in // LoadSnapshotRequest is loaded into the environment. rpc LoadSnapshot(LoadSnapshotRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:loadSnapshot" body: "*" }; option (google.longrunning.operation_info) = { response_type: "google.cloud.orchestration.airflow.service.v1.LoadSnapshotResponse" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Triggers database failover (only for highly resilient environments). rpc DatabaseFailover(DatabaseFailoverRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{environment=projects/*/locations/*/environments/*}:databaseFailover" body: "*" }; option (google.longrunning.operation_info) = { response_type: "google.cloud.orchestration.airflow.service.v1.DatabaseFailoverResponse" metadata_type: "google.cloud.orchestration.airflow.service.v1.OperationMetadata" }; } // Fetches database properties. rpc FetchDatabaseProperties(FetchDatabasePropertiesRequest) returns (FetchDatabasePropertiesResponse) { option (google.api.http) = { get: "/v1/{environment=projects/*/locations/*/environments/*}:fetchDatabaseProperties" }; } } // Create a new environment. message CreateEnvironmentRequest { // The parent must be of the form // "projects/{projectId}/locations/{locationId}". string parent = 1; // The environment to create. Environment environment = 2; } // Get an environment. message GetEnvironmentRequest { // The resource name of the environment to get, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string name = 1; } // List environments in a project and location. message ListEnvironmentsRequest { // List environments in the given project and location, in the form: // "projects/{projectId}/locations/{locationId}" string parent = 1; // The maximum number of environments to return. int32 page_size = 2; // The next_page_token value returned from a previous List request, if any. string page_token = 3; } // The environments in a project and location. message ListEnvironmentsResponse { // The list of environments returned by a ListEnvironmentsRequest. repeated Environment environments = 1; // The page token used to query for the next page if one exists. string next_page_token = 2; } // Delete an environment. message DeleteEnvironmentRequest { // The environment to delete, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string name = 1; } // Update an environment. message UpdateEnvironmentRequest { // The relative resource name of the environment to update, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string name = 2; // A patch environment. Fields specified by the `updateMask` will be copied // from the patch environment into the environment under update. Environment environment = 1; // Required. A comma-separated list of paths, relative to `Environment`, of // fields to update. // For example, to set the version of scikit-learn to install in the // environment to 0.19.0 and to remove an existing installation of // numpy, the `updateMask` parameter would include the following two // `paths` values: "config.softwareConfig.pypiPackages.scikit-learn" and // "config.softwareConfig.pypiPackages.numpy". The included patch // environment would specify the scikit-learn version as follows: // // { // "config":{ // "softwareConfig":{ // "pypiPackages":{ // "scikit-learn":"==0.19.0" // } // } // } // } // // Note that in the above example, any existing PyPI packages // other than scikit-learn and numpy will be unaffected. // // Only one update type may be included in a single request's `updateMask`. // For example, one cannot update both the PyPI packages and // labels in the same request. However, it is possible to update multiple // members of a map field simultaneously in the same request. For example, // to set the labels "label1" and "label2" while clearing "label3" (assuming // it already exists), one can // provide the paths "labels.label1", "labels.label2", and "labels.label3" // and populate the patch environment as follows: // // { // "labels":{ // "label1":"new-label1-value" // "label2":"new-label2-value" // } // } // // Note that in the above example, any existing labels that are not // included in the `updateMask` will be unaffected. // // It is also possible to replace an entire map field by providing the // map field's path in the `updateMask`. The new value of the field will // be that which is provided in the patch environment. For example, to // delete all pre-existing user-specified PyPI packages and // install botocore at version 1.7.14, the `updateMask` would contain // the path "config.softwareConfig.pypiPackages", and // the patch environment would be the following: // // { // "config":{ // "softwareConfig":{ // "pypiPackages":{ // "botocore":"==1.7.14" // } // } // } // } // // **Note:** Only the following fields can be updated: // // * `config.softwareConfig.pypiPackages` // * Replace all custom custom PyPI packages. If a replacement // package map is not included in `environment`, all custom // PyPI packages are cleared. It is an error to provide both // this mask and a mask specifying an individual package. // * `config.softwareConfig.pypiPackages.`packagename // * Update the custom PyPI package *packagename*, // preserving other packages. To delete the package, include it in // `updateMask`, and omit the mapping for it in // `environment.config.softwareConfig.pypiPackages`. It is an error // to provide both a mask of this form and the // `config.softwareConfig.pypiPackages` mask. // * `labels` // * Replace all environment labels. If a replacement labels map is not // included in `environment`, all labels are cleared. It is an error to // provide both this mask and a mask specifying one or more individual // labels. // * `labels.`labelName // * Set the label named *labelName*, while preserving other // labels. To delete the label, include it in `updateMask` and omit its // mapping in `environment.labels`. It is an error to provide both a // mask of this form and the `labels` mask. // * `config.nodeCount` // * Horizontally scale the number of nodes in the environment. An integer // greater than or equal to 3 must be provided in the `config.nodeCount` // field. Supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. // * `config.webServerNetworkAccessControl` // * Replace the environment's current `WebServerNetworkAccessControl`. // * `config.softwareConfig.airflowConfigOverrides` // * Replace all Apache Airflow config overrides. If a replacement config // overrides map is not included in `environment`, all config overrides // are cleared. // It is an error to provide both this mask and a mask specifying one or // more individual config overrides. // * `config.softwareConfig.airflowConfigOverrides.`section-name // * Override the Apache Airflow config property *name* in the // section named *section*, preserving other properties. To // delete the property override, include it in `updateMask` and omit its // mapping in // `environment.config.softwareConfig.airflowConfigOverrides`. // It is an error to provide both a mask of this form and the // `config.softwareConfig.airflowConfigOverrides` mask. // * `config.softwareConfig.envVariables` // * Replace all environment variables. If a replacement environment // variable map is not included in `environment`, all custom environment // variables are cleared. // * `config.softwareConfig.schedulerCount` // * Horizontally scale the number of schedulers in Airflow. A positive // integer not greater than the number of nodes must be provided in the // `config.softwareConfig.schedulerCount` field. Supported for Cloud // Composer environments in versions composer-1.*.*-airflow-2.*.*. // * `config.databaseConfig.machineType` // * Cloud SQL machine type used by Airflow database. // It has to be one of: db-n1-standard-2, db-n1-standard-4, // db-n1-standard-8 or db-n1-standard-16. Supported for Cloud Composer // environments in versions composer-1.*.*-airflow-*.*.*. // * `config.webServerConfig.machineType` // * Machine type on which Airflow web server is running. // It has to be one of: composer-n1-webserver-2, composer-n1-webserver-4 // or composer-n1-webserver-8. Supported for Cloud Composer environments // in versions composer-1.*.*-airflow-*.*.*. google.protobuf.FieldMask update_mask = 3; } // Execute Airflow Command request. message ExecuteAirflowCommandRequest { // The resource name of the environment in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}". string environment = 1; // Airflow command. string command = 2; // Airflow subcommand. string subcommand = 3; // Parameters for the Airflow command/subcommand as an array of arguments. // It may contain positional arguments like `["my-dag-id"]`, key-value // parameters like `["--foo=bar"]` or `["--foo","bar"]`, // or other flags like `["-f"]`. repeated string parameters = 4; } // Response to ExecuteAirflowCommandRequest. message ExecuteAirflowCommandResponse { // The unique ID of the command execution for polling. string execution_id = 1; // The name of the pod where the command is executed. string pod = 2; // The namespace of the pod where the command is executed. string pod_namespace = 3; // Error message. Empty if there was no error. string error = 4; } // Stop Airflow Command request. message StopAirflowCommandRequest { // The resource name of the environment in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}". string environment = 1; // The unique ID of the command execution. string execution_id = 2; // The name of the pod where the command is executed. string pod = 3; // The namespace of the pod where the command is executed. string pod_namespace = 4; // If true, the execution is terminated forcefully (SIGKILL). If false, the // execution is stopped gracefully, giving it time for cleanup. bool force = 5; } // Response to StopAirflowCommandRequest. message StopAirflowCommandResponse { // Whether the execution is still running. bool is_done = 1; // Output message from stopping execution request. repeated string output = 2; } // Poll Airflow Command request. message PollAirflowCommandRequest { // The resource name of the environment in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1; // The unique ID of the command execution. string execution_id = 2; // The name of the pod where the command is executed. string pod = 3; // The namespace of the pod where the command is executed. string pod_namespace = 4; // Line number from which new logs should be fetched. int32 next_line_number = 5; } // Response to PollAirflowCommandRequest. message PollAirflowCommandResponse { // Contains information about a single line from logs. message Line { // Number of the line. int32 line_number = 1; // Text content of the log line. string content = 2; } // Information about how a command ended. message ExitInfo { // The exit code from the command execution. int32 exit_code = 1; // Error message. Empty if there was no error. string error = 2; } // Output from the command execution. It may not contain the full output // and the caller may need to poll for more lines. repeated Line output = 1; // Whether the command execution has finished and there is no more output. bool output_end = 2; // The result exit status of the command. ExitInfo exit_info = 3; } // Create user workloads Secret request. message CreateUserWorkloadsSecretRequest { // Required. The environment name to create a Secret for, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string parent = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; // Required. User workloads Secret to create. UserWorkloadsSecret user_workloads_secret = 2 [(google.api.field_behavior) = REQUIRED]; } // Get user workloads Secret request. message GetUserWorkloadsSecretRequest { // Required. The resource name of the Secret to get, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/UserWorkloadsSecret" } ]; } // List user workloads Secrets request. message ListUserWorkloadsSecretsRequest { // Required. List Secrets in the given environment, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string parent = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; // Optional. The maximum number of Secrets to return. int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The next_page_token value returned from a previous List request, // if any. string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; } // Update user workloads Secret request. message UpdateUserWorkloadsSecretRequest { // Optional. User workloads Secret to override. UserWorkloadsSecret user_workloads_secret = 1 [(google.api.field_behavior) = OPTIONAL]; } // Delete user workloads Secret request. message DeleteUserWorkloadsSecretRequest { // Required. The Secret to delete, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/UserWorkloadsSecret" } ]; } // Create user workloads ConfigMap request. message CreateUserWorkloadsConfigMapRequest { // Required. The environment name to create a ConfigMap for, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string parent = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; // Required. User workloads ConfigMap to create. UserWorkloadsConfigMap user_workloads_config_map = 2 [(google.api.field_behavior) = REQUIRED]; } // Get user workloads ConfigMap request. message GetUserWorkloadsConfigMapRequest { // Required. The resource name of the ConfigMap to get, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/UserWorkloadsConfigMap" } ]; } // List user workloads ConfigMaps request. message ListUserWorkloadsConfigMapsRequest { // Required. List ConfigMaps in the given environment, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string parent = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; // Optional. The maximum number of ConfigMaps to return. int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The next_page_token value returned from a previous List request, // if any. string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; } // Update user workloads ConfigMap request. message UpdateUserWorkloadsConfigMapRequest { // Optional. User workloads ConfigMap to override. UserWorkloadsConfigMap user_workloads_config_map = 1 [(google.api.field_behavior) = OPTIONAL]; } // Delete user workloads ConfigMap request. message DeleteUserWorkloadsConfigMapRequest { // Required. The ConfigMap to delete, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/UserWorkloadsConfigMap" } ]; } // User workloads Secret used by Airflow tasks that run with Kubernetes executor // or KubernetesPodOperator. message UserWorkloadsSecret { option (google.api.resource) = { type: "composer.googleapis.com/UserWorkloadsSecret" pattern: "projects/{project}/locations/{location}/environments/{environment}/userWorkloadsSecrets/{user_workloads_secret}" plural: "userWorkloadsSecrets" singular: "userWorkloadsSecret" }; // Identifier. The resource name of the Secret, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsSecrets/{userWorkloadsSecretId}" string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Optional. The "data" field of Kubernetes Secret, organized in key-value // pairs, which can contain sensitive values such as a password, a token, or a // key. The values for all keys have to be base64-encoded strings. For details // see: https://kubernetes.io/docs/concepts/configuration/secret/ map data = 2 [(google.api.field_behavior) = OPTIONAL]; } // The user workloads Secrets for a given environment. message ListUserWorkloadsSecretsResponse { // The list of Secrets returned by a ListUserWorkloadsSecretsRequest. repeated UserWorkloadsSecret user_workloads_secrets = 1; // The page token used to query for the next page if one exists. string next_page_token = 2; } // User workloads ConfigMap used by Airflow tasks that run with Kubernetes // executor or KubernetesPodOperator. message UserWorkloadsConfigMap { option (google.api.resource) = { type: "composer.googleapis.com/UserWorkloadsConfigMap" pattern: "projects/{project}/locations/{location}/environments/{environment}/userWorkloadsConfigMaps/{user_workloads_config_map}" plural: "userWorkloadsConfigMaps" singular: "userWorkloadsConfigMap" }; // Identifier. The resource name of the ConfigMap, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}/userWorkloadsConfigMaps/{userWorkloadsConfigMapId}" string name = 1 [(google.api.field_behavior) = IDENTIFIER]; // Optional. The "data" field of Kubernetes ConfigMap, organized in key-value // pairs. For details see: // https://kubernetes.io/docs/concepts/configuration/configmap/ map data = 2 [(google.api.field_behavior) = OPTIONAL]; } // The user workloads ConfigMaps for a given environment. message ListUserWorkloadsConfigMapsResponse { // The list of ConfigMaps returned by a ListUserWorkloadsConfigMapsRequest. repeated UserWorkloadsConfigMap user_workloads_config_maps = 1; // The page token used to query for the next page if one exists. string next_page_token = 2; } // Request for listing workloads in a Cloud Composer environment. message ListWorkloadsRequest { // Required. The environment name to get workloads for, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string parent = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; // Optional. The maximum number of environments to return. int32 page_size = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The next_page_token value returned from a previous List request, // if any. string page_token = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The list filter. // Currently only supports equality on the type field. The value of a field // specified in the filter expression must be one ComposerWorkloadType enum // option. It's possible to get multiple types using "OR" operator, e.g.: // "type=SCHEDULER OR type=CELERY_WORKER". If not specified, all items are // returned. string filter = 4 [(google.api.field_behavior) = OPTIONAL]; } // Response to ListWorkloadsRequest. message ListWorkloadsResponse { // Information about a single workload. message ComposerWorkload { // Name of a workload. string name = 1; // Type of a workload. ComposerWorkloadType type = 2; // Output only. Status of a workload. ComposerWorkloadStatus status = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Supported workload types. enum ComposerWorkloadType { // Not able to determine the type of the workload. COMPOSER_WORKLOAD_TYPE_UNSPECIFIED = 0; // Celery worker. CELERY_WORKER = 1; // Kubernetes worker. KUBERNETES_WORKER = 2; // Workload created by Kubernetes Pod Operator. KUBERNETES_OPERATOR_POD = 3; // Airflow scheduler. SCHEDULER = 4; // Airflow Dag processor. DAG_PROCESSOR = 5; // Airflow triggerer. TRIGGERER = 6; // Airflow web server UI. WEB_SERVER = 7; // Redis. REDIS = 8; } // Workload status. message ComposerWorkloadStatus { // Output only. Workload state. ComposerWorkloadState state = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Text to provide more descriptive status. string status_message = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Detailed message of the status. string detailed_status_message = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Workload states. enum ComposerWorkloadState { // Not able to determine the status of the workload. COMPOSER_WORKLOAD_STATE_UNSPECIFIED = 0; // Workload is in pending state and has not yet started. PENDING = 1; // Workload is running fine. OK = 2; // Workload is running but there are some non-critical problems. WARNING = 3; // Workload is not running due to an error. ERROR = 4; // Workload has finished execution with success. SUCCEEDED = 5; // Workload has finished execution with failure. FAILED = 6; } // The list of environment workloads. repeated ComposerWorkload workloads = 1; // The page token used to query for the next page if one exists. string next_page_token = 2; } // Request to create a snapshot of a Cloud Composer environment. message SaveSnapshotRequest { // The resource name of the source environment in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1; // Location in a Cloud Storage where the snapshot is going to be stored, e.g.: // "gs://my-bucket/snapshots". string snapshot_location = 2; } // Response to SaveSnapshotRequest. message SaveSnapshotResponse { // The fully-resolved Cloud Storage path of the created snapshot, // e.g.: // "gs://my-bucket/snapshots/project_location_environment_timestamp". // This field is populated only if the snapshot creation was successful. string snapshot_path = 1; } // Request to load a snapshot into a Cloud Composer environment. message LoadSnapshotRequest { // The resource name of the target environment in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1; // A Cloud Storage path to a snapshot to load, e.g.: // "gs://my-bucket/snapshots/project_location_environment_timestamp". string snapshot_path = 2; // Whether or not to skip installing Pypi packages when loading the // environment's state. bool skip_pypi_packages_installation = 3; // Whether or not to skip setting environment variables when loading the // environment's state. bool skip_environment_variables_setting = 4; // Whether or not to skip setting Airflow overrides when loading the // environment's state. bool skip_airflow_overrides_setting = 5; // Whether or not to skip copying Cloud Storage data when loading the // environment's state. bool skip_gcs_data_copying = 6; } // Response to LoadSnapshotRequest. message LoadSnapshotResponse {} // Request to trigger database failover (only for highly resilient // environments). message DatabaseFailoverRequest { // Target environment: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1; } // Response for DatabaseFailoverRequest. message DatabaseFailoverResponse {} // Request to fetch properties of environment's database. message FetchDatabasePropertiesRequest { // Required. The resource name of the environment, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { type: "composer.googleapis.com/Environment" } ]; } // Response for FetchDatabasePropertiesRequest. message FetchDatabasePropertiesResponse { // The Compute Engine zone that the instance is currently serving from. string primary_gce_zone = 1; // The Compute Engine zone that the failover instance is currently serving // from for a regional Cloud SQL instance. string secondary_gce_zone = 2; // The availability status of the failover replica. A false status indicates // that the failover replica is out of sync. The primary instance can only // fail over to the failover replica when the status is true. bool is_failover_replica_available = 3; } // The configuration for data storage in the environment. message StorageConfig { // Optional. The name of the Cloud Storage bucket used by the environment. No // `gs://` prefix. string bucket = 1 [(google.api.field_behavior) = OPTIONAL]; } // Configuration information for an environment. message EnvironmentConfig { // The size of the Cloud Composer environment. enum EnvironmentSize { // The size of the environment is unspecified. ENVIRONMENT_SIZE_UNSPECIFIED = 0; // The environment size is small. ENVIRONMENT_SIZE_SMALL = 1; // The environment size is medium. ENVIRONMENT_SIZE_MEDIUM = 2; // The environment size is large. ENVIRONMENT_SIZE_LARGE = 3; } // Resilience mode of the Cloud Composer Environment. enum ResilienceMode { // Default mode doesn't change environment parameters. RESILIENCE_MODE_UNSPECIFIED = 0; // Enabled High Resilience mode, including Cloud SQL HA. HIGH_RESILIENCE = 1; } // Output only. The Kubernetes Engine cluster used to run this environment. string gke_cluster = 1; // Output only. The Cloud Storage prefix of the DAGs for this environment. // Although Cloud Storage objects reside in a flat namespace, a hierarchical // file tree can be simulated using "/"-delimited object name prefixes. DAG // objects for this environment reside in a simulated directory with the given // prefix. string dag_gcs_prefix = 2; // The number of nodes in the Kubernetes Engine cluster that will be // used to run this environment. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. int32 node_count = 3; // The configuration settings for software inside the environment. SoftwareConfig software_config = 4; // The configuration used for the Kubernetes Engine cluster. NodeConfig node_config = 5; // The configuration used for the Private IP Cloud Composer environment. PrivateEnvironmentConfig private_environment_config = 7; // Optional. The network-level access control policy for the Airflow web // server. If unspecified, no network-level access restrictions will be // applied. WebServerNetworkAccessControl web_server_network_access_control = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration settings for Cloud SQL instance used internally // by Apache Airflow software. DatabaseConfig database_config = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration settings for the Airflow web server App Engine // instance. WebServerConfig web_server_config = 10 [(google.api.field_behavior) = OPTIONAL]; // Optional. The encryption options for the Cloud Composer environment // and its dependencies. Cannot be updated. EncryptionConfig encryption_config = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. The maintenance window is the period when Cloud Composer // components may undergo maintenance. It is defined so that maintenance is // not executed during peak hours or critical time periods. // // The system will not be under maintenance for every occurrence of this // window, but when maintenance is planned, it will be scheduled // during the window. // // The maintenance window period must encompass at least 12 hours per week. // This may be split into multiple chunks, each with a size of // at least 4 hours. // // If this value is omitted, the default value for maintenance window is // applied. By default, maintenance windows are from 00:00:00 to 04:00:00 // (GMT) on Friday, Saturday, and Sunday every week. MaintenanceWindow maintenance_window = 12 [(google.api.field_behavior) = OPTIONAL]; // Optional. The workloads configuration settings for the GKE cluster // associated with the Cloud Composer environment. The GKE cluster runs // Airflow scheduler, web server and workers workloads. // // This field is supported for Cloud Composer environments in versions // composer-2.*.*-airflow-*.*.* and newer. WorkloadsConfig workloads_config = 15 [(google.api.field_behavior) = OPTIONAL]; // Optional. The size of the Cloud Composer environment. // // This field is supported for Cloud Composer environments in versions // composer-2.*.*-airflow-*.*.* and newer. EnvironmentSize environment_size = 16 [(google.api.field_behavior) = OPTIONAL]; // Output only. The URI of the Apache Airflow Web UI hosted within this // environment (see [Airflow web // interface](/composer/docs/how-to/accessing/airflow-web-interface)). string airflow_uri = 6; // Output only. The 'bring your own identity' variant of the URI of the Apache // Airflow Web UI hosted within this environment, to be accessed with external // identities using workforce identity federation (see [Access environments // with workforce identity // federation](/composer/docs/composer-2/access-environments-with-workforce-identity-federation)). string airflow_byoid_uri = 20 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The configuration options for GKE cluster master authorized // networks. By default master authorized networks feature is: // - in case of private environment: enabled with no external networks // allowlisted. // - in case of public environment: disabled. MasterAuthorizedNetworksConfig master_authorized_networks_config = 17 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Recovery settings configuration of an environment. // // This field is supported for Cloud Composer environments in versions // composer-2.*.*-airflow-*.*.* and newer. RecoveryConfig recovery_config = 18 [(google.api.field_behavior) = OPTIONAL]; // Optional. Resilience mode of the Cloud Composer Environment. // // This field is supported for Cloud Composer environments in versions // composer-2.2.0-airflow-*.*.* and newer. ResilienceMode resilience_mode = 19 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration setting for Airflow database data retention // mechanism. DataRetentionConfig data_retention_config = 21 [(google.api.field_behavior) = OPTIONAL]; } // Network-level access control policy for the Airflow web server. message WebServerNetworkAccessControl { // Allowed IP range with user-provided description. message AllowedIpRange { // IP address or range, defined using CIDR notation, of requests that this // rule applies to. // Examples: `192.168.1.1` or `192.168.0.0/16` or `2001:db8::/32` // or `2001:0db8:0000:0042:0000:8a2e:0370:7334`. // // IP range prefixes should be properly truncated. For example, // `1.2.3.4/24` should be truncated to `1.2.3.0/24`. Similarly, for IPv6, // `2001:db8::1/32` should be truncated to `2001:db8::/32`. string value = 1; // Optional. User-provided description. It must contain at most 300 // characters. string description = 2 [(google.api.field_behavior) = OPTIONAL]; } // A collection of allowed IP ranges with descriptions. repeated AllowedIpRange allowed_ip_ranges = 1; } // The configuration of Cloud SQL instance that is used by the Apache Airflow // software. message DatabaseConfig { // Optional. Cloud SQL machine type used by Airflow database. // It has to be one of: db-n1-standard-2, db-n1-standard-4, db-n1-standard-8 // or db-n1-standard-16. If not specified, db-n1-standard-2 will be used. // Supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string machine_type = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Compute Engine zone where the Airflow database is created. If // zone is provided, it must be in the region selected for the environment. If // zone is not provided, a zone is automatically selected. The zone can only // be set during environment creation. Supported for Cloud Composer // environments in versions composer-2.*.*-airflow-*.*.*. string zone = 2 [(google.api.field_behavior) = OPTIONAL]; } // The configuration settings for the Airflow web server App Engine instance. // Supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.* message WebServerConfig { // Optional. Machine type on which Airflow web server is running. // It has to be one of: composer-n1-webserver-2, composer-n1-webserver-4 or // composer-n1-webserver-8. // If not specified, composer-n1-webserver-2 will be used. // Value custom is returned only in response, if Airflow web server parameters // were manually changed to a non-standard values. string machine_type = 1 [(google.api.field_behavior) = OPTIONAL]; } // The encryption options for the Cloud Composer environment // and its dependencies.Supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. message EncryptionConfig { // Optional. Customer-managed Encryption Key available through Google's Key // Management Service. Cannot be updated. If not specified, Google-managed key // will be used. string kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL]; } // The configuration settings for Cloud Composer maintenance window. // The following example: // // ``` // { // "startTime":"2019-08-01T01:00:00Z" // "endTime":"2019-08-01T07:00:00Z" // "recurrence":"FREQ=WEEKLY;BYDAY=TU,WE" // } // ``` // // would define a maintenance window between 01 and 07 hours UTC during // each Tuesday and Wednesday. message MaintenanceWindow { // Required. Start time of the first recurrence of the maintenance window. google.protobuf.Timestamp start_time = 1 [(google.api.field_behavior) = REQUIRED]; // Required. Maintenance window end time. It is used only to calculate the // duration of the maintenance window. The value for end-time must be in the // future, relative to `start_time`. google.protobuf.Timestamp end_time = 2 [(google.api.field_behavior) = REQUIRED]; // Required. Maintenance window recurrence. Format is a subset of // [RFC-5545](https://tools.ietf.org/html/rfc5545) `RRULE`. The only allowed // values for `FREQ` field are `FREQ=DAILY` and `FREQ=WEEKLY;BYDAY=...` // Example values: `FREQ=WEEKLY;BYDAY=TU,WE`, `FREQ=DAILY`. string recurrence = 3 [(google.api.field_behavior) = REQUIRED]; } // Specifies the selection and configuration of software inside the environment. message SoftwareConfig { // Web server plugins mode of the Cloud Composer environment. enum WebServerPluginsMode { // Default mode. WEB_SERVER_PLUGINS_MODE_UNSPECIFIED = 0; // Web server plugins are not supported. PLUGINS_DISABLED = 1; // Web server plugins are supported. PLUGINS_ENABLED = 2; } // The version of the software running in the environment. // This encapsulates both the version of Cloud Composer functionality and the // version of Apache Airflow. It must match the regular expression // `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+(\.[0-9]+(\.[0-9]+)?)?)`. // When used as input, the server also checks if the provided version is // supported and denies the request for an unsupported version. // // The Cloud Composer portion of the image version is a full // [semantic version](https://semver.org), or an alias in the form of major // version number or `latest`. When an alias is provided, the server replaces // it with the current Cloud Composer version that satisfies the alias. // // The Apache Airflow portion of the image version is a full semantic version // that points to one of the supported Apache Airflow versions, or an alias in // the form of only major or major.minor versions specified. When an alias is // provided, the server replaces it with the latest Apache Airflow version // that satisfies the alias and is supported in the given Cloud Composer // version. // // In all cases, the resolved image version is stored in the same field. // // See also [version // list](/composer/docs/concepts/versioning/composer-versions) and [versioning // overview](/composer/docs/concepts/versioning/composer-versioning-overview). string image_version = 1; // Optional. Apache Airflow configuration properties to override. // // Property keys contain the section and property names, separated by a // hyphen, for example "core-dags_are_paused_at_creation". Section names must // not contain hyphens ("-"), opening square brackets ("["), or closing // square brackets ("]"). The property name must not be empty and must not // contain an equals sign ("=") or semicolon (";"). Section and property names // must not contain a period ("."). Apache Airflow configuration property // names must be written in // [snake_case](https://en.wikipedia.org/wiki/Snake_case). Property values can // contain any character, and can be written in any lower/upper case format. // // Certain Apache Airflow configuration property values are // [blocked](/composer/docs/concepts/airflow-configurations), // and cannot be overridden. map airflow_config_overrides = 2; // Optional. Custom Python Package Index (PyPI) packages to be installed in // the environment. // // Keys refer to the lowercase package name such as "numpy" // and values are the lowercase extras and version specifier such as // "==1.12.0", "[devel,gcp_api]", or "[devel]>=1.8.2, <1.9.2". To specify a // package without pinning it to a version specifier, use the empty string as // the value. map pypi_packages = 3; // Optional. Additional environment variables to provide to the Apache Airflow // scheduler, worker, and webserver processes. // // Environment variable names must match the regular expression // `[a-zA-Z_][a-zA-Z0-9_]*`. They cannot specify Apache Airflow // software configuration overrides (they cannot match the regular expression // `AIRFLOW__[A-Z0-9_]+__[A-Z0-9_]+`), and they cannot match any of the // following reserved names: // // * `AIRFLOW_HOME` // * `C_FORCE_ROOT` // * `CONTAINER_NAME` // * `DAGS_FOLDER` // * `GCP_PROJECT` // * `GCS_BUCKET` // * `GKE_CLUSTER_NAME` // * `SQL_DATABASE` // * `SQL_INSTANCE` // * `SQL_PASSWORD` // * `SQL_PROJECT` // * `SQL_REGION` // * `SQL_USER` map env_variables = 4; // Optional. The major version of Python used to run the Apache Airflow // scheduler, worker, and webserver processes. // // Can be set to '2' or '3'. If not specified, the default is '3'. Cannot be // updated. // // This field is only supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. Environments in newer versions always use // Python major version 3. string python_version = 6; // Optional. The number of schedulers for Airflow. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-2.*.*. int32 scheduler_count = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration for Cloud Data Lineage integration. CloudDataLineageIntegration cloud_data_lineage_integration = 8 [(google.api.field_behavior) = OPTIONAL]; // Optional. Whether or not the web server uses custom plugins. // If unspecified, the field defaults to `PLUGINS_ENABLED`. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. WebServerPluginsMode web_server_plugins_mode = 10 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for controlling how IPs are allocated in the // GKE cluster running the Apache Airflow software. message IPAllocationPolicy { // Optional. Whether or not to enable Alias IPs in the GKE cluster. // If `true`, a VPC-native cluster is created. // // This field is only supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. Environments in newer versions always use // VPC-native GKE clusters. bool use_ip_aliases = 1 [(google.api.field_behavior) = OPTIONAL]; // Configuration of allocating IP addresses for pods in the GKE cluster. oneof cluster_ip_allocation { // Optional. The name of the GKE cluster's secondary range used to allocate // IP addresses to pods. // // For Cloud Composer environments in versions composer-1.*.*-airflow-*.*.*, // this field is applicable only when `use_ip_aliases` is true. string cluster_secondary_range_name = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The IP address range used to allocate IP addresses to pods in // the GKE cluster. // // For Cloud Composer environments in versions composer-1.*.*-airflow-*.*.*, // this field is applicable only when `use_ip_aliases` is true. // // Set to blank to have GKE choose a range with the default size. // // Set to /netmask (e.g. `/14`) to have GKE choose a range with a specific // netmask. // // Set to a // [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) // notation (e.g. `10.96.0.0/14`) from the RFC-1918 private networks (e.g. // `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`) to pick a specific range // to use. string cluster_ipv4_cidr_block = 4 [(google.api.field_behavior) = OPTIONAL]; } // Configuration of allocating IP addresses for services in the GKE cluster. oneof services_ip_allocation { // Optional. The name of the services' secondary range used to allocate // IP addresses to the GKE cluster. // // For Cloud Composer environments in versions composer-1.*.*-airflow-*.*.*, // this field is applicable only when `use_ip_aliases` is true. string services_secondary_range_name = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The IP address range of the services IP addresses in this // GKE cluster. // // For Cloud Composer environments in versions composer-1.*.*-airflow-*.*.*, // this field is applicable only when `use_ip_aliases` is true. // // Set to blank to have GKE choose a range with the default size. // // Set to /netmask (e.g. `/14`) to have GKE choose a range with a specific // netmask. // // Set to a // [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) // notation (e.g. `10.96.0.0/14`) from the RFC-1918 private networks (e.g. // `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`) to pick a specific range // to use. string services_ipv4_cidr_block = 5 [(google.api.field_behavior) = OPTIONAL]; } } // The configuration information for the Kubernetes Engine nodes running // the Apache Airflow software. message NodeConfig { // Optional. The Compute Engine [zone](/compute/docs/regions-zones) in which // to deploy the VMs used to run the Apache Airflow software, specified as a // [relative resource // name](/apis/design/resource_names#relative_resource_name). For example: // "projects/{projectId}/zones/{zoneId}". // // This `location` must belong to the enclosing environment's project and // location. If both this field and `nodeConfig.machineType` are specified, // `nodeConfig.machineType` must belong to this `location`; if both are // unspecified, the service will pick a zone in the Compute Engine region // corresponding to the Cloud Composer location, and propagate that choice to // both fields. If only one field (`location` or `nodeConfig.machineType`) is // specified, the location information from the specified field will be // propagated to the unspecified field. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string location = 1; // Optional. The Compute Engine // [machine type](/compute/docs/machine-types) used for cluster instances, // specified as a // [relative resource // name](/apis/design/resource_names#relative_resource_name). For example: // "projects/{projectId}/zones/{zoneId}/machineTypes/{machineTypeId}". // // The `machineType` must belong to the enclosing environment's project and // location. If both this field and `nodeConfig.location` are specified, // this `machineType` must belong to the `nodeConfig.location`; if both are // unspecified, the service will pick a zone in the Compute Engine region // corresponding to the Cloud Composer location, and propagate that choice to // both fields. If exactly one of this field and `nodeConfig.location` is // specified, the location information from the specified field will be // propagated to the unspecified field. // // The `machineTypeId` must not be a [shared-core machine // type](/compute/docs/machine-types#sharedcore). // // If this field is unspecified, the `machineTypeId` defaults // to "n1-standard-1". // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string machine_type = 2; // Optional. The Compute Engine network to be used for machine // communications, specified as a // [relative resource // name](/apis/design/resource_names#relative_resource_name). For example: // "projects/{projectId}/global/networks/{networkId}". // // If unspecified, the "default" network ID in the environment's project is // used. If a [Custom Subnet Network](/vpc/docs/vpc#vpc_networks_and_subnets) // is provided, `nodeConfig.subnetwork` must also be provided. For // [Shared VPC](/vpc/docs/shared-vpc) subnetwork requirements, see // `nodeConfig.subnetwork`. string network = 3; // Optional. The Compute Engine subnetwork to be used for machine // communications, specified as a // [relative resource // name](/apis/design/resource_names#relative_resource_name). For example: // "projects/{projectId}/regions/{regionId}/subnetworks/{subnetworkId}" // // If a subnetwork is provided, `nodeConfig.network` must also be provided, // and the subnetwork must belong to the enclosing environment's project and // location. string subnetwork = 4; // Optional. The disk size in GB used for node VMs. Minimum size is 30GB. // If unspecified, defaults to 100GB. Cannot be updated. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. int32 disk_size_gb = 5; // Optional. The set of Google API scopes to be made available on all // node VMs. If `oauth_scopes` is empty, defaults to // ["https://www.googleapis.com/auth/cloud-platform"]. Cannot be updated. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. repeated string oauth_scopes = 6; // Optional. The Google Cloud Platform Service Account to be used by the node // VMs. If a service account is not specified, the "default" Compute Engine // service account is used. Cannot be updated. string service_account = 7; // Optional. The list of instance tags applied to all node VMs. Tags are used // to identify valid sources or targets for network firewalls. Each tag within // the list must comply with [RFC1035](https://www.ietf.org/rfc/rfc1035.txt). // Cannot be updated. repeated string tags = 8; // Optional. The configuration for controlling how IPs are allocated in the // GKE cluster. IPAllocationPolicy ip_allocation_policy = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. Deploys 'ip-masq-agent' daemon set in the GKE cluster and defines // nonMasqueradeCIDRs equals to pod IP range so IP masquerading is used for // all destination addresses, except between pods traffic. // // See: // https://cloud.google.com/kubernetes-engine/docs/how-to/ip-masquerade-agent bool enable_ip_masq_agent = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. Network Attachment that Cloud Composer environment is connected // to, which provides connectivity with a user's VPC network. Takes precedence // over network and subnetwork settings. If not provided, but network and // subnetwork are defined during environment, it will be provisioned. If not // provided and network and subnetwork are also empty, then connectivity to // user's VPC network is disabled. Network attachment must be provided in // format // projects/{project}/regions/{region}/networkAttachments/{networkAttachment}. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. string composer_network_attachment = 12 [(google.api.field_behavior) = OPTIONAL]; // Optional. The IP range in CIDR notation to use internally by Cloud // Composer. IP addresses are not reserved - and the same range can be used by // multiple Cloud Composer environments. In case of overlap, IPs from this // range will not be accessible in the user's VPC network. Cannot be updated. // If not specified, the default value of '100.64.128.0/20' is used. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. string composer_internal_ipv4_cidr_block = 13 [(google.api.field_behavior) = OPTIONAL]; } // Configuration options for the private GKE cluster in a Cloud Composer // environment. message PrivateClusterConfig { // Optional. If `true`, access to the public endpoint of the GKE cluster is // denied. bool enable_private_endpoint = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The CIDR block from which IPv4 range for GKE master will be // reserved. If left blank, the default value of '172.16.0.0/23' is used. string master_ipv4_cidr_block = 2 [(google.api.field_behavior) = OPTIONAL]; // Output only. The IP range in CIDR notation to use for the hosted master // network. This range is used for assigning internal IP addresses to the GKE // cluster master or set of masters and to the internal load balancer virtual // IP. This range must not overlap with any other ranges in use within the // cluster's network. string master_ipv4_reserved_range = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Configuration options for networking connections in the Composer 2 // environment. message NetworkingConfig { // Represents connection type between Composer environment in Customer // Project and the corresponding Tenant project, from a predefined list // of available connection modes. enum ConnectionType { // No specific connection type was requested, so the environment uses // the default value corresponding to the rest of its configuration. CONNECTION_TYPE_UNSPECIFIED = 0; // Requests the use of VPC peerings for connecting the Customer and Tenant // projects. VPC_PEERING = 1; // Requests the use of Private Service Connect for connecting the Customer // and Tenant projects. PRIVATE_SERVICE_CONNECT = 2; } // Optional. Indicates the user requested specifc connection type between // Tenant and Customer projects. You cannot set networking connection type in // public IP environment. ConnectionType connection_type = 1 [(google.api.field_behavior) = OPTIONAL]; } // The configuration information for configuring a Private IP Cloud Composer // environment. message PrivateEnvironmentConfig { // Optional. If `true`, a Private IP Cloud Composer environment is created. // If this field is set to true, `IPAllocationPolicy.use_ip_aliases` must be // set to true for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. bool enable_private_environment = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. If `true`, builds performed during operations that install Python // packages have only private connectivity to Google services (including // Artifact Registry) and VPC network (if either `NodeConfig.network` and // `NodeConfig.subnetwork` fields or `NodeConfig.composer_network_attachment` // field are specified). If `false`, the builds also have access to the // internet. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. bool enable_private_builds_only = 11 [(google.api.field_behavior) = OPTIONAL]; // Optional. Configuration for the private GKE cluster for a Private IP // Cloud Composer environment. PrivateClusterConfig private_cluster_config = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. The CIDR block from which IP range for web server will be // reserved. Needs to be disjoint from // `private_cluster_config.master_ipv4_cidr_block` and // `cloud_sql_ipv4_cidr_block`. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string web_server_ipv4_cidr_block = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The CIDR block from which IP range in tenant project will be // reserved for Cloud SQL. Needs to be disjoint from // `web_server_ipv4_cidr_block`. string cloud_sql_ipv4_cidr_block = 4 [(google.api.field_behavior) = OPTIONAL]; // Output only. The IP range reserved for the tenant project's App Engine VMs. // // This field is supported for Cloud Composer environments in versions // composer-1.*.*-airflow-*.*.*. string web_server_ipv4_reserved_range = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. The CIDR block from which IP range for Cloud Composer Network in // tenant project will be reserved. Needs to be disjoint from // private_cluster_config.master_ipv4_cidr_block and // cloud_sql_ipv4_cidr_block. // // This field is supported for Cloud Composer environments in versions // composer-2.*.*-airflow-*.*.* and newer. string cloud_composer_network_ipv4_cidr_block = 7 [(google.api.field_behavior) = OPTIONAL]; // Output only. The IP range reserved for the tenant project's Cloud Composer // network. // // This field is supported for Cloud Composer environments in versions // composer-2.*.*-airflow-*.*.* and newer. string cloud_composer_network_ipv4_reserved_range = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. When enabled, IPs from public (non-RFC1918) ranges can be used // for `IPAllocationPolicy.cluster_ipv4_cidr_block` and // `IPAllocationPolicy.service_ipv4_cidr_block`. bool enable_privately_used_public_ips = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. When specified, the environment will use Private Service Connect // instead of VPC peerings to connect to Cloud SQL in the Tenant Project, // and the PSC endpoint in the Customer Project will use an IP address from // this subnetwork. string cloud_composer_connection_subnetwork = 9 [(google.api.field_behavior) = OPTIONAL]; // Optional. Configuration for the network connections configuration in the // environment. NetworkingConfig networking_config = 10 [(google.api.field_behavior) = OPTIONAL]; } // The Kubernetes workloads configuration for GKE cluster associated with the // Cloud Composer environment. Supported for Cloud Composer environments in // versions composer-2.*.*-airflow-*.*.* and newer. message WorkloadsConfig { // Configuration for resources used by Airflow schedulers. message SchedulerResource { // Optional. CPU request and limit for a single Airflow scheduler replica. float cpu = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Memory (GB) request and limit for a single Airflow scheduler // replica. float memory_gb = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Storage (GB) request and limit for a single Airflow scheduler // replica. float storage_gb = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The number of schedulers. int32 count = 4 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for resources used by Airflow web server. message WebServerResource { // Optional. CPU request and limit for Airflow web server. float cpu = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Memory (GB) request and limit for Airflow web server. float memory_gb = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Storage (GB) request and limit for Airflow web server. float storage_gb = 3 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for resources used by Airflow workers. message WorkerResource { // Optional. CPU request and limit for a single Airflow worker replica. float cpu = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Memory (GB) request and limit for a single Airflow worker // replica. float memory_gb = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Storage (GB) request and limit for a single Airflow worker // replica. float storage_gb = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Minimum number of workers for autoscaling. int32 min_count = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. Maximum number of workers for autoscaling. int32 max_count = 5 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for resources used by Airflow triggerers. message TriggererResource { // Optional. The number of triggerers. int32 count = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. CPU request and limit for a single Airflow triggerer replica. float cpu = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Memory (GB) request and limit for a single Airflow triggerer // replica. float memory_gb = 3 [(google.api.field_behavior) = OPTIONAL]; } // Configuration for resources used by Airflow DAG processors. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. message DagProcessorResource { // Optional. CPU request and limit for a single Airflow DAG processor // replica. float cpu = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Memory (GB) request and limit for a single Airflow DAG // processor replica. float memory_gb = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Storage (GB) request and limit for a single Airflow DAG // processor replica. float storage_gb = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. The number of DAG processors. If not provided or set to 0, a // single DAG processor instance will be created. int32 count = 4 [(google.api.field_behavior) = OPTIONAL]; } // Optional. Resources used by Airflow schedulers. SchedulerResource scheduler = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. Resources used by Airflow web server. WebServerResource web_server = 2 [(google.api.field_behavior) = OPTIONAL]; // Optional. Resources used by Airflow workers. WorkerResource worker = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Resources used by Airflow triggerers. TriggererResource triggerer = 4 [(google.api.field_behavior) = OPTIONAL]; // Optional. Resources used by Airflow DAG processors. // // This field is supported for Cloud Composer environments in versions // composer-3.*.*-airflow-*.*.* and newer. DagProcessorResource dag_processor = 5 [(google.api.field_behavior) = OPTIONAL]; } // The Recovery settings of an environment. message RecoveryConfig { // Optional. The configuration for scheduled snapshot creation mechanism. ScheduledSnapshotsConfig scheduled_snapshots_config = 1 [(google.api.field_behavior) = OPTIONAL]; } // The configuration for scheduled snapshot creation mechanism. message ScheduledSnapshotsConfig { // Optional. Whether scheduled snapshots creation is enabled. bool enabled = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The Cloud Storage location for storing automatically created // snapshots. string snapshot_location = 6 [(google.api.field_behavior) = OPTIONAL]; // Optional. The cron expression representing the time when snapshots creation // mechanism runs. This field is subject to additional validation around // frequency of execution. string snapshot_creation_schedule = 3 [(google.api.field_behavior) = OPTIONAL]; // Optional. Time zone that sets the context to interpret // snapshot_creation_schedule. string time_zone = 5 [(google.api.field_behavior) = OPTIONAL]; } // Configuration options for the master authorized networks feature. Enabled // master authorized networks will disallow all external traffic to access // Kubernetes master through HTTPS except traffic from the given CIDR blocks, // Google Compute Engine Public IPs and Google Prod IPs. message MasterAuthorizedNetworksConfig { // CIDR block with an optional name. message CidrBlock { // User-defined name that identifies the CIDR block. string display_name = 1; // CIDR block that must be specified in CIDR notation. string cidr_block = 2; } // Whether or not master authorized networks feature is enabled. bool enabled = 1; // Up to 50 external networks that could access Kubernetes master through // HTTPS. repeated CidrBlock cidr_blocks = 2; } // Configuration for Cloud Data Lineage integration. message CloudDataLineageIntegration { // Optional. Whether or not Cloud Data Lineage integration is enabled. bool enabled = 1 [(google.api.field_behavior) = OPTIONAL]; } // An environment for running orchestration tasks. message Environment { option (google.api.resource) = { type: "composer.googleapis.com/Environment" pattern: "projects/{project}/locations/{location}/environments/{environment}" }; // State of the environment. enum State { // The state of the environment is unknown. STATE_UNSPECIFIED = 0; // The environment is in the process of being created. CREATING = 1; // The environment is currently running and healthy. It is ready for use. RUNNING = 2; // The environment is being updated. It remains usable but cannot receive // additional update requests or be deleted at this time. UPDATING = 3; // The environment is undergoing deletion. It cannot be used. DELETING = 4; // The environment has encountered an error and cannot be used. ERROR = 5; } // The resource name of the environment, in the form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" // // EnvironmentId must start with a lowercase letter followed by up to 63 // lowercase letters, numbers, or hyphens, and cannot end with a hyphen. string name = 1; // Configuration parameters for this environment. EnvironmentConfig config = 2; // Output only. The UUID (Universally Unique IDentifier) associated with this // environment. This value is generated when the environment is created. string uuid = 3; // The current state of the environment. State state = 4; // Output only. The time at which this environment was created. google.protobuf.Timestamp create_time = 5; // Output only. The time at which this environment was last modified. google.protobuf.Timestamp update_time = 6; // Optional. User-defined labels for this environment. // The labels map can contain no more than 64 entries. Entries of the labels // map are UTF8 strings that comply with the following restrictions: // // * Keys must conform to regexp: [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62} // * Values must conform to regexp: [\p{Ll}\p{Lo}\p{N}_-]{0,63} // * Both keys and values are additionally constrained to be <= 128 bytes in // size. map labels = 7; // Output only. Reserved for future use. bool satisfies_pzs = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Reserved for future use. bool satisfies_pzi = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Optional. Storage configuration for this environment. StorageConfig storage_config = 9 [(google.api.field_behavior) = OPTIONAL]; } // Request to check whether image upgrade will succeed. message CheckUpgradeRequest { // Required. The resource name of the environment to check upgrade for, in the // form: // "projects/{projectId}/locations/{locationId}/environments/{environmentId}" string environment = 1 [(google.api.field_behavior) = REQUIRED]; // Optional. The version of the software running in the environment. // This encapsulates both the version of Cloud Composer functionality and the // version of Apache Airflow. It must match the regular expression // `composer-([0-9]+(\.[0-9]+\.[0-9]+(-preview\.[0-9]+)?)?|latest)-airflow-([0-9]+(\.[0-9]+(\.[0-9]+)?)?)`. // When used as input, the server also checks if the provided version is // supported and denies the request for an unsupported version. // // The Cloud Composer portion of the image version is a full // [semantic version](https://semver.org), or an alias in the form of major // version number or `latest`. When an alias is provided, the server replaces // it with the current Cloud Composer version that satisfies the alias. // // The Apache Airflow portion of the image version is a full semantic version // that points to one of the supported Apache Airflow versions, or an alias in // the form of only major or major.minor versions specified. When an alias is // provided, the server replaces it with the latest Apache Airflow version // that satisfies the alias and is supported in the given Cloud Composer // version. // // In all cases, the resolved image version is stored in the same field. // // See also [version // list](/composer/docs/concepts/versioning/composer-versions) and [versioning // overview](/composer/docs/concepts/versioning/composer-versioning-overview). string image_version = 2 [(google.api.field_behavior) = OPTIONAL]; } // Message containing information about the result of an upgrade check // operation. message CheckUpgradeResponse { // Whether there were python modules conflict during image build. enum ConflictResult { // It is unknown whether build had conflicts or not. CONFLICT_RESULT_UNSPECIFIED = 0; // There were python packages conflicts. CONFLICT = 1; // There were no python packages conflicts. NO_CONFLICT = 2; } // Output only. Url for a docker build log of an upgraded image. string build_log_uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Whether build has succeeded or failed on modules conflicts. ConflictResult contains_pypi_modules_conflict = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. Extract from a docker image build log containing information // about pypi modules conflicts. string pypi_conflict_build_log_extract = 3 [(google.api.field_behavior) = OUTPUT_ONLY]; // Composer image for which the build was happening. string image_version = 5; // Pypi dependencies specified in the environment configuration, at the time // when the build was triggered. map pypi_dependencies = 6; } // The configuration setting for Airflow database data retention mechanism. message DataRetentionConfig { // Optional. The retention policy for airflow metadata database. AirflowMetadataRetentionPolicyConfig airflow_metadata_retention_config = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. The configuration settings for task logs retention TaskLogsRetentionConfig task_logs_retention_config = 2 [(google.api.field_behavior) = OPTIONAL]; } // The configuration setting for Task Logs. message TaskLogsRetentionConfig { // The definition of task_logs_storage_mode. enum TaskLogsStorageMode { // This configuration is not specified by the user. TASK_LOGS_STORAGE_MODE_UNSPECIFIED = 0; // Store task logs in Cloud Logging and in the environment's Cloud Storage // bucket. CLOUD_LOGGING_AND_CLOUD_STORAGE = 1; // Store task logs in Cloud Logging only. CLOUD_LOGGING_ONLY = 2; } // Optional. The mode of storage for Airflow workers task logs. TaskLogsStorageMode storage_mode = 2 [(google.api.field_behavior) = OPTIONAL]; } // The policy for airflow metadata database retention. message AirflowMetadataRetentionPolicyConfig { // Describes retention policy. enum RetentionMode { // Default mode doesn't change environment parameters. RETENTION_MODE_UNSPECIFIED = 0; // Retention policy is enabled. RETENTION_MODE_ENABLED = 1; // Retention policy is disabled. RETENTION_MODE_DISABLED = 2; } // Optional. Retention can be either enabled or disabled. RetentionMode retention_mode = 1 [(google.api.field_behavior) = OPTIONAL]; // Optional. How many days data should be retained for. int32 retention_days = 2 [(google.api.field_behavior) = OPTIONAL]; }