Add Connector proto

flyteidl2/core/metrics.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "google/protobuf/struct.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// ExecutionMetrics is a collection of metrics that are collected during the execution of a Flyte task.
+message ExecutionMetricResult {
+  // The metric this data represents. e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG.
+  string metric = 1;
+
+  // The result data in prometheus range query result format
+  // https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats.
+  // This may include multiple time series, differentiated by their metric labels.
+  // Start time is greater of (execution attempt start, 48h ago)
+  // End time is lesser of (execution attempt end, now)
+  google.protobuf.Struct data = 2;
+}

flyteidl2/core/security.proto

@@ -50,6 +50,19 @@ message Secret {
   string env_var = 5;
 }
 
+message Connection {
+  // The task type that the connection is used for.
+  string task_type = 1;
+  // The credentials to use for the connection, such as API keys, OAuth2 tokens, etc.
+  // The key is the name of the secret, and it's defined in the flytekit.
+  // flytekit uses the key to locate the desired secret within the map.
+  map<string, string> secrets = 2;
+
+  // The configuration to use for the connection, such as the endpoint, account name, etc.
+  // The key is the name of the config, and it's defined in the flytekit.
+  map<string, string> configs = 3;
+}
+
 // OAuth2Client encapsulates OAuth2 Client Credentials to be used when making calls on behalf of that task.
 message OAuth2Client {
   // client_id is the public id for the client to use. The system will not perform any pre-auth validation that the

flyteidl2/plugins/connector.proto

@@ -0,0 +1,241 @@
+syntax = "proto3";
+
+package flyteidl2.plugins;
+
+import "flyteidl2/core/execution.proto";
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/literals.proto";
+import "flyteidl2/core/metrics.proto";
+import "flyteidl2/core/security.proto";
+import "flyteidl2/core/tasks.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl/admin";
+
+// Represents a subset of runtime task execution metadata that are relevant to external plugins.
+message TaskExecutionMetadata {
+  // ID of the task execution
+
+  core.TaskExecutionIdentifier task_execution_id = 1;
+  // k8s namespace where the task is executed in
+  string namespace = 2;
+  // Labels attached to the task execution
+  map<string, string> labels = 3;
+  // Annotations attached to the task execution
+  map<string, string> annotations = 4;
+  // k8s service account associated with the task execution
+  string k8s_service_account = 5;
+  // Environment variables attached to the task execution
+  map<string, string> environment_variables = 6;
+  // Represents the maximum number of attempts allowed for a task.
+  // If a task fails, it can be retried up to this maximum number of attempts.
+  int32 max_attempts = 7;
+  // Indicates whether the task execution can be interrupted.
+  // If set to true, the task can be stopped before completion.
+  bool interruptible = 8;
+  // Specifies the threshold for failure count at which the interruptible property
+  // will take effect. If the number of consecutive task failures exceeds this threshold,
+  // interruptible behavior will be activated.
+  int32 interruptible_failure_threshold = 9;
+  // Identity of user running this task execution
+  core.Identity identity = 10;
+}
+
+// Represents a request structure to create task.
+message CreateTaskRequest {
+  // The inputs required to start the execution. All required inputs must be
+  // included in this map. If not required and not provided, defaults apply.
+  // +optional
+  core.LiteralMap inputs = 1;
+  // Template of the task that encapsulates all the metadata of the task.
+  core.TaskTemplate template = 2;
+  // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+  string output_prefix = 3;
+  // subset of runtime task execution metadata.
+  TaskExecutionMetadata task_execution_metadata = 4;
+  // Connection (secret and config) required by the connector.
+  // Connector will use the secret and config in the taskTemplate if it's None.
+  // +optional
+  core.Connection connection = 5;
+}
+
+// Represents a create response structure.
+message CreateTaskResponse {
+  // ResourceMeta is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+  bytes resource_meta = 1;
+}
+
+message CreateRequestHeader {
+  // Template of the task that encapsulates all the metadata of the task.
+  core.TaskTemplate template = 1;
+  // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+  string output_prefix = 2;
+  // subset of runtime task execution metadata.
+  TaskExecutionMetadata task_execution_metadata = 3;
+  // MaxDatasetSizeBytes is the maximum size of the dataset that can be generated by the task.
+  int64 max_dataset_size_bytes = 4;
+  // Connection (secret and config) required by the connector.
+  // Connector will use the secret and config in the taskTemplate if it's None.
+  // +optional
+  core.Connection connection = 5;
+}
+
+message ExecuteTaskSyncRequest {
+  oneof part {
+    CreateRequestHeader header = 1;
+    core.LiteralMap inputs = 2;
+  }
+}
+
+message ExecuteTaskSyncResponseHeader {
+  Resource resource = 1;
+}
+
+message ExecuteTaskSyncResponse {
+  // Metadata is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+  // Resource is for synchronous task execution.
+  oneof res {
+    ExecuteTaskSyncResponseHeader header = 1;
+    core.LiteralMap outputs = 2;
+  }
+}
+
+// A message used to fetch a job resource from flyte connector server.
+message GetTaskRequest {
+  // Metadata about the resource to be pass to the connector.
+  bytes resource_meta = 1;
+  // A predefined yet extensible Task type identifier.
+  TaskCategory task_category = 2;
+  // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+  string output_prefix = 3;
+  // Connection (secret and config) required by the connector.
+  // Connector will use the secret and config in the taskTemplate if it's None.
+  // +optional
+  core.Connection connection = 4;
+}
+
+// Response to get an individual task resource.
+message GetTaskResponse {
+  Resource resource = 1;
+}
+
+message Resource {
+  // The outputs of the execution. It's typically used by sql task. connector service will create a
+  // Structured dataset pointing to the query result table.
+  // +optional
+  core.LiteralMap outputs = 1;
+  // A descriptive message for the current state. e.g. waiting for cluster.
+  string message = 2;
+  // log information for the task execution.
+  repeated core.TaskLog log_links = 3;
+  // The phase of the execution is used to determine the phase of the plugin's execution.
+  core.TaskExecution.Phase phase = 4;
+  // Custom data specific to the connector.
+  google.protobuf.Struct custom_info = 5;
+}
+
+// A message used to delete a task.
+message DeleteTaskRequest {
+  // Metadata about the resource to be pass to the connector.
+  bytes resource_meta = 1;
+  // A predefined yet extensible Task type identifier.
+  TaskCategory task_category = 2;
+  // Connection (secret and config) required by the connector.
+  // Connector will use the secret and config in the taskTemplate if it's None.
+  // +optional
+  core.Connection connection = 3;
+}
+
+// Response to delete a task.
+message DeleteTaskResponse {}
+
+// A message containing the connector metadata.
+message Connector {
+  // Name is the developer-assigned name of the connector.
+  string name = 1;
+  // Supported_task_categories are the categories of the tasks that the connector can handle.
+  repeated TaskCategory supported_task_categories = 2;
+}
+
+message TaskCategory {
+  // The name of the task type.
+  string name = 1;
+  // The version of the task type.
+  int32 version = 2;
+}
+
+// A request to get an connector.
+message GetConnectorRequest {
+  // The name of the connector.
+  string name = 1;
+}
+
+// A response containing an connector.
+message GetConnectorResponse {
+  Connector connector = 1;
+}
+
+// A request to list all connectors.
+message ListConnectorsRequest {}
+
+// A response containing a list of connectors.
+message ListConnectorsResponse {
+  repeated Connector connectors = 1;
+}
+
+// A request to get the metrics from a task execution.
+message GetTaskMetricsRequest {
+  // Metadata is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+  bytes resource_meta = 1;
+  // The metrics to query. If empty, will return a default set of metrics.
+  // e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG
+  repeated string queries = 2;
+  // Start timestamp, inclusive.
+  google.protobuf.Timestamp start_time = 3;
+  // End timestamp, inclusive..
+  google.protobuf.Timestamp end_time = 4;
+  // Query resolution step width in duration format or float number of seconds.
+  google.protobuf.Duration step = 5;
+  // A predefined yet extensible Task type identifier.
+  TaskCategory task_category = 6;
+}
+
+// A response containing a list of metrics for a task execution.
+message GetTaskMetricsResponse {
+  // The execution metric results.
+  repeated core.ExecutionMetricResult results = 1;
+}
+
+// A request to get the log from a task execution.
+message GetTaskLogsRequest {
+  // Metadata is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+  bytes resource_meta = 1;
+  // Number of lines to return.
+  uint64 lines = 2;
+  // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+  // in a query. If there are no more results, this value will be empty.
+  string token = 3;
+  // A predefined yet extensible Task type identifier.
+  TaskCategory task_category = 4;
+}
+
+message GetTaskLogsResponseHeader {
+  // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+  // in a query. If there are no more results, this value will be empty.
+  string token = 1;
+}
+
+message GetTaskLogsResponseBody {
+  // The execution log results.
+  repeated string results = 1;
+}
+
+// A response containing the logs for a task execution.
+message GetTaskLogsResponse {
+  oneof part {
+    GetTaskLogsResponseHeader header = 1;
+    GetTaskLogsResponseBody body = 2;
+  }
+}