File: C:/Users/fred/anaconda3/Library/include/aws/s3/s3_client.h
#ifndef AWS_S3_CLIENT_H
#define AWS_S3_CLIENT_H
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/
#include <aws/auth/signing_config.h>
#include <aws/io/retry_strategy.h>
#include <aws/s3/s3.h>
struct aws_allocator;
struct aws_http_stream;
struct aws_http_message;
struct aws_http_headers;
struct aws_tls_connection_options;
struct aws_input_stream;
struct aws_s3_client;
struct aws_s3_request;
struct aws_s3_meta_request;
struct aws_s3_meta_request_result;
struct aws_uri;
struct aws_string;
/**
* A Meta Request represents a group of generated requests that are being done on behalf of the
* original request. For example, one large GetObject request can be transformed into a series
* of ranged GetObject requests that are executed in parallel to improve throughput.
*
* The aws_s3_meta_request_type is a hint of transformation to be applied.
*/
enum aws_s3_meta_request_type {
/**
* The Default meta request type sends any request to S3 as-is (with no transformation). For example,
* it can be used to pass a CreateBucket request.
*/
AWS_S3_META_REQUEST_TYPE_DEFAULT,
/**
* The GetObject request will be split into a series of ranged GetObject requests that are
* executed in parallel to improve throughput, when possible.
*/
AWS_S3_META_REQUEST_TYPE_GET_OBJECT,
/**
* The PutObject request will be split into MultiPart uploads that are executed in parallel
* to improve throughput, when possible.
*/
AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
/**
* The CopyObject meta request performs a multi-part copy
* using multiple S3 UploadPartCopy requests in parallel, or bypasses
* a CopyObject request to S3 if the object size is not large enough for
* a multipart upload.
*/
AWS_S3_META_REQUEST_TYPE_COPY_OBJECT,
AWS_S3_META_REQUEST_TYPE_MAX,
};
/**
* Invoked to provide response headers received during execution of the meta request, both for
* success and error HTTP status codes.
*
* Return AWS_OP_SUCCESS to continue processing the request.
* Return AWS_OP_ERR to indicate failure and cancel the request.
*/
typedef int(aws_s3_meta_request_headers_callback_fn)(
struct aws_s3_meta_request *meta_request,
const struct aws_http_headers *headers,
int response_status,
void *user_data);
/**
* Invoked to provide the response body as it is received.
*
* Note: If you set `enable_read_backpressure` true on the S3 client,
* you must maintain the flow-control window.
* The flow-control window shrinks as you receive body data via this callback.
* Whenever the flow-control window reaches 0 you will stop downloading data.
* Use aws_s3_meta_request_increment_read_window() to increment the window and keep data flowing.
* Maintain a larger window to keep up a high download throughput,
* parts cannot download in parallel unless the window is large enough to hold multiple parts.
* Maintain a smaller window to limit the amount of data buffered in memory.
*
* If `manual_window_management` is false, you do not need to maintain the flow-control window.
* No back-pressure is applied and data arrives as fast as possible.
*
* Return AWS_OP_SUCCESS to continue processing the request.
* Return AWS_OP_ERR to indicate failure and cancel the request.
*/
typedef int(aws_s3_meta_request_receive_body_callback_fn)(
/* The meta request that the callback is being issued for. */
struct aws_s3_meta_request *meta_request,
/* The body data for this chunk of the object. */
const struct aws_byte_cursor *body,
/* The byte index of the object that this refers to. For example, for an HTTP message that has a range header, the
first chunk received will have a range_start that matches the range header's range-start.*/
uint64_t range_start,
/* User data specified by aws_s3_meta_request_options.*/
void *user_data);
/**
* Invoked when the entire meta request execution is complete.
*/
typedef void(aws_s3_meta_request_finish_fn)(
struct aws_s3_meta_request *meta_request,
const struct aws_s3_meta_request_result *meta_request_result,
void *user_data);
/**
* Information sent in the meta_request progress callback.
*/
struct aws_s3_meta_request_progress {
/* Bytes transferred since the previous progress update */
uint64_t bytes_transferred;
/* Length of the entire meta request operation */
uint64_t content_length;
};
/**
* Invoked to report progress of multi-part upload and copy object requests.
*/
typedef void(aws_s3_meta_request_progress_fn)(
struct aws_s3_meta_request *meta_request,
const struct aws_s3_meta_request_progress *progress,
void *user_data);
typedef void(aws_s3_meta_request_shutdown_fn)(void *user_data);
typedef void(aws_s3_client_shutdown_complete_callback_fn)(void *user_data);
enum aws_s3_meta_request_tls_mode {
AWS_MR_TLS_ENABLED,
AWS_MR_TLS_DISABLED,
};
enum aws_s3_meta_request_compute_content_md5 {
AWS_MR_CONTENT_MD5_DISABLED,
AWS_MR_CONTENT_MD5_ENABLED,
};
enum aws_s3_checksum_algorithm {
AWS_SCA_NONE = 0,
AWS_SCA_INIT,
AWS_SCA_CRC32C = AWS_SCA_INIT,
AWS_SCA_CRC32,
AWS_SCA_SHA1,
AWS_SCA_SHA256,
AWS_SCA_END = AWS_SCA_SHA256,
};
enum aws_s3_checksum_location {
AWS_SCL_NONE = 0,
AWS_SCL_HEADER,
AWS_SCL_TRAILER,
};
/* Keepalive properties are TCP only.
* If interval or timeout are zero, then default values are used.
*/
struct aws_s3_tcp_keep_alive_options {
uint16_t keep_alive_interval_sec;
uint16_t keep_alive_timeout_sec;
/* If set, sets the number of keep alive probes allowed to fail before the connection is considered
* lost. If zero OS defaults are used. On Windows, this option is meaningless until Windows 10 1703.*/
uint16_t keep_alive_max_failed_probes;
};
/* Options for a new client. */
struct aws_s3_client_config {
/* When set, this will cap the number of active connections. When 0, the client will determine this value based on
* throughput_target_gbps. (Recommended) */
uint32_t max_active_connections_override;
/* Region that the S3 bucket lives in. */
struct aws_byte_cursor region;
/* Client bootstrap used for common staples such as event loop group, host resolver, etc.. s*/
struct aws_client_bootstrap *client_bootstrap;
/* How tls should be used while performing the request
* If this is ENABLED:
* If tls_connection_options is not-null, then those tls options will be used
* If tls_connection_options is NULL, then default tls options will be used
* If this is DISABLED:
* No tls options will be used, regardless of tls_connection_options value.
*/
enum aws_s3_meta_request_tls_mode tls_mode;
/* TLS Options to be used for each connection, if tls_mode is ENABLED. When compiling with BYO_CRYPTO, and tls_mode
* is ENABLED, this is required. Otherwise, this is optional. */
struct aws_tls_connection_options *tls_connection_options;
/* Signing options to be used for each request. Specify NULL to not sign requests. */
struct aws_signing_config_aws *signing_config;
/* Size of parts the files will be downloaded or uploaded in. */
size_t part_size;
/* If the part size needs to be adjusted for service limits, this is the maximum size it will be adjusted to.. */
size_t max_part_size;
/* Throughput target in Gbps that we are trying to reach. */
double throughput_target_gbps;
/* Retry strategy to use. If NULL, a default retry strategy will be used. */
struct aws_retry_strategy *retry_strategy;
/**
* TODO: move MD5 config to checksum config.
* For multi-part upload, content-md5 will be calculated if the AWS_MR_CONTENT_MD5_ENABLED is specified
* or initial request has content-md5 header.
* For single-part upload, keep the content-md5 in the initial request unchanged. */
enum aws_s3_meta_request_compute_content_md5 compute_content_md5;
/* Callback and associated user data for when the client has completed its shutdown process. */
aws_s3_client_shutdown_complete_callback_fn *shutdown_callback;
void *shutdown_callback_user_data;
/**
* Optional.
* Proxy configuration for http connection.
*/
struct aws_http_proxy_options *proxy_options;
/**
* Optional.
* Configuration for fetching proxy configuration from environment.
* By Default proxy_ev_settings.aws_http_proxy_env_var_type is set to AWS_HPEV_ENABLE which means read proxy
* configuration from environment.
* Only works when proxy_options is not set. If both are set, configuration from proxy_options is used.
*/
struct proxy_env_var_settings *proxy_ev_settings;
/**
* Optional.
* If set to 0, default value is used.
*/
uint32_t connect_timeout_ms;
/**
* Optional.
* Set keepalive to periodically transmit messages for detecting a disconnected peer.
*/
struct aws_s3_tcp_keep_alive_options *tcp_keep_alive_options;
/**
* Optional.
* Configuration options for connection monitoring.
* If the transfer speed falls below the specified minimum_throughput_bytes_per_second, the operation is aborted.
*/
struct aws_http_connection_monitoring_options *monitoring_options;
/**
* Enable backpressure and prevent response data from downloading faster than you can handle it.
*
* If false (default), no backpressure is applied and data will download as fast as possible.
*
* If true, each meta request has a flow-control window that shrinks as
* response body data is downloaded (headers do not affect the window).
* `initial_read_window` determines the starting size of each meta request's window.
* You will stop downloading data whenever the flow-control window reaches 0
* You must call aws_s3_meta_request_increment_read_window() to keep data flowing.
*
* WARNING: This feature is experimental.
* Currently, backpressure is only applied to GetObject requests which are split into multiple parts,
* and you may still receive some data after the window reaches 0.
*/
bool enable_read_backpressure;
/**
* The starting size of each meta request's flow-control window, in bytes.
* Ignored unless `enable_read_backpressure` is true.
*/
size_t initial_read_window;
};
struct aws_s3_checksum_config {
/**
* The location of client added checksum header.
*
* If AWS_SCL_NONE. No request payload checksum will be add and calculated.
*
* If AWS_SCL_HEADER, the checksum will be calculated by client and added related header to the request sent.
*
* If AWS_SCL_TRAILER, the payload will be aws_chunked encoded, The checksum will be calculate while reading the
* payload by client. Related header will be added to the trailer part of the encoded payload. Note the payload of
* the original request cannot be aws-chunked encoded already. Otherwise, error will be raised.
*/
enum aws_s3_checksum_location location;
/**
* The checksum algorithm used.
* Must be set if location is not AWS_SCL_NONE. Must be AWS_SCA_NONE if location is AWS_SCL_NONE.
*/
enum aws_s3_checksum_algorithm checksum_algorithm;
/**
* Enable checksum mode header will be attached to get requests, this will tell s3 to send back checksums headers if
* they exist. Calculate the corresponding checksum on the response bodies. The meta request will finish with a did
* validate field and set the error code to AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH if the calculated
* checksum, and checksum found in the response header do not match.
*/
bool validate_response_checksum;
/**
* Optional array of `enum aws_s3_checksum_algorithm`.
*
* Ignored when validate_response_checksum is not set.
* If not set all the algorithms will be selected as default behavior.
* Owned by the caller.
*
* The list of algorithms for user to pick up when validate the checksum. Client will pick up the algorithm from the
* list with the priority based on performance, and the algorithm sent by server. The priority based on performance
* is [CRC32C, CRC32, SHA1, SHA256].
*
* If the response checksum was validated by client, the result will indicate which algorithm was picked.
*/
struct aws_array_list *validate_checksum_algorithms;
};
/* Options for a new meta request, ie, file transfer that will be handled by the high performance client. */
struct aws_s3_meta_request_options {
/* TODO: The meta request options cannot control the request to be split or not. Should consider to add one */
/* The type of meta request we will be trying to accelerate. */
enum aws_s3_meta_request_type type;
/* Signing options to be used for each request created for this meta request. If NULL, options in the client will
* be used. If not NULL, these options will override the client options. */
const struct aws_signing_config_aws *signing_config;
/* Initial HTTP message that defines what operation we are doing. */
struct aws_http_message *message;
/**
* Optional.
* if set, the flexible checksum will be performed by client based on the config.
*/
const struct aws_s3_checksum_config *checksum_config;
/* User data for all callbacks. */
void *user_data;
/**
* Optional.
* Invoked to provide response headers received during execution of the meta request.
* See `aws_s3_meta_request_headers_callback_fn`.
*/
aws_s3_meta_request_headers_callback_fn *headers_callback;
/**
* Invoked to provide the response body as it is received.
* See `aws_s3_meta_request_receive_body_callback_fn`.
*/
aws_s3_meta_request_receive_body_callback_fn *body_callback;
/**
* Invoked when the entire meta request execution is complete.
* See `aws_s3_meta_request_finish_fn`.
*/
aws_s3_meta_request_finish_fn *finish_callback;
/* Callback for when the meta request has completely cleaned up. */
aws_s3_meta_request_shutdown_fn *shutdown_callback;
/**
* Invoked to report progress of the meta request execution.
* Currently, the progress callback is invoked only for the CopyObject meta request type.
* TODO: support this callback for all the types of meta requests
* See `aws_s3_meta_request_progress_fn`
*/
aws_s3_meta_request_progress_fn *progress_callback;
/**
* Optional.
* The endpoint for the meta request to connect to. If not set, then tls settings from client will
* determine the port, and host header from `message` will determine the host for the connection.
* Note: must match the host header from `message`
*/
struct aws_uri *endpoint;
/**
* Optional.
* For meta requests that support pause/resume (e.g. PutObject), the resume token returned by
* aws_s3_meta_request_pause() can be provided here.
* Note: If PutObject request specifies a checksum algorithm, client will calculate checksums while skipping parts
* from the buffer and compare them them to previously uploaded part checksums.
*/
const struct aws_byte_cursor *resume_token;
};
/* Result details of a meta request.
*
* If error_code is AWS_ERROR_SUCCESS, then response_status will match the response_status passed earlier by the header
* callback and error_response_headers and error_response_body will be NULL.
*
* If error_code is equal to AWS_ERROR_S3_INVALID_RESPONSE_STATUS, then error_response_headers, error_response_body, and
* response_status will be populated by the failed request.
*
* For all other error codes, response_status will be 0, and the error_response variables will be NULL.
*/
struct aws_s3_meta_request_result {
/* HTTP Headers for the failed request that triggered finish of the meta request. NULL if no request failed. */
struct aws_http_headers *error_response_headers;
/* Response body for the failed request that triggered finishing of the meta request. NUll if no request failed.*/
struct aws_byte_buf *error_response_body;
/* Response status of the failed request or of the entire meta request. */
int response_status;
/* Only set for GET request.
* Was the server side checksum compared against a calculated checksum of the response body. This may be false
* even if validate_get_response_checksum was set because the object was uploaded without a checksum, or was
* uploaded as a multipart object.
*
* If the object to get is multipart object, the part checksum MAY be validated if the part size to get matches the
* part size uploaded. In that case, if any part mismatch the checksum received, the meta request will failed with
* checksum mismatch. However, even if the parts checksum were validated, this will NOT be set to true, as the
* checksum for the whole meta request was NOT validated.
**/
bool did_validate;
/* algorithm used to validate checksum */
enum aws_s3_checksum_algorithm validation_algorithm;
/* Final error code of the meta request. */
int error_code;
};
AWS_EXTERN_C_BEGIN
AWS_S3_API
struct aws_s3_client *aws_s3_client_new(
struct aws_allocator *allocator,
const struct aws_s3_client_config *client_config);
AWS_S3_API
void aws_s3_client_acquire(struct aws_s3_client *client);
AWS_S3_API
void aws_s3_client_release(struct aws_s3_client *client);
AWS_S3_API
struct aws_s3_meta_request *aws_s3_client_make_meta_request(
struct aws_s3_client *client,
const struct aws_s3_meta_request_options *options);
/**
* Increment the flow-control window, so that response data continues downloading.
*
* If the client was created with `enable_read_backpressure` set true,
* each meta request has a flow-control window that shrinks as response
* body data is downloaded (headers do not affect the size of the window).
* The client's `initial_read_window` determines the starting size of each meta request's window.
* If a meta request's flow-control window reaches 0, no further data will be downloaded.
* If the `initial_read_window` is 0, the request will not start until the window is incremented.
* Maintain a larger window to keep up a high download throughput,
* parts cannot download in parallel unless the window is large enough to hold multiple parts.
* Maintain a smaller window to limit the amount of data buffered in memory.
*
* If `enable_read_backpressure` is false this call will have no effect,
* no backpressure is being applied and data is being downloaded as fast as possible.
*
* WARNING: This feature is experimental.
* Currently, backpressure is only applied to GetObject requests which are split into multiple parts,
* and you may still receive some data after the window reaches 0.
*/
AWS_S3_API
void aws_s3_meta_request_increment_read_window(struct aws_s3_meta_request *meta_request, uint64_t bytes);
AWS_S3_API
void aws_s3_meta_request_cancel(struct aws_s3_meta_request *meta_request);
/**
* In order to pause an ongoing upload, call aws_s3_meta_request_pause(). It will return a resume token that can be
* persisted and used to resume the upload. To resume an upload that was paused, supply the resume token in the meta
* request options structure member aws_s3_meta_request_options.persistable_state.
* The upload can be resumed either from the same client or a different one.
* Resume token is opaque with format varying based on operation.
* Clients should not parse the token. For format details refer to pause method comments for a given operation.
* Resume token will be set to null in case of failures.
* @param meta_request pointer to the aws_s3_meta_request of the upload to be paused
* @param resume_token outputs the json string with the state that can be used to resume the operation.
* @return
*/
AWS_S3_API
int aws_s3_meta_request_pause(struct aws_s3_meta_request *meta_request, struct aws_string **out_resume_token);
AWS_S3_API
void aws_s3_meta_request_acquire(struct aws_s3_meta_request *meta_request);
AWS_S3_API
void aws_s3_meta_request_release(struct aws_s3_meta_request *meta_request);
AWS_S3_API
void aws_s3_init_default_signing_config(
struct aws_signing_config_aws *signing_config,
const struct aws_byte_cursor region,
struct aws_credentials_provider *credentials_provider);
AWS_EXTERN_C_END
#endif /* AWS_S3_CLIENT_H */