Smaws_Client_Glue.Typestype nonrec metric_counts = Smaws_Lib.CoreTypes.Int64.ttype nonrec version_id = Smaws_Lib.CoreTypes.Int64.ttype nonrec xml_classifier = {row_tag : row_tag option;The XML tag designating the element that contains each record in an XML document being parsed. This can't identify a self-closing element (closed by />). An empty row element that contains only attributes can be parsed as long as it ends with a closing tag (for example,
is okay, but
is not).
*)version : version_id option;The version of this classifier.
*)last_updated : timestamp option;The time that this classifier was last updated.
*)creation_time : timestamp option;The time that this classifier was registered.
*)classification : classification;An identifier of the data format that the classifier matches.
*)name : name_string;The name of the classifier.
*)}A classifier for XML content.
type nonrec workflow_run_properties = (id_string * generic_string) listtype nonrec workflow_run_statistics = {waiting_actions : integer_value option;Indicates the count of job runs in WAITING state in the workflow run.
*)errored_actions : integer_value option;Indicates the count of job runs in the ERROR state in the workflow run.
*)running_actions : integer_value option;Total number Actions in running state.
*)succeeded_actions : integer_value option;Total number of Actions that have succeeded.
*)stopped_actions : integer_value option;Total number of Actions that have stopped.
*)failed_actions : integer_value option;Total number of Actions that have failed.
*)timeout_actions : integer_value option;Total number of Actions that timed out.
*)total_actions : integer_value option;Total number of Actions in the workflow run.
*)}Workflow run statistics provides statistics about the workflow run.
type nonrec generic_map = (generic_string * generic_string) listtype nonrec notification_property = {notify_delay_after : notify_delay_after option;After a job run starts, the number of minutes to wait before sending a job run delay notification.
*)}Specifies configuration properties of a notification.
type nonrec action = {crawler_name : name_string option;The name of the crawler to be used with this action.
*)notification_property : notification_property option;Specifies configuration properties of a job run notification.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this action.
timeout : timeout option;The JobRun timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status. This overrides the timeout value set in the parent job.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)arguments : generic_map option;The job arguments used when this trigger fires. For this job run, they replace the default arguments set in the job definition itself.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the key-value pairs that Glue consumes to set up your job, see the Special Parameters Used by Glue topic in the developer guide.
*)job_name : name_string option;The name of a job to be run.
*)}Defines an action to be initiated by a trigger.
type nonrec action_list = action listtype nonrec condition = {crawl_state : crawl_state option;The state of the crawler to which this condition applies.
*)crawler_name : name_string option;The name of the crawler to which this condition applies.
*)state : job_run_state option;The condition state. Currently, the only job states that a trigger can listen for are SUCCEEDED, STOPPED, FAILED, and TIMEOUT. The only crawler states that a trigger can listen for are SUCCEEDED, FAILED, and CANCELLED.
job_name : name_string option;The name of the job whose JobRuns this condition applies to, and on which this trigger waits.
logical_operator : logical_operator option;A logical operator.
*)}Defines a condition under which a trigger fires.
type nonrec condition_list = condition listtype nonrec predicate = {conditions : condition_list option;A list of the conditions that determine when the trigger will fire.
*)logical : logical option;An optional field if only one condition is listed. If multiple conditions are listed, then this field is required.
*)}Defines the predicate of the trigger, which determines when it fires.
type nonrec event_batching_condition = {batch_window : batch_window option;Window of time in seconds after which EventBridge event trigger fires. Window starts when first event is received.
*)batch_size : batch_size;Number of events that must be received from Amazon EventBridge before EventBridge event trigger fires.
*)}Batch condition that must be met (specified number of events received or batch time window expired) before EventBridge event trigger fires.
type nonrec trigger = {event_batching_condition : event_batching_condition option;Batch condition that must be met (specified number of events received or batch time window expired) before EventBridge event trigger fires.
*)predicate : predicate option;The predicate of this trigger, which defines when it will fire.
*)actions : action_list option;The actions initiated by this trigger.
*)schedule : generic_string option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
description : description_string option;A description of this trigger.
*)state : trigger_state option;The current state of the trigger.
*)type_ : trigger_type option;The type of trigger that this is.
*)id : id_string option;Reserved for future use.
*)workflow_name : name_string option;The name of the workflow associated with the trigger.
*)name : name_string option;The name of the trigger.
*)}Information about a specific trigger.
type nonrec trigger_node_details = {trigger : trigger option;The information of the trigger represented by the trigger node.
*)}The details of a Trigger node present in the workflow.
type nonrec predecessor = {run_id : id_string option;The job-run ID of the predecessor job run.
*)job_name : name_string option;The name of the job definition used by the predecessor job run.
*)}A job run that was used in the predicate of a conditional trigger that triggered this job run.
type nonrec predecessor_list = predecessor listtype nonrec job_run = {execution_role_session_policy : orchestration_policy_json_string option;This inline session policy to the StartJobRun API allows you to dynamically restrict the permissions of the specified execution role for the scope of the job, without requiring the creation of additional IAM roles.
*)state_detail : orchestration_message_string option;This field holds details that pertain to the state of a job run. The field is nullable.
For example, when a job run is in a WAITING state as a result of job run queuing, the field has the reason why the job run is in that state.
*)profile_name : name_string option;The name of an Glue usage profile associated with the job run.
*)maintenance_window : maintenance_window option;This field specifies a day of the week and hour for a maintenance window for streaming jobs. Glue periodically performs maintenance activities. During these maintenance windows, Glue will need to restart your streaming jobs.
Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.
*)execution_class : execution_class option;Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.
The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.
Only jobs with Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX. The flexible execution class is available for Spark jobs.
dpu_seconds : nullable_double option;This field can be set for either job runs with execution class FLEX or when Auto Scaling is enabled, and represents the total time each executor ran during the lifecycle of a job run in seconds, multiplied by a DPU factor (1 for G.1X, 2 for G.2X, or 0.25 for G.025X workers). This value may be different than the executionEngineRuntime * MaxCapacity as in the case of Auto Scaling jobs, as the number of executors running at a given time may be less than the MaxCapacity. Therefore, it is possible that the value of DPUSeconds is less than executionEngineRuntime * MaxCapacity.
glue_version : glue_version_string option;In Spark jobs, GlueVersion determines the versions of Apache Spark and Python that Glue available in a job. The Python version indicates the version supported for jobs of type Spark.
Ray jobs should set GlueVersion to 4.0 or greater. However, the versions of Ray, Python and additional libraries available in your Ray job are determined by the Runtime parameter of the Job command.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
Jobs that are created without specifying a Glue version default to Glue 5.1.
*)notification_property : notification_property option;Specifies configuration properties of a job run notification.
*)log_group_name : generic_string option;The name of the log group for secure logging that can be server-side encrypted in Amazon CloudWatch using KMS. This name can be /aws-glue/jobs/, in which case the default encryption is NONE. If you add a role name and SecurityConfiguration name (in other words, /aws-glue/jobs-yourRoleName-yourSecurityConfigurationName/), then that security configuration is used to encrypt the log group.
security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this job run.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a job runs.
worker_type : worker_type option;The type of predefined worker that is allocated when a job runs. Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.max_capacity : nullable_double option;For Glue version 1.0 or earlier jobs, using the standard worker type, the number of Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
For Glue version 2.0+ jobs, you cannot specify a Maximum capacity. Instead, you should specify a Worker type and the Number of workers.
Do not set MaxCapacity if using WorkerType and NumberOfWorkers.
The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL job:
JobCommand.Name="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.JobCommand.Name="glueetl") or Apache Spark streaming ETL job (JobCommand.Name="gluestreaming"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.timeout : timeout option;The JobRun timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status. This value overrides the timeout value set in the parent job.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)execution_time : execution_time option;The amount of time (in seconds) that the job run consumed resources.
*)allocated_capacity : integer_value option;This field is deprecated. Use MaxCapacity instead.
The number of Glue data processing units (DPUs) allocated to this JobRun. From 2 to 100 DPUs can be allocated; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
*)predecessor_runs : predecessor_list option;A list of predecessors to this job run.
*)error_message : error_string option;An error message associated with this job run.
*)arguments : generic_map option;The job arguments associated with this run. For this job run, they replace the default arguments set in the job definition itself.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Secrets Manager or other secret management mechanism if you intend to keep them within the Job.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the arguments you can provide to this field when configuring Spark jobs, see the Special Parameters Used by Glue topic in the developer guide.
For information about the arguments you can provide to this field when configuring Ray jobs, see Using job parameters in Ray jobs in the developer guide.
*)job_run_state : job_run_state option;The current state of the job run. For more information about the statuses of jobs that have terminated abnormally, see Glue Job Run Statuses.
*)completed_on : timestamp_value option;The date and time that this job run completed.
*)last_modified_on : timestamp_value option;The last time that this job run was modified.
*)started_on : timestamp_value option;The date and time at which this job run was started.
*)job_run_queuing_enabled : nullable_boolean option;Specifies whether job run queuing is enabled for the job run.
A value of true means job run queuing is enabled for the job run. If false or not populated, the job run will not be considered for queueing.
*)job_mode : job_mode option;A mode that describes how a job was created. Valid values are:
SCRIPT - The job was created using the Glue Studio script editor.VISUAL - The job was created using the Glue Studio visual editor.NOTEBOOK - The job was created using an interactive sessions notebook.When the JobMode field is missing or null, SCRIPT is assigned as the default value.
job_name : name_string option;The name of the job definition being used in this run.
*)trigger_name : name_string option;The name of the trigger that started this job run.
*)previous_run_id : id_string option;The ID of the previous run of this job. For example, the JobRunId specified in the StartJobRun action.
attempt : attempt_count option;The number of the attempt to run this job.
*)id : id_string option;The ID of this job run.
*)}Contains information about a job run.
type nonrec job_run_list = job_run listtype nonrec job_node_details = {job_runs : job_run_list option;The information for the job runs represented by the job node.
*)}The details of a Job node present in the workflow.
type nonrec crawl = {log_stream : log_stream option;The log stream associated with the crawl.
*)log_group : log_group option;The log group associated with the crawl.
*)error_message : description_string option;The error message associated with the crawl.
*)completed_on : timestamp_value option;The date and time on which the crawl completed.
*)started_on : timestamp_value option;The date and time on which the crawl started.
*)state : crawl_state option;The state of the crawler.
*)}The details of a crawl in the workflow.
type nonrec crawl_list = crawl listtype nonrec crawler_node_details = {crawls : crawl_list option;A list of crawls represented by the crawl node.
*)}The details of a Crawler node present in the workflow.
type nonrec node = {crawler_details : crawler_node_details option;Details of the crawler when the node represents a crawler.
*)job_details : job_node_details option;Details of the Job when the node represents a Job.
*)trigger_details : trigger_node_details option;Details of the Trigger when the node represents a Trigger.
*)unique_id : name_string option;The unique Id assigned to the node within the workflow.
*)name : name_string option;The name of the Glue component represented by the node.
*)type_ : node_type option;The type of Glue component represented by the node.
*)}A node represents an Glue component (trigger, crawler, or job) on a workflow graph.
type nonrec node_list = node listtype nonrec edge = {destination_id : name_string option;The unique of the node within the workflow where the edge ends.
*)source_id : name_string option;The unique of the node within the workflow where the edge starts.
*)}An edge represents a directed connection between two Glue components that are part of the workflow the edge belongs to.
type nonrec edge_list = edge listtype nonrec workflow_graph = {edges : edge_list option;A list of all the directed connections between the nodes belonging to the workflow.
*)nodes : node_list option;A list of the the Glue components belong to the workflow represented as nodes.
*)}A workflow graph represents the complete workflow containing all the Glue components present in the workflow and all the directed connections between them.
type nonrec starting_event_batch_condition = {batch_window : nullable_integer option;Duration of the batch window in seconds.
*)batch_size : nullable_integer option;Number of events in the batch.
*)}The batch condition that started the workflow run. Either the number of events in the batch size arrived, in which case the BatchSize member is non-zero, or the batch window expired, in which case the BatchWindow member is non-zero.
type nonrec workflow_run = {starting_event_batch_condition : starting_event_batch_condition option;The batch condition that started the workflow run.
*)graph : workflow_graph option;The graph representing all the Glue components that belong to the workflow as nodes and directed connections between them as edges.
*)statistics : workflow_run_statistics option;The statistics of the run.
*)error_message : error_string option;This error message describes any error that may have occurred in starting the workflow run. Currently the only error message is "Concurrent runs exceeded for workflow: foo."
status : workflow_run_status option;The status of the workflow run.
*)completed_on : timestamp_value option;The date and time when the workflow run completed.
*)started_on : timestamp_value option;The date and time when the workflow run was started.
*)workflow_run_properties : workflow_run_properties option;The workflow run properties which were set during the run.
*)previous_run_id : id_string option;The ID of the previous workflow run.
*)workflow_run_id : id_string option;The ID of this workflow run.
*)name : name_string option;Name of the workflow that was run.
*)}A workflow run is an execution of a workflow providing all the runtime information.
type nonrec blueprint_details = {run_id : id_string option;The run ID for this blueprint.
*)blueprint_name : orchestration_name_string option;The name of the blueprint.
*)}The details of a blueprint.
type nonrec workflow = {blueprint_details : blueprint_details option;This structure indicates the details of the blueprint that this particular workflow is created from.
*)max_concurrent_runs : nullable_integer option;You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs. If you leave this parameter blank, there is no limit to the number of concurrent workflow runs.
*)graph : workflow_graph option;The graph representing all the Glue components that belong to the workflow as nodes and directed connections between them as edges.
*)last_run : workflow_run option;The information about the last execution of the workflow.
*)last_modified_on : timestamp_value option;The date and time when the workflow was last modified.
*)created_on : timestamp_value option;The date and time when the workflow was created.
*)default_run_properties : workflow_run_properties option;A collection of properties to be used as part of each execution of the workflow. The run properties are made available to each job in the workflow. A job can modify the properties for the next jobs in the flow.
*)description : generic_string option;A description of the workflow.
*)name : name_string option;The name of the workflow.
*)}A workflow is a collection of multiple dependent Glue jobs and crawlers that are run to complete a complex ETL task. A workflow manages the execution and monitoring of all its jobs and crawlers.
type nonrec workflows = workflow listtype nonrec workflow_runs = workflow_run listtype nonrec workflow_names = name_string listtype nonrec error_detail = {error_message : description_string option;A message describing the error.
*)error_code : name_string option;The code associated with this error.
*)}Contains details about an error.
type nonrec view_validation = {error : error_detail option;An error associated with the validation.
*)state : resource_state option;The state of the validation.
*)update_time : timestamp option;The time of the last update.
*)view_validation_text : view_text_string option;The SELECT query that defines the view, as provided by the customer.
dialect_version : view_dialect_version_string option;The version of the dialect of the query engine. For example, 3.0.0.
*)dialect : view_dialect option;The dialect of the query engine.
*)}A structure that contains information for an analytical engine to validate a view, prior to persisting the view metadata. Used in the case of direct UpdateTable or CreateTable API calls.
type nonrec view_validation_list = view_validation listtype nonrec view_sub_objects_list = arn_string listtype nonrec table_version_id = Smaws_Lib.CoreTypes.Int64.ttype nonrec view_sub_object_version_ids_list = table_version_id listtype nonrec view_representation = {is_stale : nullable_boolean option;Dialects marked as stale are no longer valid and must be updated before they can be queried in their respective query engines.
*)validation_connection : name_string option;The name of the connection to be used to validate the specific representation of the view.
*)view_expanded_text : view_text_string option;The expanded SQL for the view. This SQL is used by engines while processing a query on a view. Engines may perform operations during view creation to transform ViewOriginalText to ViewExpandedText. For example:
SELECT * from table1 -> SELECT * from db1.table1view_original_text : view_text_string option;The SELECT query provided by the customer during CREATE VIEW DDL. This SQL is not used during a query on a view (ViewExpandedText is used instead). ViewOriginalText is used for cases like SHOW CREATE VIEW where users want to see the original DDL command that created the view.
dialect_version : view_dialect_version_string option;The version of the dialect of the query engine. For example, 3.0.0.
*)dialect : view_dialect option;The dialect of the query engine.
*)}A structure that contains the dialect of the view, and the query that defines the view.
type nonrec view_representation_list = view_representation listtype nonrec view_representation_input = {view_expanded_text : view_text_string option;A string that represents the SQL query that describes the view with expanded resource ARNs
*)validation_connection : name_string option;The name of the connection to be used to validate the specific representation of the view.
*)view_original_text : view_text_string option;A string that represents the original SQL query that describes the view.
*)dialect_version : view_dialect_version_string option;A parameter that specifies the version of the engine of a specific representation.
*)dialect : view_dialect option;A parameter that specifies the engine type of a specific representation.
*)}A structure containing details of a representation to update or create a Lake Formation view.
type nonrec view_representation_input_list = view_representation_input listtype nonrec refresh_seconds = Smaws_Lib.CoreTypes.Int64.ttype nonrec view_definition_input = {sub_object_version_ids : view_sub_object_version_ids_list option;List of the Apache Iceberg table versions referenced by the materialized view.
*)sub_objects : view_sub_objects_list option;A list of base table ARNs that make up the view.
*)last_refresh_type : last_refresh_type option;The type of the materialized view's last refresh. Valid values: Full, Incremental.
refresh_seconds : refresh_seconds option;Auto refresh interval in seconds for the materialized view. If not specified, the view will not automatically refresh.
*)view_version_token : version_string option;The version ID of the Apache Iceberg table.
*)view_version_id : table_version_id option;The ID value that identifies this view's version. For materialized views, the version ID is the Apache Iceberg table's snapshot ID.
*)representations : view_representation_input_list option;A list of structures that contains the dialect of the view, and the query that defines the view.
*)definer : arn_string option;The definer of a view in SQL.
*)is_protected : nullable_boolean option;You can set this flag as true to instruct the engine not to push user-provided operations into the logical plan of the view during query planning. However, setting this flag does not guarantee that the engine will comply. Refer to the engine's documentation to understand the guarantees provided, if any.
*)}A structure containing details for creating or updating an Glue view.
type nonrec view_definition = {representations : view_representation_list option;A list of representations.
*)sub_object_version_ids : view_sub_object_version_ids_list option;List of the Apache Iceberg table versions referenced by the materialized view.
*)sub_objects : view_sub_objects_list option;A list of table Amazon Resource Names (ARNs).
*)last_refresh_type : last_refresh_type option;Sets the method used for the most recent refresh.
*)refresh_seconds : refresh_seconds option;Auto refresh interval in seconds for the materialized view. If not specified, the view will not automatically refresh.
*)view_version_token : hash_string option;The version ID of the Apache Iceberg table.
*)view_version_id : table_version_id option;The ID value that identifies this view's version. For materialized views, the version ID is the Apache Iceberg table's snapshot ID.
*)definer : arn_string option;The definer of a view in SQL.
*)is_protected : nullable_boolean option;You can set this flag as true to instruct the engine not to push user-provided operations into the logical plan of the view during query planning. However, setting this flag does not guarantee that the engine will comply. Refer to the engine's documentation to understand the guarantees provided, if any.
*)}A structure containing details for representations.
type nonrec version_mismatch_exception = {message : message_string option;A message describing the problem.
*)}There was a version conflict.
type nonrec version_long_number = Smaws_Lib.CoreTypes.Int64.ttype nonrec value_string_list = value_string listtype nonrec validation_exception = {message : message_string option;A message describing the problem.
*)}A value could not be validated.
type nonrec resource_uri = {uri : ur_i option;The URI for accessing the resource.
*)resource_type : resource_type option;The type of the resource.
*)}The URIs for function resources.
type nonrec resource_uri_list = resource_uri listtype nonrec user_defined_function = {catalog_id : catalog_id_string option;The ID of the Data Catalog in which the function resides.
*)resource_uris : resource_uri_list option;The resource URIs for the function.
*)create_time : timestamp option;The time at which the function was created.
*)owner_type : principal_type option;The owner type.
*)function_type : function_type option;The type of the function.
*)owner_name : name_string option;The owner of the function.
*)class_name : name_string option;The Java class that contains the function code.
*)database_name : name_string option;The name of the catalog database that contains the function.
*)function_name : name_string option;The name of the function.
*)}Represents the equivalent of a Hive user-defined function (UDF) definition.
type nonrec user_defined_function_list = user_defined_function listtype nonrec user_defined_function_input = {resource_uris : resource_uri_list option;The resource URIs for the function.
*)owner_type : principal_type option;The owner type.
*)function_type : function_type option;The type of the function.
*)owner_name : name_string option;The owner of the function.
*)class_name : name_string option;The Java class that contains the function code.
*)function_name : name_string option;The name of the function.
*)}A structure used to create or update a user-defined function.
type nonrec usage_profile_definition = {last_modified_on : timestamp_value option;The date and time when the usage profile was last modified.
*)created_on : timestamp_value option;The date and time when the usage profile was created.
*)description : description_string option;A description of the usage profile.
*)name : name_string option;The name of the usage profile.
*)}Describes an Glue usage profile.
type nonrec usage_profile_definition_list = usage_profile_definition listtype nonrec enclosed_in_string_properties_min_one =
enclosed_in_string_property listtype nonrec upsert_redshift_target_options = {upsert_keys : enclosed_in_string_properties_min_one option;The keys used to determine whether to perform an update or insert.
*)connection_name : enclosed_in_string_property option;The name of the connection to use to write to Redshift.
*)table_location : enclosed_in_string_property option;The physical location of the Redshift table.
*)}The options to configure an upsert operation when writing to a Redshift target .
type nonrec update_xml_classifier_request = {row_tag : row_tag option;The XML tag designating the element that contains each record in an XML document being parsed. This cannot identify a self-closing element (closed by />). An empty row element that contains only attributes can be parsed as long as it ends with a closing tag (for example,
is okay, but
is not).
*)classification : classification option;An identifier of the data format that the classifier matches.
*)name : name_string;The name of the classifier.
*)}Specifies an XML classifier to be updated.
type nonrec update_workflow_response = {name : name_string option;The name of the workflow which was specified in input.
*)}type nonrec update_workflow_request = {max_concurrent_runs : nullable_integer option;You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs. If you leave this parameter blank, there is no limit to the number of concurrent workflow runs.
*)default_run_properties : workflow_run_properties option;A collection of properties to be used as part of each execution of the workflow.
Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.
*)description : workflow_description_string option;The description of the workflow.
*)name : name_string;Name of the workflow to be updated.
*)}type nonrec operation_timeout_exception = {message : message_string option;A message describing the problem.
*)}The operation timed out.
type nonrec invalid_input_exception = {from_federation_source : nullable_boolean option;Indicates whether or not the exception relates to a federated source.
*)message : message_string option;A message describing the problem.
*)}The input provided was not valid.
type nonrec internal_service_exception = {message : message_string option;A message describing the problem.
*)}An internal service error occurred.
type nonrec entity_not_found_exception = {from_federation_source : nullable_boolean option;Indicates whether or not the exception relates to a federated source.
*)message : message_string option;A message describing the problem.
*)}A specified entity does not exist
type nonrec concurrent_modification_exception = {message : message_string option;A message describing the problem.
*)}Two processes are trying to modify a resource simultaneously.
type nonrec update_user_defined_function_request = {function_input : user_defined_function_input;A FunctionInput object that redefines the function in the Data Catalog.
function_name : name_string;The name of the function.
*)database_name : name_string;The name of the catalog database where the function to be updated is located.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the function to be updated is located. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec glue_encryption_exception = {message : message_string option;The message describing the problem.
*)}An encryption operation failed.
type nonrec update_usage_profile_response = {name : name_string option;The name of the usage profile that was updated.
*)}type nonrec allowed_values_string_list = config_value_string listtype nonrec configuration_object = {max_value : config_value_string option;A maximum allowed value for the parameter.
*)min_value : config_value_string option;A minimum allowed value for the parameter.
*)allowed_values : allowed_values_string_list option;A list of allowed values for the parameter.
*)default_value : config_value_string option;A default value for the parameter.
*)}Specifies the values that an admin sets for each job or session parameter configured in a Glue usage profile.
type nonrec configuration_map = (name_string * configuration_object) listtype nonrec profile_configuration = {job_configuration : configuration_map option;A key-value map of configuration parameters for Glue jobs.
*)session_configuration : configuration_map option;A key-value map of configuration parameters for Glue sessions.
*)}Specifies the job and session values that an admin configures in an Glue usage profile.
type nonrec update_usage_profile_request = {configuration : profile_configuration;A ProfileConfiguration object specifying the job and session values for the profile.
description : description_string option;A description of the usage profile.
*)name : name_string;The name of the usage profile.
*)}type nonrec operation_not_supported_exception = {message : message_string option;A message describing the problem.
*)}The operation is not available in the region.
type nonrec update_trigger_response = {trigger : trigger option;The resulting trigger definition.
*)}type nonrec trigger_update = {event_batching_condition : event_batching_condition option;Batch condition that must be met (specified number of events received or batch time window expired) before EventBridge event trigger fires.
*)predicate : predicate option;The predicate of this trigger, which defines when it will fire.
*)actions : action_list option;The actions initiated by this trigger.
*)schedule : generic_string option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
description : description_string option;A description of this trigger.
*)name : name_string option;Reserved for future use.
*)}A structure used to provide information used to update a trigger. This object updates the previous trigger definition by overwriting it completely.
type nonrec update_trigger_request = {trigger_update : trigger_update;The new values with which to update the trigger.
*)name : name_string;The name of the trigger to update.
*)}type nonrec parameters_map = (key_string * parameters_map_value) listtype nonrec column = {parameters : parameters_map option;These key-value pairs define properties associated with the column.
*)comment : comment_string option;A free-form text comment.
*)type_ : column_type_string option;The data type of the Column.
name : name_string;The name of the Column.
}A column in a Table.
type nonrec column_list = column listtype nonrec location_string_list = location_string listtype nonrec ser_de_info = {parameters : parameters_map option;These key-value pairs define initialization parameters for the SerDe.
*)serialization_library : name_string option;Usually the class that implements the SerDe. An example is org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.
name : name_string option;Name of the SerDe.
*)}Information about a serialization/deserialization program (SerDe) that serves as an extractor and loader.
type nonrec name_string_list = name_string listtype nonrec order = {sort_order : integer_flag;Indicates that the column is sorted in ascending order (== 1), or in descending order (==0).
column : name_string;The name of the column.
*)}Specifies the sort order of a sorted column.
type nonrec order_list = order listtype nonrec column_value_string_list = column_values_string listtype nonrec location_map = (column_values_string * column_values_string) listtype nonrec skewed_info = {skewed_column_value_location_maps : location_map option;A mapping of skewed values to the columns that contain them.
*)skewed_column_values : column_value_string_list option;A list of values that appear so frequently as to be considered skewed.
*)skewed_column_names : name_string_list option;A list of names of columns that contain skewed values.
*)}Specifies skewed values in a table. Skewed values are those that occur with very high frequency.
type nonrec schema_id = {registry_name : schema_registry_name_string option;The name of the schema registry that contains the schema.
*)schema_name : schema_registry_name_string option;The name of the schema. One of SchemaArn or SchemaName has to be provided.
schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema. One of SchemaArn or SchemaName has to be provided.
}The unique ID of the schema in the Glue schema registry.
type nonrec schema_reference = {schema_version_number : version_long_number option;The version number of the schema.
*)schema_version_id : schema_version_id_string option;The unique ID assigned to a version of the schema. Either this or the SchemaId has to be provided.
schema_id : schema_id option;A structure that contains schema identity fields. Either this or the SchemaVersionId has to be provided.
}An object that references a schema stored in the Glue Schema Registry.
type nonrec storage_descriptor = {schema_reference : schema_reference option;An object that references a schema stored in the Glue Schema Registry.
When creating a table, you can pass an empty list of columns for the schema, and instead use a schema reference.
*)stored_as_sub_directories : boolean_ option;True if the table data is stored in subdirectories, or False if not.
skewed_info : skewed_info option;The information about values that appear frequently in a column (skewed values).
*)parameters : parameters_map option;The user-supplied properties in key-value form.
*)sort_columns : order_list option;A list specifying the sort order of each bucket in the table.
*)bucket_columns : name_string_list option;A list of reducer grouping columns, clustering columns, and bucketing columns in the table.
*)serde_info : ser_de_info option;The serialization/deserialization (SerDe) information.
*)number_of_buckets : integer option;Must be specified if the table contains any dimension columns.
*)compressed : boolean_ option;True if the data in the table is compressed, or False if not.
output_format : format_string option;The output format: SequenceFileOutputFormat (binary), or IgnoreKeyTextOutputFormat, or a custom format.
input_format : format_string option;The input format: SequenceFileInputFormat (binary), or TextInputFormat, or a custom format.
additional_locations : location_string_list option;A list of locations that point to the path where a Delta table is located.
*)location : location_string option;The physical location of the table. By default, this takes the form of the warehouse location, followed by the database location in the warehouse, followed by the table name.
*)columns : column_list option;A list of the Columns in the table.
}Describes the physical storage of table data.
type nonrec table_identifier = {region : name_string option;Region of the target table.
*)name : name_string option;The name of the target table.
*)database_name : name_string option;The name of the catalog database that contains the target table.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the table resides.
*)}A structure that describes a target table for resource linking.
type nonrec table_input = {view_definition : view_definition_input option;A structure that contains all the information that defines the view, including the dialect or dialects for the view, and the query.
*)target_table : table_identifier option;A TableIdentifier structure that describes a target table for resource linking.
parameters : parameters_map option;These key-value pairs define properties associated with the table.
*)table_type : table_type_string option;The type of this table. Glue will create tables with the EXTERNAL_TABLE type. Other services, such as Athena, may create tables with additional table types.
Glue related table types:
EXTERNAL_TABLE Hive compatible attribute - indicates a non-Hive managed table.
GOVERNED Used by Lake Formation. The Glue Data Catalog understands GOVERNED.
view_expanded_text : view_text_string option;Included for Apache Hive compatibility. Not used in the normal course of Glue operations.
*)view_original_text : view_text_string option;Included for Apache Hive compatibility. Not used in the normal course of Glue operations. If the table is a VIRTUAL_VIEW, certain Athena configuration encoded in base64.
partition_keys : column_list option;A list of columns by which the table is partitioned. Only primitive types are supported as partition keys.
When you create a table used by Amazon Athena, and you do not specify any partitionKeys, you must at least set the value of partitionKeys to an empty list. For example:
"PartitionKeys": []
storage_descriptor : storage_descriptor option;A storage descriptor containing information about the physical storage of this table.
*)retention : non_negative_integer option;The retention time for this table.
*)last_analyzed_time : timestamp option;The last time that column statistics were computed for this table.
*)last_access_time : timestamp option;The last time that the table was accessed.
*)owner : name_string option;The table owner. Included for Apache Hive compatibility. Not used in the normal course of Glue operations.
*)description : description_string option;A description of the table.
*)name : name_string;The table name. For Hive compatibility, this is folded to lowercase when it is stored.
*)}A structure used to define a table.
type nonrec integer_list = integer listtype nonrec iceberg_struct_field = {write_default : iceberg_document option;Default value used to populate the field's value for any records written after the field was added to the schema, if the writer does not supply the field's value. This can be changed through schema evolution.
*)initial_default : iceberg_document option;Default value used to populate the field's value for all records that were written before the field was added to the schema. This enables backward compatibility when adding new fields to existing Iceberg tables.
*)doc : comment_string option;Optional documentation or description text that provides additional context about the purpose and usage of this field.
*)required : boolean_;Indicates whether this field is required (non-nullable) or optional (nullable) in the table schema.
*)type_ : iceberg_document;The data type definition for this field, specifying the structure and format of the data it contains.
*)name : column_name_string;The name of the field as it appears in the table schema and query operations.
*)id : integer;The unique identifier assigned to this field within the Iceberg table schema, used for schema evolution and field tracking.
*)}Defines a single field within an Iceberg table schema, including its identifier, name, data type, nullability, and documentation.
type nonrec iceberg_struct_field_list = iceberg_struct_field listtype nonrec iceberg_schema = {fields : iceberg_struct_field_list;The list of field definitions that make up the table schema, including field names, types, and metadata.
*)type_ : iceberg_struct_type_enum option;The root type of the schema structure, typically "struct" for Iceberg table schemas.
*)identifier_field_ids : integer_list option;The list of field identifiers that uniquely identify records in the table, used for row-level operations and deduplication.
*)schema_id : integer option;The unique identifier for this schema version within the Iceberg table's schema evolution history.
*)}Defines the schema structure for an Iceberg table, including field definitions, data types, and schema metadata.
type nonrec iceberg_partition_field = {field_id : integer option;The unique identifier assigned to this partition field within the Iceberg table's partition specification.
*)name : column_name_string;The name of the partition field as it will appear in the partitioned table structure.
*)transform : iceberg_transform_string;The transformation function applied to the source field to create the partition, such as identity, bucket, truncate, year, month, day, or hour.
*)source_id : integer;The identifier of the source field from the table schema that this partition field is based on.
*)}Defines a single partition field within an Iceberg partition specification, including the source field, transformation function, partition name, and unique identifier.
type nonrec iceberg_partition_spec_field_list = iceberg_partition_field listtype nonrec iceberg_partition_spec = {spec_id : integer option;The unique identifier for this partition specification within the Iceberg table's metadata history.
*)fields : iceberg_partition_spec_field_list;The list of partition fields that define how the table data should be partitioned, including source fields and their transformations.
*)}Defines the partitioning specification for an Iceberg table, determining how table data will be organized and partitioned for optimal query performance.
type nonrec iceberg_sort_field = {null_order : iceberg_null_order;The ordering behavior for null values in this field, specifying whether nulls should appear first or last in the sort order.
*)direction : iceberg_sort_direction;The sort direction for this field, either ascending or descending.
*)transform : iceberg_transform_string;The transformation function applied to the source field before sorting, such as identity, bucket, or truncate.
*)source_id : integer;The identifier of the source field from the table schema that this sort field is based on.
*)}Defines a single field within an Iceberg sort order specification, including the source field, transformation, sort direction, and null value ordering.
type nonrec iceberg_sort_order_field_list = iceberg_sort_field listtype nonrec iceberg_sort_order = {fields : iceberg_sort_order_field_list;The list of fields and their sort directions that define the ordering criteria for the Iceberg table data.
*)order_id : integer;The unique identifier for this sort order specification within the Iceberg table's metadata.
*)}Defines the sort order specification for an Iceberg table, determining how data should be ordered within partitions to optimize query performance.
type nonrec string_to_string_map = (nullable_string * nullable_string) listtype nonrec iceberg_encrypted_key = {properties : string_to_string_map option;A string to string map of additional metadata used by the table's encryption scheme. These properties provide additional context and configuration for the encryption key implementation.
*)encrypted_by_id : encryption_key_id_string option;Optional ID of the key used to encrypt or wrap the key metadata in Iceberg table encryption. This field references another encryption key that was used to encrypt the current key's metadata.
*)encrypted_key_metadata : encrypted_key_metadata_string;Encrypted key and metadata, base64 encoded. The format of encrypted key metadata is determined by the table's encryption scheme and can be a wrapped format specific to the table's KMS provider.
*)key_id : encryption_key_id_string;Unique identifier of the encryption key used for Iceberg table encryption. This ID is used to reference the key in table metadata and track which key was used to encrypt specific data.
*)}Encryption key structure used for Iceberg table encryption. Contains the key ID, encrypted key metadata, optional reference to the encrypting key, and additional properties for the table's encryption scheme.
type nonrec iceberg_table_update = {key_id : encryption_key_id_string option;Identifier of the encryption key involved in an Iceberg table update operation. References the specific key being added to or removed from the table's encryption configuration.
*)encryption_key : iceberg_encrypted_key option;Encryption key information associated with an Iceberg table update operation. Used when adding or removing encryption keys from the table metadata during table evolution.
*)action : iceberg_update_action option;The type of update action to be performed on the Iceberg table. Defines the specific operation such as adding schema, setting current schema, adding partition spec, or managing encryption keys.
*)properties : string_to_string_map option;Updated key-value pairs of table properties and configuration settings for the Iceberg table.
*)location : location_string;The updated S3 location where the Iceberg table data will be stored.
*)sort_order : iceberg_sort_order option;The updated sort order specification that defines how data should be ordered within partitions for optimal query performance.
*)partition_spec : iceberg_partition_spec option;The updated partitioning specification that defines how the table data should be reorganized and partitioned.
*)schema : iceberg_schema;The updated schema definition for the Iceberg table, specifying any changes to field structure, data types, or schema metadata.
*)}Defines a complete set of updates to be applied to an Iceberg table, including schema changes, partitioning modifications, sort order adjustments, location updates, and property changes.
type nonrec iceberg_table_update_list = iceberg_table_update listtype nonrec update_iceberg_table_input = {updates : iceberg_table_update_list;The list of table update operations that specify the changes to be made to the Iceberg table, including schema modifications, partition specifications, and table properties.
*)}Contains the update operations to be applied to an existing Iceberg table inGlue Data Catalog, defining the new state of the table metadata.
type nonrec update_iceberg_input = {update_iceberg_table_input : update_iceberg_table_input;The specific update operations to be applied to the Iceberg table, containing a list of updates that define the new state of the table including schema, partitions, and properties.
*)}Input parameters specific to updating Apache Iceberg tables in Glue Data Catalog, containing the update operations to be applied to an existing Iceberg table.
type nonrec update_open_table_format_input = {update_iceberg_input : update_iceberg_input option;Apache Iceberg-specific update parameters that define the table modifications to be applied, including schema changes, partition specifications, and table properties.
*)}Input parameters for updating open table format tables in GlueData Catalog, serving as a wrapper for format-specific update operations such as Apache Iceberg.
type nonrec update_table_request = {update_open_table_format_input : update_open_table_format_input option;Input parameters for updating open table format tables in GlueData Catalog, serving as a wrapper for format-specific update operations such as Apache Iceberg.
*)force : boolean_ option;A flag that can be set to true to ignore matching storage descriptor and subobject matching requirements.
*)view_update_action : view_update_action option;The operation to be performed when updating the view.
*)version_id : version_string option;The version ID at which to update the table contents.
*)transaction_id : transaction_id_string option;The transaction ID at which to update the table contents.
*)skip_archive : boolean_nullable option;By default, UpdateTable always creates an archived version of the table before updating it. However, if skipArchive is set to true, UpdateTable does not create the archived version.
table_input : table_input option;An updated TableInput object to define the metadata table in the catalog.
name : name_string option;The unique identifier for the table within the specified database that will be created in the Glue Data Catalog.
*)database_name : name_string;The name of the catalog database in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec table_optimizer_vpc_configuration = | GlueConnectionName of glue_connection_name_stringThe name of the Glue connection used for the VPC for the table optimizer.
*)An object that describes the VPC configuration for a table optimizer.
This configuration is necessary to perform optimization on tables that are in a customer VPC.
type nonrec iceberg_compaction_configuration = {delete_file_threshold : nullable_integer option;The minimum number of deletes that must be present in a data file to make it eligible for compaction. This parameter helps optimize compaction by focusing on files that contain a significant number of delete operations, which can improve query performance by removing deleted records. If an input is not provided, the default value 1 will be used.
*)min_input_files : nullable_integer option;The minimum number of data files that must be present in a partition before compaction will actually compact files. This parameter helps control when compaction is triggered, preventing unnecessary compaction operations on partitions with few files. If an input is not provided, the default value 100 will be used.
*)strategy : compaction_strategy option;The strategy to use for compaction. Valid values are:
binpack: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases.sort: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the sort_order table property.z-order: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the sort_order table property.If an input is not provided, the default value 'binpack' will be used.
*)}The configuration for an Iceberg compaction optimizer. This configuration defines parameters for optimizing the layout of data files in Iceberg tables.
type nonrec compaction_configuration = {iceberg_configuration : iceberg_compaction_configuration option;The configuration for an Iceberg compaction optimizer.
*)}The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.
type nonrec iceberg_retention_configuration = {run_rate_in_hours : nullable_integer option;The interval in hours between retention job runs. This parameter controls how frequently the retention optimizer will run to clean up expired snapshots. The value must be between 3 and 168 hours (7 days). If an input is not provided, the default value 24 will be used.
*)clean_expired_files : nullable_boolean option;If set to false, snapshots are only deleted from table metadata, and the underlying data and metadata files are not deleted.
*)number_of_snapshots_to_retain : nullable_integer option;The number of Iceberg snapshots to retain within the retention period. If an input is not provided, the corresponding Iceberg table configuration field will be used or if not present, the default value 1 will be used.
*)snapshot_retention_period_in_days : nullable_integer option;The number of days to retain the Iceberg snapshots. If an input is not provided, the corresponding Iceberg table configuration field will be used or if not present, the default value 5 will be used.
*)}The configuration for an Iceberg snapshot retention optimizer.
type nonrec retention_configuration = {iceberg_configuration : iceberg_retention_configuration option;The configuration for an Iceberg snapshot retention optimizer.
*)}The configuration for a snapshot retention optimizer.
type nonrec iceberg_orphan_file_deletion_configuration = {run_rate_in_hours : nullable_integer option;The interval in hours between orphan file deletion job runs. This parameter controls how frequently the orphan file deletion optimizer will run to clean up orphan files. The value must be between 3 and 168 hours (7 days). If an input is not provided, the default value 24 will be used.
*)location : message_string option;Specifies a directory in which to look for files (defaults to the table's location). You may choose a sub-directory rather than the top-level table location.
*)orphan_file_retention_period_in_days : nullable_integer option;The number of days that orphan files should be retained before file deletion. If an input is not provided, the default value 3 will be used.
*)}The configuration for an Iceberg orphan file deletion optimizer.
type nonrec orphan_file_deletion_configuration = {iceberg_configuration : iceberg_orphan_file_deletion_configuration option;The configuration for an Iceberg orphan file deletion optimizer.
*)}The configuration for an orphan file deletion optimizer.
type nonrec table_optimizer_configuration = {orphan_file_deletion_configuration : orphan_file_deletion_configuration option;The configuration for an orphan file deletion optimizer.
*)retention_configuration : retention_configuration option;The configuration for a snapshot retention optimizer.
*)compaction_configuration : compaction_configuration option;The configuration for a compaction optimizer. This configuration defines how data files in your table will be compacted to improve query performance and reduce storage costs.
*)vpc_configuration : table_optimizer_vpc_configuration option;A TableOptimizerVpcConfiguration object representing the VPC configuration for a table optimizer.
This configuration is necessary to perform optimization on tables that are in a customer VPC.
*)enabled : nullable_boolean option;Whether table optimization is enabled.
*)role_arn : arn_string option;A role passed by the caller which gives the service permission to update the resources associated with the optimizer on the caller's behalf.
*)}Contains details on the configuration of a table optimizer. You pass this configuration when creating or updating a table optimizer.
type nonrec update_table_optimizer_request = {table_optimizer_configuration : table_optimizer_configuration;A TableOptimizerConfiguration object representing the configuration of a table optimizer.
type_ : table_optimizer_type;The type of table optimizer.
*)table_name : name_string;The name of the table.
*)database_name : name_string;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string;The Catalog ID of the table.
*)}type nonrec throttling_exception = {message : message_string option;A message describing the problem.
*)}The throttling threshhold was exceeded.
type nonrec access_denied_exception = {message : message_string option;A message describing the problem.
*)}Access to a resource was denied.
type nonrec resource_number_limit_exceeded_exception = {message : message_string option;A message describing the problem.
*)}A resource numerical limit was exceeded.
type nonrec resource_not_ready_exception = {message : message_string option;A message describing the problem.
*)}A resource was not ready for a transaction.
type nonrec federation_source_retryable_exception = {message : message_string option;A message describing the problem.
*)}A federation source failed, but the operation may be retried.
type nonrec federation_source_exception = {message : message_string option;The message describing the problem.
*)federation_source_error_code : federation_source_error_code option;The error code of the problem.
*)}A federation source failed.
type nonrec already_exists_exception = {message : message_string option;A message describing the problem.
*)}A resource to be created or added already exists.
type nonrec update_source_control_from_job_response = {job_name : name_string option;The name of the Glue job.
*)}type nonrec update_source_control_from_job_request = {auth_token : auth_token_string option;The value of the authorization token.
*)auth_strategy : source_control_auth_strategy option;The type of authentication, which can be an authentication token stored in Amazon Web Services Secrets Manager, or a personal access token.
*)commit_id : commit_id_string option;A commit ID for a commit in the remote repository.
*)folder : name_string option;An optional folder in the remote repository.
*)branch_name : name_string option;An optional branch in the remote repository.
*)repository_owner : name_string option;The owner of the remote repository that contains the job artifacts.
*)repository_name : name_string option;The name of the remote repository that contains the job artifacts. For BitBucket providers, RepositoryName should include WorkspaceName. Use the format
/.
*)provider : source_control_provider option;The provider for the remote repository. Possible values: GITHUB, AWS_CODE_COMMIT, GITLAB, BITBUCKET.
*)job_name : name_string option;The name of the Glue job to be synchronized to or from the remote repository.
*)}type nonrec update_schema_response = {registry_name : schema_registry_name_string option;The name of the registry that contains the schema.
*)schema_name : schema_registry_name_string option;The name of the schema.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)}type nonrec schema_version_number = {version_number : version_long_number option;The version number of the schema.
*)latest_version : latest_schema_version_boolean option;The latest version available for the schema.
*)}A structure containing the schema version information.
type nonrec update_schema_input = {description : description_string option;The new description for the schema.
*)compatibility : compatibility option;The new compatibility setting for the schema.
*)schema_version_number : schema_version_number option;Version number required for check pointing. One of VersionNumber or Compatibility has to be provided.
schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName has to be provided.SchemaArn or SchemaName has to be provided.}type nonrec update_registry_response = {registry_arn : glue_resource_arn option;The Amazon Resource name (ARN) of the updated registry.
*)registry_name : schema_registry_name_string option;The name of the updated registry.
*)}type nonrec registry_id = {registry_arn : glue_resource_arn option;Arn of the registry to be updated. One of RegistryArn or RegistryName has to be provided.
registry_name : schema_registry_name_string option;Name of the registry. Used only for lookup. One of RegistryArn or RegistryName has to be provided.
}A wrapper structure that may contain the registry name and Amazon Resource Name (ARN).
type nonrec update_registry_input = {description : description_string;A description of the registry. If description is not provided, this field will not be updated.
*)registry_id : registry_id;This is a wrapper structure that may contain the registry name and Amazon Resource Name (ARN).
*)}type nonrec bounded_partition_value_list = value_string listtype nonrec partition_input = {last_analyzed_time : timestamp option;The last time at which column statistics were computed for this partition.
*)parameters : parameters_map option;These key-value pairs define partition parameters.
*)storage_descriptor : storage_descriptor option;Provides information about the physical location where the partition is stored.
*)last_access_time : timestamp option;The last time at which the partition was accessed.
*)values : value_string_list option;The values of the partition. Although this parameter is not required by the SDK, you must specify this parameter for a valid input.
The values for the keys for the new partition must be passed as an array of String objects that must be ordered in the same order as the partition keys appearing in the Amazon S3 prefix. Otherwise Glue will add the values to the wrong keys.
*)}The structure used to create and update a partition.
type nonrec update_partition_request = {partition_input : partition_input;The new partition object to update the partition to.
The Values property can't be changed. If you want to change the partition key values for a partition, delete and recreate the partition.
partition_value_list : bounded_partition_value_list;List of partition key values that define the partition to update.
*)table_name : name_string;The name of the table in which the partition to be updated is located.
*)database_name : name_string;The name of the catalog database in which the table in question resides.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partition to be updated resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec update_ml_transform_response = {transform_id : hash_string option;The unique identifier for the transform that was updated.
*)}type nonrec find_matches_parameters = {enforce_provided_labels : nullable_boolean option;The value to switch on or off to force the output to match the provided labels from users. If the value is True, the find matches transform forces the output to match the provided labels. The results override the normal conflation results. If the value is False, the find matches transform does not ensure all the labels provided are respected, and the results rely on the trained model.
Note that setting this value to true may increase the conflation execution time.
*)accuracy_cost_tradeoff : generic_bounded_double option;The value that is selected when tuning your transform for a balance between accuracy and cost. A value of 0.5 means that the system balances accuracy and cost concerns. A value of 1.0 means a bias purely for accuracy, which typically results in a higher cost, sometimes substantially higher. A value of 0.0 means a bias purely for cost, which results in a less accurate FindMatches transform, sometimes with unacceptable accuracy.
Accuracy measures how well the transform finds true positives and true negatives. Increasing accuracy requires more machine resources and cost. But it also results in increased recall.
Cost measures how many compute resources, and thus money, are consumed to run the transform.
*)precision_recall_tradeoff : generic_bounded_double option;The value selected when tuning your transform for a balance between precision and recall. A value of 0.5 means no preference; a value of 1.0 means a bias purely for precision, and a value of 0.0 means a bias for recall. Because this is a tradeoff, choosing values close to 1.0 means very low recall, and choosing values close to 0.0 results in very low precision.
The precision metric indicates how often your model is correct when it predicts a match.
The recall metric indicates that for an actual match, how often your model predicts the match.
*)primary_key_column_name : column_name_string option;The name of a column that uniquely identifies rows in the source table. Used to help identify matching records.
*)}The parameters to configure the find matches transform.
type nonrec transform_parameters = {find_matches_parameters : find_matches_parameters option;The parameters for the find matches algorithm.
*)transform_type : transform_type;The type of machine learning transform.
For information about the types of machine learning transforms, see Creating Machine Learning Transforms.
*)}The algorithm-specific parameters that are associated with the machine learning transform.
type nonrec update_ml_transform_request = {max_retries : nullable_integer option;The maximum number of times to retry a task for this transform after a task run fails.
*)timeout : timeout option;The timeout for a task run for this transform in minutes. This is the maximum time that a task run for this transform can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when this task runs.
worker_type : worker_type option;The type of predefined worker that is allocated when this task runs. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker provides 4 vCPU, 16 GB of memory and a 64GB disk, and 1 executor per worker.G.2X worker type, each worker provides 8 vCPU, 32 GB of memory and a 128GB disk, and 1 executor per worker.max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that are allocated to task runs for this transform. You can allocate from 2 to 100 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
When the WorkerType field is set to a value other than Standard, the MaxCapacity field is set automatically and becomes read-only.
glue_version : glue_version_string option;This value determines which version of Glue this machine learning transform is compatible with. Glue 1.0 is recommended for most customers. If the value is not set, the Glue compatibility defaults to Glue 0.9. For more information, see Glue Versions in the developer guide.
*)role : role_string option;The name or Amazon Resource Name (ARN) of the IAM role with the required permissions.
*)parameters : transform_parameters option;The configuration parameters that are specific to the transform type (algorithm) used. Conditionally dependent on the transform type.
*)description : description_string option;A description of the transform. The default is an empty string.
*)name : name_string option;The unique name that you gave the transform when you created it.
*)transform_id : hash_string;A unique identifier that was generated when the transform was created.
*)}type nonrec update_json_classifier_request = {json_path : json_path option;A JsonPath string defining the JSON data for the classifier to classify. Glue supports a subset of JsonPath, as described in Writing JsonPath Custom Classifiers.
name : name_string;The name of the classifier.
*)}Specifies a JSON classifier to be updated.
type nonrec update_job_response = {job_name : name_string option;Returns the name of the updated job definition.
*)}type nonrec execution_property = {max_concurrent_runs : max_concurrent_runs option;The maximum number of concurrent runs allowed for the job. The default is 1. An error is returned when this threshold is reached. The maximum value you can specify is controlled by a service limit.
*)}An execution property of a job.
type nonrec job_command = {runtime : runtime_name_string option;In Ray jobs, Runtime is used to specify the versions of Ray, Python and additional libraries available in your environment. This field is not used in other job types. For supported runtime environment values, see Supported Ray runtime environments in the Glue Developer Guide.
*)python_version : python_version_string option;The Python version being used to run a Python shell job. Allowed values are 2 or 3.
*)script_location : script_location_string option;Specifies the Amazon Simple Storage Service (Amazon S3) path to a script that runs a job.
*)name : generic_string option;The name of the job command. For an Apache Spark ETL job, this must be glueetl. For a Python shell job, it must be pythonshell. For an Apache Spark streaming ETL job, this must be gluestreaming. For a Ray job, this must be glueray.
}Specifies code that runs when a job is run.
type nonrec connection_string_list = connection_string listtype nonrec connections_list = {connections : connection_string_list option;A list of connections used by the job.
*)}Specifies the connections used by a job.
type nonrec glue_studio_schema_column = {glue_studio_type : column_type_string option;The data type of the column as defined in Glue Studio.
*)type_ : column_type_string option;The hive type for this column in the Glue Studio schema.
*)name : glue_studio_column_name_string;The name of the column in the Glue Studio schema.
*)}Specifies a single column in a Glue schema definition.
type nonrec glue_studio_schema_column_list = glue_studio_schema_column listtype nonrec glue_schema = {columns : glue_studio_schema_column_list option;Specifies the column definitions that make up a Glue schema.
*)}Specifies a user-defined schema when a schema cannot be determined by Glue.
type nonrec glue_schemas = glue_schema listtype nonrec athena_connector_source = {output_schemas : glue_schemas option;Specifies the data schema for the custom Athena source.
*)schema_name : enclosed_in_string_property;The name of the Cloudwatch log group to read from. For example, /aws-glue/jobs/output.
connection_table : enclosed_in_string_property_with_quote option;The name of the table in the data source.
*)connection_type : enclosed_in_string_property;The type of connection, such as marketplace.athena or custom.athena, designating a connection to an Amazon Athena data store.
*)connector_name : enclosed_in_string_property;The name of a connector that assists with accessing the data store in Glue Studio.
*)connection_name : enclosed_in_string_property;The name of the connection that is associated with the connector.
*)name : node_name;The name of the data source.
*)}Specifies a connector to an Amazon Athena data source.
type nonrec boxed_non_negative_long = Smaws_Lib.CoreTypes.Int64.ttype nonrec enclosed_in_string_properties = enclosed_in_string_property listtype nonrec jdbc_data_type = | VARCHAR| VARBINARY| TINYINT| TIMESTAMP_WITH_TIMEZONE| TIMESTAMP| TIME_WITH_TIMEZONE| TIME| STRUCT| SQLXML| SMALLINT| ROWID| REF_CURSOR| REF| REAL| OTHER| NVARCHAR| NUMERIC| NULL| NCLOB| NCHAR| LONGVARCHAR| LONGVARBINARY| LONGNVARCHAR| JAVA_OBJECT| INTEGER| FLOAT| DOUBLE| DISTINCT| DECIMAL| DATE| DATALINK| CLOB| CHAR| BOOLEAN| BLOB| BIT| BINARY| BIGINT| ARRAYtype nonrec jdbc_data_type_mapping = (jdbc_data_type * glue_record_type) listtype nonrec jdbc_connector_options = {data_type_mapping : jdbc_data_type_mapping option;Custom data type mapping that builds a mapping from a JDBC data type to an Glue data type. For example, the option "dataTypeMapping":{"FLOAT":"STRING"} maps data fields of JDBC type FLOAT into the Java String type by calling the ResultSet.getString() method of the driver, and uses it to build the Glue record. The ResultSet object is implemented by each driver, so the behavior is specific to the driver you use. Refer to the documentation for your JDBC driver to understand how the driver performs the conversions.
job_bookmark_keys_sort_order : enclosed_in_string_property option;Specifies an ascending or descending sort order.
*)job_bookmark_keys : enclosed_in_string_properties option;The name of the job bookmark keys on which to sort.
*)num_partitions : boxed_non_negative_long option;The number of partitions. This value, along with lowerBound (inclusive) and upperBound (exclusive), form partition strides for generated WHERE clause expressions that are used to split the partitionColumn.
upper_bound : boxed_non_negative_long option;The maximum value of partitionColumn that is used to decide partition stride.
lower_bound : boxed_non_negative_long option;The minimum value of partitionColumn that is used to decide partition stride.
partition_column : enclosed_in_string_property option;The name of an integer column that is used for partitioning. This option works only when it's included with lowerBound, upperBound, and numPartitions. This option works the same way as in the Spark SQL JDBC reader.
filter_predicate : enclosed_in_string_property option;Extra condition clause to filter data from source. For example:
BillingCity='Mountain View'
When using a query instead of a table name, you should validate that the query works with the specified filterPredicate.
}Additional connection options for the connector.
type nonrec jdbc_connector_source = {output_schemas : glue_schemas option;Specifies the data schema for the custom JDBC source.
*)query : sql_query option;The table or SQL query to get the data from. You can specify either ConnectionTable or query, but not both.
connection_table : enclosed_in_string_property_with_quote option;The name of the table in the data source.
*)additional_options : jdbc_connector_options option;Additional connection options for the connector.
*)connection_type : enclosed_in_string_property;The type of connection, such as marketplace.jdbc or custom.jdbc, designating a connection to a JDBC data store.
*)connector_name : enclosed_in_string_property;The name of a connector that assists with accessing the data store in Glue Studio.
*)connection_name : enclosed_in_string_property;The name of the connection that is associated with the connector.
*)name : node_name;The name of the data source.
*)}Specifies a connector to a JDBC data source.
type nonrec additional_options =
(enclosed_in_string_property * enclosed_in_string_property) listtype nonrec spark_connector_source = {output_schemas : glue_schemas option;Specifies data schema for the custom spark source.
*)additional_options : additional_options option;Additional connection options for the connector.
*)connection_type : enclosed_in_string_property;The type of connection, such as marketplace.spark or custom.spark, designating a connection to an Apache Spark data store.
*)connector_name : enclosed_in_string_property;The name of a connector that assists with accessing the data store in Glue Studio.
*)connection_name : enclosed_in_string_property;The name of the connection that is associated with the connector.
*)name : node_name;The name of the data source.
*)}Specifies a connector to an Apache Spark data source.
type nonrec catalog_source = {output_schemas : glue_schemas option;Specifies the data schema for the catalog source.
*)partition_predicate : enclosed_in_string_property option;Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data store.
*)}Specifies a data store in the Glue Data Catalog.
type nonrec redshift_source = {tmp_dir_iam_role : enclosed_in_string_property option;The IAM role with permissions.
*)redshift_tmp_dir : enclosed_in_string_property option;The Amazon S3 path where temporary data can be staged when copying out of the database.
*)table : enclosed_in_string_property;The database table to read from.
*)database : enclosed_in_string_property;The database to read from.
*)name : node_name;The name of the Amazon Redshift data store.
*)}Specifies an Amazon Redshift data store.
type nonrec boxed_long = Smaws_Lib.CoreTypes.Int64.ttype nonrec s3_source_additional_options = {bounded_files : boxed_long option;Sets the upper limit for the target number of files that will be processed.
*)bounded_size : boxed_long option;Sets the upper limit for the target size of the dataset in bytes that will be processed.
*)}Specifies additional connection options for the Amazon S3 data store.
type nonrec s3_catalog_source = {additional_options : s3_source_additional_options option;Specifies additional connection options.
*)partition_predicate : enclosed_in_string_property option;Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted. Set to "" – empty by default.
table : enclosed_in_string_property;The database table to read from.
*)database : enclosed_in_string_property;The database to read from.
*)name : node_name;The name of the data store.
*)}Specifies an Amazon S3 data store in the Glue Data Catalog.
type nonrec s3_direct_source_additional_options = {sample_path : enclosed_in_string_property option;If enabled, specifies the sample path.
*)enable_sample_path : boxed_boolean option;Sets option to enable a sample path.
*)bounded_files : boxed_long option;Sets the upper limit for the target number of files that will be processed.
*)bounded_size : boxed_long option;Sets the upper limit for the target size of the dataset in bytes that will be processed.
*)}Specifies additional connection options for the Amazon S3 data store.
type nonrec s3_csv_source = {output_schemas : glue_schemas option;Specifies the data schema for the S3 CSV source.
*)optimize_performance : boolean_value option;A Boolean value that specifies whether to use the advanced SIMD CSV reader along with Apache Arrow based columnar memory formats. Only available in Glue version 3.0.
*)skip_first : boxed_boolean option;A Boolean value that specifies whether to skip the first data line. The default value is False.
write_header : boxed_boolean option;A Boolean value that specifies whether to write the header to output. The default value is True.
with_header : boxed_boolean option;A Boolean value that specifies whether to treat the first line as a header. The default value is False.
multiline : boxed_boolean option;A Boolean value that specifies whether a single record can span multiple lines. This can occur when a field contains a quoted new-line character. You must set this option to True if any record spans multiple lines. The default value is False, which allows for more aggressive file-splitting during parsing.
quote_char : quote_char;Specifies the character to use for quoting. The default is a double quote: '"'. Set this to -1 to turn off quoting entirely.
escaper : enclosed_in_string_property_with_quote option;Specifies a character to use for escaping. This option is used only when reading CSV files. The default value is none. If enabled, the character which immediately follows is used as-is, except for a small set of well-known escapes (\n, \r, \t, and \0).
separator : separator;Specifies the delimiter character. The default is a comma: ",", but any other character can be specified.
*)additional_options : s3_direct_source_additional_options option;Specifies additional connection options.
*)max_files_in_band : boxed_non_negative_int option;This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.
*)max_band : boxed_non_negative_int option;This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.
*)recurse : boxed_boolean option;If set to true, recursively reads files in all subdirectories under the specified paths.
*)group_files : enclosed_in_string_property option;Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to "inPartition". To disable grouping when there are more than 50,000 files, set this parameter to "none".
group_size : enclosed_in_string_property option;The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, "groupFiles" must be set to "inPartition" for this to take effect.
exclusions : enclosed_in_string_properties option;A string containing a JSON list of Unix-style glob patterns to exclude. For example, "[\"**.pdf\"]" excludes all PDF files.
*)compression_type : compression_type option;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
paths : enclosed_in_string_properties;A list of the Amazon S3 paths to read from.
*)name : node_name;The name of the data store.
*)}Specifies a command-separated value (CSV) data store stored in Amazon S3.
type nonrec s3_json_source = {output_schemas : glue_schemas option;Specifies the data schema for the S3 JSON source.
*)multiline : boxed_boolean option;A Boolean value that specifies whether a single record can span multiple lines. This can occur when a field contains a quoted new-line character. You must set this option to True if any record spans multiple lines. The default value is False, which allows for more aggressive file-splitting during parsing.
json_path : enclosed_in_string_property option;A JsonPath string defining the JSON data.
*)additional_options : s3_direct_source_additional_options option;Specifies additional connection options.
*)max_files_in_band : boxed_non_negative_int option;This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.
*)max_band : boxed_non_negative_int option;This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.
*)recurse : boxed_boolean option;If set to true, recursively reads files in all subdirectories under the specified paths.
*)group_files : enclosed_in_string_property option;Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to "inPartition". To disable grouping when there are more than 50,000 files, set this parameter to "none".
group_size : enclosed_in_string_property option;The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, "groupFiles" must be set to "inPartition" for this to take effect.
exclusions : enclosed_in_string_properties option;A string containing a JSON list of Unix-style glob patterns to exclude. For example, "[\"**.pdf\"]" excludes all PDF files.
*)compression_type : compression_type option;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
paths : enclosed_in_string_properties;A list of the Amazon S3 paths to read from.
*)name : node_name;The name of the data store.
*)}Specifies a JSON data store stored in Amazon S3.
type nonrec s3_parquet_source = {output_schemas : glue_schemas option;Specifies the data schema for the S3 Parquet source.
*)additional_options : s3_direct_source_additional_options option;Specifies additional connection options.
*)max_files_in_band : boxed_non_negative_int option;This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.
*)max_band : boxed_non_negative_int option;This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.
*)recurse : boxed_boolean option;If set to true, recursively reads files in all subdirectories under the specified paths.
*)group_files : enclosed_in_string_property option;Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to "inPartition". To disable grouping when there are more than 50,000 files, set this parameter to "none".
group_size : enclosed_in_string_property option;The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, "groupFiles" must be set to "inPartition" for this to take effect.
exclusions : enclosed_in_string_properties option;A string containing a JSON list of Unix-style glob patterns to exclude. For example, "[\"**.pdf\"]" excludes all PDF files.
*)compression_type : parquet_compression_type option;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
paths : enclosed_in_string_properties;A list of the Amazon S3 paths to read from.
*)name : node_name;The name of the data store.
*)}Specifies an Apache Parquet data store stored in Amazon S3.
type nonrec relational_catalog_source = {table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies a Relational database data source in the Glue Data Catalog.
type nonrec ddbelt_catalog_additional_options = {dynamodb_unnest_ddb_json : boolean_value option;Specifies whether to unnest DynamoDB JSON format. When set to true, nested JSON structures in DynamoDB items are flattened.
dynamodb_export : enclosed_in_string_property option;Specifies the DynamoDB export configuration for the ELT operation.
*)}Specifies additional options for DynamoDB ELT catalog operations.
type nonrec dynamo_db_catalog_source = {additional_options : ddbelt_catalog_additional_options option;Specifies additional connection options for the DynamoDB data source.
*)pitr_enabled : boxed_boolean option;Specifies whether Point-in-Time Recovery (PITR) is enabled for the DynamoDB table. When set to true, allows reading from a specific point in time. The default value is false.
table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies a DynamoDB data source in the Glue Data Catalog.
type nonrec one_input = node_id listtype nonrec jdbc_connector_target = {output_schemas : glue_schemas option;Specifies the data schema for the JDBC target.
*)additional_options : additional_options option;Additional connection options for the connector.
*)connection_type : enclosed_in_string_property;The type of connection, such as marketplace.jdbc or custom.jdbc, designating a connection to a JDBC data target.
*)connector_name : enclosed_in_string_property;The name of a connector that will be used.
*)connection_table : enclosed_in_string_property_with_quote;The name of the table in the data target.
*)connection_name : enclosed_in_string_property;The name of the connection that is associated with the connector.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.
type nonrec spark_connector_target = {output_schemas : glue_schemas option;Specifies the data schema for the custom spark target.
*)additional_options : additional_options option;Additional connection options for the connector.
*)connection_type : enclosed_in_string_property;The type of connection, such as marketplace.spark or custom.spark, designating a connection to an Apache Spark data store.
*)connector_name : enclosed_in_string_property;The name of an Apache Spark connector.
*)connection_name : enclosed_in_string_property;The name of a connection for an Apache Spark connector.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses an Apache Spark connector.
type nonrec glue_studio_path_list = enclosed_in_string_properties listtype nonrec basic_catalog_target = {table : enclosed_in_string_property;The table that defines the schema of your output data. This table must already exist in the Data Catalog.
*)database : enclosed_in_string_property;The database that contains the table you want to use as the target. This database must already exist in the Data Catalog.
*)partition_keys : glue_studio_path_list option;The partition keys used to distribute data across multiple partitions or shards based on a specific key or set of key.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of your data target.
*)}Specifies a target that uses a Glue Data Catalog table.
type nonrec redshift_target = {upsert_redshift_options : upsert_redshift_target_options option;The set of options to configure an upsert operation when writing to a Redshift target.
*)tmp_dir_iam_role : enclosed_in_string_property option;The IAM role with permissions.
*)redshift_tmp_dir : enclosed_in_string_property option;The Amazon S3 path where temporary data can be staged when copying out of the database.
*)table : enclosed_in_string_property;The name of the table in the database to write to.
*)database : enclosed_in_string_property;The name of the database to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses Amazon Redshift.
type nonrec catalog_schema_change_policy = {update_behavior : update_catalog_behavior option;The update behavior when the crawler finds a changed schema.
*)enable_update_catalog : boxed_boolean option;Whether to use the specified update behavior when the crawler finds a changed schema.
*)}A policy that specifies update behavior for the crawler.
type nonrec auto_data_quality = {evaluation_context : enclosed_in_string_property option;The evaluation context for the automatic data quality checks. This defines the scope and parameters for the data quality evaluation.
*)is_enabled : boolean_value option;Specifies whether automatic data quality evaluation is enabled. When set to true, data quality checks are performed automatically.
}Specifies configuration options for automatic data quality evaluation in Glue jobs. This structure enables automated data quality checks and monitoring during ETL operations, helping to ensure data integrity and reliability without manual intervention.
type nonrec s3_catalog_target = {auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 catalog target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : catalog_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)database : enclosed_in_string_property;The name of the database to write to.
*)table : enclosed_in_string_property;The name of the table in the database to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.
type nonrec direct_schema_change_policy = {database : enclosed_in_string_property option;Specifies the database that the schema change policy applies to.
*)table : enclosed_in_string_property option;Specifies the table in the database that the schema change policy applies to.
*)update_behavior : update_catalog_behavior option;The update behavior when the crawler finds a changed schema.
*)enable_update_catalog : boxed_boolean option;Whether to use the specified update behavior when the crawler finds a changed schema.
*)}A policy that specifies update behavior for the crawler.
type nonrec s3_glue_parquet_target = {auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Glue Parquet target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : direct_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)number_target_partitions : number_target_partitions_string option;Specifies the number of target partitions for Parquet files when writing to Amazon S3 using Glue.
*)compression : parquet_compression_type option;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
path : enclosed_in_string_property;A single Amazon S3 path to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.
type nonrec s3_direct_target = {output_schemas : glue_schemas option;Specifies the data schema for the S3 direct target.
*)auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 direct target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : direct_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)format : target_format;Specifies the data output format for the target.
*)number_target_partitions : number_target_partitions_string option;Specifies the number of target partitions when writing data directly to Amazon S3.
*)compression : enclosed_in_string_property option;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
path : enclosed_in_string_property;A single Amazon S3 path to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a data target that writes to Amazon S3.
type mapping = {children : mappings option;Only applicable to nested data structures. If you want to change the parent structure, but also one of its children, you can fill out this data strucutre. It is also Mapping, but its FromPath will be the parent's FromPath plus the FromPath from this structure.
For the children part, suppose you have the structure:
{ "FromPath": "OuterStructure", "ToKey": "OuterStructure", "ToType": "Struct", "Dropped": false, "Chidlren": [{ "FromPath": "inner", "ToKey": "inner", "ToType": "Double", "Dropped": false, }] }
You can specify a Mapping that looks like:
{ "FromPath": "OuterStructure", "ToKey": "OuterStructure", "ToType": "Struct", "Dropped": false, "Chidlren": [{ "FromPath": "inner", "ToKey": "inner", "ToType": "Double", "Dropped": false, }] }
dropped : boxed_boolean option;If true, then the column is removed.
*)to_type : enclosed_in_string_property option;The data type that the data is to be modified to.
*)from_type : enclosed_in_string_property option;The type of the data to be modified.
*)from_path : enclosed_in_string_properties option;The table or column to be modified.
*)to_key : enclosed_in_string_property option;After the apply mapping, what the name of the column should be. Can be the same as FromPath.
}Specifies the mapping of data property keys.
and mappings = mapping listtype nonrec apply_mapping = {mapping : mappings;Specifies the mapping of data property keys in the data source to data property keys in the data target.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that maps data property keys in the data source to data property keys in the data target. You can rename keys, modify the data types for keys, and choose which keys to drop from the dataset.
type nonrec select_fields = {paths : glue_studio_path_list;A JSON path to a variable in the data structure.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that chooses the data property keys that you want to keep.
type nonrec drop_fields = {paths : glue_studio_path_list;A JSON path to a variable in the data structure.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that chooses the data property keys that you want to drop.
type nonrec rename_field = {target_path : enclosed_in_string_properties;A JSON path to a variable in the data structure for the target data.
*)source_path : enclosed_in_string_properties;A JSON path to a variable in the data structure for the source data.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that renames a single data property key.
type nonrec spigot = {prob : prob option;The probability (a decimal value with a maximum value of 1) of picking any given record. A value of 1 indicates that each row read from the dataset should be included in the sample output.
*)topk : topk option;Specifies a number of records to write starting from the beginning of the dataset.
*)path : enclosed_in_string_property;A path in Amazon S3 where the transform will write a subset of records from the dataset to a JSON file in an Amazon S3 bucket.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that writes samples of the data to an Amazon S3 bucket.
type nonrec two_inputs = node_id listtype nonrec join_column = {keys : glue_studio_path_list;The key of the column to be joined.
*)from_ : enclosed_in_string_property;The column to be joined.
*)}Specifies a column to be joined.
type nonrec join_columns = join_column listtype nonrec join = {columns : join_columns;A list of the two columns to be joined.
*)join_type : join_type;Specifies the type of join to be performed on the datasets.
*)inputs : two_inputs;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that joins two datasets into one dataset using a comparison phrase on the specified data property keys. You can use inner, outer, left, right, left semi, and left anti joins.
type nonrec split_fields = {paths : glue_studio_path_list;A JSON path to a variable in the data structure.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that splits data property keys into two DynamicFrames. The output is a collection of DynamicFrames: one with selected data property keys, and one with the remaining data property keys.
type nonrec select_from_collection = {index : non_negative_int;The index for the DynamicFrame to be selected.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that chooses one DynamicFrame from a collection of DynamicFrames. The output is the selected DynamicFrame
type nonrec fill_missing_values = {filled_path : enclosed_in_string_property option;A JSON path to a variable in the data structure for the dataset that is filled.
*)imputed_path : enclosed_in_string_property;A JSON path to a variable in the data structure for the dataset that is imputed.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that locates records in the dataset that have missing values and adds a new field with a value determined by imputation. The input data set is used to train the machine learning model that determines what the missing value should be.
type nonrec filter_value = {value : enclosed_in_string_properties;The value to be associated.
*)type_ : filter_value_type;The type of filter value.
*)}Represents a single entry in the list of values for a FilterExpression.
type nonrec filter_values = filter_value listtype nonrec filter_expression = {values : filter_values;A list of filter values.
*)negated : boxed_boolean option;Whether the expression is to be negated.
*)operation : filter_operation;The type of operation to perform in the expression.
*)}Specifies a filter expression.
type nonrec filter_expressions = filter_expression listtype nonrec filter = {filters : filter_expressions;Specifies a filter expression.
*)logical_operator : filter_logical_operator;The operator used to filter rows by comparing the key value to a specified value.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that splits a dataset into two, based on a filter condition.
type nonrec many_inputs = node_id listtype nonrec custom_code = {output_schemas : glue_schemas option;Specifies the data schema for the custom code transform.
*)class_name : enclosed_in_string_property;The name defined for the custom code node class.
*)code : extended_string;The custom code that is used to perform the data transformation.
*)inputs : many_inputs;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that uses custom code you provide to perform the data transformation. The output is a collection of DynamicFrames.
type nonrec sql_alias = {alias : enclosed_in_string_property_with_quote;A temporary name given to a table, or a column in a table.
*)from_ : node_id;A table, or a column in a table.
*)}Represents a single entry in the list of values for SqlAliases.
type nonrec sql_aliases = sql_alias listtype nonrec spark_sq_l = {output_schemas : glue_schemas option;Specifies the data schema for the SparkSQL transform.
*)sql_aliases : sql_aliases;A list of aliases. An alias allows you to specify what name to use in the SQL for a given input. For example, you have a datasource named "MyDataSource". If you specify From as MyDataSource, and Alias as SqlName, then in your SQL you can do:
select * from SqlName
and that gets data from MyDataSource.
*)sql_query : sql_query;A SQL query that must use Spark SQL syntax and return a single data set.
*)inputs : many_inputs;The data inputs identified by their node names. You can associate a table name with each input node to use in the SQL query. The name you choose must meet the Spark SQL naming restrictions.
*)name : node_name;The name of the transform node.
*)}Specifies a transform where you enter a SQL query using Spark SQL syntax to transform the data. The output is a single DynamicFrame.
type nonrec kinesis_streaming_source_options = {fanout_consumer_ar_n : enclosed_in_string_property option;The Amazon Resource Name (ARN) of the Kinesis Data Streams enhanced fan-out consumer. When specified, enables enhanced fan-out for dedicated throughput and lower latency data consumption.
*)starting_timestamp : iso8601_date_time option;The timestamp of the record in the Kinesis data stream to start reading data from. The possible values are a timestamp string in UTC format of the pattern yyyy-mm-ddTHH:MM:SSZ (where Z represents a UTC timezone offset with a +/-. For example: "2023-04-04T08:00:00+08:00").
emit_consumer_lag_metrics : enclosed_in_string_property option;When this option is set to 'true', for each batch, it will emit the metrics for the duration between the oldest record received by the stream and the time it arrives in Glue to CloudWatch. The metric's name is "glue.driver.streaming.maxConsumerLagInMs". The default value is 'false'. This option is supported in Glue version 4.0 or later.
*)add_record_timestamp : enclosed_in_string_property option;When this option is set to 'true', the data output will contain an additional column named "__src_timestamp" that indicates the time when the corresponding record received by the stream. The default value is 'false'. This option is supported in Glue version 4.0 or later.
*)role_session_name : enclosed_in_string_property option;An identifier for the session assuming the role using AWS STS. You must use this parameter when accessing a data stream in a different account. Used in conjunction with "awsSTSRoleARN".
role_arn : enclosed_in_string_property option;The Amazon Resource Name (ARN) of the role to assume using AWS Security Token Service (AWS STS). This role must have permissions for describe or read record operations for the Kinesis data stream. You must use this parameter when accessing a data stream in a different account. Used in conjunction with "awsSTSSessionName".
stream_arn : enclosed_in_string_property option;The Amazon Resource Name (ARN) of the Kinesis data stream.
*)avoid_empty_batches : boxed_boolean option;Avoids creating an empty microbatch job by checking for unread data in the Kinesis data stream before the batch is started. The default value is "False".
max_retry_interval_ms : boxed_non_negative_long option;The maximum cool-off time period (specified in ms) between two retries of a Kinesis Data Streams API call. The default value is 10000.
retry_interval_ms : boxed_non_negative_long option;The cool-off time period (specified in ms) before retrying the Kinesis Data Streams API call. The default value is 1000.
num_retries : boxed_non_negative_int option;The maximum number of retries for Kinesis Data Streams API requests. The default value is 3.
describe_shard_interval : boxed_non_negative_long option;The minimum time interval between two ListShards API calls for your script to consider resharding. The default value is 1s.
idle_time_between_reads_in_ms : boxed_non_negative_long option;The minimum time delay between two consecutive getRecords operations, specified in ms. The default value is 1000. This option is only configurable for Glue version 2.0 and above.
add_idle_time_between_reads : boxed_boolean option;Adds a time delay between two consecutive getRecords operations. The default value is "False". This option is only configurable for Glue version 2.0 and above.
max_record_per_read : boxed_non_negative_long option;The maximum number of records to fetch from the Kinesis data stream in each getRecords operation. The default value is 10000.
max_fetch_records_per_shard : boxed_non_negative_long option;The maximum number of records to fetch per shard in the Kinesis data stream per microbatch. Note: The client can exceed this limit if the streaming job has already read extra records from Kinesis (in the same get-records call). If MaxFetchRecordsPerShard needs to be strict then it needs to be a multiple of MaxRecordPerRead. The default value is 100000.
max_fetch_time_in_ms : boxed_non_negative_long option;The maximum time spent for the job executor to read records for the current batch from the Kinesis data stream, specified in milliseconds (ms). Multiple GetRecords API calls may be made within this time. The default value is 1000.
starting_position : starting_position option;The starting position in the Kinesis data stream to read data from. The possible values are "latest", "trim_horizon", "earliest", or a timestamp string in UTC format in the pattern yyyy-mm-ddTHH:MM:SSZ (where Z represents a UTC timezone offset with a +/-. For example: "2023-04-04T08:00:00-04:00"). The default value is "latest".
Note: Using a value that is a timestamp string in UTC format for "startingPosition" is supported only for Glue version 4.0 or later.
*)delimiter : enclosed_in_string_property option;Specifies the delimiter character.
*)classification : enclosed_in_string_property option;An optional classification.
*)stream_name : enclosed_in_string_property option;The name of the Kinesis data stream.
*)endpoint_url : enclosed_in_string_property option;The URL of the Kinesis endpoint.
*)}Additional options for the Amazon Kinesis streaming data source.
type nonrec polling_time = Smaws_Lib.CoreTypes.Int64.ttype nonrec positive_long = Smaws_Lib.CoreTypes.Int64.ttype nonrec streaming_data_preview_options = {record_polling_limit : positive_long option;The limit to the number of records polled.
*)polling_time : polling_time option;The polling time in milliseconds.
*)}Specifies options related to data preview for viewing a sample of your data.
type nonrec direct_kinesis_source = {data_preview_options : streaming_data_preview_options option;Additional options for data preview.
*)streaming_options : kinesis_streaming_source_options option;Additional options for the Kinesis streaming data source.
*)detect_schema : boxed_boolean option;Whether to automatically determine the schema from the incoming data.
*)window_size : boxed_positive_int option;The amount of time to spend processing each micro batch.
*)name : node_name;The name of the data source.
*)}Specifies a direct Amazon Kinesis data source.
type nonrec kafka_streaming_source_options = {starting_timestamp : iso8601_date_time option;The timestamp of the record in the Kafka topic to start reading data from. The possible values are a timestamp string in UTC format of the pattern yyyy-mm-ddTHH:MM:SSZ (where Z represents a UTC timezone offset with a +/-. For example: "2023-04-04T08:00:00+08:00").
Only one of StartingTimestamp or StartingOffsets must be set.
emit_consumer_lag_metrics : enclosed_in_string_property option;When this option is set to 'true', for each batch, it will emit the metrics for the duration between the oldest record received by the topic and the time it arrives in Glue to CloudWatch. The metric's name is "glue.driver.streaming.maxConsumerLagInMs". The default value is 'false'. This option is supported in Glue version 4.0 or later.
*)add_record_timestamp : enclosed_in_string_property option;When this option is set to 'true', the data output will contain an additional column named "__src_timestamp" that indicates the time when the corresponding record received by the topic. The default value is 'false'. This option is supported in Glue version 4.0 or later.
*)include_headers : boxed_boolean option;Whether to include the Kafka headers. When the option is set to "true", the data output will contain an additional column named "glue_streaming_kafka_headers" with type Array[Struct(key: String, value: String)]. The default value is "false". This option is available in Glue version 3.0 or later only.
min_partitions : boxed_non_negative_int option;The desired minimum number of partitions to read from Kafka. The default value is null, which means that the number of spark partitions is equal to the number of Kafka partitions.
*)max_offsets_per_trigger : boxed_non_negative_long option;The rate limit on the maximum number of offsets that are processed per trigger interval. The specified total number of offsets is proportionally split across topicPartitions of different volumes. The default value is null, which means that the consumer reads all offsets until the known latest offset.
retry_interval_ms : boxed_non_negative_long option;The time in milliseconds to wait before retrying to fetch Kafka offsets. The default value is 10.
num_retries : boxed_non_negative_int option;The number of times to retry before failing to fetch Kafka offsets. The default value is 3.
poll_timeout_ms : boxed_non_negative_long option;The timeout in milliseconds to poll data from Kafka in Spark job executors. The default value is 512.
ending_offsets : enclosed_in_string_property option;The end point when a batch query is ended. Possible values are either "latest" or a JSON string that specifies an ending offset for each TopicPartition.
starting_offsets : enclosed_in_string_property option;The starting position in the Kafka topic to read data from. The possible values are "earliest" or "latest". The default value is "latest".
delimiter : enclosed_in_string_property option;Specifies the delimiter character.
*)classification : enclosed_in_string_property option;An optional classification.
*)subscribe_pattern : enclosed_in_string_property option;A Java regex string that identifies the topic list to subscribe to. You must specify at least one of "topicName", "assign" or "subscribePattern".
assign : enclosed_in_string_property option;The specific TopicPartitions to consume. You must specify at least one of "topicName", "assign" or "subscribePattern".
topic_name : enclosed_in_string_property option;The topic name as specified in Apache Kafka. You must specify at least one of "topicName", "assign" or "subscribePattern".
connection_name : enclosed_in_string_property option;The name of the connection.
*)security_protocol : enclosed_in_string_property option;The protocol used to communicate with brokers. The possible values are "SSL" or "PLAINTEXT".
bootstrap_servers : enclosed_in_string_property option;A list of bootstrap server URLs, for example, as b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094. This option must be specified in the API call or defined in the table metadata in the Data Catalog.
}Additional options for streaming.
type nonrec direct_kafka_source = {data_preview_options : streaming_data_preview_options option;Specifies options related to data preview for viewing a sample of your data.
*)detect_schema : boxed_boolean option;Whether to automatically determine the schema from the incoming data.
*)window_size : boxed_positive_int option;The amount of time to spend processing each micro batch.
*)streaming_options : kafka_streaming_source_options option;Specifies the streaming options.
*)name : node_name;The name of the data store.
*)}Specifies an Apache Kafka data store.
type nonrec catalog_kinesis_source = {data_preview_options : streaming_data_preview_options option;Additional options for data preview.
*)streaming_options : kinesis_streaming_source_options option;Additional options for the Kinesis streaming data source.
*)database : enclosed_in_string_property;The name of the database to read from.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)detect_schema : boxed_boolean option;Whether to automatically determine the schema from the incoming data.
*)window_size : boxed_positive_int option;The amount of time to spend processing each micro batch.
*)name : node_name;The name of the data source.
*)}Specifies a Kinesis data source in the Glue Data Catalog.
type nonrec catalog_kafka_source = {data_preview_options : streaming_data_preview_options option;Specifies options related to data preview for viewing a sample of your data.
*)streaming_options : kafka_streaming_source_options option;Specifies the streaming options.
*)database : enclosed_in_string_property;The name of the database to read from.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)detect_schema : boxed_boolean option;Whether to automatically determine the schema from the incoming data.
*)window_size : boxed_positive_int option;The amount of time to spend processing each micro batch.
*)name : node_name;The name of the data store.
*)}Specifies an Apache Kafka data store in the Data Catalog.
type nonrec null_check_box_list = {is_neg_one : boxed_boolean option;Specifies that an integer value of -1 is considered as a null value.
*)is_null_string : boxed_boolean option;Specifies that a value spelling out the word 'null' is considered as a null value.
*)is_empty : boxed_boolean option;Specifies that an empty string is considered as a null value.
*)}Represents whether certain values are recognized as null values for removal.
type nonrec datatype = {label : generic_limited_string;A label assigned to the datatype.
*)id : generic_limited_string;The datatype of the value.
*)}A structure representing the datatype of the value.
type nonrec null_value_field = {datatype : datatype;The datatype of the value.
*)value : enclosed_in_string_property;The value of the null placeholder.
*)}Represents a custom null value such as a zeros or other value being used as a null placeholder unique to the dataset.
type nonrec null_value_fields = null_value_field listtype nonrec drop_null_fields = {null_text_list : null_value_fields option;A structure that specifies a list of NullValueField structures that represent a custom null value such as zero or other value being used as a null placeholder unique to the dataset.
The DropNullFields transform removes custom null values only if both the value of the null placeholder and the datatype match the data.
null_check_box_list : null_check_box_list option;A structure that represents whether certain values are recognized as null values for removal.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that removes columns from the dataset if all values in the column are 'null'. By default, Glue Studio will recognize null objects, but some values such as empty strings, strings that are "null", -1 integers or other placeholders such as zeros, are not automatically recognized as nulls.
type nonrec merge = {primary_keys : glue_studio_path_list;The list of primary key fields to match records from the source and staging dynamic frames.
*)source : node_id;The source DynamicFrame that will be merged with a staging DynamicFrame.
inputs : two_inputs;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that merges a DynamicFrame with a staging DynamicFrame based on the specified primary keys to identify records. Duplicate records (records with the same primary keys) are not de-duplicated.
type nonrec union = {union_type : union_type;Indicates the type of Union transform.
Specify ALL to join all rows from data sources to the resulting DynamicFrame. The resulting union does not remove duplicate rows.
Specify DISTINCT to remove duplicate rows in the resulting DynamicFrame.
inputs : two_inputs;The node ID inputs to the transform.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that combines the rows from two or more datasets into a single result.
type nonrec pii_detection = {detection_sensitivity : enclosed_in_string_property option;The sensitivity level for PII detection. Higher sensitivity levels detect more potential PII but may result in more false positives.
*)detection_parameters : enclosed_in_string_property option;Additional parameters for configuring PII detection behavior and sensitivity settings.
*)num_right_chars_to_exclude : boxed_positive_int option;The number of characters to exclude from redaction on the right side of detected PII content. This allows preserving context around the sensitive data.
*)num_left_chars_to_exclude : boxed_positive_int option;The number of characters to exclude from redaction on the left side of detected PII content. This allows preserving context around the sensitive data.
*)match_pattern : enclosed_in_string_property option;A regular expression pattern used to identify additional PII content beyond the standard detection algorithms.
*)redact_char : enclosed_in_string_property option;The character used to replace detected PII content when redaction is enabled. The default redaction character is *.
redact_text : enclosed_in_string_property option;Specifies whether to redact the detected PII text. When set to true, PII content is replaced with redaction characters.
mask_value : mask_value option;Indicates the value that will replace the detected entity.
*)threshold_fraction : boxed_double_fraction option;Indicates the fraction of the data that must be met in order for a column to be identified as PII data.
*)sample_fraction : boxed_double_fraction option;Indicates the fraction of the data to sample when scanning for PII entities.
*)output_column_name : enclosed_in_string_property option;Indicates the output column name that will contain any entity type detected in that row.
*)entity_types_to_detect : enclosed_in_string_properties;Indicates the types of entities the PIIDetection transform will identify as PII data.
PII type entities include: PERSON_NAME, DATE, USA_SNN, EMAIL, USA_ITIN, USA_PASSPORT_NUMBER, PHONE_NUMBER, BANK_ACCOUNT, IP_ADDRESS, MAC_ADDRESS, USA_CPT_CODE, USA_HCPCS_CODE, USA_NATIONAL_DRUG_CODE, USA_MEDICARE_BENEFICIARY_IDENTIFIER, USA_HEALTH_INSURANCE_CLAIM_NUMBER,CREDIT_CARD,USA_NATIONAL_PROVIDER_IDENTIFIER,USA_DEA_NUMBER,USA_DRIVING_LICENSE
*)pii_type : pii_type;Indicates the type of PIIDetection transform.
*)inputs : one_input;The node ID inputs to the transform.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that identifies, removes or masks PII data.
type nonrec aggregate_operation = {agg_func : agg_function;Specifies the aggregation function to apply.
Possible aggregation functions include: avg countDistinct, count, first, last, kurtosis, max, min, skewness, stddev_samp, stddev_pop, sum, sumDistinct, var_samp, var_pop
*)column : enclosed_in_string_properties;Specifies the column on the data set on which the aggregation function will be applied.
*)}Specifies the set of parameters needed to perform aggregation in the aggregate transform.
type nonrec aggregate_operations = aggregate_operation listtype nonrec aggregate = {aggs : aggregate_operations;Specifies the aggregate functions to be performed on specified fields.
*)groups : glue_studio_path_list;Specifies the fields to group by.
*)inputs : one_input;Specifies the fields and rows to use as inputs for the aggregate transform.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that groups rows by chosen fields and computes the aggregated value by specified function.
type nonrec limited_string_list = generic_limited_string listtype nonrec limited_path_list = limited_string_list listtype nonrec drop_duplicates = {columns : limited_path_list option;The name of the columns to be merged or removed if repeating.
*)inputs : one_input;The data inputs identified by their node names.
*)name : node_name;The name of the transform node.
*)}Specifies a transform that removes rows of repeating data from a data set.
type nonrec governed_catalog_target = {schema_change_policy : catalog_schema_change_policy option;A policy that specifies update behavior for the governed catalog.
*)database : enclosed_in_string_property;The name of the database to write to.
*)table : enclosed_in_string_property;The name of the table in the database to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.
type nonrec governed_catalog_source = {additional_options : s3_source_additional_options option;Specifies additional connection options.
*)partition_predicate : enclosed_in_string_property option;Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted. Set to "" – empty by default.
table : enclosed_in_string_property;The database table to read from.
*)database : enclosed_in_string_property;The database to read from.
*)name : node_name;The name of the data store.
*)}Specifies the data store in the governed Glue Data Catalog.
type nonrec microsoft_sql_server_catalog_source = {table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies a Microsoft SQL server data source in the Glue Data Catalog.
type nonrec my_sql_catalog_source = {table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies a MySQL data source in the Glue Data Catalog.
type nonrec oracle_sql_catalog_source = {table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies an Oracle data source in the Glue Data Catalog.
type nonrec postgre_sql_catalog_source = {table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the data source.
*)}Specifies a PostgresSQL data source in the Glue Data Catalog.
type nonrec microsoft_sql_server_catalog_target = {table : enclosed_in_string_property;The name of the table in the database to write to.
*)database : enclosed_in_string_property;The name of the database to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses Microsoft SQL.
type nonrec my_sql_catalog_target = {table : enclosed_in_string_property;The name of the table in the database to write to.
*)database : enclosed_in_string_property;The name of the database to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses MySQL.
type nonrec oracle_sql_catalog_target = {table : enclosed_in_string_property;The name of the table in the database to write to.
*)database : enclosed_in_string_property;The name of the database to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses Oracle SQL.
type nonrec postgre_sql_catalog_target = {table : enclosed_in_string_property;The name of the table in the database to write to.
*)database : enclosed_in_string_property;The name of the database to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that uses Postgres SQL.
type nonrec group_filters = {logical_operator : filter_logical_operator;The logical operator used to combine the filters in this group. Determines whether all filters must match (AND) or any filter can match (OR).
*)filters : filter_expressions;A list of filter expressions that define the conditions for this group.
*)group_name : generic_limited_string;The name of the filter group.
*)}Specifies a group of filters with a logical operator that determines how the filters are combined to evaluate routing conditions.
type nonrec group_filters_list = group_filters listtype nonrec route = {group_filters_list : group_filters_list;A list of group filters that define the routing conditions and criteria for directing data to different output paths.
*)inputs : one_input;The input connection for the route node.
*)name : node_name;The name of the route node.
*)}Specifies a route node that directs data to different output paths based on defined filtering conditions.
type nonrec transform_config_parameter = {is_optional : boxed_boolean option;Specifies whether the parameter is optional or not in the config file of the dynamic transform.
*)list_type : param_type option;Specifies the list type of the parameter in the config file of the dynamic transform.
*)value : enclosed_in_string_properties option;Specifies the value of the parameter in the config file of the dynamic transform.
*)validation_message : enclosed_in_string_property option;Specifies the validation message in the config file of the dynamic transform.
*)validation_rule : enclosed_in_string_property option;Specifies the validation rule in the config file of the dynamic transform.
*)type_ : param_type;Specifies the parameter type in the config file of the dynamic transform.
*)name : enclosed_in_string_property;Specifies the name of the parameter in the config file of the dynamic transform.
*)}Specifies the parameters in the config file of the dynamic transform.
type nonrec transform_config_parameter_list = transform_config_parameter listtype nonrec dynamic_transform = {output_schemas : glue_schemas option;Specifies the data schema for the dynamic transform.
*)version : enclosed_in_string_property option;This field is not used and will be deprecated in future release.
*)path : enclosed_in_string_property;Specifies the path of the dynamic transform source and config files.
*)function_name : enclosed_in_string_property;Specifies the name of the function of the dynamic transform.
*)parameters : transform_config_parameter_list option;Specifies the parameters of the dynamic transform.
*)inputs : one_input;Specifies the inputs for the dynamic transform that are required.
*)transform_name : enclosed_in_string_property;Specifies the name of the dynamic transform as it appears in the Glue Studio visual editor.
*)name : enclosed_in_string_property;Specifies the name of the dynamic transform.
*)}Specifies the set of parameters needed to perform the dynamic transform.
type nonrec dq_results_publishing_options = {results_publishing_enabled : boxed_boolean option;Enable publishing for your data quality results.
*)cloud_watch_metrics_enabled : boxed_boolean option;Enable metrics for your data quality results.
*)results_s3_prefix : enclosed_in_string_property option;The Amazon S3 prefix prepended to the results.
*)evaluation_context : generic_limited_string option;The context of the evaluation.
*)}Options to configure how your data quality evaluation results are published.
type nonrec dq_stop_job_on_failure_options = {stop_job_on_failure_timing : dq_stop_job_on_failure_timing option;When to stop job if your data quality evaluation fails. Options are Immediate or AfterDataLoad.
*)}Options to configure how your job will stop if your data quality evaluation fails.
type nonrec evaluate_data_quality = {stop_job_on_failure_options : dq_stop_job_on_failure_options option;Options to configure how your job will stop if your data quality evaluation fails.
*)publishing_options : dq_results_publishing_options option;Options to configure how your results are published.
*)output : dq_transform_output option;The output of your data quality evaluation.
*)ruleset : dqdl_string;The ruleset for your data quality evaluation.
*)inputs : one_input;The inputs of your data quality evaluation.
*)name : node_name;The name of the data quality evaluation.
*)}Specifies your data quality evaluation criteria.
type nonrec s3_catalog_hudi_source = {output_schemas : glue_schemas option;Specifies the data schema for the Hudi source.
*)additional_hudi_options : additional_options option;Specifies additional connection options.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Hudi data source.
*)}Specifies a Hudi data source that is registered in the Glue Data Catalog. The Hudi data source must be stored in Amazon S3.
type nonrec catalog_hudi_source = {output_schemas : glue_schemas option;Specifies the data schema for the Hudi source.
*)additional_hudi_options : additional_options option;Specifies additional connection options.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Hudi data source.
*)}Specifies a Hudi data source that is registered in the Glue Data Catalog.
type nonrec s3_hudi_source = {output_schemas : glue_schemas option;Specifies the data schema for the Hudi source.
*)additional_options : s3_direct_source_additional_options option;Specifies additional options for the connector.
*)additional_hudi_options : additional_options option;Specifies additional connection options.
*)paths : enclosed_in_string_properties;A list of the Amazon S3 paths to read from.
*)name : node_name;The name of the Hudi source.
*)}Specifies a Hudi data source stored in Amazon S3.
type nonrec s3_hudi_catalog_target = {output_schemas : glue_schemas option;Specifies the data schema for the S3 Hudi catalog target.
*)auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Hudi catalog target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : catalog_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)additional_options : additional_options;Specifies additional connection options for the connector.
*)database : enclosed_in_string_property;The name of the database to write to.
*)table : enclosed_in_string_property;The name of the table in the database to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that writes to a Hudi data source in the Glue Data Catalog.
type nonrec s3_hudi_direct_target = {auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Hudi direct target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : direct_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)additional_options : additional_options;Specifies additional connection options for the connector.
*)format : target_format;Specifies the data output format for the target.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)number_target_partitions : number_target_partitions_string option;Specifies the number of target partitions for distributing Hudi dataset files across Amazon S3.
*)compression : hudi_target_compression_type;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
path : enclosed_in_string_property;The Amazon S3 path of your Hudi data source to write to.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that writes to a Hudi data source in Amazon S3.
type nonrec direct_jdbc_source = {output_schemas : glue_schemas option;Specifies the data schema for the direct JDBC source.
*)redshift_tmp_dir : enclosed_in_string_property option;The temp directory of the JDBC Redshift source.
*)connection_type : jdbc_connection_type;The connection type of the JDBC source.
*)connection_name : enclosed_in_string_property;The connection name of the JDBC source.
*)table : enclosed_in_string_property;The table of the JDBC source connection.
*)database : enclosed_in_string_property;The database of the JDBC source connection.
*)name : node_name;The name of the JDBC source connection.
*)}Specifies the direct JDBC source connection.
type nonrec s3_catalog_delta_source = {output_schemas : glue_schemas option;Specifies the data schema for the Delta Lake source.
*)additional_delta_options : additional_options option;Specifies additional connection options.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Delta Lake data source.
*)}Specifies a Delta Lake data source that is registered in the Glue Data Catalog. The data source must be stored in Amazon S3.
type nonrec catalog_delta_source = {output_schemas : glue_schemas option;Specifies the data schema for the Delta Lake source.
*)additional_delta_options : additional_options option;Specifies additional connection options.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Delta Lake data source.
*)}Specifies a Delta Lake data source that is registered in the Glue Data Catalog.
type nonrec s3_delta_source = {output_schemas : glue_schemas option;Specifies the data schema for the Delta Lake source.
*)additional_options : s3_direct_source_additional_options option;Specifies additional options for the connector.
*)additional_delta_options : additional_options option;Specifies additional connection options.
*)paths : enclosed_in_string_properties;A list of the Amazon S3 paths to read from.
*)name : node_name;The name of the Delta Lake source.
*)}Specifies a Delta Lake data source stored in Amazon S3.
type nonrec s3_delta_catalog_target = {output_schemas : glue_schemas option;Specifies the data schema for the S3 Delta catalog target.
*)auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Delta catalog target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : catalog_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)additional_options : additional_options option;Specifies additional connection options for the connector.
*)database : enclosed_in_string_property;The name of the database to write to.
*)table : enclosed_in_string_property;The name of the table in the database to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that writes to a Delta Lake data source in the Glue Data Catalog.
type nonrec s3_delta_direct_target = {auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Delta direct target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : direct_schema_change_policy option;A policy that specifies update behavior for the crawler.
*)additional_options : additional_options option;Specifies additional connection options for the connector.
*)format : target_format;Specifies the data output format for the target.
*)number_target_partitions : number_target_partitions_string option;Specifies the number of target partitions for distributing Delta Lake dataset files across Amazon S3.
*)compression : delta_target_compression_type;Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are "gzip" and "bzip").
path : enclosed_in_string_property;The Amazon S3 path of your Delta Lake data source to write to.
*)partition_keys : glue_studio_path_list option;Specifies native partitioning using a sequence of keys.
*)inputs : one_input;The nodes that are inputs to the data target.
*)name : node_name;The name of the data target.
*)}Specifies a target that writes to a Delta Lake data source in Amazon S3.
type nonrec option_ = {description : enclosed_in_string_property option;Specifies the description of the option.
*)label : enclosed_in_string_property option;Specifies the label of the option.
*)value : enclosed_in_string_property option;Specifies the value of the option.
*)}Specifies an option value.
type nonrec amazon_redshift_advanced_option = {value : generic_string option;The value for the additional connection option.
*)key : generic_string option;The key for the additional connection option.
*)}Specifies an optional value when connecting to the Redshift cluster.
type nonrec amazon_redshift_advanced_options =
amazon_redshift_advanced_option listtype nonrec option_list = option_ listtype nonrec amazon_redshift_node_data = {selected_columns : option_list option;The list of column names used to determine a matching record when doing a MERGE or APPEND with upsert.
*)staging_table : generic_string option;The name of the temporary staging table that is used when doing a MERGE or APPEND with upsert.
*)table_schema : option_list option;The array of schema output for a given node.
*)crawler_connection : generic_string option;Specifies the name of the connection that is associated with the catalog table used.
*)merge_clause : generic_string option;The SQL used in a custom merge to deal with matching records.
*)merge_when_not_matched : generic_limited_string option;The action used when to detemine how a MERGE in a Redshift sink will be handled when an existing record doesn't match a new record.
*)merge_when_matched : generic_limited_string option;The action used when to detemine how a MERGE in a Redshift sink will be handled when an existing record matches a new record.
*)merge_action : generic_limited_string option;The action used when to detemine how a MERGE in a Redshift sink will be handled.
*)upsert : boolean_value option;The action used on Redshift sinks when doing an APPEND.
*)table_prefix : generic_limited_string option;Specifies the prefix to a table.
*)action : generic_string option;Specifies how writing to a Redshift cluser will occur.
*)post_action : generic_string option;The SQL used before a MERGE or APPEND with upsert is run.
*)pre_action : generic_string option;The SQL used before a MERGE or APPEND with upsert is run.
*)sample_query : generic_string option;The SQL used to fetch the data from a Redshift sources when the SourceType is 'query'.
*)advanced_options : amazon_redshift_advanced_options option;Optional values when connecting to the Redshift cluster.
*)iam_role : option_ option;Optional. The role name use when connection to S3. The IAM role ill default to the role on the job when left blank.
*)temp_dir : enclosed_in_string_property option;The Amazon S3 path where temporary data can be staged when copying out of the database.
*)catalog_redshift_table : generic_string option;The database table to read from.
*)catalog_redshift_schema : generic_string option;The Redshift schema name when working with a data catalog.
*)catalog_table : option_ option;The Glue Data Catalog table name when working with a data catalog.
*)catalog_database : option_ option;The name of the Glue Data Catalog database when working with a data catalog.
*)table : option_ option;The Redshift table name when working with a direct connection.
*)schema : option_ option;The Redshift schema name when working with a direct connection.
*)connection : option_ option;The Glue connection to the Redshift cluster.
*)source_type : generic_limited_string option;The source type to specify whether a specific table is the source or a custom query.
*)access_type : generic_limited_string option;The access type for the Redshift connection. Can be a direct connection or catalog connections.
*)}Specifies an Amazon Redshift node.
type nonrec amazon_redshift_source = {data : amazon_redshift_node_data option;Specifies the data of the Amazon Reshift source node.
*)name : node_name option;The name of the Amazon Redshift source.
*)}Specifies an Amazon Redshift source.
type nonrec amazon_redshift_target = {inputs : one_input option;The nodes that are inputs to the data target.
*)data : amazon_redshift_node_data option;Specifies the data of the Amazon Redshift target node.
*)name : node_name option;The name of the Amazon Redshift target.
*)}Specifies an Amazon Redshift target.
type nonrec dqdl_aliases = (node_name * enclosed_in_string_property) listtype nonrec dq_additional_options =
(additional_option_keys * generic_string) listtype nonrec evaluate_data_quality_multi_frame = {stop_job_on_failure_options : dq_stop_job_on_failure_options option;Options to configure how your job will stop if your data quality evaluation fails.
*)additional_options : dq_additional_options option;Options to configure runtime behavior of the transform.
*)publishing_options : dq_results_publishing_options option;Options to configure how your results are published.
*)ruleset : dqdl_string;The ruleset for your data quality evaluation.
*)additional_data_sources : dqdl_aliases option;The aliases of all data sources except primary.
*)inputs : many_inputs;The inputs of your data quality evaluation. The first input in this list is the primary data source.
*)name : node_name;The name of the data quality evaluation.
*)}Specifies your data quality evaluation criteria.
type nonrec recipe_reference = {recipe_version : recipe_version;The RecipeVersion of the DataBrew recipe.
*)recipe_arn : enclosed_in_string_property;The ARN of the DataBrew recipe.
*)}A reference to a Glue DataBrew recipe.
type nonrec parameter_map = (parameter_name * parameter_value) listtype nonrec recipe_action = {parameters : parameter_map option;The parameters of the recipe action.
*)operation : operation;The operation of the recipe action.
*)}Actions defined in the Glue Studio data preparation recipe node.
type nonrec condition_expression = {target_column : target_column;The target column of the condition expressions.
*)value : databrew_condition_value option;The value of the condition expression.
*)condition : databrew_condition;The condition of the condition expression.
*)}Condition expression defined in the Glue Studio data preparation recipe node.
type nonrec condition_expression_list = condition_expression listtype nonrec recipe_step = {condition_expressions : condition_expression_list option;The condition expressions for the recipe step.
*)action : recipe_action;The transformation action of the recipe step.
*)}A recipe step used in a Glue Studio data preparation recipe node.
type nonrec recipe_steps = recipe_step listtype nonrec recipe = {recipe_steps : recipe_steps option;Transform steps used in the recipe node.
*)recipe_reference : recipe_reference option;A reference to the DataBrew recipe used by the node.
*)inputs : one_input;The nodes that are inputs to the recipe node, identified by id.
*)name : node_name;The name of the Glue Studio node.
*)}A Glue Studio node that uses a Glue DataBrew recipe in Glue jobs.
type nonrec snowflake_node_data = {table_schema : option_list option;Manually defines the target schema for the node. A list of structures with value , label and description keys. Each structure defines a column.
auto_pushdown : boolean_value option;Specifies whether automatic query pushdown is enabled. If pushdown is enabled, then when a query is run on Spark, if part of the query can be "pushed down" to the Snowflake server, it is pushed down. This improves performance of some queries.
*)selected_columns : option_list option;Specifies the columns combined to identify a record when detecting matches for merges and upserts. A list of structures with value, label and description keys. Each structure describes a column.
staging_table : generic_string option;The name of a staging table used when performing merge or upsert append actions. Data is written to this table, then moved to table by a generated postaction.
merge_clause : generic_string option;A SQL statement that specifies a custom merge behavior.
*)merge_when_not_matched : generic_limited_string option;Specifies how to process records that do not match preexisting data when merging. Valid values: insert, none.
merge_when_matched : generic_limited_string option;Specifies how to resolve records that match preexisting data when merging. Valid values: update, delete.
merge_action : generic_limited_string option;Specifies a merge action. Valid values: simple, custom. If simple, merge behavior is defined by MergeWhenMatched and MergeWhenNotMatched. If custom, defined by MergeClause.
upsert : boolean_value option;Used when Action is append. Specifies the resolution behavior when a row already exists. If true, preexisting rows will be updated. If false, those rows will be inserted.
action : generic_string option;Specifies what action to take when writing to a table with preexisting data. Valid values: append, merge, truncate, drop.
post_action : generic_string option;A SQL string run after the Snowflake connector performs its standard actions.
*)pre_action : generic_string option;A SQL string run before the Snowflake connector performs its standard actions.
*)sample_query : generic_string option;A SQL string used to retrieve data with the query sourcetype.
additional_options : additional_options option;Specifies additional options passed to the Snowflake connector. If options are specified elsewhere in this node, this will take precedence.
*)iam_role : option_ option;Not currently used.
*)temp_dir : enclosed_in_string_property option;Not currently used.
*)database : generic_string option;Specifies a Snowflake database for your node to use.
*)table : generic_string option;Specifies a Snowflake table for your node to use.
*)schema : generic_string option;Specifies a Snowflake database schema for your node to use.
*)connection : option_ option;Specifies a Glue Data Catalog Connection to a Snowflake endpoint.
*)source_type : generic_limited_string option;Specifies how retrieved data is specified. Valid values: "table", "query".
}Specifies configuration for Snowflake nodes in Glue Studio.
type nonrec snowflake_source = {output_schemas : glue_schemas option;Specifies user-defined schemas for your output data.
*)data : snowflake_node_data;Configuration for the Snowflake data source.
*)name : node_name;The name of the Snowflake data source.
*)}Specifies a Snowflake data source.
type nonrec snowflake_target = {inputs : one_input option;The nodes that are inputs to the data target.
*)data : snowflake_node_data;Specifies the data of the Snowflake target node.
*)name : node_name;The name of the Snowflake target.
*)}Specifies a Snowflake target.
type nonrec connector_options = (generic_string * generic_string) listtype nonrec connector_data_source = {output_schemas : glue_schemas option;Specifies the data schema for this source.
*)data : connector_options;A map specifying connection options for the node. You can find standard connection options for the corresponding connection type in the Connection parameters section of the Glue documentation.
*)connection_type : enclosed_in_string_property;The connectionType, as provided to the underlying Glue library. This node type supports the following connection types:
opensearchazuresqlazurecosmosbigquerysaphanateradataverticaname : node_name;The name of this source node.
*)}Specifies a source generated with standard connection options.
type nonrec connector_data_target = {inputs : one_input option;The nodes that are inputs to the data target.
*)data : connector_options;A map specifying connection options for the node. You can find standard connection options for the corresponding connection type in the Connection parameters section of the Glue documentation.
*)connection_type : enclosed_in_string_property;The connectionType, as provided to the underlying Glue library. This node type supports the following connection types:
opensearchazuresqlazurecosmosbigquerysaphanateradataverticaname : node_name;The name of this target node.
*)}Specifies a target generated with standard connection options.
type nonrec s3_catalog_iceberg_source = {output_schemas : glue_schemas option;Specifies the data schema for the Iceberg source.
*)additional_iceberg_options : additional_options option;Specifies additional connection options for the Iceberg data source.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Iceberg data source.
*)}Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog. The Iceberg data source must be stored in Amazon S3.
type nonrec catalog_iceberg_source = {output_schemas : glue_schemas option;Specifies the data schema for the Iceberg source.
*)additional_iceberg_options : additional_options option;Specifies additional connection options for the Iceberg data source.
*)table : enclosed_in_string_property;The name of the table in the database to read from.
*)database : enclosed_in_string_property;The name of the database to read from.
*)name : node_name;The name of the Iceberg data source.
*)}Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog.
type nonrec s3_iceberg_catalog_target = {auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Iceberg catalog target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : catalog_schema_change_policy option;The policy for handling schema changes in the catalog target.
*)additional_options : additional_options option;Specifies additional connection options for the Iceberg catalog target.
*)database : enclosed_in_string_property;The name of the database to write to.
*)table : enclosed_in_string_property;The name of the table to write to in the catalog.
*)partition_keys : glue_studio_path_list option;A list of partition keys for the Iceberg table.
*)inputs : one_input;The input connection for the Iceberg catalog target.
*)name : node_name;The name of the Iceberg catalog target.
*)}Specifies an Apache Iceberg catalog target that writes data to Amazon S3 and registers the table in the Glue Data Catalog.
type nonrec s3_iceberg_direct_target = {output_schemas : glue_schemas option;Specifies the data schema for the S3 Iceberg direct target.
*)number_target_partitions : number_target_partitions_string option;Sets the number of target partitions for distributing Iceberg table files across S3.
*)compression : iceberg_target_compression_type;Specifies the compression codec used for Iceberg table files in S3.
*)auto_data_quality : auto_data_quality option;schema_change_policy : direct_schema_change_policy option;Defines how schema changes are handled when writing data to the Iceberg table.
*)additional_options : additional_options option;Provides additional configuration options for customizing the Iceberg table behavior.
*)format : target_format;Specifies the file format used for storing Iceberg table data (e.g., Parquet, ORC).
*)path : enclosed_in_string_property;Defines the S3 location where the Iceberg table data will be stored.
*)partition_keys : glue_studio_path_list option;Specifies the columns used to partition the Iceberg table data in S3.
*)inputs : one_input;Defines the single input source that provides data to this Iceberg target.
*)name : node_name;Specifies the unique identifier for the Iceberg target node in your data pipeline.
*)}Specifies a target that writes to an Iceberg data source in Amazon S3.
type nonrec s3_excel_source = {output_schemas : glue_schemas option;The Glue schemas to apply to the processed data.
*)number_rows : boxed_long option;The number of rows to process from each Excel file.
*)additional_options : s3_direct_source_additional_options option;Additional configuration options for S3 direct source processing.
*)max_files_in_band : boxed_non_negative_int option;The maximum number of files to process in each band.
*)max_band : boxed_non_negative_int option;The maximum number of processing bands to use.
*)recurse : boxed_boolean option;Indicates whether to recursively process subdirectories.
*)group_files : enclosed_in_string_property option;Specifies how files should be grouped for processing.
*)group_size : enclosed_in_string_property option;Defines the size of file groups for batch processing.
*)exclusions : enclosed_in_string_properties option;Patterns to exclude specific files or paths from processing.
*)compression_type : parquet_compression_type option;The compression format used for the Excel files.
*)paths : enclosed_in_string_properties;The S3 paths where the Excel files are located.
*)name : node_name;The name of the S3 Excel data source.
*)}Specifies an S3 Excel data source.
type nonrec s3_hyper_direct_target = {output_schemas : glue_schemas option;Specifies the data schema for the S3 Hyper direct target.
*)auto_data_quality : auto_data_quality option;Specifies whether to automatically enable data quality evaluation for the S3 Hyper direct target. When set to true, data quality checks are performed automatically during the write operation.
schema_change_policy : direct_schema_change_policy option;Defines how schema changes are handled during write operations.
*)compression : hyper_target_compression_type option;The compression type to apply to the output data.
*)path : enclosed_in_string_property;The S3 location where the output data will be written.
*)partition_keys : glue_studio_path_list option;Defines the partitioning strategy for the output data.
*)format : target_format option;Specifies the data output format for the HyperDirect target.
*)inputs : one_input;Specifies the input source for the HyperDirect target.
*)name : node_name;The unique identifier for the HyperDirect target node.
*)}Specifies a HyperDirect data target that writes to Amazon S3.
type nonrec ddbelt_connection_options = {dynamodb_sts_role_arn : enclosed_in_string_property option;The Amazon Resource Name (ARN) of the Amazon Web Services Security Token Service (STS) role to assume for accessing DynamoDB and S3 resources during the ELT operation. This role must have the necessary permissions to read from the DynamoDB table and write to the intermediate S3 bucket.
*)dynamodb_s3_bucket_owner : enclosed_in_string_property option;The Amazon Web Services account ID of the owner of the S3 bucket specified in DynamodbS3Bucket. This parameter is required when the S3 bucket is owned by a different Amazon Web Services account than the one running the ELT job, enabling cross-account access to the intermediate storage bucket.
dynamodb_s3_prefix : enclosed_in_string_property option;The S3 object key prefix for files stored in the intermediate S3 bucket during the DynamoDB ELT process. This prefix helps organize and identify the temporary files created during data extraction.
*)dynamodb_s3_bucket : enclosed_in_string_property option;The name of the Amazon S3 bucket used for intermediate storage during the DynamoDB ELT process. This bucket is used to temporarily store exported DynamoDB data before it is processed by the ELT job.
*)dynamodb_table_arn : enclosed_in_string_property;The Amazon Resource Name (ARN) of the DynamoDB table to extract data from. This parameter specifies the source table for the ELT operation.
*)dynamodb_unnest_ddb_json : boolean_value option;A boolean value that specifies whether to unnest DynamoDB JSON format during data extraction. When set to true, the connector will flatten nested JSON structures from DynamoDB items. When set to false, the original DynamoDB JSON structure is preserved.
dynamodb_export : ddb_export_type option;Specifies the export type for DynamoDB data extraction. This parameter determines how data is exported from the DynamoDB table during the ELT process.
*)}Specifies connection options for DynamoDB ELT (Extract, Load, Transform) operations. This structure contains configuration parameters for connecting to and extracting data from DynamoDB tables using the ELT connector.
type nonrec dynamo_dbelt_connector_source = {output_schemas : glue_schemas option;Specifies the data schema for the DynamoDB ELT connector source.
*)connection_options : ddbelt_connection_options option;The connection options for the DynamoDB ELT connector source.
*)name : node_name;The name of the DynamoDB ELT connector source.
*)}Specifies a DynamoDB ELT connector source for extracting data from DynamoDB tables.
type nonrec code_gen_configuration_node = {dynamo_dbelt_connector_source : dynamo_dbelt_connector_source option;Specifies a DynamoDB ELT connector source for extracting data from DynamoDB tables.
*)s3_hyper_direct_target : s3_hyper_direct_target option;Defines configuration parameters for writing data to Amazon S3 using HyperDirect optimization.
*)s3_excel_source : s3_excel_source option;Defines configuration parameters for reading Excel files from Amazon S3.
*)s3_iceberg_direct_target : s3_iceberg_direct_target option;Defines configuration parameters for writing data to Amazon S3 as an Apache Iceberg table.
*)s3_iceberg_catalog_target : s3_iceberg_catalog_target option;Specifies an Apache Iceberg catalog target that writes data to Amazon S3 and registers the table in the Glue Data Catalog.
*)catalog_iceberg_source : catalog_iceberg_source option;Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog.
*)s3_catalog_iceberg_source : s3_catalog_iceberg_source option;Specifies an Apache Iceberg data source that is registered in the Glue Data Catalog. The Iceberg data source must be stored in Amazon S3.
*)connector_data_target : connector_data_target option;Specifies a target generated with standard connection options.
*)connector_data_source : connector_data_source option;Specifies a source generated with standard connection options.
*)snowflake_target : snowflake_target option;Specifies a target that writes to a Snowflake data source.
*)snowflake_source : snowflake_source option;Specifies a Snowflake data source.
*)recipe : recipe option;Specifies a Glue DataBrew recipe node.
*)evaluate_data_quality_multi_frame : evaluate_data_quality_multi_frame option;Specifies your data quality evaluation criteria. Allows multiple input data and returns a collection of Dynamic Frames.
*)amazon_redshift_target : amazon_redshift_target option;Specifies a target that writes to a data target in Amazon Redshift.
*)amazon_redshift_source : amazon_redshift_source option;Specifies a target that writes to a data source in Amazon Redshift.
*)s3_delta_direct_target : s3_delta_direct_target option;Specifies a target that writes to a Delta Lake data source in Amazon S3.
*)s3_delta_catalog_target : s3_delta_catalog_target option;Specifies a target that writes to a Delta Lake data source in the Glue Data Catalog.
*)s3_delta_source : s3_delta_source option;Specifies a Delta Lake data source stored in Amazon S3.
*)catalog_delta_source : catalog_delta_source option;Specifies a Delta Lake data source that is registered in the Glue Data Catalog.
*)s3_catalog_delta_source : s3_catalog_delta_source option;Specifies a Delta Lake data source that is registered in the Glue Data Catalog. The data source must be stored in Amazon S3.
*)direct_jdbc_source : direct_jdbc_source option;s3_hudi_direct_target : s3_hudi_direct_target option;Specifies a target that writes to a Hudi data source in Amazon S3.
*)s3_hudi_catalog_target : s3_hudi_catalog_target option;Specifies a target that writes to a Hudi data source in the Glue Data Catalog.
*)s3_hudi_source : s3_hudi_source option;Specifies a Hudi data source stored in Amazon S3.
*)catalog_hudi_source : catalog_hudi_source option;Specifies a Hudi data source that is registered in the Glue Data Catalog.
*)s3_catalog_hudi_source : s3_catalog_hudi_source option;Specifies a Hudi data source that is registered in the Glue Data Catalog. The data source must be stored in Amazon S3.
*)evaluate_data_quality : evaluate_data_quality option;Specifies your data quality evaluation criteria.
*)dynamic_transform : dynamic_transform option;Specifies a custom visual transform created by a user.
*)route : route option;Specifies a route node that directs data to different output paths based on defined filtering conditions.
*)postgre_sql_catalog_target : postgre_sql_catalog_target option;Specifies a target that uses Postgres SQL.
*)oracle_sql_catalog_target : oracle_sql_catalog_target option;Specifies a target that uses Oracle SQL.
*)my_sql_catalog_target : my_sql_catalog_target option;Specifies a target that uses MySQL.
*)microsoft_sql_server_catalog_target : microsoft_sql_server_catalog_target
option;Specifies a target that uses Microsoft SQL.
*)postgre_sql_catalog_source : postgre_sql_catalog_source option;Specifies a PostgresSQL data source in the Glue Data Catalog.
*)oracle_sql_catalog_source : oracle_sql_catalog_source option;Specifies an Oracle data source in the Glue Data Catalog.
*)my_sql_catalog_source : my_sql_catalog_source option;Specifies a MySQL data source in the Glue Data Catalog.
*)microsoft_sql_server_catalog_source : microsoft_sql_server_catalog_source
option;Specifies a Microsoft SQL server data source in the Glue Data Catalog.
*)governed_catalog_source : governed_catalog_source option;Specifies a data source in a goverened Data Catalog.
*)governed_catalog_target : governed_catalog_target option;Specifies a data target that writes to a goverened catalog.
*)drop_duplicates : drop_duplicates option;Specifies a transform that removes rows of repeating data from a data set.
*)aggregate : aggregate option;Specifies a transform that groups rows by chosen fields and computes the aggregated value by specified function.
*)pii_detection : pii_detection option;Specifies a transform that identifies, removes or masks PII data.
*)union : union option;Specifies a transform that combines the rows from two or more datasets into a single result.
*)merge : merge option;Specifies a transform that merges a DynamicFrame with a staging DynamicFrame based on the specified primary keys to identify records. Duplicate records (records with the same primary keys) are not de-duplicated.
drop_null_fields : drop_null_fields option;Specifies a transform that removes columns from the dataset if all values in the column are 'null'. By default, Glue Studio will recognize null objects, but some values such as empty strings, strings that are "null", -1 integers or other placeholders such as zeros, are not automatically recognized as nulls.
*)catalog_kafka_source : catalog_kafka_source option;Specifies an Apache Kafka data store in the Data Catalog.
*)catalog_kinesis_source : catalog_kinesis_source option;Specifies a Kinesis data source in the Glue Data Catalog.
*)direct_kafka_source : direct_kafka_source option;Specifies an Apache Kafka data store.
*)direct_kinesis_source : direct_kinesis_source option;Specifies a direct Amazon Kinesis data source.
*)spark_sq_l : spark_sq_l option;Specifies a transform where you enter a SQL query using Spark SQL syntax to transform the data. The output is a single DynamicFrame.
custom_code : custom_code option;Specifies a transform that uses custom code you provide to perform the data transformation. The output is a collection of DynamicFrames.
*)filter : filter option;Specifies a transform that splits a dataset into two, based on a filter condition.
*)fill_missing_values : fill_missing_values option;Specifies a transform that locates records in the dataset that have missing values and adds a new field with a value determined by imputation. The input data set is used to train the machine learning model that determines what the missing value should be.
*)select_from_collection : select_from_collection option;Specifies a transform that chooses one DynamicFrame from a collection of DynamicFrames. The output is the selected DynamicFrame
split_fields : split_fields option;Specifies a transform that splits data property keys into two DynamicFrames. The output is a collection of DynamicFrames: one with selected data property keys, and one with the remaining data property keys.
join : join option;Specifies a transform that joins two datasets into one dataset using a comparison phrase on the specified data property keys. You can use inner, outer, left, right, left semi, and left anti joins.
*)spigot : spigot option;Specifies a transform that writes samples of the data to an Amazon S3 bucket.
*)rename_field : rename_field option;Specifies a transform that renames a single data property key.
*)drop_fields : drop_fields option;Specifies a transform that chooses the data property keys that you want to drop.
*)select_fields : select_fields option;Specifies a transform that chooses the data property keys that you want to keep.
*)apply_mapping : apply_mapping option;Specifies a transform that maps data property keys in the data source to data property keys in the data target. You can rename keys, modify the data types for keys, and choose which keys to drop from the dataset.
*)s3_direct_target : s3_direct_target option;Specifies a data target that writes to Amazon S3.
*)s3_glue_parquet_target : s3_glue_parquet_target option;Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.
*)s3_catalog_target : s3_catalog_target option;Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.
*)redshift_target : redshift_target option;Specifies a target that uses Amazon Redshift.
*)catalog_target : basic_catalog_target option;Specifies a target that uses a Glue Data Catalog table.
*)spark_connector_target : spark_connector_target option;Specifies a target that uses an Apache Spark connector.
*)jdbc_connector_target : jdbc_connector_target option;Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.
*)dynamo_db_catalog_source : dynamo_db_catalog_source option;Specifies a DynamoDBC Catalog data store in the Glue Data Catalog.
*)relational_catalog_source : relational_catalog_source option;Specifies a relational catalog data store in the Glue Data Catalog.
*)s3_parquet_source : s3_parquet_source option;Specifies an Apache Parquet data store stored in Amazon S3.
*)s3_json_source : s3_json_source option;Specifies a JSON data store stored in Amazon S3.
*)s3_csv_source : s3_csv_source option;Specifies a command-separated value (CSV) data store stored in Amazon S3.
*)s3_catalog_source : s3_catalog_source option;Specifies an Amazon S3 data store in the Glue Data Catalog.
*)redshift_source : redshift_source option;Specifies an Amazon Redshift data store.
*)catalog_source : catalog_source option;Specifies a data store in the Glue Data Catalog.
*)spark_connector_source : spark_connector_source option;Specifies a connector to an Apache Spark data source.
*)jdbc_connector_source : jdbc_connector_source option;Specifies a connector to a JDBC data source.
*)athena_connector_source : athena_connector_source option;Specifies a connector to an Amazon Athena data source.
*)}CodeGenConfigurationNode enumerates all valid Node types. One and only one of its member variables can be populated.
type nonrec code_gen_configuration_nodes =
(node_id * code_gen_configuration_node) listtype nonrec source_control_details = {auth_token : generic512_char_string option;The value of an authorization token.
*)auth_strategy : source_control_auth_strategy option;The type of authentication, which can be an authentication token stored in Amazon Web Services Secrets Manager, or a personal access token.
*)last_commit_id : generic512_char_string option;The last commit ID for a commit in the remote repository.
*)folder : generic512_char_string option;An optional folder in the remote repository.
*)branch : generic512_char_string option;An optional branch in the remote repository.
*)owner : generic512_char_string option;The owner of the remote repository that contains the job artifacts.
*)repository : generic512_char_string option;The name of the remote repository that contains the job artifacts.
*)provider : source_control_provider option;The provider for the remote repository.
*)}The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository.
type nonrec job_update = {maintenance_window : maintenance_window option;This field specifies a day of the week and hour for a maintenance window for streaming jobs. Glue periodically performs maintenance activities. During these maintenance windows, Glue will need to restart your streaming jobs.
Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.
*)source_control_details : source_control_details option;The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository.
*)execution_class : execution_class option;Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.
The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.
Only jobs with Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX. The flexible execution class is available for Spark jobs.
code_gen_configuration_nodes : code_gen_configuration_nodes option;The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.
*)glue_version : glue_version_string option;In Spark jobs, GlueVersion determines the versions of Apache Spark and Python that Glue available in a job. The Python version indicates the version supported for jobs of type Spark.
Ray jobs should set GlueVersion to 4.0 or greater. However, the versions of Ray, Python and additional libraries available in your Ray job are determined by the Runtime parameter of the Job command.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
If not provided in the update request, the Glue version retains the value from the existing job definition.
*)notification_property : notification_property option;Specifies the configuration properties of a job notification.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this job.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a job runs.
worker_type : worker_type option;The type of predefined worker that is allocated when a job runs. Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs. For more information, see Defining job properties for Spark jobs
*)max_capacity : nullable_double option;For Glue version 1.0 or earlier jobs, using the standard worker type, the number of Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
For Glue version 2.0+ jobs, you cannot specify a Maximum capacity. Instead, you should specify a Worker type and the Number of workers.
Do not set MaxCapacity if using WorkerType and NumberOfWorkers.
The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL job:
JobCommand.Name="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.JobCommand.Name="glueetl") or Apache Spark streaming ETL job (JobCommand.Name="gluestreaming"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.timeout : timeout option;The job timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)allocated_capacity : integer_value option;This field is deprecated. Use MaxCapacity instead.
The number of Glue data processing units (DPUs) to allocate to this job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
*)max_retries : max_retries option;The maximum number of times to retry this job if it fails.
*)connections : connections_list option;The connections used for this job.
*)non_overridable_arguments : generic_map option;Arguments for this job that are not overridden when providing job arguments in a job run, specified as name-value pairs.
*)default_arguments : generic_map option;The default arguments for every run of this job, specified as name-value pairs.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Secrets Manager or other secret management mechanism if you intend to keep them within the Job.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the arguments you can provide to this field when configuring Spark jobs, see the Special Parameters Used by Glue topic in the developer guide.
For information about the arguments you can provide to this field when configuring Ray jobs, see Using job parameters in Ray jobs in the developer guide.
*)command : job_command option;The JobCommand that runs this job (required).
execution_property : execution_property option;An ExecutionProperty specifying the maximum number of concurrent runs allowed for this job.
role : role_string option;The name or Amazon Resource Name (ARN) of the IAM role associated with this job (required).
*)log_uri : uri_string option;This field is reserved for future use.
*)description : description_string option;Description of the job being defined.
*)job_run_queuing_enabled : nullable_boolean option;Specifies whether job run queuing is enabled for the job runs for this job.
A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
If this field does not match the value set in the job run, then the value from the job run field will be used.
*)job_mode : job_mode option;A mode that describes how a job was created. Valid values are:
SCRIPT - The job was created using the Glue Studio script editor.VISUAL - The job was created using the Glue Studio visual editor.NOTEBOOK - The job was created using an interactive sessions notebook.When the JobMode field is missing or null, SCRIPT is assigned as the default value.
}Specifies information used to update an existing job definition. The previous job definition is completely overwritten by this information.
type nonrec update_job_request = {job_update : job_update;Specifies the values with which to update the job definition. Unspecified configuration is removed or reset to default values.
*)job_name : name_string;The name of the job definition to update.
*)}type nonrec update_job_from_source_control_response = {job_name : name_string option;The name of the Glue job.
*)}type nonrec update_job_from_source_control_request = {auth_token : auth_token_string option;The value of the authorization token.
*)auth_strategy : source_control_auth_strategy option;The type of authentication, which can be an authentication token stored in Amazon Web Services Secrets Manager, or a personal access token.
*)commit_id : commit_id_string option;A commit ID for a commit in the remote repository.
*)folder : name_string option;An optional folder in the remote repository.
*)branch_name : name_string option;An optional branch in the remote repository.
*)repository_owner : name_string option;The owner of the remote repository that contains the job artifacts.
*)repository_name : name_string option;The name of the remote repository that contains the job artifacts. For BitBucket providers, RepositoryName should include WorkspaceName. Use the format
/.
*)provider : source_control_provider option;The provider for the remote repository. Possible values: GITHUB, AWS_CODE_COMMIT, GITLAB, BITBUCKET.
*)job_name : name_string option;The name of the Glue job to be synchronized to or from the remote repository.
*)}type nonrec source_table_fields_list = string128 listtype nonrec primary_key_list = string128 listtype nonrec source_table_config = {record_update_field : string128 option;Incremental pull timestamp-based field. Currently unsupported.
*)primary_key : primary_key_list option;Provide the primary key set for this table. Currently supported specifically for SAP EntityOf entities upon request. Contact Amazon Web Services Support to make this feature available.
filter_predicate : string128 option;A condition clause used for row-level filtering. Currently unsupported.
*)fields : source_table_fields_list option;A list of fields used for column-level filtering. Currently unsupported.
*)}Properties used by the source leg to process data from the source.
type nonrec integration_partition = {conversion_spec : string128 option;Specifies the timestamp format of the source data. Valid values are:
epoch_sec - Unix epoch timestamp in secondsepoch_milli - Unix epoch timestamp in millisecondsiso - ISO 8601 formatted timestampOnly specify ConversionSpec when using timestamp-based partition functions (year, month, day, or hour). Glue Zero-ETL uses this parameter to correctly transform source data into timestamp format before partitioning.
Do not use high-cardinality columns with the identity partition function. High-cardinality columns include:
LastModifiedTimestamp, CreatedDate)Using high-cardinality columns with identity partitioning creates many small partitions, which can significantly degrade ingestion performance.
*)function_spec : string128 option;Specifies the function used to partition data on the target. The accepted values for this parameter are:
identity - Uses source values directly without transformationyear - Extracts the year from timestamp values (e.g., 2023)month - Extracts the month from timestamp values (e.g., 2023-01)day - Extracts the day from timestamp values (e.g., 2023-01-15)hour - Extracts the hour from timestamp values (e.g., 2023-01-15-14)field_name : string128 option;The field name used to partition data on the target. Avoid using columns that have unique values for each row (for example, `LastModifiedTimestamp`, `SystemModTimeStamp`) as the partition column. These columns are not suitable for partitioning because they create a large number of small partitions, which can lead to performance issues.
*)}A structure that describes how data is partitioned on the target.
type nonrec integration_partition_spec_list = integration_partition listtype nonrec target_table_config = {target_table_name : string128 option;The optional name of a target table.
*)partition_spec : integration_partition_spec_list option;Determines the file layout on the target.
*)unnest_spec : unnest_spec option;Specifies how nested objects are flattened to top-level elements. Valid values are: "TOPLEVEL", "FULL", or "NOUNNEST".
*)}Properties used by the target leg to partition the data on the target.
type nonrec update_integration_table_properties_request = {target_table_config : target_table_config option;A structure for the target table configuration.
*)source_table_config : source_table_config option;A structure for the source table configuration.
*)table_name : string128;The name of the table to be replicated.
*)resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec resource_not_found_exception = {message : integration_error_message option;A message describing the problem.
*)}The resource could not be found.
type nonrec internal_server_exception = {message : integration_error_message option;A message describing the problem.
*)}An internal server error occurred.
type nonrec source_processing_properties = {role_arn : string128 option;The IAM role to access the Glue connection.
*)}The resource properties associated with the integration source.
type nonrec target_processing_properties = {event_bus_arn : string2048 option;The ARN of an Eventbridge event bus to receive the integration status notification.
*)connection_name : string128 option;The Glue network connection to configure the Glue job running in the customer VPC.
*)kms_arn : string2048 option;The ARN of the KMS key used for encryption.
*)role_arn : string128 option;The IAM role to access the Glue database.
*)}The resource properties associated with the integration target.
type nonrec update_integration_resource_property_response = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_property_arn : string512 option;The resource ARN created through this create API. The format is something like arn:aws:glue:::integrationresourceproperty/*
*)resource_arn : string512 option;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec update_integration_resource_property_request = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec update_grok_classifier_request = {custom_patterns : custom_patterns option;Optional custom grok patterns used by this classifier.
*)grok_pattern : grok_pattern option;The grok pattern used by this classifier.
*)classification : classification option;An identifier of the data format that the classifier matches, such as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
*)name : name_string;The name of the GrokClassifier.
}Specifies a grok classifier to update when passed to UpdateClassifier.
type nonrec identity_center_scopes_list = identity_center_scope listtype nonrec update_glue_identity_center_configuration_request = {user_background_sessions_enabled : nullable_boolean option;Specifies whether users can run background sessions when using Identity Center authentication with Glue services.
*)scopes : identity_center_scopes_list option;A list of Identity Center scopes that define the updated permissions and access levels for the Glue configuration.
*)}Request to update an existing Glue Identity Center configuration.
type nonrec update_glossary_term_response = {long_description : glossary_long_description option;The long description of the glossary term.
*)short_description : glossary_short_description option;The short description of the glossary term.
*)name : glossary_term_name option;The name of the glossary term.
*)glossary_id : glossary_id option;The unique identifier of the glossary containing this term.
*)id : glossary_term_id option;The unique identifier of the glossary term.
*)}type nonrec update_glossary_term_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)long_description : glossary_long_description option;The updated long description of the glossary term.
*)short_description : glossary_short_description option;The updated short description of the glossary term.
*)name : glossary_term_name option;The updated name of the glossary term.
*)identifier : glossary_term_id;The unique identifier of the glossary term to update.
*)}type nonrec update_glossary_response = {description : metadata_description option;The description of the glossary.
*)name : glossary_name option;The name of the glossary.
*)id : glossary_id option;The unique identifier of the glossary.
*)}type nonrec update_glossary_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)description : metadata_description option;The updated description of the glossary.
*)name : glossary_name option;The updated name of the glossary.
*)identifier : glossary_id;The unique identifier of the glossary to update.
*)}type nonrec public_keys_list = generic_string listtype nonrec dev_endpoint_custom_libraries = {extra_jars_s3_path : generic_string option;The path to one or more Java .jar files in an S3 bucket that should be loaded in your DevEndpoint.
You can only use pure Java/Scala libraries with a DevEndpoint.
extra_python_libs_s3_path : generic_string option;The paths to one or more Python libraries in an Amazon Simple Storage Service (Amazon S3) bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma.
You can only use pure Python libraries with a DevEndpoint. Libraries that rely on C extensions, such as the pandas Python data analysis library, are not currently supported.
}Custom libraries to be loaded into a development endpoint.
type nonrec string_list = generic_string listtype nonrec map_value = (generic_string * generic_string) listtype nonrec update_dev_endpoint_request = {add_arguments : map_value option;The map of arguments to add the map of arguments used to configure the DevEndpoint.
Valid arguments are:
"--enable-glue-datacatalog": ""You can specify a version of Python support for development endpoints by using the Arguments parameter in the CreateDevEndpoint or UpdateDevEndpoint APIs. If no arguments are provided, the version defaults to Python 2.
delete_arguments : string_list option;The list of argument keys to be deleted from the map of arguments used to configure the DevEndpoint.
update_etl_libraries : boolean_value option;True if the list of custom libraries to be loaded in the development endpoint needs to be updated, or False if otherwise.
custom_libraries : dev_endpoint_custom_libraries option;Custom Python or Java libraries to be loaded in the DevEndpoint.
delete_public_keys : public_keys_list option;The list of public keys to be deleted from the DevEndpoint.
add_public_keys : public_keys_list option;The list of public keys for the DevEndpoint to use.
public_key : generic_string option;The public key for the DevEndpoint to use.
endpoint_name : generic_string;The name of the DevEndpoint to be updated.
}type nonrec data_lake_principal = {data_lake_principal_identifier : data_lake_principal_string option;An identifier for the Lake Formation principal.
*)}The Lake Formation principal.
type nonrec permission_list = permission listtype nonrec principal_permissions = {permissions : permission_list option;The permissions that are granted to the principal.
*)principal : data_lake_principal option;The principal who is granted permissions.
*)}Permissions granted to a principal.
type nonrec principal_permissions_list = principal_permissions listtype nonrec database_identifier = {region : name_string option;Region of the target database.
*)database_name : name_string option;The name of the catalog database.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the database resides.
*)}A structure that describes a target database for resource linking.
type nonrec federated_database = {connection_type : name_string option;The type of connection used to access the federated database, such as JDBC, ODBC, or other supported connection protocols.
*)connection_name : name_string option;The name of the connection to the external metastore.
*)identifier : federation_identifier option;A unique identifier for the federated database.
*)}A database that points to an entity outside the Glue Data Catalog.
type nonrec database_input = {federated_database : federated_database option;A FederatedDatabase structure that references an entity outside the Glue Data Catalog.
target_database : database_identifier option;A DatabaseIdentifier structure that describes a target database for resource linking.
create_table_default_permissions : principal_permissions_list option;Creates a set of default permissions on the table for principals. Used by Lake Formation. Not used in the normal course of Glue operations.
*)parameters : parameters_map option;These key-value pairs define parameters and properties of the database.
These key-value pairs define parameters and properties of the database.
*)location_uri : ur_i option;The location of the database (for example, an HDFS path).
*)description : description_string option;A description of the database.
*)name : name_string;The name of the database. For Hive compatibility, this is folded to lowercase when it is stored.
*)}The structure used to create or update a database.
type nonrec update_database_request = {database_input : database_input;A DatabaseInput object specifying the new definition of the metadata database in the catalog.
name : name_string;The name of the database to update in the catalog. For Hive compatibility, this is folded to lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the metadata database resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec update_data_quality_ruleset_response = {ruleset : data_quality_ruleset_string option;A Data Quality Definition Language (DQDL) ruleset. For more information, see the Glue developer guide.
*)description : description_string option;A description of the ruleset.
*)name : name_string option;The name of the data quality ruleset.
*)}type nonrec update_data_quality_ruleset_request = {ruleset : data_quality_ruleset_string option;A Data Quality Definition Language (DQDL) ruleset. For more information, see the Glue developer guide.
*)description : description_string option;A description of the ruleset.
*)name : name_string;The name of the data quality ruleset.
*)}type nonrec idempotent_parameter_mismatch_exception = {message : message_string option;A message describing the problem.
*)}The same unique identifier was associated with two different records.
type nonrec csv_header = name_string listtype nonrec custom_datatypes = name_string listtype nonrec update_csv_classifier_request = {serde : csv_serde_option option;Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are OpenCSVSerDe, LazySimpleSerDe, and None. You can specify the None value when you want the crawler to do the detection.
custom_datatypes : custom_datatypes option;Specifies a list of supported custom datatypes.
*)custom_datatype_configured : nullable_boolean option;Specifies the configuration of custom datatypes.
*)allow_single_column : nullable_boolean option;Enables the processing of files that contain only one column.
*)disable_value_trimming : nullable_boolean option;Specifies not to trim values before identifying the type of column values. The default value is true.
*)header : csv_header option;A list of strings representing column names.
*)contains_header : csv_header_option option;Indicates whether the CSV file contains a header.
*)quote_symbol : csv_quote_symbol option;A custom symbol to denote what combines content into a single column value. It must be different from the column delimiter.
*)delimiter : csv_column_delimiter option;A custom symbol to denote what separates each column entry in the row.
*)name : name_string;The name of the classifier.
*)}Specifies a custom CSV classifier to be updated.
type nonrec update_crawler_schedule_request = {schedule : cron_expression option;The updated cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
crawler_name : name_string;The name of the crawler whose schedule to update.
*)}type nonrec scheduler_transitioning_exception = {message : message_string option;A message describing the problem.
*)}The specified scheduler is transitioning.
type nonrec path_list = path listtype nonrec s3_target = {dlq_event_queue_arn : event_queue_arn option;A valid Amazon dead-letter SQS ARN. For example, arn:aws:sqs:region:account:deadLetterQueue.
event_queue_arn : event_queue_arn option;A valid Amazon SQS ARN. For example, arn:aws:sqs:region:account:sqs.
sample_size : nullable_integer option;Sets the number of files in each leaf folder to be crawled when crawling sample files in a dataset. If not set, all the files are crawled. A valid value is an integer between 1 and 249.
*)connection_name : connection_name option;The name of a connection which allows a job or crawler to access data in Amazon S3 within an Amazon Virtual Private Cloud environment (Amazon VPC).
*)exclusions : path_list option;A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.
*)path : path option;The path to the Amazon S3 target.
*)}Specifies a data store in Amazon Simple Storage Service (Amazon S3).
type nonrec s3_target_list = s3_target listtype nonrec enable_additional_metadata = jdbc_metadata_entry listtype nonrec jdbc_target = {enable_additional_metadata : enable_additional_metadata option;Specify a value of RAWTYPES or COMMENTS to enable additional metadata in table responses. RAWTYPES provides the native-level datatype. COMMENTS provides comments associated with a column or table in the database.
If you do not need additional metadata, keep the field empty.
*)exclusions : path_list option;A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.
*)path : path option;The path of the JDBC target.
*)connection_name : connection_name option;The name of the connection to use to connect to the JDBC target.
*)}Specifies a JDBC data store to crawl.
type nonrec jdbc_target_list = jdbc_target listtype nonrec mongo_db_target = {scan_all : nullable_boolean option;Indicates whether to scan all the records, or to sample rows from the table. Scanning all the records can take a long time when the table is not a high throughput table.
A value of true means to scan all records, while a value of false means to sample the records. If no value is specified, the value defaults to true.
path : path option;The path of the Amazon DocumentDB or MongoDB target (database/collection).
*)connection_name : connection_name option;The name of the connection to use to connect to the Amazon DocumentDB or MongoDB target.
*)}Specifies an Amazon DocumentDB or MongoDB data store to crawl.
type nonrec mongo_db_target_list = mongo_db_target listtype nonrec dynamo_db_target = {scan_rate : nullable_double option;The percentage of the configured read capacity units to use by the Glue crawler. Read capacity units is a term defined by DynamoDB, and is a numeric value that acts as rate limiter for the number of reads that can be performed on that table per second.
The valid values are null or a value between 0.1 to 1.5. A null value is used when user does not provide a value, and defaults to 0.5 of the configured Read Capacity Unit (for provisioned tables), or 0.25 of the max configured Read Capacity Unit (for tables using on-demand mode).
*)scan_all : nullable_boolean option;Indicates whether to scan all the records, or to sample rows from the table. Scanning all the records can take a long time when the table is not a high throughput table.
A value of true means to scan all records, while a value of false means to sample the records. If no value is specified, the value defaults to true.
path : path option;The name of the DynamoDB table to crawl.
*)}Specifies an Amazon DynamoDB table to crawl.
type nonrec dynamo_db_target_list = dynamo_db_target listtype nonrec catalog_tables_list = name_string listtype nonrec catalog_target = {dlq_event_queue_arn : event_queue_arn option;A valid Amazon dead-letter SQS ARN. For example, arn:aws:sqs:region:account:deadLetterQueue.
event_queue_arn : event_queue_arn option;A valid Amazon SQS ARN. For example, arn:aws:sqs:region:account:sqs.
connection_name : connection_name option;The name of the connection for an Amazon S3-backed Data Catalog table to be a target of the crawl when using a Catalog connection type paired with a NETWORK Connection type.
tables : catalog_tables_list;A list of the tables to be synchronized.
*)database_name : name_string;The name of the database to be synchronized.
*)}Specifies an Glue Data Catalog target.
type nonrec catalog_target_list = catalog_target listtype nonrec delta_target = {create_native_delta_table : nullable_boolean option;Specifies whether the crawler will create native tables, to allow integration with query engines that support querying of the Delta transaction log directly.
*)write_manifest : nullable_boolean option;Specifies whether to write the manifest files to the Delta table path.
*)connection_name : connection_name option;The name of the connection to use to connect to the Delta table target.
*)delta_tables : path_list option;A list of the Amazon S3 paths to the Delta tables.
*)}Specifies a Delta data store to crawl one or more Delta tables.
type nonrec delta_target_list = delta_target listtype nonrec iceberg_target = {maximum_traversal_depth : nullable_integer option;The maximum depth of Amazon S3 paths that the crawler can traverse to discover the Iceberg metadata folder in your Amazon S3 path. Used to limit the crawler run time.
*)exclusions : path_list option;A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.
*)connection_name : connection_name option;The name of the connection to use to connect to the Iceberg target.
*)paths : path_list option;One or more Amazon S3 paths that contains Iceberg metadata folders as s3://bucket/prefix.
}Specifies an Apache Iceberg data source where Iceberg tables are stored in Amazon S3.
type nonrec iceberg_target_list = iceberg_target listtype nonrec hudi_target = {maximum_traversal_depth : nullable_integer option;The maximum depth of Amazon S3 paths that the crawler can traverse to discover the Hudi metadata folder in your Amazon S3 path. Used to limit the crawler run time.
*)exclusions : path_list option;A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.
*)connection_name : connection_name option;The name of the connection to use to connect to the Hudi target. If your Hudi files are stored in buckets that require VPC authorization, you can set their connection properties here.
*)paths : path_list option;An array of Amazon S3 location strings for Hudi, each indicating the root folder with which the metadata files for a Hudi table resides. The Hudi folder may be located in a child folder of the root folder.
The crawler will scan all folders underneath a path for a Hudi folder.
*)}Specifies an Apache Hudi data source.
type nonrec hudi_target_list = hudi_target listtype nonrec crawler_targets = {hudi_targets : hudi_target_list option;Specifies Apache Hudi data store targets.
*)iceberg_targets : iceberg_target_list option;Specifies Apache Iceberg data store targets.
*)delta_targets : delta_target_list option;Specifies Delta data store targets.
*)catalog_targets : catalog_target_list option;Specifies Glue Data Catalog targets.
*)dynamo_db_targets : dynamo_db_target_list option;Specifies Amazon DynamoDB targets.
*)mongo_db_targets : mongo_db_target_list option;Specifies Amazon DocumentDB or MongoDB targets.
*)jdbc_targets : jdbc_target_list option;Specifies JDBC targets.
*)s3_targets : s3_target_list option;Specifies Amazon Simple Storage Service (Amazon S3) targets.
*)}Specifies data stores to crawl.
type nonrec classifier_name_list = name_string listtype nonrec schema_change_policy = {delete_behavior : delete_behavior option;The deletion behavior when the crawler finds a deleted object.
*)update_behavior : update_behavior option;The update behavior when the crawler finds a changed schema.
*)}A policy that specifies update and deletion behaviors for the crawler.
type nonrec recrawl_policy = {recrawl_behavior : recrawl_behavior option;Specifies whether to crawl the entire dataset again or to crawl only folders that were added since the last crawler run.
A value of CRAWL_EVERYTHING specifies crawling the entire dataset again.
A value of CRAWL_NEW_FOLDERS_ONLY specifies crawling only folders that were added since the last crawler run.
A value of CRAWL_EVENT_MODE specifies crawling only the changes identified by Amazon S3 events.
}When crawling an Amazon S3 data source after the first crawl is complete, specifies whether to crawl the entire dataset again or to crawl only folders that were added since the last crawler run. For more information, see Incremental Crawls in Glue in the developer guide.
type nonrec lineage_configuration = {crawler_lineage_settings : crawler_lineage_settings option;Specifies whether data lineage is enabled for the crawler. Valid values are:
}Specifies data lineage configuration settings for the crawler.
type nonrec lake_formation_configuration = {account_id : account_id option;Required for cross account crawls. For same account crawls as the target data, this can be left as null.
*)use_lake_formation_credentials : nullable_boolean option;Specifies whether to use Lake Formation credentials for the crawler instead of the IAM role credentials.
*)}Specifies Lake Formation configuration settings for the crawler.
type nonrec update_crawler_request = {crawler_security_configuration : crawler_security_configuration option;The name of the SecurityConfiguration structure to be used by this crawler.
configuration : crawler_configuration option;Crawler configuration information. This versioned JSON string allows users to specify aspects of a crawler's behavior. For more information, see Setting crawler configuration options.
*)lake_formation_configuration : lake_formation_configuration option;Specifies Lake Formation configuration settings for the crawler.
*)lineage_configuration : lineage_configuration option;Specifies data lineage configuration settings for the crawler.
*)recrawl_policy : recrawl_policy option;A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.
*)schema_change_policy : schema_change_policy option;The policy for the crawler's update and deletion behavior.
*)table_prefix : table_prefix option;The table prefix used for catalog tables that are created.
*)classifiers : classifier_name_list option;A list of custom classifiers that the user has registered. By default, all built-in classifiers are included in a crawl, but these custom classifiers always override the default classifiers for a given classification.
*)schedule : cron_expression option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
targets : crawler_targets option;A list of targets to crawl.
*)description : description_string_removable option;A description of the new crawler.
*)database_name : database_name option;The Glue database where results are stored, such as: arn:aws:daylight:us-east-1::database/sometable/*.
role : role option;The IAM role or Amazon Resource Name (ARN) of an IAM role that is used by the new crawler to access customer resources.
*)name : name_string;Name of the new crawler.
*)}type nonrec crawler_running_exception = {message : message_string option;A message describing the problem.
*)}The operation cannot be performed because the crawler is already running.
type nonrec connection_type = | VERTICA| TPCDS| TIMESTREAM| TERADATANOS| TERADATA| SYNAPSE| SQLSERVER| SAPHANA| POSTGRESQL| PIPEDRIVE| ORACLE| OPENSEARCH| OKTA| MYSQL| MONDAY| MICROSOFTDYNAMICS365CRM| KUSTOMER| HBASE| GOOGLECLOUDSTORAGE| DYNAMODB| DOMO| DOCUMENTDB| DB2AS400| DB2| DATALAKEGEN2| CMDB| CLOUDWATCHMETRICS| CLOUDWATCH| CLOUDERAIMPALA| CLOUDERAHIVE| BLACKBAUD| BIGQUERY| AZURESQL| AZURECOSMOS| SENDGRID| PIPEDIVE| CIRCLECI| PRODUCTBOARD| PENDO| GITLAB| MAILCHIMP| BLACKBAUDRAISEREDGENXT| MICROSOFTTEAMS| MICROSOFTDYNAMIC365FINANCEANDOPS| DYNATRACE| SAPCONCUR| SALESFORCECOMMERCECLOUD| GOOGLESEARCHCONSOLE| ZOOM| FRESHSALES| DOCUSIGNMONITOR| TWILIO| FRESHDESK| FACEBOOKPAGEINSIGHTS| QUICKBOOKS| PAYPAL| SNAPCHATADS| INTERCOM| WOOCOMMERCE| DATADOG| SMARTSHEET| STRIPE| ASANA| MIXPANEL| LINKEDIN| SLACK| ADOBEANALYTICS| SALESFORCEMARKETINGCLOUD| SALESFORCEPARDOT| ZOHOCRM| INSTAGRAMADS| FACEBOOKADS| HUBSPOT| NETSUITEERP| JIRACLOUD| ZENDESK| SAPODATA| MARKETO| SERVICENOW| GOOGLEANALYTICS4| GOOGLESHEETS| GOOGLEADS| VIEW_VALIDATION_ATHENA| VIEW_VALIDATION_REDSHIFT| SALESFORCE| CUSTOM| MARKETPLACE| NETWORK| KAFKA| MONGODB| SFTP| JDBCtype nonrec match_criteria = name_string listtype nonrec connection_property_key = | DATABASE| CLUSTER_IDENTIFIER| WORKGROUP_NAME| REGION| ROLE_ARN| ENDPOINT_TYPE| ENDPOINT| CONNECTOR_CLASS_NAME| CONNECTOR_TYPE| CONNECTOR_URL| SECRET_ID| KAFKA_SASL_GSSAPI_PRINCIPAL| KAFKA_SASL_GSSAPI_SERVICE| KAFKA_SASL_GSSAPI_KRB5_CONF| KAFKA_SASL_GSSAPI_KEYTAB| ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD| KAFKA_SASL_SCRAM_SECRETS_ARN| KAFKA_SASL_SCRAM_PASSWORD| KAFKA_SASL_SCRAM_USERNAME| ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD| KAFKA_SASL_PLAIN_PASSWORD| KAFKA_SASL_PLAIN_USERNAME| KAFKA_SASL_MECHANISM| ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD| ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD| KAFKA_CLIENT_KEY_PASSWORD| KAFKA_CLIENT_KEYSTORE_PASSWORD| KAFKA_CLIENT_KEYSTORE| KAFKA_SKIP_CUSTOM_CERT_VALIDATION| KAFKA_CUSTOM_CERT| KAFKA_SSL_ENABLED| KAFKA_BOOTSTRAP_SERVERS| CONNECTION_URL| CUSTOM_JDBC_CERT_STRING| SKIP_CUSTOM_JDBC_CERT_VALIDATION| CUSTOM_JDBC_CERT| JDBC_ENFORCE_SSL| JDBC_CONNECTION_URL| INSTANCE_ID| CONFIG_FILES| JDBC_ENGINE_VERSION| JDBC_ENGINE| JDBC_DRIVER_CLASS_NAME| JDBC_DRIVER_JAR_URI| ENCRYPTED_PASSWORD| PASSWORD| USER_NAME| PORT| HOSTtype nonrec connection_properties =
(connection_property_key * value_string) listtype nonrec property_map = (property_key * property_value) listtype nonrec security_group_id_list = name_string listtype nonrec physical_connection_requirements = {availability_zone : name_string option;The connection's Availability Zone.
*)security_group_id_list : security_group_id_list option;The security group ID list used by the connection.
*)subnet_id : name_string option;The subnet ID used by the connection.
*)}The OAuth client app in GetConnection response.
type nonrec o_auth2_client_application = {aws_managed_client_application_reference : aws_managed_client_application_reference
option;The reference to the SaaS-side client app that is Amazon Web Services managed.
*)user_managed_client_application_client_id : user_managed_client_application_client_id
option;The client application clientID if the ClientAppType is USER_MANAGED.
}The OAuth2 client app used for the connection.
type nonrec token_url_parameters_map =
(token_url_parameter_key * token_url_parameter_value) listThe set of properties required for the the OAuth2 AUTHORIZATION_CODE grant type workflow.
type nonrec o_auth2_credentials = {jwt_token : jwt_token option;The JSON Web Token (JWT) used when the authentication type is OAuth2.
*)refresh_token : refresh_token option;The refresh token used when the authentication type is OAuth2.
*)access_token : access_token option;The access token used when the authentication type is OAuth2.
*)user_managed_client_application_client_secret : user_managed_client_application_client_secret
option;The client application client secret if the client application is user managed.
*)}The credentials used when the authentication type is OAuth2 authentication.
type nonrec o_auth2_properties_input = {o_auth2_credentials : o_auth2_credentials option;The credentials used when the authentication type is OAuth2 authentication.
*)token_url_parameters_map : token_url_parameters_map option;A map of parameters that are added to the token GET request.
token_url : token_url option;The URL of the provider's authentication server, to exchange an authorization code for an access token.
*)o_auth2_client_application : o_auth2_client_application option;The client application type in the CreateConnection request. For example, AWS_MANAGED or USER_MANAGED.
o_auth2_grant_type : o_auth2_grant_type option;The OAuth2 grant type in the CreateConnection request. For example, AUTHORIZATION_CODE, JWT_BEARER, or CLIENT_CREDENTIALS.
}A structure containing properties for OAuth2 in the CreateConnection request.
type nonrec basic_authentication_credentials = {password : password option;The password to connect to the data source.
*)username : username option;The username to connect to the data source.
*)}For supplying basic auth credentials when not providing a SecretArn value.
type nonrec credential_map = (credential_key * credential_value) listtype nonrec authentication_configuration_input = {custom_authentication_credentials : credential_map option;The credentials used when the authentication type is custom authentication.
*)basic_authentication_credentials : basic_authentication_credentials option;The credentials used when the authentication type is basic authentication.
*)kms_key_arn : kms_key_arn option;The ARN of the KMS key used to encrypt the connection. Only taken an as input in the request and stored in the Secret Manager.
*)secret_arn : secret_arn option;The secret manager ARN to store credentials in the CreateConnection request.
*)o_auth2_properties : o_auth2_properties_input option;The properties for OAuth2 authentication in the CreateConnection request.
*)authentication_type : authentication_type option;A structure containing the authentication configuration in the CreateConnection request.
*)}A structure containing the authentication configuration in the CreateConnection request.
type nonrec compute_environment_list = compute_environment listtype nonrec connection_input = {validate_for_compute_environments : compute_environment_list option;The compute environments that the specified connection properties are validated against.
*)validate_credentials : boolean_ option;A flag to validate the credentials during create connection. Default is true.
*)authentication_configuration : authentication_configuration_input option;The authentication properties of the connection.
*)physical_connection_requirements : physical_connection_requirements option;The physical connection requirements, such as virtual private cloud (VPC) and SecurityGroup, that are needed to successfully make this connection.
python_properties : property_map option;Connection properties specific to the Python compute environment.
*)athena_properties : property_map option;Connection properties specific to the Athena compute environment.
*)spark_properties : property_map option;Connection properties specific to the Spark compute environment.
*)connection_properties : connection_properties;These key-value pairs define parameters for the connection.
*)match_criteria : match_criteria option;A list of criteria that can be used in selecting this connection.
*)connection_type : connection_type;The type of the connection. Currently, these types are supported:
JDBC - Designates a connection to a database through Java Database Connectivity (JDBC).
JDBC Connections use the following ConnectionParameters.
HOST, PORT, JDBC_ENGINE) or JDBC_CONNECTION_URL.USERNAME, PASSWORD) or SECRET_ID.JDBC_ENFORCE_SSL, CUSTOM_JDBC_CERT, CUSTOM_JDBC_CERT_STRING, SKIP_CUSTOM_JDBC_CERT_VALIDATION. These parameters are used to configure SSL with JDBC.KAFKA - Designates a connection to an Apache Kafka streaming platform.
KAFKA Connections use the following ConnectionParameters.
KAFKA_BOOTSTRAP_SERVERS.KAFKA_SSL_ENABLED, KAFKA_CUSTOM_CERT, KAFKA_SKIP_CUSTOM_CERT_VALIDATION. These parameters are used to configure SSL with KAFKA.KAFKA_CLIENT_KEYSTORE, KAFKA_CLIENT_KEYSTORE_PASSWORD, KAFKA_CLIENT_KEY_PASSWORD, ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD, ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD. These parameters are used to configure TLS client configuration with SSL in KAFKA.KAFKA_SASL_MECHANISM. Can be specified as SCRAM-SHA-512, GSSAPI, or AWS_MSK_IAM.KAFKA_SASL_SCRAM_USERNAME, KAFKA_SASL_SCRAM_PASSWORD, ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD. These parameters are used to configure SASL/SCRAM-SHA-512 authentication with KAFKA.KAFKA_SASL_GSSAPI_KEYTAB, KAFKA_SASL_GSSAPI_KRB5_CONF, KAFKA_SASL_GSSAPI_SERVICE, KAFKA_SASL_GSSAPI_PRINCIPAL. These parameters are used to configure SASL/GSSAPI authentication with KAFKA.MONGODB - Designates a connection to a MongoDB document database.
MONGODB Connections use the following ConnectionParameters.
CONNECTION_URL.USERNAME, PASSWORD) or SECRET_ID.VIEW_VALIDATION_REDSHIFT - Designates a connection used for view validation by Amazon Redshift.VIEW_VALIDATION_ATHENA - Designates a connection used for view validation by Amazon Athena.NETWORK - Designates a network connection to a data source within an Amazon Virtual Private Cloud environment (Amazon VPC).
NETWORK Connections do not require ConnectionParameters. Instead, provide a PhysicalConnectionRequirements.
MARKETPLACE - Uses configuration settings contained in a connector purchased from Amazon Web Services Marketplace to read from and write to data stores that are not natively supported by Glue.
MARKETPLACE Connections use the following ConnectionParameters.
CONNECTOR_TYPE, CONNECTOR_URL, CONNECTOR_CLASS_NAME, CONNECTION_URL.JDBC CONNECTOR_TYPE connections: All of (USERNAME, PASSWORD) or SECRET_ID.CUSTOM - Uses configuration settings contained in a custom connector to read from and write to data stores that are not natively supported by Glue.For more information on the connection parameters needed for a particular connector, see the documentation for the connector in Adding an Glue connectionin the Glue User Guide.
SFTP is not supported.
For more information about how optional ConnectionProperties are used to configure features in Glue, consult Glue connection properties.
For more information about how optional ConnectionProperties are used to configure features in Glue Studio, consult Using connectors and connections.
*)description : description_string option;The description of the connection.
*)name : name_string;The name of the connection.
*)}A structure that is used to specify a connection to create or update.
type nonrec update_connection_request = {connection_input : connection_input;A ConnectionInput object that redefines the connection in question.
name : name_string;The name of the connection definition to update.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the connection resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec column_name_list = name_string listtype nonrec update_column_statistics_task_settings_request = {security_configuration : name_string option;Name of the security configuration that is used to encrypt CloudWatch logs.
*)catalog_i_d : name_string option;The ID of the Data Catalog in which the database resides.
*)sample_size : sample_size_percentage option;The percentage of data to sample.
*)column_name_list : column_name_list option;A list of column names for which to run statistics.
*)schedule : cron_expression option;A schedule for running the column statistics, specified in CRON syntax.
*)role : name_string option;The role used for running the column statistics.
*)table_name : name_string;The name of the table for which to generate column statistics.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec non_negative_long = Smaws_Lib.CoreTypes.Int64.ttype nonrec boolean_column_statistics_data = {number_of_nulls : non_negative_long;The number of null values in the column.
*)number_of_falses : non_negative_long;The number of false values in the column.
*)number_of_trues : non_negative_long;The number of true values in the column.
*)}Defines column statistics supported for Boolean data columns.
type nonrec date_column_statistics_data = {number_of_distinct_values : non_negative_long;The number of distinct values in a column.
*)number_of_nulls : non_negative_long;The number of null values in the column.
*)maximum_value : timestamp option;The highest value in the column.
*)minimum_value : timestamp option;The lowest value in the column.
*)}Defines column statistics supported for timestamp data columns.
type nonrec decimal_number = {scale : integer;The scale that determines where the decimal point falls in the unscaled value.
*)unscaled_value : blob;The unscaled numeric value.
*)}Contains a numeric value in decimal format.
type nonrec decimal_column_statistics_data = {number_of_distinct_values : non_negative_long;The number of distinct values in a column.
*)number_of_nulls : non_negative_long;The number of null values in the column.
*)maximum_value : decimal_number option;The highest value in the column.
*)minimum_value : decimal_number option;The lowest value in the column.
*)}Defines column statistics supported for fixed-point number data columns.
type nonrec double_column_statistics_data = {number_of_distinct_values : non_negative_long;The number of distinct values in a column.
*)number_of_nulls : non_negative_long;The number of null values in the column.
*)maximum_value : double option;The highest value in the column.
*)minimum_value : double option;The lowest value in the column.
*)}Defines column statistics supported for floating-point number data columns.
type nonrec long = Smaws_Lib.CoreTypes.Int64.ttype nonrec long_column_statistics_data = {number_of_distinct_values : non_negative_long;The number of distinct values in a column.
*)number_of_nulls : non_negative_long;The number of null values in the column.
*)maximum_value : long option;The highest value in the column.
*)minimum_value : long option;The lowest value in the column.
*)}Defines column statistics supported for integer data columns.
type nonrec string_column_statistics_data = {number_of_distinct_values : non_negative_long;The number of distinct values in a column.
*)number_of_nulls : non_negative_long;The number of null values in the column.
*)average_length : non_negative_double;The average string length in the column.
*)maximum_length : non_negative_long;The size of the longest string in the column.
*)}Defines column statistics supported for character sequence data values.
type nonrec binary_column_statistics_data = {number_of_nulls : non_negative_long;The number of null values in the column.
*)average_length : non_negative_double;The average bit sequence length in the column.
*)maximum_length : non_negative_long;The size of the longest bit sequence in the column.
*)}Defines column statistics supported for bit sequence data values.
type nonrec column_statistics_data = {binary_column_statistics_data : binary_column_statistics_data option;Binary column statistics data.
*)string_column_statistics_data : string_column_statistics_data option;String column statistics data.
*)long_column_statistics_data : long_column_statistics_data option;Long column statistics data.
*)double_column_statistics_data : double_column_statistics_data option;Double column statistics data.
*)decimal_column_statistics_data : decimal_column_statistics_data option;Decimal column statistics data. UnscaledValues within are Base64-encoded binary objects storing big-endian, two's complement representations of the decimal's unscaled value.
*)date_column_statistics_data : date_column_statistics_data option;Date column statistics data.
*)boolean_column_statistics_data : boolean_column_statistics_data option;Boolean column statistics data.
*)type_ : column_statistics_type;The type of column statistics data.
*)}Contains the individual types of column statistics data. Only one data object should be set and indicated by the Type attribute.
type nonrec column_statistics = {statistics_data : column_statistics_data;A ColumnStatisticData object that contains the statistics data values.
analyzed_time : timestamp;The timestamp of when column statistics were generated.
*)column_type : type_string;The data type of the column.
*)column_name : name_string;Name of column which statistics belong to.
*)}Represents the generated column-level statistics for a table or partition.
type nonrec update_column_statistics_list = column_statistics listtype nonrec column_statistics_error = {error : error_detail option;An error message with the reason for the failure of an operation.
*)column_statistics : column_statistics option;The ColumnStatistics of the column.
}Encapsulates a ColumnStatistics object that failed and the reason for failure.
type nonrec column_statistics_errors = column_statistics_error listtype nonrec update_column_statistics_for_table_response = {errors : column_statistics_errors option;List of ColumnStatisticsErrors.
*)}type nonrec update_column_statistics_for_table_request = {column_statistics_list : update_column_statistics_list;A list of the column statistics.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec update_column_statistics_for_partition_response = {errors : column_statistics_errors option;Error occurred during updating column statistics data.
*)}type nonrec update_column_statistics_for_partition_request = {column_statistics_list : update_column_statistics_list;A list of the column statistics.
*)partition_values : value_string_list;A list of partition values identifying the partition.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec update_classifier_request = {csv_classifier : update_csv_classifier_request option;A CsvClassifier object with updated fields.
json_classifier : update_json_classifier_request option;A JsonClassifier object with updated fields.
xml_classifier : update_xml_classifier_request option;An XMLClassifier object with updated fields.
grok_classifier : update_grok_classifier_request option;A GrokClassifier object with updated fields.
}type nonrec federated_catalog = {connection_type : name_string option;The type of connection used to access the federated catalog, specifying the protocol or method for connection to the external data source.
*)connection_name : name_string option;The name of the connection to an external data source, for example a Redshift-federated catalog.
*)identifier : federation_identifier option;A unique identifier for the federated catalog.
*)}A catalog that points to an entity outside the Glue Data Catalog.
type nonrec target_redshift_catalog = {catalog_arn : resource_arn_string;The Amazon Resource Name (ARN) of the catalog resource.
*)}A structure that describes a target catalog for resource linking.
type nonrec data_lake_access_properties = {catalog_type : name_string option;Specifies a federated catalog type for the native catalog resource. The currently supported type is aws:redshift.
kms_key : resource_arn_string option;An encryption key that will be used for the staging bucket that will be created along with the catalog.
*)data_transfer_role : iam_role_arn option;A role that will be assumed by Glue for transferring data into/out of the staging bucket during a query.
*)data_lake_access : boolean_ option;Turns on or off data lake access for Apache Spark applications that access Amazon Redshift databases in the Data Catalog from any non-Redshift engine, such as Amazon Athena, Amazon EMR, or Glue ETL.
*)}Input properties to configure data lake access for your catalog resource in the Glue Data Catalog.
type nonrec iceberg_optimization_properties = {orphan_file_deletion : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg orphan file deletion operations, which identify and remove files that are no longer referenced by the table metadata.
*)retention : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg table retention operations, which manage the lifecycle of table snapshots to control storage costs.
*)compaction : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg table compaction operations, which optimize the layout of data files to improve query performance.
*)role_arn : iam_role_arn option;The Amazon Resource Name (ARN) of the IAM role that will be assumed to perform Iceberg table optimization operations.
*)}A structure that specifies Iceberg table optimization properties for the catalog, including configurations for compaction, retention, and orphan file deletion operations.
type nonrec catalog_properties = {custom_properties : parameters_map option;Additional key-value properties for the catalog, such as column statistics optimizations.
*)iceberg_optimization_properties : iceberg_optimization_properties option;A structure that specifies Iceberg table optimization properties for the catalog. This includes configuration for compaction, retention, and orphan file deletion operations that can be applied to Iceberg tables in this catalog.
*)data_lake_access_properties : data_lake_access_properties option;A DataLakeAccessProperties object that specifies properties to configure data lake access for your catalog resource in the Glue Data Catalog.
}A structure that specifies data lake access properties and other custom properties.
type nonrec catalog_input = {overwrite_child_resource_permissions_with_default : overwrite_child_resource_permissions_with_default_enum
option;Overwrites existing Amazon Web Services Lake Formation permissions with CatalogInput$CreateTableDefaultPermissions and CatalogInput$CreateDatabaseDefaultPermissions for all child resources.
allow_full_table_external_data_access : allow_full_table_external_data_access_enum
option;Allows third-party engines to access data in Amazon S3 locations that are registered with Lake Formation.
*)create_database_default_permissions : principal_permissions_list option;An array of PrincipalPermissions objects. Creates a set of default permissions on the database(s) for principals. Used by Amazon Web Services Lake Formation. Typically should be explicitly set as an empty list.
create_table_default_permissions : principal_permissions_list option;An array of PrincipalPermissions objects. Creates a set of default permissions on the table(s) for principals. Used by Amazon Web Services Lake Formation. Typically should be explicitly set as an empty list.
catalog_properties : catalog_properties option;A CatalogProperties object that specifies data lake access properties and other custom properties.
target_redshift_catalog : target_redshift_catalog option;A TargetRedshiftCatalog object that describes a target catalog for resource linking.
parameters : parameters_map option;A map array of key-value pairs that define the parameters and properties of the catalog.
*)federated_catalog : federated_catalog option;A FederatedCatalog object. A FederatedCatalog structure that references an entity outside the Glue Data Catalog, for example a Redshift database.
description : description_string option;Description string, not more than 2048 bytes long, matching the URI address multi-line string pattern. A description of the catalog.
*)}A structure that describes catalog properties.
type nonrec update_catalog_request = {catalog_input : catalog_input;A CatalogInput object specifying the new properties of an existing catalog.
catalog_id : catalog_id_string;The ID of the catalog.
*)}type nonrec update_blueprint_response = {name : name_string option;Returns the name of the blueprint that was updated.
*)}type nonrec update_blueprint_request = {blueprint_location : orchestration_s3_location;Specifies a path in Amazon S3 where the blueprint is published.
*)description : generic512_char_string option;A description of the blueprint.
*)name : orchestration_name_string;The name of the blueprint.
*)}type nonrec illegal_blueprint_state_exception = {message : message_string option;A message describing the problem.
*)}The blueprint is in an invalid state to perform a requested operation.
type nonrec update_asset_response = {updated_at : updated_at option;The timestamp at which the asset was last updated.
*)description : asset_description option;The description of the asset.
*)name : asset_name option;The name of the asset.
*)id : asset_id;The unique identifier of the asset.
*)}type nonrec update_asset_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)description : asset_description option;The new description of the asset.
*)name : asset_name option;The new name of the asset.
*)identifier : asset_id;The unique identifier of the asset to update.
*)}type nonrec tag_keys_list = tag_key listtype nonrec untag_resource_request = {resource_arn : glue_resource_arn;The Amazon Resource Name (ARN) of the resource from which to remove the tags.
*)}type nonrec partition = {catalog_id : catalog_id_string option;The ID of the Data Catalog in which the partition resides.
*)last_analyzed_time : timestamp option;The last time at which column statistics were computed for this partition.
*)parameters : parameters_map option;These key-value pairs define partition parameters.
*)storage_descriptor : storage_descriptor option;Provides information about the physical location where the partition is stored.
*)last_access_time : timestamp option;The last time at which the partition was accessed.
*)creation_time : timestamp option;The time at which the partition was created.
*)table_name : name_string option;The name of the database table in which to create the partition.
*)database_name : name_string option;The name of the catalog database in which to create the partition.
*)values : value_string_list option;The values of the partition.
*)}Represents a slice of table data.
type nonrec unfiltered_partition = {is_registered_with_lake_formation : boolean_ option;A Boolean value indicating that the partition location is registered with Lake Formation.
*)partition : partition option;The partition object.
*)}A partition that contains unfiltered metadata.
type nonrec unfiltered_partition_list = unfiltered_partition listtype nonrec trigger_name_list = name_string listtype nonrec trigger_list = trigger listtype nonrec transform_sort_criteria = {sort_direction : sort_direction_type;The sort direction to be used in the sorting criteria that are associated with the machine learning transform.
*)column : transform_sort_column_type;The column to be used in the sorting criteria that are associated with the machine learning transform.
*)}The sorting criteria that are associated with the machine learning transform.
type nonrec schema_column = {data_type : column_type_string option;The type of data in the column.
*)name : column_name_string option;The name of the column.
*)}A key-value pair representing a column and data type that this transform can run against. The Schema parameter of the MLTransform may contain up to 100 of these structures.
type nonrec transform_schema = schema_column listtype nonrec glue_table_additional_options =
(name_string * description_string) listtype nonrec glue_table = {additional_options : glue_table_additional_options option;Additional options for the table. Currently there are two keys supported:
pushDownPredicate: to filter on partitions without having to list and read all the files in your dataset.catalogPartitionPredicate: to use server-side partition pruning using partition indexes in the Glue Data Catalog.connection_name : name_string option;The name of the connection to the Glue Data Catalog.
*)catalog_id : name_string option;A unique identifier for the Glue Data Catalog.
*)table_name : name_string;A table name in the Glue Data Catalog.
*)database_name : name_string;A database name in the Glue Data Catalog.
*)}The database and table in the Glue Data Catalog that is used for input or output data.
type nonrec glue_tables = glue_table listtype nonrec records_count = Smaws_Lib.CoreTypes.Int64.ttype nonrec confusion_matrix = {num_false_negatives : records_count option;The number of matches in the data that the transform didn't find, in the confusion matrix for your transform.
*)num_true_negatives : records_count option;The number of nonmatches in the data that the transform correctly rejected, in the confusion matrix for your transform.
*)num_false_positives : records_count option;The number of nonmatches in the data that the transform incorrectly classified as a match, in the confusion matrix for your transform.
*)num_true_positives : records_count option;The number of matches in the data that the transform correctly found, in the confusion matrix for your transform.
*)}The confusion matrix shows you what your transform is predicting accurately and what types of errors it is making.
For more information, see Confusion matrix in Wikipedia.
type nonrec column_importance = {importance : generic_bounded_double option;The column importance score for the column, as a decimal.
*)column_name : name_string option;The name of a column.
*)}A structure containing the column name and column importance score for a column.
Column importance helps you understand how columns contribute to your model, by identifying which columns in your records are more important than others.
type nonrec column_importance_list = column_importance listtype nonrec find_matches_metrics = {column_importances : column_importance_list option;A list of ColumnImportance structures containing column importance metrics, sorted in order of descending importance.
confusion_matrix : confusion_matrix option;The confusion matrix shows you what your transform is predicting accurately and what types of errors it is making.
For more information, see Confusion matrix in Wikipedia.
*)f1 : generic_bounded_double option;The maximum F1 metric indicates the transform's accuracy between 0 and 1, where 1 is the best accuracy.
For more information, see F1 score in Wikipedia.
*)recall : generic_bounded_double option;The recall metric indicates that for an actual match, how often your transform predicts the match. Specifically, it measures how well the transform finds true positives from the total records in the source data.
For more information, see Precision and recall in Wikipedia.
*)precision : generic_bounded_double option;The precision metric indicates when often your transform is correct when it predicts a match. Specifically, it measures how well the transform finds true positives from the total true positives possible.
For more information, see Precision and recall in Wikipedia.
*)area_under_pr_curve : generic_bounded_double option;The area under the precision/recall curve (AUPRC) is a single number measuring the overall quality of the transform, that is independent of the choice made for precision vs. recall. Higher values indicate that you have a more attractive precision vs. recall tradeoff.
For more information, see Precision and recall in Wikipedia.
*)}The evaluation metrics for the find matches algorithm. The quality of your machine learning transform is measured by getting your transform to predict some matches and comparing the results to known matches from the same dataset. The quality metrics are based on a subset of your data, so they are not precise.
type nonrec evaluation_metrics = {find_matches_metrics : find_matches_metrics option;The evaluation metrics for the find matches algorithm.
*)transform_type : transform_type;The type of machine learning transform.
*)}Evaluation metrics provide an estimate of the quality of your machine learning transform.
type nonrec ml_user_data_encryption = {kms_key_id : name_string option;The ID for the customer-provided KMS key.
*)ml_user_data_encryption_mode : ml_user_data_encryption_mode_string;The encryption mode applied to user data. Valid values are:
}The encryption-at-rest settings of the transform that apply to accessing user data.
type nonrec transform_encryption = {task_run_security_configuration_name : name_string option;The name of the security configuration.
*)ml_user_data_encryption : ml_user_data_encryption option;An MLUserDataEncryption object containing the encryption mode and customer-provided KMS key ID.
}The encryption-at-rest settings of the transform that apply to accessing user data. Machine learning transforms can access user data encrypted in Amazon S3 using KMS.
Additionally, imported labels and trained transforms can now be encrypted using a customer provided KMS key.
type nonrec ml_transform = {transform_encryption : transform_encryption option;The encryption-at-rest settings of the transform that apply to accessing user data. Machine learning transforms can access user data encrypted in Amazon S3 using KMS.
*)max_retries : nullable_integer option;The maximum number of times to retry after an MLTaskRun of the machine learning transform fails.
timeout : timeout option;The timeout in minutes of the machine learning transform.
*)number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a task of the transform runs.
If WorkerType is set, then NumberOfWorkers is required (and vice versa).
worker_type : worker_type option;The type of predefined worker that is allocated when a task of this transform runs. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker provides 4 vCPU, 16 GB of memory and a 64GB disk, and 1 executor per worker.G.2X worker type, each worker provides 8 vCPU, 32 GB of memory and a 128GB disk, and 1 executor per worker.MaxCapacity is a mutually exclusive option with NumberOfWorkers and WorkerType.
NumberOfWorkers or WorkerType is set, then MaxCapacity cannot be set.MaxCapacity is set then neither NumberOfWorkers or WorkerType can be set.WorkerType is set, then NumberOfWorkers is required (and vice versa).MaxCapacity and NumberOfWorkers must both be at least 1.max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that are allocated to task runs for this transform. You can allocate from 2 to 100 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
MaxCapacity is a mutually exclusive option with NumberOfWorkers and WorkerType.
NumberOfWorkers or WorkerType is set, then MaxCapacity cannot be set.MaxCapacity is set then neither NumberOfWorkers or WorkerType can be set.WorkerType is set, then NumberOfWorkers is required (and vice versa).MaxCapacity and NumberOfWorkers must both be at least 1.When the WorkerType field is set to a value other than Standard, the MaxCapacity field is set automatically and becomes read-only.
glue_version : glue_version_string option;This value determines which version of Glue this machine learning transform is compatible with. Glue 1.0 is recommended for most customers. If the value is not set, the Glue compatibility defaults to Glue 0.9. For more information, see Glue Versions in the developer guide.
*)role : role_string option;The name or Amazon Resource Name (ARN) of the IAM role with the required permissions. The required permissions include both Glue service role permissions to Glue resources, and Amazon S3 permissions required by the transform.
schema : transform_schema option;A map of key-value pairs representing the columns and data types that this transform can run against. Has an upper bound of 100 columns.
*)label_count : label_count option;A count identifier for the labeling files generated by Glue for this transform. As you create a better transform, you can iteratively download, label, and upload the labeling file.
*)evaluation_metrics : evaluation_metrics option;An EvaluationMetrics object. Evaluation metrics provide an estimate of the quality of your machine learning transform.
parameters : transform_parameters option;A TransformParameters object. You can use parameters to tune (customize) the behavior of the machine learning transform by specifying what data it learns from and your preference on various tradeoffs (such as precious vs. recall, or accuracy vs. cost).
input_record_tables : glue_tables option;A list of Glue table definitions used by the transform.
*)last_modified_on : timestamp option;A timestamp. The last point in time when this machine learning transform was modified.
*)created_on : timestamp option;A timestamp. The time and date that this machine learning transform was created.
*)status : transform_status_type option;The current status of the machine learning transform.
*)description : description_string option;A user-defined, long-form description text for the machine learning transform. Descriptions are not guaranteed to be unique and can be changed at any time.
*)name : name_string option;A user-defined name for the machine learning transform. Names are not guaranteed unique and can be changed at any time.
*)transform_id : hash_string option;The unique transform ID that is generated for the machine learning transform. The ID is guaranteed to be unique and does not change.
*)}A structure for a machine learning transform.
type nonrec transform_list = ml_transform listtype nonrec transform_id_list = hash_string listtype nonrec transform_filter_criteria = {schema : transform_schema option;Filters on datasets with a specific schema. The
Mapobject is an array of key-value pairs representing the schema this transform accepts, where Column is the name of a column, and Type is the type of the data such as an integer or string. Has an upper bound of 100 columns.
last_modified_after : timestamp option;Filter on transforms last modified after this date.
*)last_modified_before : timestamp option;Filter on transforms last modified before this date.
*)created_after : timestamp option;The time and date after which the transforms were created.
*)created_before : timestamp option;The time and date before which the transforms were created.
*)glue_version : glue_version_string option;This value determines which version of Glue this machine learning transform is compatible with. Glue 1.0 is recommended for most customers. If the value is not set, the Glue compatibility defaults to Glue 0.9. For more information, see Glue Versions in the developer guide.
*)status : transform_status_type option;Filters the list of machine learning transforms by the last known status of the transforms (to indicate whether a transform can be used or not). One of "NOT_READY", "READY", or "DELETING".
*)transform_type : transform_type option;The type of machine learning transform that is used to filter the machine learning transforms.
*)name : name_string option;A unique transform name that is used to filter the machine learning transforms.
*)}The criteria used to filter the machine learning transforms.
type nonrec timestamped_inclusion_annotation = {last_modified_on : timestamp option;The timestamp when the inclusion annotation was last modified.
*)value : inclusion_annotation_value option;The inclusion annotation value.
*)}A timestamped inclusion annotation.
type nonrec timestamp_filter = {recorded_after : timestamp option;The timestamp after which statistics should be included in the results.
*)recorded_before : timestamp option;The timestamp before which statistics should be included in the results.
*)}A timestamp filter.
type nonrec test_connection_input = {authentication_configuration : authentication_configuration_input option;A structure containing the authentication configuration in the TestConnection request. Required for a connection to Salesforce using OAuth authentication.
*)connection_properties : connection_properties;The key-value pairs that define parameters for the connection.
JDBC connections use the following connection properties:
HOST, PORT, JDBC_ENGINE) or JDBC_CONNECTION_URL.USERNAME, PASSWORD) or SECRET_ID.JDBC_ENFORCE_SSL, CUSTOM_JDBC_CERT, CUSTOM_JDBC_CERT_STRING, SKIP_CUSTOM_JDBC_CERT_VALIDATION. These parameters are used to configure SSL with JDBC.SALESFORCE connections require the AuthenticationConfiguration member to be configured.
connection_type : connection_type;The type of connection to test. This operation is only available for the JDBC or SALESFORCE connection types.
}A structure that is used to specify testing a connection to a service.
type nonrec test_connection_request = {test_connection_input : test_connection_input option;A structure that is used to specify testing a connection to a service.
*)catalog_id : catalog_id_string option;The catalog ID where the connection resides.
*)connection_name : name_string option;Optional. The name of the connection to test. If only name is provided, the operation will get the connection and use that for testing.
*)}type nonrec conflict_exception = {message : message_string option;A message describing the problem.
*)}The CreatePartitions API was called on a table that has indexes enabled.
type nonrec task_run_sort_criteria = {sort_direction : sort_direction_type;The sort direction to be used to sort the list of task runs for the machine learning transform.
*)column : task_run_sort_column_type;The column to be used to sort the list of task runs for the machine learning transform.
*)}The sorting criteria that are used to sort the list of task runs for the machine learning transform.
type nonrec import_labels_task_run_properties = {replace : replace_boolean option;Indicates whether to overwrite your existing labels.
*)input_s3_path : uri_string option;The Amazon Simple Storage Service (Amazon S3) path from where you will import the labels.
*)}Specifies configuration properties for an importing labels task run.
type nonrec export_labels_task_run_properties = {output_s3_path : uri_string option;The Amazon Simple Storage Service (Amazon S3) path where you will export the labels.
*)}Specifies configuration properties for an exporting labels task run.
type nonrec labeling_set_generation_task_run_properties = {output_s3_path : uri_string option;The Amazon Simple Storage Service (Amazon S3) path where you will generate the labeling set.
*)}Specifies configuration properties for a labeling set generation task run.
type nonrec find_matches_task_run_properties = {job_run_id : hash_string option;The job run ID for the Find Matches task run.
*)job_name : name_string option;The name assigned to the job for the Find Matches task run.
*)job_id : hash_string option;The job ID for the Find Matches task run.
*)}Specifies configuration properties for a Find Matches task run.
type nonrec task_run_properties = {find_matches_task_run_properties : find_matches_task_run_properties option;The configuration properties for a find matches task run.
*)labeling_set_generation_task_run_properties : labeling_set_generation_task_run_properties
option;The configuration properties for a labeling set generation task run.
*)export_labels_task_run_properties : export_labels_task_run_properties option;The configuration properties for an exporting labels task run.
*)import_labels_task_run_properties : import_labels_task_run_properties option;The configuration properties for an importing labels task run.
*)task_type : task_type option;The type of task run.
*)}The configuration properties for the task run.
type nonrec task_run = {execution_time : execution_time option;The amount of time (in seconds) that the task run consumed resources.
*)completed_on : timestamp option;The last point in time that the requested task run was completed.
*)last_modified_on : timestamp option;The last point in time that the requested task run was updated.
*)started_on : timestamp option;The date and time that this task run started.
*)error_string : generic_string option;The list of error strings associated with this task run.
*)properties : task_run_properties option;Specifies configuration properties associated with this task run.
*)log_group_name : generic_string option;The names of the log group for secure logging, associated with this task run.
*)status : task_status_type option;The current status of the requested task run.
*)task_run_id : hash_string option;The unique identifier for this task run.
*)transform_id : hash_string option;The unique identifier for the transform.
*)}The sampling parameters that are associated with the machine learning transform.
type nonrec task_run_list = task_run listtype nonrec task_run_filter_criteria = {started_after : timestamp option;Filter on task runs started after this date.
*)started_before : timestamp option;Filter on task runs started before this date.
*)status : task_status_type option;The current status of the task run.
*)task_run_type : task_type option;The type of task run.
*)}The criteria that are used to filter the task runs for the machine learning transform.
type nonrec target_resource_not_found = {message : integration_error_message option;A message describing the problem.
*)}The target resource could not be found.
type nonrec tag_resource_request = {resource_arn : glue_resource_arn;The ARN of the Glue resource to which to add the tags. For more information about Glue resource ARNs, see the Glue ARN string pattern.
*)}type nonrec tag = {value : tag_value option;The tag value. The value is optional when you create a tag on an object. The value is case-sensitive, and must not contain the prefix aws.
*)key : tag_key option;The tag key. The key is required when you create a tag on an object. The key is case-sensitive, and must not contain the prefix aws.
*)}The Tag object represents a label that you can assign to an Amazon Web Services resource. Each tag consists of a key and an optional value, both of which you define.
For more information about tags, and controlling access to resources in Glue, see Amazon Web Services Tags in Glue and Specifying Glue Resource ARNs in the developer guide.
type nonrec table_version_error = {error_detail : error_detail option;The details about the error.
*)version_id : version_string option;The ID value of the version in question. A VersionID is a string representation of an integer. Each version is incremented by 1.
table_name : name_string option;The name of the table in question.
*)}An error record for table-version operations.
type nonrec table_version_errors = table_version_error listtype nonrec federated_table = {connection_type : name_string option;The type of connection used to access the federated table, specifying the protocol or method for connecting to the external data source.
*)connection_name : name_string option;The name of the connection to the external metastore.
*)database_identifier : federation_identifier option;A unique identifier for the federated database.
*)identifier : federation_identifier option;A unique identifier for the federated table.
*)}A table that points to an entity outside the Glue Data Catalog.
type nonrec iceberg_schema_list = iceberg_schema listtype nonrec iceberg_partition_spec_list = iceberg_partition_spec listtype nonrec iceberg_sort_order_list = iceberg_sort_order listtype nonrec iceberg_table_metadata = {default_sort_order_id : integer option;The identifier of the sort order that is currently used by default when writing new data to the Iceberg table.
*)sort_orders : iceberg_sort_order_list option;The list of sort order specifications that have been associated with the Iceberg table over its history.
*)last_partition_id : integer option;The highest partition field identifier that has been assigned across the table's partition specifications.
*)default_spec_id : integer option;The identifier of the partition specification that is currently used by default when writing new data to the Iceberg table.
*)partition_specs : iceberg_partition_spec_list option;The list of partition specifications that have been associated with the Iceberg table over its history, supporting partition evolution.
*)last_column_id : integer option;The highest column identifier that has been assigned in the Iceberg table's schema, used to ensure unique IDs as new columns are added.
*)current_schema_id : integer option;The identifier of the schema that is currently active for the Iceberg table. Matches an entry in Schemas.
schemas : iceberg_schema_list option;The list of schemas that have been associated with the Iceberg table over its history, supporting schema evolution.
*)properties : string_to_string_map option;A map of key-value pairs that define table-level properties and configuration settings for the Iceberg table.
*)location : location_string option;The base S3 location where the Iceberg table's data and metadata files are stored.
*)table_uuid : table_id_string option;The unique identifier (UUID) for the Iceberg table, assigned when the table is created and used to track the table across metadata updates.
*)format_version : version_string option;The Apache Iceberg table format version, such as 1 or 2. Determines the set of features and on-disk layout supported by the table.
}The Apache Iceberg table metadata, including format version, table identifier, schemas, partition specifications, sort orders, and table properties. This structure captures the current state of an Iceberg table's metadata as managed by the Glue Data Catalog.
type status_details = {view_validations : view_validation_list option;A list of ViewValidation objects that contain information for an analytical engine to validate a view.
requested_change : table option;A Table object representing the requested changes.
}A structure containing information about an asynchronous change to a table.
and table = {status : table_status option;Indicates the the state of an asynchronous change to a table.
*)iceberg_table_metadata : iceberg_table_metadata option;The latest Apache Iceberg table metadata for the table, including format version, schemas, partition specifications, and sort orders. This field is populated for Iceberg tables and reflects the current state of the table's Iceberg metadata.
*)is_materialized_view : nullable_boolean option;Indicates a table is a MaterializedView.
is_multi_dialect_view : nullable_boolean option;Specifies whether the view supports the SQL dialects of one or more different query engines and can therefore be read by those engines.
*)view_definition : view_definition option;A structure that contains all the information that defines the view, including the dialect or dialects for the view, and the query.
*)federated_table : federated_table option;A FederatedTable structure that references an entity outside the Glue Data Catalog.
version_id : version_string option;The ID of the table version.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the table resides.
*)target_table : table_identifier option;A TableIdentifier structure that describes a target table for resource linking.
is_registered_with_lake_formation : boolean_ option;Indicates whether the table has been registered with Lake Formation.
*)created_by : name_string option;The person or entity who created the table.
*)parameters : parameters_map option;These key-value pairs define properties associated with the table.
*)table_type : table_type_string option;The type of this table. Glue will create tables with the EXTERNAL_TABLE type. Other services, such as Athena, may create tables with additional table types.
Glue related table types:
EXTERNAL_TABLE Hive compatible attribute - indicates a non-Hive managed table.
GOVERNED Used by Lake Formation. The Glue Data Catalog understands GOVERNED.
view_expanded_text : view_text_string option;Included for Apache Hive compatibility. Not used in the normal course of Glue operations.
*)view_original_text : view_text_string option;Included for Apache Hive compatibility. Not used in the normal course of Glue operations. If the table is a VIRTUAL_VIEW, certain Athena configuration encoded in base64.
partition_keys : column_list option;A list of columns by which the table is partitioned. Only primitive types are supported as partition keys.
When you create a table used by Amazon Athena, and you do not specify any partitionKeys, you must at least set the value of partitionKeys to an empty list. For example:
"PartitionKeys": []
storage_descriptor : storage_descriptor option;A storage descriptor containing information about the physical storage of this table.
*)retention : non_negative_integer option;The retention time for this table.
*)last_analyzed_time : timestamp option;The last time that column statistics were computed for this table.
*)last_access_time : timestamp option;The last time that the table was accessed. This is usually taken from HDFS, and might not be reliable.
*)update_time : timestamp option;The last time that the table was updated.
*)create_time : timestamp option;The time when the table definition was created in the Data Catalog.
*)owner : name_string option;The owner of the table.
*)description : description_string option;A description of the table.
*)database_name : name_string option;The name of the database where the table metadata resides. For Hive compatibility, this must be all lowercase.
*)name : name_string;The table name. For Hive compatibility, this must be entirely lowercase.
*)}Represents a collection of related data organized in columns and rows.
and table_status = {details : status_details option;A StatusDetails object with information about the requested change.
error : error_detail option;An error that will only appear when the state is "FAILED". This is a parent level exception message, there may be different Errors for each dialect.
state : resource_state option;A generic status for the change in progress, such as QUEUED, IN_PROGRESS, SUCCESS, or FAILED.
*)action : resource_action option;Indicates which action was called on the table, currently only CREATE or UPDATE.
update_time : timestamp option;An ISO 8601 formatted date string indicating the time that the state was last updated.
*)request_time : timestamp option;An ISO 8601 formatted date string indicating the time that the change was initiated.
*)updated_by : name_string option;The ARN of the user to last manually alter the asynchronous change (requesting cancellation, etc).
*)requested_by : name_string option;The ARN of the user who requested the asynchronous change.
*)}A structure containing information about the state of an asynchronous change to a table.
type nonrec table_version = {version_id : version_string option;The ID value that identifies this table version. A VersionId is a string representation of an integer. Each version is incremented by 1.
table : table option;The table in question.
*)}Specifies a version of a table.
type nonrec run_metrics = {job_duration_in_hour : message_string option;The duration of the job in hours.
*)number_of_dpus : message_string option;The number of DPUs consumed by the job, rounded up to the nearest whole number.
*)number_of_files_compacted : message_string option;The number of files removed by the compaction job run.
*)number_of_bytes_compacted : message_string option;The number of bytes removed by the compaction job run.
*)}Metrics for the optimizer run.
This structure is deprecated. See the individual metric members for compaction, retention, and orphan file deletion.
type nonrec iceberg_compaction_metrics = {job_duration_in_hour : dpu_duration_in_hour option;The duration of the job in hours.
*)number_of_dpus : dpu_counts option;The number of DPUs consumed by the job, rounded up to the nearest whole number.
*)dpu_hours : dpu_hours option;The number of DPU hours consumed by the job.
*)number_of_files_compacted : metric_counts option;The number of files removed by the compaction job run.
*)number_of_bytes_compacted : metric_counts option;The number of bytes removed by the compaction job run.
*)}Compaction metrics for Iceberg for the optimizer run.
type nonrec compaction_metrics = {iceberg_metrics : iceberg_compaction_metrics option;A structure containing the Iceberg compaction metrics for the optimizer run.
*)}A structure that contains compaction metrics for the optimizer run.
type nonrec iceberg_retention_metrics = {job_duration_in_hour : dpu_duration_in_hour option;The duration of the job in hours.
*)number_of_dpus : dpu_counts option;The number of DPUs consumed by the job, rounded up to the nearest whole number.
*)dpu_hours : dpu_hours option;The number of DPU hours consumed by the job.
*)number_of_manifest_lists_deleted : metric_counts option;The number of manifest lists deleted by the retention job run.
*)number_of_manifest_files_deleted : metric_counts option;The number of manifest files deleted by the retention job run.
*)number_of_data_files_deleted : metric_counts option;The number of data files deleted by the retention job run.
*)}Snapshot retention metrics for Iceberg for the optimizer run.
type nonrec retention_metrics = {iceberg_metrics : iceberg_retention_metrics option;A structure containing the Iceberg retention metrics for the optimizer run.
*)}A structure that contains retention metrics for the optimizer run.
type nonrec iceberg_orphan_file_deletion_metrics = {job_duration_in_hour : dpu_duration_in_hour option;The duration of the job in hours.
*)number_of_dpus : dpu_counts option;The number of DPUs consumed by the job, rounded up to the nearest whole number.
*)dpu_hours : dpu_hours option;The number of DPU hours consumed by the job.
*)number_of_orphan_files_deleted : metric_counts option;The number of orphan files deleted by the orphan file deletion job run.
*)}Orphan file deletion metrics for Iceberg for the optimizer run.
type nonrec orphan_file_deletion_metrics = {iceberg_metrics : iceberg_orphan_file_deletion_metrics option;A structure containing the Iceberg orphan file deletion metrics for the optimizer run.
*)}A structure that contains orphan file deletion metrics for the optimizer run.
type nonrec table_optimizer_run = {orphan_file_deletion_metrics : orphan_file_deletion_metrics option;An OrphanFileDeletionMetrics object containing metrics for the optimizer run.
retention_metrics : retention_metrics option;A RetentionMetrics object containing metrics for the optimizer run.
compaction_strategy : compaction_strategy option;The strategy used for the compaction run. Indicates which algorithm was applied to determine how files were selected and combined during the compaction process. Valid values are:
binpack: Combines small files into larger files, typically targeting sizes over 100MB, while applying any pending deletes. This is the recommended compaction strategy for most use cases.sort: Organizes data based on specified columns which are sorted hierarchically during compaction, improving query performance for filtered operations. This strategy is recommended when your queries frequently filter on specific columns. To use this strategy, you must first define a sort order in your Iceberg table properties using the sort_order table property.z-order: Optimizes data organization by blending multiple attributes into a single scalar value that can be used for sorting, allowing efficient querying across multiple dimensions. This strategy is recommended when you need to query data across multiple dimensions simultaneously. To use this strategy, you must first define a sort order in your Iceberg table properties using the sort_order table property.compaction_metrics : compaction_metrics option;A CompactionMetrics object containing metrics for the optimizer run.
error : message_string option;An error that occured during the optimizer run.
*)metrics : run_metrics option;A RunMetrics object containing metrics for the optimizer run.
This member is deprecated. See the individual metric members for compaction, retention, and orphan file deletion.
*)end_timestamp : table_optimizer_run_timestamp option;Represents the epoch timestamp at which the compaction job ended.
*)start_timestamp : table_optimizer_run_timestamp option;Represents the epoch timestamp at which the compaction job was started within Lake Formation.
*)event_type : table_optimizer_event_type option;An event type representing the status of the table optimizer run.
*)}Contains details for a table optimizer run.
type nonrec table_optimizer_runs = table_optimizer_run listtype nonrec table_optimizer = {configuration_source : configuration_source option;Specifies the source of the optimizer configuration. This indicates how the table optimizer was configured and which entity or service initiated the configuration.
*)last_run : table_optimizer_run option;A TableOptimizerRun object representing the last run of the table optimizer.
configuration : table_optimizer_configuration option;A TableOptimizerConfiguration object that was specified when creating or updating a table optimizer.
type_ : table_optimizer_type option;The type of table optimizer. The valid values are:
compaction: for managing compaction with a table optimizer.retention: for managing the retention of snapshot with a table optimizer.orphan_file_deletion: for managing the deletion of orphan files with a table optimizer.}Contains details about an optimizer associated with a table.
type nonrec table_list = table listtype nonrec table_error = {error_detail : error_detail option;The details about the error.
*)table_name : name_string option;The name of the table. For Hive compatibility, this must be entirely lowercase.
*)}An error record for table operations.
type nonrec table_errors = table_error listtype nonrec table_attributes_list = table_attributes listtype nonrec supported_dialect = {dialect_version : view_dialect_version_string option;The version of the dialect of the query engine. For example, 3.0.0.
*)dialect : view_dialect option;The dialect of the query engine.
*)}A structure specifying the dialect and dialect version used by the query engine.
type nonrec stop_workflow_run_request = {run_id : id_string;The ID of the workflow run to stop.
*)name : name_string;The name of the workflow to stop.
*)}type nonrec illegal_workflow_state_exception = {message : message_string option;A message describing the problem.
*)}The workflow is in an invalid state to perform a requested operation.
type nonrec stop_trigger_response = {name : name_string option;The name of the trigger that was stopped.
*)}type nonrec stop_session_response = {id : name_string option;Returns the Id of the stopped session.
*)}type nonrec stop_session_request = {request_origin : orchestration_name_string option;The origin of the request.
*)id : name_string;The ID of the session to be stopped.
*)}type nonrec illegal_session_state_exception = {message : message_string option;A message describing the problem.
*)}The session is in an invalid state to perform a requested operation.
type nonrec stop_materialized_view_refresh_task_run_request = {table_name : name_string;The name of the table to generate statistics.
*)database_name : name_string;The name of the database where the table resides.
*)catalog_id : name_string;The ID of the Data Catalog where the table reside. If none is supplied, the account ID is used by default.
*)}Exception thrown when a task is already in stopping state.
type nonrec materialized_view_refresh_task_not_running_exception = {message : message_string option;}Exception thrown when stopping a task that is not in running state.
type nonrec stop_crawler_schedule_request = {crawler_name : name_string;Name of the crawler whose schedule state to set.
*)}type nonrec scheduler_not_running_exception = {message : message_string option;A message describing the problem.
*)}The specified scheduler is not running.
type nonrec crawler_stopping_exception = {message : message_string option;A message describing the problem.
*)}The specified crawler is stopping.
type nonrec crawler_not_running_exception = {message : message_string option;A message describing the problem.
*)}The specified crawler is not running.
type nonrec stop_column_statistics_task_run_schedule_request = {table_name : name_string;The name of the table for which to stop a column statistic task run schedule.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec stop_column_statistics_task_run_request = {table_name : name_string;The name of the table.
*)database_name : database_name;The name of the database where the table resides.
*)}type nonrec column_statistics_task_stopping_exception = {message : message_string option;A message describing the problem.
*)}An exception thrown when you try to stop a task run.
type nonrec column_statistics_task_not_running_exception = {message : message_string option;A message describing the problem.
*)}An exception thrown when you try to stop a task run when there is no task running.
type nonrec run_identifier = {job_run_id : hash_string option;The Job Run ID.
*)run_id : hash_string option;The Run ID.
*)}A run identifier.
type nonrec reference_datasets_list = name_string listtype nonrec statistic_properties_map = (name_string * description_string) listtype nonrec statistic_summary = {inclusion_annotation : timestamped_inclusion_annotation option;The inclusion annotation for the statistic.
*)recorded_on : timestamp option;The timestamp when the statistic was recorded.
*)statistic_properties : statistic_properties_map option;A StatisticPropertiesMap, which contains a NameString and DescriptionString
referenced_datasets : reference_datasets_list option;The list of datasets referenced by the statistic.
*)columns_referenced : column_name_list option;The list of columns referenced by the statistic.
*)evaluation_level : statistic_evaluation_level option;The evaluation level of the statistic. Possible values: Dataset, Column, Multicolumn.
double_value : double option;The value of the statistic.
*)statistic_name : statistic_name_string option;The name of the statistic.
*)run_identifier : run_identifier option;The Run Identifier
*)profile_id : hash_string option;The Profile ID.
*)statistic_id : hash_string option;The Statistic ID.
*)}Summary information about a statistic.
type nonrec statistic_summary_list = statistic_summary listtype nonrec statistic_model_result = {inclusion_annotation : inclusion_annotation_value option;The inclusion annotation.
*)date : timestamp option;The date.
*)actual_value : nullable_double option;The actual value.
*)predicted_value : nullable_double option;The predicted value.
*)upper_bound : nullable_double option;The upper bound.
*)lower_bound : nullable_double option;The lower bound.
*)}The statistic model result.
type nonrec statistic_model_results = statistic_model_result listtype nonrec statistic_annotation = {inclusion_annotation : timestamped_inclusion_annotation option;The inclusion annotation applied to the statistic.
*)statistic_recorded_on : timestamp option;The timestamp when the annotated statistic was recorded.
*)statistic_id : hash_string option;The Statistic ID.
*)profile_id : hash_string option;The Profile ID.
*)}A Statistic Annotation.
type nonrec statement_output_data = {text_plain : generic_string option;The code execution output in text format.
*)}The code execution output in JSON format.
type nonrec orchestration_string_list = generic_string listtype nonrec statement_output = {traceback : orchestration_string_list option;The traceback of the output.
*)error_value : generic_string option;The error value of the output.
*)error_name : generic_string option;The name of the error in the output.
*)status : statement_state option;The status of the code execution output.
*)execution_count : integer_value option;The execution count of the output.
*)data : statement_output_data option;The code execution output.
*)}The code execution output in JSON format.
type nonrec long_value = Smaws_Lib.CoreTypes.Int64.ttype nonrec statement = {completed_on : long_value option;The unix time and date that the job definition was completed.
*)started_on : long_value option;The unix time and date that the job definition was started.
*)progress : double_value option;The code execution progress.
*)output : statement_output option;The output in JSON.
*)state : statement_state option;The state while request is actioned.
*)code : generic_string option;The execution code of the statement.
*)id : integer_value option;The ID of the statement.
*)}The statement or request for a particular action to occur in a session.
type nonrec statement_list = statement listtype nonrec start_workflow_run_request = {run_properties : workflow_run_properties option;The workflow run properties for the new workflow run.
Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.
*)name : name_string;The name of the workflow to start.
*)}type nonrec concurrent_runs_exceeded_exception = {message : message_string option;A message describing the problem.
*)}Too many jobs are being run concurrently.
type nonrec start_trigger_response = {name : name_string option;The name of the trigger that was started.
*)}type nonrec start_materialized_view_refresh_task_run_response = {materialized_view_refresh_task_run_id : uui_dv4 option;The identifier for the materialized view refresh task run.
*)}type nonrec start_materialized_view_refresh_task_run_request = {full_refresh : nullable_boolean option;Specifies whether this is a full refresh of the task run.
*)table_name : name_string;The name of the table to generate run the materialized view refresh task.
*)database_name : name_string;The name of the database where the table resides.
*)catalog_id : name_string;The ID of the Data Catalog where the table reside. If none is supplied, the account ID is used by default.
*)}Exception thrown when a task is already in running state.
type nonrec start_ml_labeling_set_generation_task_run_response = {task_run_id : hash_string option;The unique run identifier that is associated with this task run.
*)}type nonrec start_ml_labeling_set_generation_task_run_request = {output_s3_path : uri_string;The Amazon Simple Storage Service (Amazon S3) path where you generate the labeling set.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec start_ml_evaluation_task_run_response = {task_run_id : hash_string option;The unique identifier associated with this run.
*)}type nonrec start_ml_evaluation_task_run_request = {transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec ml_transform_not_ready_exception = {message : message_string option;A message describing the problem.
*)}The machine learning transform is not ready to run.
type nonrec start_job_run_response = {job_run_id : id_string option;The ID assigned to this job run.
*)}type nonrec start_job_run_request = {execution_role_session_policy : orchestration_policy_json_string option;This inline session policy to the StartJobRun API allows you to dynamically restrict the permissions of the specified execution role for the scope of the job, without requiring the creation of additional IAM roles.
*)execution_class : execution_class option;Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.
The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.
Only jobs with Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX. The flexible execution class is available for Spark jobs.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a job runs.
worker_type : worker_type option;The type of predefined worker that is allocated when a job runs. Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.notification_property : notification_property option;Specifies configuration properties of a job run notification.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this job run.
max_capacity : nullable_double option;For Glue version 1.0 or earlier jobs, using the standard worker type, the number of Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
For Glue version 2.0+ jobs, you cannot specify a Maximum capacity. Instead, you should specify a Worker type and the Number of workers.
Do not set MaxCapacity if using WorkerType and NumberOfWorkers.
The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL job:
JobCommand.Name="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.JobCommand.Name="glueetl") or Apache Spark streaming ETL job (JobCommand.Name="gluestreaming"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.timeout : timeout option;The JobRun timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status. This value overrides the timeout value set in the parent job.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)allocated_capacity : integer_value option;This field is deprecated. Use MaxCapacity instead.
The number of Glue data processing units (DPUs) to allocate to this JobRun. You can allocate a minimum of 2 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
*)arguments : generic_map option;The job arguments associated with this run. For this job run, they replace the default arguments set in the job definition itself.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Secrets Manager or other secret management mechanism if you intend to keep them within the Job.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the arguments you can provide to this field when configuring Spark jobs, see the Special Parameters Used by Glue topic in the developer guide.
For information about the arguments you can provide to this field when configuring Ray jobs, see Using job parameters in Ray jobs in the developer guide.
*)job_run_id : id_string option;The ID of a previous JobRun to retry.
job_run_queuing_enabled : nullable_boolean option;Specifies whether job run queuing is enabled for the job run.
A value of true means job run queuing is enabled for the job run. If false or not populated, the job run will not be considered for queueing.
*)job_name : name_string;The name of the job definition to use.
*)}type nonrec start_import_labels_task_run_response = {task_run_id : hash_string option;The unique identifier for the task run.
*)}type nonrec start_import_labels_task_run_request = {replace_all_labels : replace_boolean option;Indicates whether to overwrite your existing labels.
*)input_s3_path : uri_string;The Amazon Simple Storage Service (Amazon S3) path from where you import the labels.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec start_export_labels_task_run_response = {task_run_id : hash_string option;The unique identifier for the task run.
*)}type nonrec start_export_labels_task_run_request = {output_s3_path : uri_string;The Amazon S3 path where you export the labels.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec start_data_quality_ruleset_evaluation_run_response = {run_id : hash_string option;The unique run identifier associated with this run.
*)}type nonrec data_quality_glue_table = {pre_processing_query : pre_processing_query_string option;SQL Query of SparkSQL format that can be used to pre-process the data for the table in Glue Data Catalog, before running the Data Quality Operation.
*)additional_options : glue_table_additional_options option;Additional options for the table. Currently there are two keys supported:
pushDownPredicate: to filter on partitions without having to list and read all the files in your dataset.catalogPartitionPredicate: to use server-side partition pruning using partition indexes in the Glue Data Catalog.connection_name : name_string option;The name of the connection to the Glue Data Catalog.
*)catalog_id : name_string option;A unique identifier for the Glue Data Catalog.
*)table_name : name_string;A table name in the Glue Data Catalog.
*)database_name : name_string;A database name in the Glue Data Catalog.
*)}The database and table in the Glue Data Catalog that is used for input or output data for Data Quality Operations.
type nonrec data_source = {data_quality_glue_table : data_quality_glue_table option;An Glue table for Data Quality Operations.
*)glue_table : glue_table option;An Glue table.
*)}A data source (an Glue table) for which you want data quality results.
type nonrec data_quality_evaluation_run_additional_run_options = {custom_log_group_prefix : generic_string option;A custom prefix for the CloudWatch log group names. When specified, evaluation run logs are written to
/errorand
/outputinstead of the default /aws-glue/data-quality/error and /aws-glue/data-quality/output log groups.
composite_rule_evaluation_method : dq_composite_rule_evaluation_method option;Set the evaluation method for composite rules in the ruleset to ROW/COLUMN
*)results_s3_prefix : uri_string option;Prefix for Amazon S3 to store results.
*)cloud_watch_metrics_enabled : nullable_boolean option;Whether or not to enable CloudWatch metrics.
*)}Additional run options you can specify for an evaluation run.
type nonrec ruleset_names = name_string listtype nonrec data_source_map = (name_string * data_source) listtype nonrec start_data_quality_ruleset_evaluation_run_request = {additional_data_sources : data_source_map option;A map of reference strings to additional data sources you can specify for an evaluation run.
*)ruleset_names : ruleset_names;A list of ruleset names.
*)additional_run_options : data_quality_evaluation_run_additional_run_options
option;Additional run options you can specify for an evaluation run.
*)client_token : hash_string option;Used for idempotency and is recommended to be set to a random ID (such as a UUID) to avoid creating or starting multiple instances of the same resource.
*)timeout : timeout option;The timeout for a run in minutes. This is the maximum time that a run can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of G.1X workers to be used in the run. The default is 5.
role : role_string;An IAM role supplied to encrypt the results of the run.
*)data_source : data_source;The data source (Glue table) associated with this run.
*)}type nonrec start_data_quality_rule_recommendation_run_response = {run_id : hash_string option;The unique run identifier associated with this run.
*)}type nonrec start_data_quality_rule_recommendation_run_request = {client_token : hash_string option;Used for idempotency and is recommended to be set to a random ID (such as a UUID) to avoid creating or starting multiple instances of the same resource.
*)data_quality_security_configuration : name_string option;The name of the security configuration created with the data quality encryption option.
*)created_ruleset_name : name_string option;A name for the ruleset.
*)timeout : timeout option;The timeout for a run in minutes. This is the maximum time that a run can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of G.1X workers to be used in the run. The default is 5.
role : role_string;An IAM role supplied to encrypt the results of the run.
*)data_source : data_source;The data source (Glue table) associated with this run.
*)}The request of the Data Quality rule recommendation request.
type nonrec start_crawler_schedule_request = {crawler_name : name_string;Name of the crawler to schedule.
*)}type nonrec scheduler_running_exception = {message : message_string option;A message describing the problem.
*)}The specified scheduler is already running.
type nonrec no_schedule_exception = {message : message_string option;A message describing the problem.
*)}There is no applicable schedule.
type nonrec start_column_statistics_task_run_schedule_request = {table_name : name_string;The name of the table for which to start a column statistic task run schedule.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec start_column_statistics_task_run_response = {column_statistics_task_run_id : hash_string option;The identifier for the column statistics task run.
*)}type nonrec start_column_statistics_task_run_request = {security_configuration : name_string option;Name of the security configuration that is used to encrypt CloudWatch logs for the column stats task run.
*)catalog_i_d : name_string option;The ID of the Data Catalog where the table reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)sample_size : sample_size_percentage option;The percentage of rows used to generate statistics. If none is supplied, the entire table will be used to generate stats.
*)role : name_string;The IAM role that the service assumes to generate statistics.
*)column_name_list : column_name_list option;A list of the column names to generate statistics. If none is supplied, all column names for the table will be used by default.
*)table_name : name_string;The name of the table to generate statistics.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec column_statistics_task_running_exception = {message : message_string option;A message describing the problem.
*)}An exception thrown when you try to start another job while running a column stats generation job.
type nonrec start_blueprint_run_response = {run_id : id_string option;The run ID for this blueprint run.
*)}type nonrec start_blueprint_run_request = {role_arn : orchestration_iam_role_arn;Specifies the IAM role used to create the workflow.
*)parameters : blueprint_parameters option;Specifies the parameters as a BlueprintParameters object.
blueprint_name : orchestration_name_string;The name of the blueprint.
*)}type nonrec list_of_string = Smaws_Lib.Smithy_api.Types.string_ listtype nonrec connector_property = {property_type : property_type;The data type of this property
*)property_location : property_location option;Specifies where this property should be included in REST requests, such as in headers, query parameters, or request body.
*)allowed_values : list_of_string option;A list of AllowedValue objects representing the values allowed for the property.
default_value : Smaws_Lib.Smithy_api.Types.string_ option;The default value for the property.
*)required : bool_;Indicates whether the property is required.
*)key_override : connector_property_key option;A key name to use when sending this property in API requests, if different from the display name.
*)name : property_name;The name of the property.
*)}Defines a property configuration for connection types, default values, and where the property should be used in requests.
type nonrec connector_property_list = connector_property listtype nonrec response_configuration = {error_path : json_path_string option;The JSON path expression that identifies where error information is located within API responses when requests fail.
*)result_path : json_path_string;The JSON path expression that identifies where the actual result data is located within the API response.
*)}Configuration that defines how to parse JSON responses from REST API calls, including paths to result data and error information.
type nonrec response_extraction_mapping = {header_key : connector_property_key option;The name of an HTTP response header from which to extract the value.
*)content_path : json_path_string option;A JSON path expression that specifies how to extract a value from the response body content.
*)}Configuration that defines how to extract values from HTTP response content or headers for use in subsequent requests or parameter mapping.
type nonrec extracted_parameter = {value : response_extraction_mapping option;The JSON path or extraction mapping that defines how to extract the parameter value from API responses.
*)property_location : property_location option;Specifies where this extracted parameter should be placed in subsequent requests, such as in headers, query parameters, or request body.
*)default_value : default_value option;The default value to use if the parameter cannot be extracted from the response.
*)key : connector_property_key option;The parameter key name that will be used in subsequent requests.
*)}Parameter extraction configuration that defines how to extract and map values from API responses to request parameters.
type nonrec cursor_configuration = {limit_parameter : extracted_parameter option;The parameter name used to specify the maximum number of results to return per page.
*)next_page : extracted_parameter;The parameter name or JSON path that contains the cursor or token for retrieving the next page of results.
*)}Cursor-based pagination configuration that defines how to handle pagination using cursor tokens or next page identifiers.
type nonrec offset_configuration = {limit_parameter : extracted_parameter;The parameter name used to specify the maximum number of results to return per page.
*)offset_parameter : extracted_parameter;The parameter name used to specify the starting position or offset for retrieving results.
*)}Offset-based pagination configuration that defines how to handle pagination using numeric offsets and limits.
type nonrec pagination_configuration = {offset_configuration : offset_configuration option;Configuration for offset-based pagination, where the API uses numeric offsets and limits to control which results are returned.
*)cursor_configuration : cursor_configuration option;Configuration for cursor-based pagination, where the API provides a cursor or token to retrieve the next page of results.
*)}Configuration that defines how to handle paginated responses from REST APIs, supporting different pagination strategies used by various services.
type nonrec source_configuration = {pagination_configuration : pagination_configuration option;Configuration for handling paginated responses from the REST API, supporting both cursor-based and offset-based pagination strategies.
*)response_configuration : response_configuration option;Configuration that defines how to parse and extract data from API responses, including success and error handling.
*)request_parameters : connector_property_list option;Configuration for request parameters that should be included in API calls, such as query parameters, headers, or body content.
*)request_path : path_string option;The URL path for the REST endpoint, which may include parameter placeholders that will be replaced with actual values during requests.
*)request_method : http_method option;The HTTP method to use for requests to this endpoint, such as GET, POST.
*)}Configuration that defines how to make requests to endpoints, including request methods, paths, parameters, and response handling.
type nonrec sort_criterion = {sort : sort option;An ascending or descending sort.
*)field_name : value_string option;The name of the field on which to sort.
*)}Specifies a field to sort by and a sort order.
type nonrec sort_criteria = sort_criterion listtype nonrec session_command = {python_version : python_version_string option;Specifies the Python version. The Python version indicates the version supported for jobs of type Spark.
*)name : name_string option;Specifies the name of the SessionCommand. Can be 'glueetl' or 'gluestreaming'.
*)}The SessionCommand that runs the job.
type nonrec orchestration_arguments_map =
(orchestration_name_string * orchestration_arguments_value) listtype nonrec session = {session_type : session_type option;The type of the session.
*)profile_name : name_string option;The name of an Glue usage profile associated with the session.
*)idle_timeout : idle_timeout option;The number of minutes when idle before the session times out.
*)dpu_seconds : nullable_double option;The DPUs consumed by the session (formula: ExecutionTime * MaxCapacity).
*)execution_time : nullable_double option;The total time the session ran for.
*)completed_on : timestamp_value option;The date and time that this session is completed.
*)worker_type : worker_type option;The type of predefined worker that is allocated when a session runs. Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark sessions. Accepts the value Z.2X for Ray sessions.
number_of_workers : nullable_integer option;The number of workers of a defined WorkerType to use for the session.
glue_version : glue_version_string option;The Glue version determines the versions of Apache Spark and Python that Glue supports. The GlueVersion must be greater than 2.0.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with the session.
*)max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that can be allocated when the job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB memory.
*)progress : double_value option;The code execution progress of the session.
*)connections : connections_list option;The number of connections used for the session.
*)default_arguments : orchestration_arguments_map option;A map array of key-value pairs. Max is 75 pairs.
*)command : session_command option;The command object.See SessionCommand.
*)role : orchestration_role_arn option;The name or Amazon Resource Name (ARN) of the IAM role associated with the Session.
*)description : description_string option;The description of the session.
*)error_message : description_string option;The error message displayed during the session.
*)status : session_status option;The session status.
*)created_on : timestamp_value option;The time and date when the session was created.
*)id : name_string option;The ID of the session.
*)}The period in which a remote Spark runtime environment is running.
type nonrec session_list = session listtype nonrec session_id_list = name_string listtype nonrec session_endpoint = {auth_token_expiration_time : timestamp_value;The time at which the authentication token expires.
*)auth_token : sensitive_string;The authentication token to include in requests to the Spark Connect endpoint.
*)url : spark_connect_endpoint_url;The Spark Connect endpoint URL for the session.
*)}Contains the Spark Connect endpoint details for an interactive session, including the URL and authentication credentials.
type nonrec session_busy_exception = {message : orchestration_message_string option;A message describing the problem.
*)}The session is currently busy processing another request and cannot accept new operations.
type nonrec selected_fields = entity_field_name listtype nonrec segment = {total_segments : total_segments_integer;The total number of segments.
*)segment_number : non_negative_integer;The zero-based index number of the segment. For example, if the total number of segments is 4, SegmentNumber values range from 0 through 3.
}Defines a non-overlapping region of a table's partitions, allowing multiple requests to be run in parallel.
type nonrec s3_encryption = {kms_key_arn : kms_key_arn option;The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.
*)s3_encryption_mode : s3_encryption_mode option;The encryption mode to use for Amazon S3 data.
*)}Specifies how Amazon Simple Storage Service (Amazon S3) data should be encrypted.
type nonrec s3_encryption_list = s3_encryption listtype nonrec cloud_watch_encryption = {kms_key_arn : kms_key_arn option;The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.
*)cloud_watch_encryption_mode : cloud_watch_encryption_mode option;The encryption mode to use for CloudWatch data.
*)}Specifies how Amazon CloudWatch data should be encrypted.
type nonrec job_bookmarks_encryption = {kms_key_arn : kms_key_arn option;The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.
*)job_bookmarks_encryption_mode : job_bookmarks_encryption_mode option;The encryption mode to use for job bookmarks data.
*)}Specifies how job bookmark data should be encrypted.
type nonrec data_quality_encryption = {kms_key_arn : kms_key_arn option;The Amazon Resource Name (ARN) of the KMS key to be used to encrypt the data.
*)data_quality_encryption_mode : data_quality_encryption_mode option;The encryption mode to use for encrypting Data Quality assets. These assets include data quality rulesets, results, statistics, anomaly detection models and observations.
Valid values are SSEKMS for encryption using a customer-managed KMS key, or DISABLED.
}Specifies how Data Quality assets in your account should be encrypted.
type nonrec encryption_configuration = {data_quality_encryption : data_quality_encryption option;The encryption configuration for Glue Data Quality assets.
*)job_bookmarks_encryption : job_bookmarks_encryption option;The encryption configuration for job bookmarks.
*)cloud_watch_encryption : cloud_watch_encryption option;The encryption configuration for Amazon CloudWatch.
*)s3_encryption : s3_encryption_list option;The encryption configuration for Amazon Simple Storage Service (Amazon S3) data.
*)}Specifies an encryption configuration.
type nonrec security_configuration = {encryption_configuration : encryption_configuration option;The encryption configuration associated with this security configuration.
*)created_time_stamp : timestamp_value option;The time at which this security configuration was created.
*)name : name_string option;The name of the security configuration.
*)}Specifies a security configuration.
type nonrec security_configuration_list = security_configuration listtype nonrec search_tables_response = {table_list : table_list option;A list of the requested Table objects. The SearchTables response returns only the tables that you have access to.
next_token : token option;A continuation token, present if the current list segment is not the last.
*)}type nonrec property_predicate = {comparator : comparator option;The comparator used to compare this property to others.
*)value : value_string option;The value of the property.
*)key : value_string option;The key of the property.
*)}Defines a property predicate.
type nonrec search_property_predicates = property_predicate listtype nonrec search_tables_request = {include_status_details : boolean_nullable option;Specifies whether to include status details related to a request to create or update an Glue Data Catalog view.
*)max_results : page_size option;The maximum number of tables to return in a single response.
*)sort_criteria : sort_criteria option;A list of criteria for sorting the results by a field name, in an ascending or descending order.
*)search_text : value_string option;A string used for a text search.
Specifying a value in quotes filters based on an exact match to the value.
*)filters : search_property_predicates option;A list of key-value pairs, and a comparator used to filter the search results. Returns all entities matching the predicate.
The Comparator member of the PropertyPredicate struct is used only for time fields, and can be omitted for other field types. Also, when comparing string values, such as when Key=Name, a fuzzy match algorithm is used. The Key field (for example, the value of the Name field) is split on certain punctuation characters, for example, -, :, #, etc. into tokens. Then each token is exact-match compared with the Value member of PropertyPredicate. For example, if Key=Name and Value=link, tables named customer-link and xx-link-yy are returned, but xxlinkyy is not returned.
next_token : token option;A continuation token, included if this is a continuation call.
*)catalog_id : catalog_id_string option;A unique identifier, consisting of
{i account_id} .
*)}type nonrec search_sort = {order : search_sort_order option;The sort order. Valid values are ASCENDING and DESCENDING.
attribute : search_attribute;The attribute to sort by.
*)}The sort criteria for search results.
type nonrec search_result_item = {asset_type_id : asset_type_id option;The identifier of the asset type for the matched asset.
*)updated_at : updated_at option;The timestamp at which the matched asset was last updated.
*)asset_description : asset_description option;The description of the matched asset.
*)asset_name : search_result_name option;The name of the matched asset.
*)id : asset_id option;The unique identifier of the matched asset.
*)}A single search result item representing a matched asset.
type nonrec search_result_item_list = search_result_item listtype nonrec search_map_filter_value = | StringValue of search_filter_string_valueA string filter value.
*)A map filter value. Currently supports string comparison only.
type nonrec search_map_filter = {value : search_map_filter_value;The value to compare against.
*)key : search_map_key;The key within the map attribute to filter on.
*)attribute : search_attribute;The map attribute name to filter on.
*)}A filter on a map attribute's key-value pair.
type nonrec search_filter_long_value = Smaws_Lib.CoreTypes.Int64.ttype nonrec search_filter_value = | LongValue of search_filter_long_valueA long integer filter value.
*)| StringValue of search_filter_string_valueA string filter value.
*)A filter value. Exactly one of stringValue or longValue must be specified.
type nonrec search_attribute_filter = {value : search_filter_value option;The value to compare against.
*)operator : search_filter_operator;The comparison operator. Valid values are equals, greaterThan, greaterThanOrEquals, lessThan, lessThanOrEquals, and notExists.
attribute : search_attribute;The attribute name to filter on.
*)}A filter that compares an attribute value using an operator.
type search_filter_clause = | MapFilter of search_map_filterA filter on a map attribute's key-value pair.
*)| AttributeFilter of search_attribute_filterA filter on a single attribute value.
*)| OrAnyFilters of search_filter_clause_listA list of filter clauses where at least one must match (logical OR).
*)| AndAllFilters of search_filter_clause_listA list of filter clauses that must all match (logical AND).
*)A filter clause that supports nested boolean logic. Exactly one of andAllFilters, orAnyFilters, attributeFilter, or mapFilter must be specified.
and search_filter_clause_list = search_filter_clause listtype nonrec search_assets_output = {next_token : search_next_token option;A continuation token, present if the current segment is not the last.
*)items : search_result_item_list option;The list of assets matching the search criteria.
*)}type nonrec search_assets_input = {filter_clause : search_filter_clause option;The filter clause to apply to the search. Supports nested AND/OR logic with attribute-level and map-level filters.
*)sort : search_sort option;The sort criteria for the search results.
*)next_token : search_next_token option;A continuation token, if this is a continuation call.
*)max_results : search_max_results option;The maximum number of results to return in the response.
*)search_text : search_text option;The text to search for. At least one of searchText or filterClause must be provided.
}type nonrec schema_version_list_item = {created_time : created_timestamp option;The date and time the schema version was created.
*)status : schema_version_status option;The status of the schema version.
*)version_number : version_long_number option;The version number of the schema.
*)schema_version_id : schema_version_id_string option;The unique identifier of the schema version.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)}An object containing the details about a schema version.
type nonrec schema_version_list = schema_version_list_item listtype nonrec error_details = {error_message : error_message_string option;The error message for an error.
*)error_code : error_code_string option;The error code for an error.
*)}An object containing error details.
type nonrec schema_version_error_item = {error_details : error_details option;The details of the error for the schema version.
*)version_number : version_long_number option;The version number of the schema.
*)}An object that contains the error details for an operation on a schema version.
type nonrec schema_version_error_list = schema_version_error_item listtype nonrec schema_list_item = {updated_time : updated_timestamp option;The date and time that a schema was updated.
*)created_time : created_timestamp option;The date and time that a schema was created.
*)schema_status : schema_status option;The status of the schema.
*)description : description_string option;A description for the schema.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) for the schema.
*)schema_name : schema_registry_name_string option;The name of the schema.
*)registry_name : schema_registry_name_string option;the name of the registry where the schema resides.
*)}An object that contains minimal details for a schema.
type nonrec schema_list_definition = schema_list_item listtype nonrec schema_checkpoint_number = Smaws_Lib.CoreTypes.Int64.ttype nonrec schedule = {state : schedule_state option;The state of the schedule.
*)schedule_expression : cron_expression option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
}A scheduling object using a cron statement to schedule an event.
type nonrec run_statement_response = {id : integer_value option;Returns the Id of the statement that was run.
*)}type nonrec run_statement_request = {request_origin : orchestration_name_string option;The origin of the request.
*)code : orchestration_statement_code_string;The statement code to be run.
*)session_id : name_string;The Session Id of the statement to be run.
*)}type nonrec rule_metrics_map = (name_string * nullable_double) listtype nonrec node_id_list = name_string listtype nonrec resume_workflow_run_response = {node_ids : node_id_list option;A list of the node IDs for the nodes that were actually restarted.
*)run_id : id_string option;The new ID assigned to the resumed workflow run. Each resume of a workflow run will have a new run ID.
*)}type nonrec resume_workflow_run_request = {node_ids : node_id_list;A list of the node IDs for the nodes you want to restart. The nodes that are to be restarted must have a run attempt in the original run.
*)run_id : id_string;The ID of the workflow run to resume.
*)name : name_string;The name of the workflow to resume.
*)}type nonrec field_definition = {field_data_type : field_data_type;The data type of the field.
*)name : Smaws_Lib.Smithy_api.Types.string_;The name of the field in the entity schema.
*)}Defines a field in an entity schema for REST connector data sources, specifying the field name and data type.
type nonrec field_definition_map =
(field_definition_map_key_string * field_definition) listA map of field definitions that define the schema structure for an entity in a REST connector data source.
type nonrec entity_configuration = {schema : field_definition_map option;The schema definition for this entity, including field names, types, and other metadata that describes the structure of the data.
*)source_configuration : source_configuration option;The source configuration that defines how to make requests to access this entity's data through the REST API.
*)}Configuration that defines how to interact with a specific data entity through the REST API, including its access patterns and schema definition.
type nonrec entity_configuration_map =
(entity_configuration_map_key_string * entity_configuration) listA map of entity configurations that associates entity names with their corresponding configuration settings for REST API interactions.
type nonrec rest_configuration = {entity_configurations : entity_configuration_map option;A map of entity configurations that define how to interact with different data entities available through the REST API, including their schemas and access patterns.
*)validation_endpoint_configuration : source_configuration option;Configuration for the endpoint used to validate connection credentials and test connectivity during connection creation.
*)global_source_configuration : source_configuration option;Global configuration settings that apply to all REST API requests for this connection type, including common request methods, paths, and parameters.
*)}Configuration that defines HTTP request and response handling, validation endpoints, and entity configurations for REST API interactions.
type nonrec job_bookmark_entry = {job_bookmark : json_value option;The bookmark itself.
*)run_id : run_id option;The run ID number.
*)previous_run_id : run_id option;The unique run identifier associated with the previous job run.
*)attempt : integer_value option;The attempt ID number.
*)run : integer_value option;The run ID number.
*)version : integer_value option;The version of the job.
*)job_name : job_name option;The name of the job in question.
*)}Defines a point that a job can resume processing.
type nonrec reset_job_bookmark_response = {job_bookmark_entry : job_bookmark_entry option;The reset bookmark entry.
*)}type nonrec remove_schema_version_metadata_response = {metadata_value : metadata_value_string option;The value of the metadata key.
*)metadata_key : metadata_key_string option;The metadata key.
*)schema_version_id : schema_version_id_string option;The version ID for the schema version.
*)version_number : version_long_number option;The version number of the schema.
*)latest_version : latest_schema_version_boolean option;The latest version of the schema.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)schema_name : schema_registry_name_string option;The name of the schema.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)}type nonrec metadata_key_value_pair = {metadata_value : metadata_value_string option;A metadata key’s corresponding value.
*)metadata_key : metadata_key_string option;A metadata key.
*)}A structure containing a key value pair for metadata.
type nonrec remove_schema_version_metadata_input = {metadata_key_value : metadata_key_value_pair;The value of the metadata key.
*)schema_version_id : schema_version_id_string option;The unique version ID of the schema version.
*)schema_version_number : schema_version_number option;The version number of the schema.
*)schema_id : schema_id option;A wrapper structure that may contain the schema name and Amazon Resource Name (ARN).
*)}type nonrec registry_list_item = {updated_time : updated_timestamp option;The date the registry was updated.
*)created_time : created_timestamp option;The data the registry was created.
*)status : registry_status option;The status of the registry.
*)description : description_string option;A description of the registry.
*)registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registry.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)}A structure containing the details for a registry.
type nonrec registry_list_definition = registry_list_item listtype nonrec register_schema_version_response = {status : schema_version_status option;The status of the schema version.
*)version_number : version_long_number option;The version of this schema (for sync flow only, in case this is the first version).
*)schema_version_id : schema_version_id_string option;The unique ID that represents the version of this schema.
*)}type nonrec register_schema_version_input = {schema_definition : schema_definition_string;The schema definition using the DataFormat setting for the SchemaName.
schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName and RegistryName has to be provided.SchemaArn or SchemaName and RegistryName has to be provided.}type nonrec register_connection_type_response = {connection_type_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registered connection type. This unique identifier can be used to reference the connection type in other Glue operations.
*)}Contains the Amazon Resource Name (ARN) of the newly registered connection type.
type nonrec connection_properties_configuration = {additional_request_parameters : connector_property_list option;Key-value pairs of additional request parameters that may be needed during connection creation, such as API versions or service-specific configuration options.
*)url : connector_property option;The base instance URL for the endpoint that this connection type will connect to.
*)}Configuration that defines the base URL and additional request parameters needed during connection creation.
type nonrec authentication_types = authentication_type listtype nonrec client_credentials_properties = {token_url_parameters : connector_property_list option;Additional parameters to include in token URL requests as key-value pairs.
*)scope : connector_property option;The OAuth2 scope that defines the level of access requested for the client credentials flow.
*)client_secret : connector_property option;The OAuth2 client secret provided by the authorization server.
*)client_id : connector_property option;The OAuth2 client identifier provided by the authorization server.
*)content_type : content_type option;The content type to use for token requests, such as application/x-www-form-urlencoded or application/json.
*)request_method : http_method option;The HTTP method to use when making token requests, typically POST.
*)token_url : connector_property option;The token endpoint URL where the client will request access tokens using client credentials.
*)}OAuth2 client credentials configuration that defines the properties needed for the Client Credentials grant type flow.
type nonrec jwt_bearer_properties = {token_url_parameters : connector_property_list option;Additional parameters to include in token URL requests as key-value pairs.
*)jwt_token : connector_property option;The JWT token to be used in the bearer token grant flow for authentication.
*)content_type : content_type option;The content type to use for JWT bearer token requests, such as application/x-www-form-urlencoded or application/json.
*)request_method : http_method option;The HTTP method to use when making JWT bearer token requests, typically POST.
*)token_url : connector_property option;The token endpoint URL where the JWT bearer token will be exchanged for an access token.
*)}JWT bearer token configuration that defines the properties needed for the JWT Bearer grant type flow.
OAuth2 authorization code configuration that defines the properties needed for the Authorization Code grant type flow.
type nonrec connector_o_auth2_properties = {jwt_bearer_properties : jwt_bearer_properties option;Configuration properties specific to the OAuth2 JWT Bearer grant type flow.
*)client_credentials_properties : client_credentials_properties option;Configuration properties specific to the OAuth2 Client Credentials grant type flow.
*)o_auth2_grant_type : connector_o_auth2_grant_type;The OAuth2 grant type to use for authentication, such as CLIENT_CREDENTIALS, JWT_BEARER, or AUTHORIZATION_CODE.
*)}OAuth2 configuration container that defines the authentication properties and flow-specific configurations for OAuth2-based connections.
type nonrec basic_authentication_properties = {password : connector_property option;The password property name to use for Basic authentication credentials.
*)username : connector_property option;The username property name to use for Basic authentication credentials.
*)}Basic authentication configuration that defines the username and password properties for HTTP Basic authentication.
type nonrec custom_authentication_properties = {authentication_parameters : connector_property_list;A map of custom authentication parameters that define the specific authentication mechanism and required properties.
*)}Custom authentication configuration that allows for flexible authentication mechanisms beyond standard Basic and OAuth2 flows.
type nonrec connector_authentication_configuration = {custom_authentication_properties : custom_authentication_properties option;basic_authentication_properties : basic_authentication_properties option;o_auth2_properties : connector_o_auth2_properties option;authentication_types : authentication_types;A list of authentication types supported by this connection type, such as Basic, OAuth2, or Custom authentication methods.
*)}Configuration that defines the supported authentication types and required properties for the connection type.
type nonrec register_connection_type_request = {rest_configuration : rest_configuration;Defines the HTTP request and response configuration, validation endpoint, and entity configurations for REST API interactions.
*)connector_authentication_configuration : connector_authentication_configuration;Defines the supported authentication types and required properties for this connection type, including Basic, OAuth2, and Custom authentication methods.
*)connection_properties : connection_properties_configuration;Defines the base URL and additional request parameters needed during connection creation for this connection type.
*)description : description option;A description of the connection type. Can be up to 2048 characters and provides details about the purpose and functionality of the connection type.
*)integration_type : integration_type;The integration type for the connection. Currently only "REST" protocol is supported.
*)connection_type : name_string;The name of the connection type. Must be between 1 and 255 characters and must be prefixed with "REST-" to indicate it is a REST-based connector.
*)}type nonrec records = record listtype nonrec additional_context_map = (context_key * context_value) listtype nonrec query_session_context = {additional_context : additional_context_map option;An opaque string-string map passed by the query engine.
*)cluster_id : nullable_string option;An identifier string for the consumer cluster.
*)query_start_time : timestamp option;A timestamp provided by the query engine for when the query started.
*)query_id : hash_string option;A unique identifier generated by the query engine for the query.
*)}A structure used as a protocol between query engines and Lake Formation or Glue. Contains both a Lake Formation generated authorization identifier and information from the request's authorization context.
type nonrec other_metadata_value_list_item = {created_time : created_timestamp option;The time at which the entry was created.
*)metadata_value : metadata_value_string option;The metadata key’s corresponding value for the other metadata belonging to the same metadata key.
*)}A structure containing other metadata for a schema version belonging to the same metadata key.
type nonrec other_metadata_value_list = other_metadata_value_list_item listtype nonrec metadata_info = {other_metadata_value_list : other_metadata_value_list option;Other metadata belonging to the same metadata key.
*)created_time : created_timestamp option;The time at which the entry was created.
*)metadata_value : metadata_value_string option;The metadata key’s corresponding value.
*)}A structure containing metadata information for a schema version.
type nonrec metadata_info_map = (metadata_key_string * metadata_info) listtype nonrec query_schema_version_metadata_response = {next_token : schema_registry_token_string option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)schema_version_id : schema_version_id_string option;The unique version ID of the schema version.
*)metadata_info_map : metadata_info_map option;A map of a metadata key and associated values.
*)}type nonrec metadata_list = metadata_key_value_pair listtype nonrec query_schema_version_metadata_input = {next_token : schema_registry_token_string option;A continuation token, if this is a continuation call.
*)max_results : query_schema_version_metadata_max_results option;Maximum number of results required per page. If the value is not supplied, this will be defaulted to 25 per page.
*)metadata_list : metadata_list option;Search key-value pairs for metadata, if they are not provided all the metadata information will be fetched.
*)schema_version_id : schema_version_id_string option;The unique version ID of the schema version.
*)schema_version_number : schema_version_number option;The version number of the schema.
*)schema_id : schema_id option;A wrapper structure that may contain the schema name and Amazon Resource Name (ARN).
*)}type nonrec put_workflow_run_properties_request = {run_properties : workflow_run_properties;The properties to put for the specified run.
Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.
*)run_id : id_string;The ID of the workflow run for which the run properties should be updated.
*)name : name_string;Name of the workflow which was run.
*)}type nonrec put_schema_version_metadata_response = {metadata_value : metadata_value_string option;The value of the metadata key.
*)metadata_key : metadata_key_string option;The metadata key.
*)schema_version_id : schema_version_id_string option;The unique version ID of the schema version.
*)version_number : version_long_number option;The version number of the schema.
*)latest_version : latest_schema_version_boolean option;The latest version of the schema.
*)registry_name : schema_registry_name_string option;The name for the registry.
*)schema_name : schema_registry_name_string option;The name for the schema.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) for the schema.
*)}type nonrec put_schema_version_metadata_input = {metadata_key_value : metadata_key_value_pair;The metadata key's corresponding value.
*)schema_version_id : schema_version_id_string option;The unique version ID of the schema version.
*)schema_version_number : schema_version_number option;The version number of the schema.
*)schema_id : schema_id option;The unique ID for the schema.
*)}type nonrec put_resource_policy_response = {policy_hash : hash_string option;A hash of the policy that has just been set. This must be included in a subsequent call that overwrites or updates this policy.
*)}type nonrec put_resource_policy_request = {enable_hybrid : enable_hybrid_values option;If 'TRUE', indicates that you are using both methods to grant cross-account access to Data Catalog resources:
PutResourePolicyMust be set to 'TRUE' if you have already used the Management Console to grant cross-account access, otherwise the call fails. Default is 'FALSE'.
policy_exists_condition : exist_condition option;A value of MUST_EXIST is used to update a policy. A value of NOT_EXIST is used to create a new policy. If a value of NONE or a null value is used, the call does not depend on the existence of a policy.
policy_hash_condition : hash_string option;The hash value returned when the previous policy was set using PutResourcePolicy. Its purpose is to prevent concurrent modifications of a policy. Do not use this parameter if no previous policy has been set.
resource_arn : glue_resource_arn option;Do not use. For internal use only.
*)policy_in_json : policy_json_string;Contains the policy document to set, in JSON format.
*)}type nonrec condition_check_failure_exception = {message : message_string option;A message describing the problem.
*)}A specified condition was not satisfied.
type nonrec put_form_type_response = {schema : form_type_schema option;The Smithy IDL schema of the form type.
*)name : form_type_name option;The name of the form type.
*)id : form_type_id option;The identifier of the form type.
*)}type nonrec put_form_type_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)schema : form_type_schema;The Smithy IDL schema definition for the form type.
*)name : form_type_name;The name of the form type. Must start with an uppercase letter.
*)}type nonrec put_data_quality_profile_annotation_request = {inclusion_annotation : inclusion_annotation_value;The inclusion annotation value to apply to the profile.
*)profile_id : hash_string;The ID of the data quality monitoring profile to annotate.
*)}type nonrec encryption_at_rest = {catalog_encryption_service_role : iam_role_arn option;The role that Glue assumes to encrypt and decrypt the Data Catalog objects on the caller's behalf.
*)sse_aws_kms_key_id : name_string option;The ID of the KMS key to use for encryption at rest.
*)catalog_encryption_mode : catalog_encryption_mode;The encryption-at-rest mode for encrypting Data Catalog data.
*)}Specifies the encryption-at-rest configuration for the Data Catalog.
type nonrec connection_password_encryption = {aws_kms_key_id : name_string option;An KMS key that is used to encrypt the connection password.
If connection password protection is enabled, the caller of CreateConnection and UpdateConnection needs at least kms:Encrypt permission on the specified KMS key, to encrypt passwords before storing them in the Data Catalog.
You can set the decrypt permission to enable or restrict access on the password key according to your security requirements.
*)return_connection_password_encrypted : boolean_;When the ReturnConnectionPasswordEncrypted flag is set to "true", passwords remain encrypted in the responses of GetConnection and GetConnections. This encryption takes effect independently from catalog encryption.
}The data structure used by the Data Catalog to encrypt the password as part of CreateConnection or UpdateConnection and store it in the ENCRYPTED_PASSWORD field in the connection properties. You can enable catalog encryption or only password encryption.
When a CreationConnection request arrives containing a password, the Data Catalog first encrypts the password using your KMS key. It then encrypts the whole connection object again if catalog encryption is also enabled.
This encryption requires that you set KMS key permissions to enable or restrict access on the password key according to your security requirements. For example, you might want only administrators to have decrypt permission on the password key.
type nonrec data_catalog_encryption_settings = {connection_password_encryption : connection_password_encryption option;When connection password protection is enabled, the Data Catalog uses a customer-provided key to encrypt the password as part of CreateConnection or UpdateConnection and store it in the ENCRYPTED_PASSWORD field in the connection properties. You can enable catalog encryption or only password encryption.
encryption_at_rest : encryption_at_rest option;Specifies the encryption-at-rest configuration for the Data Catalog.
*)}Contains configuration information for maintaining Data Catalog security.
type nonrec put_data_catalog_encryption_settings_request = {data_catalog_encryption_settings : data_catalog_encryption_settings;The security configuration to set.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog to set the security configuration for. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec put_attachment_response = {form_type_id : form_type_id option;The identifier of the form type for this attachment.
*)attachment_name : attachment_name option;The name of the attachment.
*)item_identifier : item_identifier option;The identifier of the item within the iterable form, if applicable.
*)iterable_form_name : iterable_form_name option;The name of the iterable form, if the attachment targets an item.
*)asset_identifier : asset_id option;The unique identifier of the asset.
*)}type nonrec put_attachment_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)form_type_id : form_type_id;The identifier of the form type for this attachment.
*)content : form_content;The JSON content of the form, conforming to the schema of the specified form type.
*)attachment_name : attachment_name;The name of the attachment.
*)item_identifier : item_identifier option;The identifier of the item within the iterable form. Required when iterableFormName is specified.
iterable_form_name : iterable_form_name option;The name of the iterable form. When specified along with itemIdentifier, the attachment targets an item within the iterable form rather than the asset itself.
asset_identifier : asset_id;The unique identifier of the asset to attach the form to.
*)}type nonrec asset_type_form_reference = {form_type_identifier : form_type_id;The identifier of the referenced form type.
*)}A reference to a form type that is included in an asset type.
type nonrec asset_type_forms_map =
(asset_type_form_key * asset_type_form_reference) listtype nonrec put_asset_type_response = {forms : asset_type_forms_map option;The forms that make up the asset type, keyed by form name.
*)name : asset_type_name option;The name of the asset type.
*)id : asset_type_id option;The identifier of the asset type.
*)}type nonrec put_asset_type_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)forms : asset_type_forms_map;The forms that make up the asset type, keyed by form name. Each entry references the form type that defines the form's schema.
*)name : asset_type_name;The name of the asset type.
*)}type nonrec asset_form_entry = {content : form_content option;The JSON content of the form, conforming to the schema of the specified form type.
*)form_type_id : form_type_id option;The identifier of the form type that defines this form's schema.
*)}A form on an asset, consisting of the form type identifier and its JSON content.
type nonrec asset_form_map = (asset_form_key * asset_form_entry) listtype nonrec put_asset_response = {forms : asset_form_map option;The forms attached to the asset, keyed by form name.
*)created_at : created_at option;The timestamp at which the asset was created.
*)description : asset_description option;The description of the asset.
*)name : asset_name;The name of the asset.
*)id : asset_id;The unique identifier of the asset.
*)}type nonrec put_asset_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)forms : asset_form_map;The forms to set on the asset, keyed by form name. Each entry specifies the form type and its JSON content.
*)description : asset_description option;The description of the asset.
*)name : asset_name;The name of the asset.
*)identifier : asset_id;The unique identifier of the asset. If an asset with this identifier already exists, it is updated.
*)asset_type_id : asset_type_id;The identifier of the asset type for the asset.
*)}type nonrec property_types = property_type listtype nonrec property_name_overrides = (property_name * property_name) listtype nonrec allowed_value = {value : allowed_value_value_string;The value allowed for the property.
*)description : allowed_value_description_string option;A description of the allowed value.
*)}An object representing a value allowed for a property.
type nonrec allowed_values = allowed_value listtype nonrec data_operations = data_operation listtype nonrec property = {property_location : property_location option;Specifies where this property should be included in REST requests, such as in headers, query parameters, or request body.
*)key_override : Smaws_Lib.Smithy_api.Types.string_ option;A key name to use when sending this property in API requests, if different from the display name.
*)data_operation_scopes : data_operations option;Indicates which data operations are applicable to the property.
*)allowed_values : allowed_values option;A list of AllowedValue objects representing the values allowed for the property.
property_types : property_types;Describes the type of property.
*)default_value : Smaws_Lib.Smithy_api.Types.string_ option;The default value for the property.
*)required : bool_;Indicates whether the property is required.
*)description : property_description_string;A description of the property.
*)name : property_name;The name of the property.
*)}An object that defines a connection type for a compute environment.
type nonrec properties_map = (property_name * property) listtype nonrec permission_type_mismatch_exception = {message : message_string option;There is a mismatch between the SupportedPermissionType used in the query request and the permissions defined on the target table.
*)}The operation timed out.
type nonrec permission_type_list = permission_type listContains a list of values defining partitions.
type nonrec partition_list = partition listtype nonrec partition_input_list = partition_input listtype nonrec key_list = name_string listtype nonrec partition_index = {index_name : name_string;The name of the partition index.
*)keys : key_list;The keys for the partition index.
*)}A structure for a partition index.
type nonrec partition_index_list = partition_index listtype nonrec key_schema_element = {type_ : column_type_string;The type of a partition key.
*)name : name_string;The name of a partition key.
*)}A partition key pair consisting of a name and a type.
type nonrec key_schema_element_list = key_schema_element listtype nonrec backfill_errored_partitions_list = partition_value_list listtype nonrec backfill_error = {partitions : backfill_errored_partitions_list option;A list of a limited number of partitions in the response.
*)code : backfill_error_code option;The error code for an error that occurred when registering partition indexes for an existing table.
*)}A list of errors that can occur when registering partition indexes for an existing table.
These errors give the details about why an index registration failed and provide a limited number of partitions in the response, so that you can fix the partitions at fault and try registering the index again. The most common set of errors that can occur are categorized as follows:
type nonrec backfill_errors = backfill_error listtype nonrec partition_index_descriptor = {backfill_errors : backfill_errors option;A list of errors that can occur when registering partition indexes for an existing table.
*)index_status : partition_index_status;The status of the partition index.
The possible statuses are:
keys : key_schema_element_list;A list of one or more keys, as KeySchemaElement structures, for the partition index.
index_name : name_string;The name of the partition index.
*)}A descriptor for a partition index in a table.
type nonrec partition_index_descriptor_list = partition_index_descriptor listtype nonrec partition_error = {error_detail : error_detail option;The details about the partition error.
*)partition_values : value_string_list option;The values that define the partition.
*)}Contains information about a partition error.
type nonrec partition_errors = partition_error listtype nonrec create_iceberg_table_input = {properties : string_to_string_map option;Key-value pairs of additional table properties and configuration settings for the Iceberg table.
*)write_order : iceberg_sort_order option;The sort order specification that defines how data should be ordered within each partition to optimize query performance.
*)partition_spec : iceberg_partition_spec option;The partitioning specification that defines how the Iceberg table data will be organized and partitioned for optimal query performance.
*)schema : iceberg_schema;The schema definition that specifies the structure, field types, and metadata for the Iceberg table.
*)location : location_string;The S3 location where the Iceberg table data will be stored.
*)}The configuration parameters required to create a new Iceberg table in the Glue Data Catalog, including table properties and metadata specifications.
type nonrec iceberg_input = {create_iceberg_table_input : create_iceberg_table_input option;The configuration parameters required to create a new Iceberg table in the Glue Data Catalog, including table properties and metadata specifications.
*)version : version_string option;The table version for the Iceberg table. Defaults to 2.
*)metadata_operation : metadata_operation;A required metadata operation. Can only be set to CREATE.
}A structure that defines an Apache Iceberg metadata table to create in the catalog.
type nonrec open_table_format_input = {iceberg_input : iceberg_input option;Specifies an IcebergInput structure that defines an Apache Iceberg metadata table.
}A structure representing an open format table.
type nonrec o_auth2_properties = {token_url_parameters_map : token_url_parameters_map option;A map of parameters that are added to the token GET request.
token_url : token_url option;The URL of the provider's authentication server, to exchange an authorization code for an access token.
*)o_auth2_client_application : o_auth2_client_application option;The client application type. For example, AWS_MANAGED or USER_MANAGED.
*)o_auth2_grant_type : o_auth2_grant_type option;The OAuth2 grant type. For example, AUTHORIZATION_CODE, JWT_BEARER, or CLIENT_CREDENTIALS.
}A structure containing properties for OAuth2 authentication.
type nonrec new_rules = name_string listtype nonrec integration_additional_encryption_context_map =
(integration_string * integration_string) listtype nonrec integration_tags_list = tag listtype nonrec integration_error = {error_message : string2048 option;A message describing the error.
*)error_code : string128 option;The code associated with this error.
*)}An error associated with a zero-ETL integration.
type nonrec integration_error_list = integration_error listtype nonrec integration_source_properties_map =
(integration_string * integration_string) listtype nonrec integration_config = {continuous_sync : continuous_sync option;Enables continuous synchronization for on-demand data extractions from SaaS applications to Amazon Web Services data services like Amazon Redshift and Amazon S3.
*)source_properties : integration_source_properties_map option;A collection of key-value pairs that specify additional properties for the integration source. These properties provide configuration options that can be used to customize the behavior of the ODB source during data integration operations.
*)refresh_interval : string128 option;Specifies the frequency at which CDC (Change Data Capture) pulls or incremental loads should occur. This parameter provides flexibility to align the refresh rate with your specific data update patterns, system load considerations, and performance optimization goals. Time increment can be set from 15 minutes to 8640 minutes (six days).
*)}Properties associated with the integration.
type nonrec modify_integration_response = {integration_config : integration_config option;The updated configuration settings for the integration.
*)data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)errors : integration_error_list option;A list of errors associated with the integration modification.
*)create_time : integration_timestamp;The time when the integration was created, in UTC.
*)status : integration_status;The status of the integration being modified.
The possible statuses are:
additional_encryption_context : integration_additional_encryption_context_map
option;An optional set of non-secret key–value pairs that contains additional contextual information for encryption.
*)kms_key_id : string2048 option;The ARN of a KMS key used for encrypting the channel.
*)integration_arn : string128;The Amazon Resource Name (ARN) for the integration.
*)description : integration_description option;A description of the integration.
*)integration_name : string128;A unique name for an integration in Glue.
*)target_arn : string512;The ARN of the target for the integration.
*)source_arn : string512;The ARN of the source for the integration.
*)}type nonrec modify_integration_request = {integration_name : string128 option;A unique name for an integration in Glue.
*)integration_config : integration_config option;The configuration settings for the integration. Currently, only the RefreshInterval can be modified.
*)data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)description : integration_description option;A description of the integration.
*)integration_identifier : string128;The Amazon Resource Name (ARN) for the integration.
*)}type nonrec invalid_state_exception = {message : message_string option;A message describing the problem.
*)}An error that indicates your data is in an invalid state.
type nonrec invalid_integration_state_fault = {message : integration_error_message option;A message describing the problem.
*)}The integration is in an invalid state.
type nonrec integration_not_found_fault = {message : integration_error_message option;A message describing the problem.
*)}The specified integration could not be found.
type nonrec integration_conflict_operation_fault = {message : integration_error_message option;A message describing the problem.
*)}The requested operation conflicts with another operation.
type nonrec milliseconds_count = Smaws_Lib.CoreTypes.Int64.ttype nonrec data_quality_metric_values = {upper_limit : nullable_double option;The upper limit of the data quality metric value according to the analysis of historical data.
*)lower_limit : nullable_double option;The lower limit of the data quality metric value according to the analysis of historical data.
*)expected_value : nullable_double option;The expected value of the data quality metric according to the analysis of historical data.
*)actual_value : nullable_double option;The actual value of the data quality metric.
*)}Describes the data quality metric value according to the analysis of historical data.
type nonrec metric_based_observation = {new_rules : new_rules option;A list of new data quality rules generated as part of the observation based on the data quality metric value.
*)metric_values : data_quality_metric_values option;An object of type DataQualityMetricValues representing the analysis of the data quality metric value.
statistic_id : hash_string option;The Statistic ID.
*)metric_name : name_string option;The name of the data quality metric used for generating the observation.
*)}Describes the metric based observation generated based on evaluated data quality metrics.
type nonrec byte_count = Smaws_Lib.CoreTypes.Int64.ttype nonrec materialized_view_refresh_task_run = {processed_bytes : byte_count option;The number of bytes the refresh task run has scanned to refresh the materialized view.
*)refresh_type : materialized_view_refresh_type option;The type of the refresh task run. Either FULL or INCREMENTAL.
*)dpu_seconds : non_negative_double option;The calculated DPU usage in seconds for all autoscaled workers.
*)error_message : description_string option;The error message for the job.
*)end_time : timestamp option;The end time of the task.
*)start_time : timestamp option;The start time of the task.
*)last_updated : timestamp option;The last point in time when this task was modified.
*)creation_time : timestamp option;The time that this task was created.
*)status : materialized_view_refresh_state option;The status of the task run.
*)role : role option;The IAM role that the service assumes to generate statistics.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is supplied, the account ID is used by default.
*)table_name : table_name option;The name of the table for which statistics is generated.
*)database_name : database_name option;The database where the table resides.
*)materialized_view_refresh_task_run_id : uui_dv4 option;The identifier of the materialized view refresh task run.
*)customer_id : account_id option;The Amazon Web Services account ID.
*)}The object that shows the details of the materialized view refresh task run.
type nonrec materialized_view_refresh_task_runs_list =
materialized_view_refresh_task_run listtype nonrec mapping_entry = {target_type : field_type option;The target type.
*)target_path : schema_path_string option;The target path.
*)target_table : table_name option;The target table.
*)source_type : field_type option;The source type.
*)source_path : schema_path_string option;The source path.
*)source_table : table_name option;The name of the source table.
*)}Defines a mapping.
type nonrec mapping_list = mapping_entry listtype nonrec code_gen_node_arg = {param : boolean_ option;True if the value is used as a parameter.
*)value : code_gen_arg_value;The value of the argument or property.
*)name : code_gen_arg_name;The name of the argument or property.
*)}An argument or property of a node.
type nonrec code_gen_node_args = code_gen_node_arg listtype nonrec location = {dynamo_d_b : code_gen_node_args option;An Amazon DynamoDB table location.
*)s3 : code_gen_node_args option;An Amazon Simple Storage Service (Amazon S3) location.
*)jdbc : code_gen_node_args option;A JDBC location.
*)}The location of resources.
type nonrec list_workflows_response = {next_token : generic_string option;A continuation token, if not all workflow names have been returned.
*)workflows : workflow_names option;List of names of workflows in the account.
*)}type nonrec list_workflows_request = {max_results : orchestration_page_size25 option;The maximum size of a list to return.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)}type nonrec list_usage_profiles_response = {next_token : orchestration_token option;A continuation token, present if the current list segment is not the last.
*)profiles : usage_profile_definition_list option;A list of usage profile (UsageProfileDefinition) objects.
}type nonrec list_usage_profiles_request = {max_results : orchestration_page_size200 option;The maximum number of usage profiles to return in a single response.
*)next_token : orchestration_token option;A continuation token, included if this is a continuation call.
*)}type nonrec list_triggers_response = {next_token : generic_string option;A continuation token, if the returned list does not contain the last metric available.
*)trigger_names : trigger_name_list option;The names of all triggers in the account, or the triggers with the specified tags.
*)}type nonrec list_triggers_request = {max_results : orchestration_page_size200 option;The maximum size of a list to return.
*)dependent_job_name : name_string option;The name of the job for which to retrieve triggers. The trigger that can start this job is returned. If there is no such trigger, all triggers are returned.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)}type nonrec list_table_optimizer_runs_response = {table_optimizer_runs : table_optimizer_runs option;A list of the optimizer runs associated with a table.
*)next_token : list_table_optimizer_runs_token option;A continuation token for paginating the returned list of optimizer runs, returned if the current segment of the list is not the last.
*)table_name : name_string option;The name of the table.
*)database_name : name_string option;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string option;The Catalog ID of the table.
*)}type nonrec list_table_optimizer_runs_request = {next_token : list_table_optimizer_runs_token option;A continuation token, if this is a continuation call.
*)max_results : max_list_table_optimizer_runs_token_results option;The maximum number of optimizer runs to return on each call.
*)type_ : table_optimizer_type;The type of table optimizer.
*)table_name : name_string;The name of the table.
*)database_name : name_string;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string;The Catalog ID of the table.
*)}type nonrec list_statements_response = {next_token : orchestration_token option;A continuation token, if not all statements have yet been returned.
*)statements : statement_list option;Returns the list of statements.
*)}type nonrec list_statements_request = {next_token : orchestration_token option;A continuation token, if this is a continuation call.
*)request_origin : orchestration_name_string option;The origin of the request to list statements.
*)session_id : name_string;The Session ID of the statements.
*)}type nonrec list_sessions_response = {next_token : orchestration_token option;The token for the next set of results, or null if there are no more result.
*)sessions : session_list option;Returns the session object.
*)ids : session_id_list option;Returns the ID of the session.
*)}type nonrec list_sessions_request = {request_origin : orchestration_name_string option;The origin of the request.
*)max_results : page_size option;The maximum number of results.
*)next_token : orchestration_token option;The token for the next set of results, or null if there are no more result.
*)}type nonrec list_schemas_response = {next_token : schema_registry_token_string option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)schemas : schema_list_definition option;An array of SchemaListItem objects containing details of each schema.
}type nonrec list_schemas_input = {next_token : schema_registry_token_string option;A continuation token, if this is a continuation call.
*)max_results : max_results_number option;Maximum number of results required per page. If the value is not supplied, this will be defaulted to 25 per page.
*)registry_id : registry_id option;A wrapper structure that may contain the registry name and Amazon Resource Name (ARN).
*)}type nonrec list_schema_versions_response = {next_token : schema_registry_token_string option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)schemas : schema_version_list option;An array of SchemaVersionList objects containing details of each schema version.
}type nonrec list_schema_versions_input = {next_token : schema_registry_token_string option;A continuation token, if this is a continuation call.
*)max_results : max_results_number option;Maximum number of results required per page. If the value is not supplied, this will be defaulted to 25 per page.
*)schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName and RegistryName has to be provided.SchemaArn or SchemaName and RegistryName has to be provided.}type nonrec list_registries_response = {next_token : schema_registry_token_string option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)registries : registry_list_definition option;An array of RegistryDetailedListItem objects containing minimal details of each registry.
}type nonrec list_registries_input = {next_token : schema_registry_token_string option;A continuation token, if this is a continuation call.
*)max_results : max_results_number option;Maximum number of results required per page. If the value is not supplied, this will be defaulted to 25 per page.
*)}type nonrec list_materialized_view_refresh_task_runs_response = {next_token : token option;A continuation token, if not all task run IDs have yet been returned.
*)materialized_view_refresh_task_runs : materialized_view_refresh_task_runs_list
option;The results of the ListMaterializedViewRefreshTaskRuns action.
*)}type nonrec list_materialized_view_refresh_task_runs_request = {next_token : token option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum size of the response.
*)table_name : name_string option;The name of the table for which statistics is generated.
*)database_name : name_string option;The database where the table resides.
*)catalog_id : name_string;The ID of the Data Catalog where the table resides. If none is supplied, the account ID is used by default.
*)}type nonrec list_ml_transforms_response = {next_token : pagination_token option;A continuation token, if the returned list does not contain the last metric available.
*)transform_ids : transform_id_list;The identifiers of all the machine learning transforms in the account, or the machine learning transforms with the specified tags.
*)}type nonrec list_ml_transforms_request = {sort : transform_sort_criteria option;A TransformSortCriteria used to sort the machine learning transforms.
filter : transform_filter_criteria option;A TransformFilterCriteria used to filter the machine learning transforms.
max_results : page_size option;The maximum size of a list to return.
*)next_token : pagination_token option;A continuation token, if this is a continuation request.
*)}type nonrec job_name_list = name_string listtype nonrec list_jobs_response = {next_token : generic_string option;A continuation token, if the returned list does not contain the last metric available.
*)job_names : job_name_list option;The names of all jobs in the account, or the jobs with the specified tags.
*)}type nonrec list_jobs_request = {max_results : page_size option;The maximum size of a list to return.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)}type nonrec glossary_term_id_list = glossary_term_id listtype nonrec iterable_form_list_item = {glossary_terms : glossary_term_id_list option;The identifiers of the glossary terms associated with the item.
*)description : item_description option;The description of the item.
*)item_name : item_name option;The name of the item.
*)item_id : item_id option;The unique identifier of the item.
*)}A summary of an item in an iterable form.
type nonrec iterable_form_list_item_list = iterable_form_list_item listtype nonrec list_iterable_forms_response = {next_token : token option;A continuation token, present if the current segment is not the last.
*)items : iterable_form_list_item_list option;The list of iterable form items.
*)}type nonrec list_iterable_forms_request = {next_token : token option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum number of results to return in the response.
*)iterable_form_name : iterable_form_name;The name of the iterable form to list items from.
*)asset_identifier : asset_id;The unique identifier of the asset.
*)}type nonrec integration_resource_property = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_property_arn : string512 option;The resource ARN created through this create API. The format is something like arn:aws:glue:::integrationresourceproperty/*
*)resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}A structure representing an integration resource property.
type nonrec integration_resource_property_list =
integration_resource_property listtype nonrec list_integration_resource_properties_response = {marker : string1024 option;This is the pagination token for the next page.
*)integration_resource_property_list : integration_resource_property_list option;A list of integration resource property meeting the filter criteria.
*)}type nonrec integration_resource_property_filter_values = string128 listtype nonrec integration_resource_property_filter = {values : integration_resource_property_filter_values option;A list of filter values.
*)name : string128 option;The name of the filter. Supported filter keys are SourceArn and TargetArn.
}A filter for integration resource properties.
type nonrec integration_resource_property_filter_list =
integration_resource_property_filter listtype nonrec list_integration_resource_properties_request = {max_records : integration_integer option;This is total number of items to be evaluated.
*)filters : integration_resource_property_filter_list option;A list of filters, supported filter Key is SourceArn and TargetArn.
marker : string1024 option;This is the pagination token for next page, initial value is null.
}type nonrec glossary_term_item = {short_description : glossary_short_description option;The short description of the glossary term.
*)name : glossary_term_name option;The name of the glossary term.
*)id : glossary_term_id option;The unique identifier of the glossary term.
*)}A summary of a glossary term.
type nonrec glossary_term_item_list = glossary_term_item listtype nonrec list_glossary_terms_response = {next_token : token option;A continuation token, present if the current segment is not the last.
*)items : glossary_term_item_list option;The list of glossary term items.
*)}type nonrec list_glossary_terms_request = {next_token : token option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum number of results to return in the response.
*)glossary_identifier : glossary_id;The unique identifier of the glossary whose terms to list.
*)}type nonrec glossary_item = {description : metadata_description option;The description of the glossary.
*)name : glossary_name option;The name of the glossary.
*)id : glossary_id option;The unique identifier of the glossary.
*)}A summary of a business glossary.
type nonrec glossary_item_list = glossary_item listtype nonrec list_glossaries_response = {next_token : token option;A continuation token, present if the current segment is not the last.
*)items : glossary_item_list option;The list of glossary items.
*)}type nonrec form_type_item = {name : form_type_name option;The name of the form type.
*)id : form_type_id option;The identifier of the form type.
*)}A summary of a form type.
type nonrec form_type_item_list = form_type_item listtype nonrec list_form_types_response = {next_token : token option;A continuation token, present if the current segment is not the last.
*)items : form_type_item_list;The list of form type items.
*)}type nonrec custom_properties =
(Smaws_Lib.Smithy_api.Types.string_ * Smaws_Lib.Smithy_api.Types.string_)
listtype nonrec entity = {custom_properties : custom_properties option;An optional map of keys which may be returned for an entity by a connector.
*)category : category option;The type of entities that are present in the response. This value depends on the source connection. For example this is SObjects for Salesforce and databases or schemas or tables for sources like Amazon Redshift.
description : entity_description option;A description of the entity.
*)is_parent_entity : is_parent_entity option;A Boolean value which helps to determine whether there are sub objects that can be listed.
*)label : entity_label option;Label used for the entity.
*)entity_name : entity_name option;The name of the entity.
*)}An entity supported by a given ConnectionType.
type nonrec entity_list = entity listtype nonrec list_entities_response = {next_token : next_token option;A continuation token, present if the current segment is not the last.
*)entities : entity_list option;A list of Entity objects.
}type nonrec list_entities_request = {data_store_api_version : api_version option;The API version of the SaaS connector.
*)next_token : next_token option;A continuation token, included if this is a continuation call.
*)parent_entity_name : entity_name option;Name of the parent entity for which you want to list the children. This parameter takes a fully-qualified path of the entity in order to list the child entities.
*)catalog_id : catalog_id_string option;The catalog ID of the catalog that contains the connection. This can be null, By default, the Amazon Web Services Account ID is the catalog ID.
*)connection_name : name_string option;A name for the connection that has required credentials to query any connection type.
*)}type nonrec dev_endpoint_name_list = name_string listtype nonrec list_dev_endpoints_response = {next_token : generic_string option;A continuation token, if the returned list does not contain the last metric available.
*)dev_endpoint_names : dev_endpoint_name_list option;The names of all the DevEndpoints in the account, or the DevEndpoints with the specified tags.
}type nonrec list_dev_endpoints_request = {max_results : page_size option;The maximum size of a list to return.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)}type nonrec list_data_quality_statistics_response = {next_token : pagination_token option;A pagination token to request the next page of results.
*)statistics : statistic_summary_list option;A StatisticSummaryList.
}type nonrec list_data_quality_statistics_request = {next_token : pagination_token option;A pagination token to request the next page of results.
*)max_results : page_size option;The maximum number of results to return in this request.
*)timestamp_filter : timestamp_filter option;A timestamp filter.
*)profile_id : hash_string option;The Profile ID.
*)statistic_id : hash_string option;The Statistic ID.
*)}type nonrec annotation_list = statistic_annotation listtype nonrec list_data_quality_statistic_annotations_response = {next_token : pagination_token option;A pagination token to retrieve the next set of results.
*)annotations : annotation_list option;A list of StatisticAnnotation applied to the Statistic
}type nonrec list_data_quality_statistic_annotations_request = {next_token : pagination_token option;A pagination token to retrieve the next set of results.
*)max_results : page_size option;The maximum number of results to return in this request.
*)timestamp_filter : timestamp_filter option;A timestamp filter.
*)profile_id : hash_string option;The Profile ID.
*)statistic_id : hash_string option;The Statistic ID.
*)}type nonrec data_quality_target_table = {catalog_id : name_string option;The catalog id where the Glue table exists.
*)database_name : name_string;The name of the database where the Glue table exists.
*)table_name : name_string;The name of the Glue table.
*)}An object representing an Glue table.
type nonrec data_quality_ruleset_list_details = {rule_count : nullable_integer option;The number of rules in the ruleset.
*)recommendation_run_id : hash_string option;When a ruleset was created from a recommendation run, this run ID is generated to link the two together.
*)target_table : data_quality_target_table option;An object representing an Glue table.
*)last_modified_on : timestamp option;The date and time the data quality ruleset was last modified.
*)created_on : timestamp option;The date and time the data quality ruleset was created.
*)description : description_string option;A description of the data quality ruleset.
*)name : name_string option;The name of the data quality ruleset.
*)}Describes a data quality ruleset returned by GetDataQualityRuleset.
type nonrec data_quality_ruleset_list = data_quality_ruleset_list_details listtype nonrec list_data_quality_rulesets_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)rulesets : data_quality_ruleset_list option;A paginated list of rulesets for the specified list of Glue tables.
*)}type nonrec data_quality_ruleset_filter_criteria = {target_table : data_quality_target_table option;The name and database name of the target table.
*)last_modified_after : timestamp option;Filter on rulesets last modified after this date.
*)last_modified_before : timestamp option;Filter on rulesets last modified before this date.
*)created_after : timestamp option;Filter on rulesets created after this date.
*)created_before : timestamp option;Filter on rulesets created before this date.
*)description : description_string option;The description of the ruleset filter criteria.
*)name : name_string option;The name of the ruleset filter criteria.
*)}The criteria used to filter data quality rulesets.
type nonrec list_data_quality_rulesets_request = {filter : data_quality_ruleset_filter_criteria option;The filter criteria.
*)max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)}type nonrec data_quality_ruleset_evaluation_run_description = {data_source : data_source option;The data source (an Glue table) associated with the run.
*)started_on : timestamp option;The date and time when the run started.
*)status : task_status_type option;The status for this run.
*)run_id : hash_string option;The unique run identifier associated with this run.
*)}Describes the result of a data quality ruleset evaluation run.
type nonrec data_quality_ruleset_evaluation_run_list =
data_quality_ruleset_evaluation_run_description listtype nonrec list_data_quality_ruleset_evaluation_runs_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)runs : data_quality_ruleset_evaluation_run_list option;A list of DataQualityRulesetEvaluationRunDescription objects representing data quality ruleset runs.
}type nonrec data_quality_ruleset_evaluation_run_filter = {ruleset_name : name_string option;Filter results by the name of the ruleset.
*)started_after : timestamp option;Filter results by runs that started after this time.
*)started_before : timestamp option;Filter results by runs that started before this time.
*)data_source : data_source;Filter based on a data source (an Glue table) associated with the run.
*)}The filter criteria.
type nonrec list_data_quality_ruleset_evaluation_runs_request = {max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)filter : data_quality_ruleset_evaluation_run_filter option;The filter criteria.
*)}type nonrec data_quality_rule_recommendation_run_description = {data_source : data_source option;The data source (Glue table) associated with the recommendation run.
*)started_on : timestamp option;The date and time when this run started.
*)status : task_status_type option;The status for this run.
*)run_id : hash_string option;The unique run identifier associated with this run.
*)}Describes the result of a data quality rule recommendation run.
type nonrec data_quality_rule_recommendation_run_list =
data_quality_rule_recommendation_run_description listtype nonrec list_data_quality_rule_recommendation_runs_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)runs : data_quality_rule_recommendation_run_list option;A list of DataQualityRuleRecommendationRunDescription objects.
}type nonrec data_quality_rule_recommendation_run_filter = {started_after : timestamp option;Filter based on time for results started after provided time.
*)started_before : timestamp option;Filter based on time for results started before provided time.
*)data_source : data_source;Filter based on a specified data source (Glue table).
*)}A filter for listing data quality recommendation runs.
type nonrec list_data_quality_rule_recommendation_runs_request = {max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)filter : data_quality_rule_recommendation_run_filter option;The filter criteria.
*)}type nonrec data_quality_result_description = {started_on : timestamp option;The time that the run started for this data quality result.
*)job_run_id : hash_string option;The job run ID associated with the data quality result.
*)job_name : name_string option;The job name associated with the data quality result.
*)data_source : data_source option;The table name associated with the data quality result.
*)result_id : hash_string option;The unique result ID for this data quality result.
*)}Describes a data quality result.
type nonrec data_quality_result_description_list =
data_quality_result_description listtype nonrec list_data_quality_results_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)results : data_quality_result_description_list;A list of DataQualityResultDescription objects.
}type nonrec data_quality_result_filter_criteria = {started_before : timestamp option;Filter results by runs that started before this time.
*)started_after : timestamp option;Filter results by runs that started after this time.
*)job_run_id : hash_string option;Filter results by the specified job run ID.
*)job_name : name_string option;Filter results by the specified job name.
*)data_source : data_source option;Filter results by the specified data source. For example, retrieving all results for an Glue table.
*)}Criteria used to return data quality results.
type nonrec list_data_quality_results_request = {max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)filter : data_quality_result_filter_criteria option;The filter criteria.
*)}type nonrec context_words = name_string listtype nonrec custom_entity_type = {context_words : context_words option;A list of context words. If none of these context words are found within the vicinity of the regular expression the data will not be detected as sensitive data.
If no context words are passed only a regular expression is checked.
*)regex_string : name_string;A regular expression string that is used for detecting sensitive data in a custom pattern.
*)name : name_string;A name for the custom pattern that allows it to be retrieved or deleted later. This name must be unique per Amazon Web Services account.
*)}An object representing a custom pattern for detecting sensitive data across the columns and rows of your structured data.
type nonrec custom_entity_types = custom_entity_type listtype nonrec list_custom_entity_types_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)custom_entity_types : custom_entity_types option;A list of CustomEntityType objects representing custom patterns.
}type nonrec list_custom_entity_types_request = {max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)}type nonrec crawler_history = {dpu_hour : non_negative_double option;The number of data processing units (DPU) used in hours for the crawl.
*)message_prefix : message_prefix option;The prefix for a CloudWatch message about this crawl.
*)log_stream : log_stream option;The log stream associated with the crawl.
*)log_group : log_group option;The log group associated with the crawl.
*)error_message : description_string option;If an error occurred, the error message associated with the crawl.
*)summary : name_string option;A run summary for the specific crawl in JSON. Contains the catalog tables and partitions that were added, updated, or deleted.
*)end_time : timestamp option;The date and time on which the crawl ended.
*)start_time : timestamp option;The date and time on which the crawl started.
*)state : crawler_history_state option;The state of the crawl.
*)crawl_id : crawl_id option;A UUID identifier for each crawl.
*)}Contains the information for a run of a crawler.
type nonrec crawler_history_list = crawler_history listtype nonrec list_crawls_response = {next_token : token option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)crawls : crawler_history_list option;A list of CrawlerHistory objects representing the crawl runs that meet your criteria.
}type nonrec crawls_filter = {field_value : generic_string option;The value provided for comparison on the crawl field.
*)filter_operator : filter_operator option;A defined comparator that operates on the value. The available operators are:
GT: Greater than.GE: Greater than or equal to.LT: Less than.LE: Less than or equal to.EQ: Equal to.NE: Not equal to.field_name : field_name option;A key used to filter the crawler runs for a specified crawler. Valid values for each of the field names are:
CRAWL_ID: A string representing the UUID identifier for a crawl.STATE: A string representing the state of the crawl.START_TIME and END_TIME: The epoch timestamp in milliseconds.DPU_HOUR: The number of data processing unit (DPU) hours used for the crawl.}A list of fields, comparators and value that you can use to filter the crawler runs for a specified crawler.
type nonrec crawls_filter_list = crawls_filter listtype nonrec list_crawls_request = {next_token : token option;A continuation token, if this is a continuation call.
*)filters : crawls_filter_list option;Filters the crawls by the criteria you specify in a list of CrawlsFilter objects.
max_results : page_size option;The maximum number of results to return. The default is 20, and maximum is 100.
*)crawler_name : name_string;The name of the crawler whose runs you want to retrieve.
*)}type nonrec crawler_name_list = name_string listtype nonrec list_crawlers_response = {next_token : token option;A continuation token, if the returned list does not contain the last metric available.
*)crawler_names : crawler_name_list option;The names of all crawlers in the account, or the crawlers with the specified tags.
*)}type nonrec compute_environments = compute_environment listtype nonrec capabilities = {supported_compute_environments : compute_environments;A list of supported compute environments.
*)supported_data_operations : data_operations;A list of supported data operations.
*)supported_authentication_types : authentication_types;A list of supported authentication types.
*)}Specifies the supported authentication types returned by the DescribeConnectionType API.
type nonrec connection_type_variant = {logo_url : url_string option;The URL of the logo associated with a connection type variant.
*)description : description option;A detailed description of the connection type variant, including its purpose, use cases, and any specific configuration requirements.
*)display_name : display_name option;The human-readable name for the connection type variant that is displayed in the Glue console.
*)connection_type_variant_name : display_name option;The unique identifier for the connection type variant. This name is used internally to identify the specific variant of a connection type.
*)}Represents a variant of a connection type in Glue. Connection type variants provide specific configurations and behaviors for different implementations of the same general connection type.
type nonrec connection_type_variant_list = connection_type_variant listtype nonrec connection_type_brief = {connection_type_variants : connection_type_variant_list option;A list of variants available for this connection type. Different variants may provide specialized configurations for specific use cases or implementations of the same general connection type.
*)logo_url : url_string option;The URL of the logo associated with a connection type.
*)capabilities : capabilities option;The supported authentication types, data interface types (compute environments), and data operations of the connector.
*)categories : list_of_string option;A list of categories that this connection type belongs to. Categories help users filter and find appropriate connection types based on their use cases.
*)description : description option;A description of the connection type.
*)vendor : vendor option;The name of the vendor or provider that created or maintains this connection type.
*)display_name : display_name option;The human-readable name for the connection type that is displayed in the Glue console.
*)connection_type : connection_type option;The name of the connection type.
*)}Brief information about a supported connection type returned by the ListConnectionTypes API.
type nonrec connection_type_list = connection_type_brief listtype nonrec list_connection_types_response = {next_token : next_token option;A continuation token, if the current list segment is not the last.
*)connection_types : connection_type_list option;A list of ConnectionTypeBrief objects containing brief information about the supported connection types.
}type nonrec list_connection_types_request = {next_token : next_token option;A continuation token, if this is a continuation call.
*)max_results : max_results option;The maximum number of results to return.
*)}type nonrec column_statistics_task_run_id_list = hash_string listtype nonrec list_column_statistics_task_runs_response = {next_token : token option;A continuation token, if not all task run IDs have yet been returned.
*)column_statistics_task_run_ids : column_statistics_task_run_id_list option;A list of column statistics task run IDs.
*)}type nonrec blueprint_names = orchestration_name_string listtype nonrec list_blueprints_response = {next_token : generic_string option;A continuation token, if not all blueprint names have been returned.
*)blueprints : blueprint_names option;List of names of blueprints in the account.
*)}type nonrec list_blueprints_request = {max_results : orchestration_page_size25 option;The maximum size of a list to return.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)}type nonrec asset_type_item = {name : asset_type_name option;The name of the asset type.
*)id : asset_type_id option;The identifier of the asset type.
*)}A summary of an asset type.
type nonrec asset_type_item_list = asset_type_item listtype nonrec list_asset_types_response = {next_token : token option;A continuation token, present if the current segment is not the last.
*)items : asset_type_item_list option;The list of asset type items.
*)}type nonrec limit = Smaws_Lib.CoreTypes.Int64.ttype nonrec last_crawl_info = {start_time : timestamp option;The time at which the crawl started.
*)message_prefix : message_prefix option;The prefix for a message about this crawl.
*)log_stream : log_stream option;The log stream for the last crawl.
*)log_group : log_group option;The log group for the last crawl.
*)error_message : description_string option;If an error occurred, the error information about the last crawl.
*)status : last_crawl_status option;Status of the last crawl.
*)}Status and error information about the most recent crawl.
type nonrec last_active_definition = {blueprint_service_location : generic_string option;Specifies a path in Amazon S3 where the blueprint is copied when you create or update the blueprint.
*)blueprint_location : generic_string option;Specifies a path in Amazon S3 where the blueprint is published by the Glue developer.
*)parameter_spec : blueprint_parameter_spec option;A JSON string specifying the parameters for the blueprint.
*)last_modified_on : timestamp_value option;The date and time the blueprint was last modified.
*)description : generic512_char_string option;The description of the blueprint.
*)}When there are multiple versions of a blueprint and the latest version has some errors, this attribute indicates the last successful blueprint definition that is available with the service.
type nonrec labels = (name_string * name_string) listtype nonrec kms_key_not_accessible_fault = {message : integration_error_message option;A message describing the problem.
*)}The KMS key specified is not accessible.
type nonrec json_classifier = {json_path : json_path;A JsonPath string defining the JSON data for the classifier to classify. Glue supports a subset of JsonPath, as described in Writing JsonPath Custom Classifiers.
version : version_id option;The version of this classifier.
*)last_updated : timestamp option;The time that this classifier was last updated.
*)creation_time : timestamp option;The time that this classifier was registered.
*)name : name_string;The name of the classifier.
*)}A classifier for JSON content.
type nonrec job = {profile_name : name_string option;The name of an Glue usage profile associated with the job.
*)maintenance_window : maintenance_window option;This field specifies a day of the week and hour for a maintenance window for streaming jobs. Glue periodically performs maintenance activities. During these maintenance windows, Glue will need to restart your streaming jobs.
Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.
*)source_control_details : source_control_details option;The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository.
*)execution_class : execution_class option;Indicates whether the job is run with a standard or flexible execution class. The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.
The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.
Only jobs with Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX. The flexible execution class is available for Spark jobs.
code_gen_configuration_nodes : code_gen_configuration_nodes option;The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.
*)glue_version : glue_version_string option;In Spark jobs, GlueVersion determines the versions of Apache Spark and Python that Glue available in a job. The Python version indicates the version supported for jobs of type Spark.
Ray jobs should set GlueVersion to 4.0 or greater. However, the versions of Ray, Python and additional libraries available in your Ray job are determined by the Runtime parameter of the Job command.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
Jobs that are created without specifying a Glue version default to Glue 5.1.
*)notification_property : notification_property option;Specifies configuration properties of a job notification.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this job.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a job runs.
worker_type : worker_type option;The type of predefined worker that is allocated when a job runs.
Glue provides multiple worker types to accommodate different workload requirements:
G Worker Types (General-purpose compute workers):
R Worker Types (Memory-optimized workers):
max_capacity : nullable_double option;For Glue version 1.0 or earlier jobs, using the standard worker type, the number of Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
For Glue version 2.0 or later jobs, you cannot specify a Maximum capacity. Instead, you should specify a Worker type and the Number of workers.
Do not set MaxCapacity if using WorkerType and NumberOfWorkers.
The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL job:
JobCommand.Name="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.JobCommand.Name="glueetl") or Apache Spark streaming ETL job (JobCommand.Name="gluestreaming"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.timeout : timeout option;The job timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)allocated_capacity : integer_value option;This field is deprecated. Use MaxCapacity instead.
The number of Glue data processing units (DPUs) allocated to runs of this job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
*)max_retries : max_retries option;The maximum number of times to retry this job after a JobRun fails.
*)connections : connections_list option;The connections used for this job.
*)non_overridable_arguments : generic_map option;Arguments for this job that are not overridden when providing job arguments in a job run, specified as name-value pairs.
*)default_arguments : generic_map option;The default arguments for every run of this job, specified as name-value pairs.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Secrets Manager or other secret management mechanism if you intend to keep them within the Job.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the arguments you can provide to this field when configuring Spark jobs, see the Special Parameters Used by Glue topic in the developer guide.
For information about the arguments you can provide to this field when configuring Ray jobs, see Using job parameters in Ray jobs in the developer guide.
*)command : job_command option;The JobCommand that runs this job.
execution_property : execution_property option;An ExecutionProperty specifying the maximum number of concurrent runs allowed for this job.
last_modified_on : timestamp_value option;The last point in time when this job definition was modified.
*)created_on : timestamp_value option;The time and date that this job definition was created.
*)role : role_string option;The name or Amazon Resource Name (ARN) of the IAM role associated with this job.
*)log_uri : uri_string option;This field is reserved for future use.
*)description : description_string option;A description of the job.
*)job_run_queuing_enabled : nullable_boolean option;Specifies whether job run queuing is enabled for the job runs for this job.
A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
If this field does not match the value set in the job run, then the value from the job run field will be used.
*)job_mode : job_mode option;A mode that describes how a job was created. Valid values are:
SCRIPT - The job was created using the Glue Studio script editor.VISUAL - The job was created using the Glue Studio visual editor.NOTEBOOK - The job was created using an interactive sessions notebook.When the JobMode field is missing or null, SCRIPT is assigned as the default value.
name : name_string option;The name you assign to this job definition.
*)}Specifies a job definition.
type nonrec job_list = job listtype nonrec iterable_form_entry = {form_type_id : form_type_id option;The form type identifier of the iterable form (for example, columns), used to retrieve its items via ListIterableForms or BatchGetIterableForms.
}An iterable form available on an asset, identified by its form type.
type nonrec iterable_form_map = (iterable_form_key * iterable_form_entry) listtype nonrec iterable_form_item = {attachments : asset_form_map option;Additional attachments on the item for more context, keyed by attachment name.
*)forms : asset_form_map option;The forms on the item, keyed by form name.
*)glossary_terms : glossary_term_id_list option;The identifiers of the glossary terms associated with the item.
*)item_name : item_name option;The name of the item.
*)item_id : item_id option;The unique identifier of the item.
*)}A full iterable form item with its forms.
type nonrec iterable_form_item_list = iterable_form_item listtype nonrec item_identifier_list = item_identifier listtype nonrec item_error = {message : item_error_message option;The error message.
*)code : item_error_code option;The error code.
*)item_identifier : item_identifier option;The identifier of the item that caused the error.
*)}An error that occurred when retrieving an iterable form item.
type nonrec item_error_list = item_error listtype nonrec integration = {data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)errors : integration_error_list option;A list of errors associated with the integration.
*)integration_config : integration_config option;Properties associated with the integration.
*)create_time : integration_timestamp;The time that the integration was created, in UTC.
*)status : integration_status;The possible statuses are:
additional_encryption_context : integration_additional_encryption_context_map
option;An optional set of non-secret key–value pairs that contains additional contextual information for encryption. This can only be provided if KMSKeyId is provided.
kms_key_id : string2048 option;The ARN of a KMS key used for encrypting the channel.
*)integration_arn : string128;The Amazon Resource Name (ARN) for the integration.
*)integration_name : string128;A unique name for the integration.
*)description : integration_description option;A description for the integration.
*)target_arn : string512;The ARN for the target of the integration.
*)source_arn : string512;The ARN for the source of the integration.
*)}Describes a zero-ETL integration.
type nonrec integrations_list = integration listtype nonrec integration_quota_exceeded_fault = {message : integration_error_message option;A message describing the problem.
*)}The data processed through your integration exceeded your quota.
type nonrec integration_filter_values = string128 listtype nonrec integration_filter = {values : integration_filter_values option;A list of filter values.
*)name : string128 option;The name of the filter.
*)}A filter that can be used when invoking a DescribeIntegrations request.
type nonrec integration_filter_list = integration_filter listtype nonrec datapoint_inclusion_annotation = {inclusion_annotation : inclusion_annotation_value option;The inclusion annotation value to apply to the statistic.
*)statistic_id : hash_string option;The Statistic ID.
*)profile_id : hash_string option;The ID of the data quality profile the statistic belongs to.
*)}An Inclusion Annotation.
type nonrec inclusion_annotation_list = datapoint_inclusion_annotation listtype nonrec inbound_integration = {errors : integration_error_list option;A list of errors associated with the integration.
*)integration_config : integration_config option;Properties associated with the integration.
*)create_time : integration_timestamp;The time that the integration was created, in UTC.
*)status : integration_status;The possible statuses are:
integration_arn : string128;The ARN of the zero-ETL integration.
*)target_arn : string512;The ARN of the target resource for the integration.
*)source_arn : string512;The ARN of the source resource for the integration.
*)}A structure for an integration that writes data into a resource.
type nonrec inbound_integrations_list = inbound_integration listtype nonrec import_catalog_to_glue_request = {catalog_id : catalog_id_string option;The ID of the catalog to import. Currently, this should be the Amazon Web Services account ID.
*)}type nonrec iceberg_optimization_properties_output = {last_updated_time : timestamp option;The timestamp when the Iceberg optimization properties were last updated.
*)orphan_file_deletion : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg orphan file deletion operations, which identify and remove files that are no longer referenced by the table metadata.
*)retention : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg table retention operations, which manage the lifecycle of table snapshots to control storage costs.
*)compaction : parameters_map option;A map of key-value pairs that specify configuration parameters for Iceberg table compaction operations, which optimize the layout of data files to improve query performance.
*)role_arn : iam_role_arn option;The Amazon Resource Name (ARN) of the IAM role that is used to perform Iceberg table optimization operations.
*)}A structure that contains the output properties of Iceberg table optimization configuration for your catalog resource in the Glue Data Catalog.
type nonrec grok_classifier = {custom_patterns : custom_patterns option;Optional custom grok patterns defined by this classifier. For more information, see custom patterns in Writing Custom Classifiers.
*)grok_pattern : grok_pattern;The grok pattern applied to a data store by this classifier. For more information, see built-in patterns in Writing Custom Classifiers.
*)version : version_id option;The version of this classifier.
*)last_updated : timestamp option;The time that this classifier was last updated.
*)creation_time : timestamp option;The time that this classifier was registered.
*)classification : classification;An identifier of the data format that the classifier matches, such as Twitter, JSON, Omniture logs, and so on.
*)name : name_string;The name of the classifier.
*)}A classifier that uses grok patterns.
type nonrec glue_policy = {update_time : timestamp option;The date and time at which the policy was last updated.
*)create_time : timestamp option;The date and time at which the policy was created.
*)policy_hash : hash_string option;Contains the hash value associated with this policy.
*)policy_in_json : policy_json_string option;Contains the requested policy document, in JSON format.
*)}A structure for returning a resource policy.
type nonrec get_workflow_runs_response = {next_token : generic_string option;A continuation token, if not all requested workflow runs have been returned.
*)runs : workflow_runs option;A list of workflow run metadata objects.
*)}type nonrec get_workflow_runs_request = {max_results : page_size option;The maximum number of workflow runs to be included in the response.
*)next_token : generic_string option;The maximum size of the response.
*)include_graph : nullable_boolean option;Specifies whether to include the workflow graph in response or not.
*)name : name_string;Name of the workflow whose metadata of runs should be returned.
*)}type nonrec get_workflow_run_response = {run : workflow_run option;The requested workflow run metadata.
*)}type nonrec get_workflow_run_request = {include_graph : nullable_boolean option;Specifies whether to include the workflow graph in response or not.
*)run_id : id_string;The ID of the workflow run.
*)name : name_string;Name of the workflow being run.
*)}type nonrec get_workflow_run_properties_response = {run_properties : workflow_run_properties option;The workflow run properties which were set during the specified run.
*)}type nonrec get_workflow_run_properties_request = {run_id : id_string;The ID of the workflow run whose run properties should be returned.
*)name : name_string;Name of the workflow which was run.
*)}type nonrec get_workflow_response = {workflow : workflow option;The resource metadata for the workflow.
*)}type nonrec get_workflow_request = {include_graph : nullable_boolean option;Specifies whether to include a graph when returning the workflow resource metadata.
*)name : name_string;The name of the workflow to retrieve.
*)}type nonrec get_user_defined_functions_response = {next_token : token option;A continuation token, if the list of functions returned does not include the last requested function.
*)user_defined_functions : user_defined_function_list option;A list of requested function definitions.
*)}type nonrec get_user_defined_functions_request = {max_results : catalog_getter_page_size option;The maximum number of functions to return in one response.
*)next_token : token option;A continuation token, if this is a continuation call.
*)function_type : function_type option;An optional function-type pattern string that filters the function definitions returned from Amazon Redshift Federated Permissions Catalog.
Specify a value of REGULAR_FUNCTION or STORED_PROCEDURE. The STORED_PROCEDURE function type is only compatible with Amazon Redshift Federated Permissions Catalog.
pattern : name_string;An optional function-name pattern string that filters the function definitions returned.
*)database_name : name_string option;The name of the catalog database where the functions are located. If none is provided, functions from all the databases across the catalog will be returned.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the functions to be retrieved are located. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_user_defined_function_response = {user_defined_function : user_defined_function option;The requested function definition.
*)}type nonrec get_user_defined_function_request = {function_name : name_string;The name of the function.
*)database_name : name_string;The name of the catalog database where the function is located.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the function to be retrieved is located. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_usage_profile_response = {last_modified_on : timestamp_value option;The date and time when the usage profile was last modified.
*)created_on : timestamp_value option;The date and time when the usage profile was created.
*)configuration : profile_configuration option;A ProfileConfiguration object specifying the job and session values for the profile.
description : description_string option;A description of the usage profile.
*)name : name_string option;The name of the usage profile.
*)}type nonrec get_usage_profile_request = {name : name_string;The name of the usage profile to retrieve.
*)}type nonrec column_row_filter = {row_filter_expression : predicate_string option;A string containing the row-level filter expression.
*)column_name : name_string option;A string containing the name of the column.
*)}A filter that uses both column-level and row-level filtering.
type nonrec column_row_filter_list = column_row_filter listtype nonrec get_unfiltered_table_metadata_response = {row_filter : predicate_string option;The filter that applies to the table. For example when applying the filter in SQL, it would go in the WHERE clause and can be evaluated by using an AND operator with any other predicates applied by the user querying the table.
permissions : permission_list option;The Lake Formation data permissions of the caller on the table. Used to authorize the call when no view context is found.
*)is_protected : boolean_ option;A flag that instructs the engine not to push user-provided operations into the logical plan of the view during query planning. However, if set this flag does not guarantee that the engine will comply. Refer to the engine's documentation to understand the guarantees provided, if any.
*)resource_arn : arn_string option;The resource ARN of the parent resource extracted from the request.
*)is_materialized_view : boolean_ option;Indicates if a table is a materialized view.
*)is_multi_dialect_view : boolean_ option;Specifies whether the view supports the SQL dialects of one or more different query engines and can therefore be read by those engines.
*)cell_filters : column_row_filter_list option;A list of column row filters.
*)is_registered_with_lake_formation : boolean_ option;A Boolean value that indicates whether the partition location is registered with Lake Formation.
*)table : table option;A Table object containing the table metadata.
*)}type nonrec audit_column_names_list = column_name_string listtype nonrec audit_context = {all_columns_requested : nullable_boolean option;All columns request for audit.
*)requested_columns : audit_column_names_list option;The requested columns for audit.
*)additional_audit_context : audit_context_string option;A string containing the additional audit context information.
*)}A structure containing the Lake Formation audit context.
type nonrec get_unfiltered_table_metadata_request = {query_session_context : query_session_context option;A structure used as a protocol between query engines and Lake Formation or Glue. Contains both a Lake Formation generated authorization identifier and information from the request's authorization context.
*)permissions : permission_list option;The Lake Formation data permissions of the caller on the table. Used to authorize the call when no view context is found.
*)supported_dialect : supported_dialect option;A structure specifying the dialect and dialect version used by the query engine.
*)root_resource_arn : arn_string option;The resource ARN of the root view in a chain of nested views.
*)parent_resource_arn : arn_string option;The resource ARN of the view.
*)supported_permission_types : permission_type_list;Indicates the level of filtering a third-party analytical engine is capable of enforcing when calling the GetUnfilteredTableMetadata API operation. Accepted values are:
COLUMN_PERMISSION - Column permissions ensure that users can access only specific columns in the table. If there are particular columns contain sensitive data, data lake administrators can define column filters that exclude access to specific columns.CELL_FILTER_PERMISSION - Cell-level filtering combines column filtering (include or exclude columns) and row filter expressions to restrict access to individual elements in the table.NESTED_PERMISSION - Nested permissions combines cell-level filtering and nested column filtering to restrict access to columns and/or nested columns in specific rows based on row filter expressions.NESTED_CELL_PERMISSION - Nested cell permissions combines nested permission with nested cell-level filtering. This allows different subsets of nested columns to be restricted based on an array of row filter expressions.Note: Each of these permission types follows a hierarchical order where each subsequent permission type includes all permission of the previous type.
Important: If you provide a supported permission type that doesn't match the user's level of permissions on the table, then Lake Formation raises an exception. For example, if the third-party engine calling the GetUnfilteredTableMetadata operation can enforce only column-level filtering, and the user has nested cell filtering applied on the table, Lake Formation throws an exception, and will not return unfiltered table metadata and data access credentials.
audit_context : audit_context option;A structure containing Lake Formation audit context information.
*)name : name_string;(Required) Specifies the name of a table for which you are requesting metadata.
*)database_name : name_string;(Required) Specifies the name of a database that contains the table.
*)catalog_id : catalog_id_string;The catalog ID where the table resides.
*)region : value_string option;Specified only if the base tables belong to a different Amazon Web Services Region.
*)}type nonrec get_unfiltered_partitions_metadata_response = {next_token : token option;A continuation token, if the returned list of partitions does not include the last one.
*)unfiltered_partitions : unfiltered_partition_list option;A list of requested partitions.
*)}type nonrec get_unfiltered_partitions_metadata_request = {query_session_context : query_session_context option;A structure used as a protocol between query engines and Lake Formation or Glue. Contains both a Lake Formation generated authorization identifier and information from the request's authorization context.
*)max_results : page_size option;The maximum number of partitions to return in a single response.
*)segment : segment option;The segment of the table's partitions to scan in this request.
*)next_token : token option;A continuation token, if this is not the first call to retrieve these partitions.
*)supported_permission_types : permission_type_list;A list of supported permission types.
*)audit_context : audit_context option;A structure containing Lake Formation audit context information.
*)expression : predicate_string option;An expression that filters the partitions to be returned.
The expression uses SQL syntax similar to the SQL WHERE filter clause. The SQL statement parser JSQLParser parses the expression.
Operators: The following are the operators that you can use in the Expression API call:
= Checks whether the values of the two operands are equal; if yes, then the condition becomes true.
Example: Assume 'variable a' holds 10 and 'variable b' holds 20.
(a = b) is not true.
< > Checks whether the values of two operands are equal; if the values are not equal, then the condition becomes true.
Example: (a < > b) is true.
> Checks whether the value of the left operand is greater than the value of the right operand; if yes, then the condition becomes true.
Example: (a > b) is not true.
< Checks whether the value of the left operand is less than the value of the right operand; if yes, then the condition becomes true.
Example: (a < b) is true.
>= Checks whether the value of the left operand is greater than or equal to the value of the right operand; if yes, then the condition becomes true.
Example: (a >= b) is not true.
<= Checks whether the value of the left operand is less than or equal to the value of the right operand; if yes, then the condition becomes true.
Example: (a <= b) is true.
AND, OR, IN, BETWEEN, LIKE, NOT, IS NULL Logical operators.
Supported Partition Key Types: The following are the supported partition keys.
stringdatetimestampintbigintlongtinyintsmallintdecimalIf an type is encountered that is not valid, an exception is thrown.
*)table_name : name_string;The name of the table that contains the partition.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string;The ID of the Data Catalog where the partitions in question reside. If none is provided, the AWS account ID is used by default.
*)region : value_string option;Specified only if the base tables belong to a different Amazon Web Services Region.
*)}type nonrec get_unfiltered_partition_metadata_request = {query_session_context : query_session_context option;A structure used as a protocol between query engines and Lake Formation or Glue. Contains both a Lake Formation generated authorization identifier and information from the request's authorization context.
*)supported_permission_types : permission_type_list;(Required) A list of supported permission types.
*)audit_context : audit_context option;A structure containing Lake Formation audit context information.
*)partition_values : value_string_list;(Required) A list of partition key values.
*)table_name : name_string;(Required) Specifies the name of a table that contains the partition.
*)database_name : name_string;(Required) Specifies the name of a database that contains the partition.
*)catalog_id : catalog_id_string;The catalog ID where the partition resides.
*)region : value_string option;Specified only if the base tables belong to a different Amazon Web Services Region.
*)}type nonrec get_triggers_response = {next_token : generic_string option;A continuation token, if not all the requested triggers have yet been returned.
*)triggers : trigger_list option;A list of triggers for the specified job.
*)}type nonrec get_triggers_request = {max_results : orchestration_page_size200 option;The maximum size of the response.
*)dependent_job_name : name_string option;The name of the job to retrieve triggers for. The trigger that can start this job is returned, and if there is no such trigger, all triggers are returned.
*)next_token : generic_string option;A continuation token, if this is a continuation call.
*)}type nonrec get_tables_response = {next_token : token option;A continuation token, present if the current list segment is not the last.
*)table_list : table_list option;A list of the requested Table objects.
}type nonrec get_tables_request = {attributes_to_get : table_attributes_list option;Specifies the table fields returned by the GetTables call. This parameter doesn’t accept an empty list. The request must include NAME.
The following are the valid combinations of values:
NAME - Names of all tables in the database.NAME, TABLE_TYPE - Names of all tables and the table types.include_status_details : boolean_nullable option;Specifies whether to include status details related to a request to create or update an Glue Data Catalog view.
*)audit_context : audit_context option;query_as_of_time : timestamp option;The time as of when to read the table contents. If not set, the most recent transaction commit time will be used. Cannot be specified along with TransactionId.
transaction_id : transaction_id_string option;The transaction ID at which to read the table contents.
*)max_results : catalog_getter_page_size option;The maximum number of tables to return in a single response.
*)next_token : token option;A continuation token, included if this is a continuation call.
*)expression : filter_string option;A regular expression pattern. If present, only those tables whose names match the pattern are returned.
*)database_name : name_string;The database in the catalog whose tables to list. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the tables reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_table_versions_list = table_version listtype nonrec get_table_versions_response = {next_token : token option;A continuation token, if the list of available versions does not include the last one.
*)table_versions : get_table_versions_list option;A list of strings identifying available versions of the specified table.
*)}type nonrec get_table_versions_request = {audit_context : audit_context option;max_results : catalog_getter_page_size option;The maximum number of table versions to return in one response.
*)next_token : token option;A continuation token, if this is not the first call.
*)table_name : name_string;The name of the table. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The database in the catalog in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the tables reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_table_version_response = {table_version : table_version option;The requested table version.
*)}type nonrec get_table_version_request = {audit_context : audit_context option;version_id : version_string option;The ID value of the table version to be retrieved. A VersionID is a string representation of an integer. Each version is incremented by 1.
table_name : name_string;The name of the table. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The database in the catalog in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the tables reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_table_response = {table : table option;The Table object that defines the specified table.
}type nonrec get_table_request = {attributes_to_get : table_attributes_list option;Specifies the table fields returned by the GetTable call. This parameter doesn't accept an empty list.
The following are the valid combinations of values:
DEFAULT - Returns the Hive-style table definition only.LATEST_ICEBERG_METADATA - Returns only the latest Apache Iceberg table metadata.DEFAULT, LATEST_ICEBERG_METADATA - Returns both the Hive-style table definition and the latest Apache Iceberg table metadata.include_status_details : boolean_nullable option;Specifies whether to include status details related to a request to create or update an Glue Data Catalog view.
*)audit_context : audit_context option;query_as_of_time : timestamp option;The time as of when to read the table contents. If not set, the most recent transaction commit time will be used. Cannot be specified along with TransactionId.
transaction_id : transaction_id_string option;The transaction ID at which to read the table contents.
*)name : name_string;The name of the table for which to retrieve the definition. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The name of the database in the catalog in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_table_optimizer_response = {table_optimizer : table_optimizer option;The optimizer associated with the specified table.
*)table_name : name_string option;The name of the table.
*)database_name : name_string option;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string option;The Catalog ID of the table.
*)}type nonrec get_table_optimizer_request = {type_ : table_optimizer_type;The type of table optimizer.
*)table_name : name_string;The name of the table.
*)database_name : name_string;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string;The Catalog ID of the table.
*)}type nonrec get_statement_request = {request_origin : orchestration_name_string option;The origin of the request.
*)id : integer_value;The Id of the statement.
*)session_id : name_string;The Session ID of the statement.
*)}type nonrec get_session_response = {session : session option;The session object is returned in the response.
*)}type nonrec get_session_request = {request_origin : orchestration_name_string option;The origin of the request.
*)id : name_string;The ID of the session.
*)}type nonrec get_session_endpoint_response = {spark_connect : session_endpoint;The Spark Connect endpoint details for the session.
*)}type nonrec get_session_endpoint_request = {session_id : name_string;The unique identifier of the interactive session.
*)}type nonrec get_security_configurations_response = {next_token : generic_string option;A continuation token, if there are more security configurations to return.
*)security_configurations : security_configuration_list option;A list of security configurations.
*)}type nonrec get_security_configurations_request = {next_token : generic_string option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum number of results to return.
*)}type nonrec get_security_configuration_response = {security_configuration : security_configuration option;The requested security configuration.
*)}type nonrec get_security_configuration_request = {name : name_string;The name of the security configuration to retrieve.
*)}type nonrec get_schema_versions_diff_response = {diff : schema_definition_diff option;The difference between schemas as a string in JsonPatch format.
*)}type nonrec get_schema_versions_diff_input = {schema_diff_type : schema_diff_type;Refers to SYNTAX_DIFF, which is the currently supported diff type.
second_schema_version_number : schema_version_number;The second of the two schema versions to be compared.
*)first_schema_version_number : schema_version_number;The first of the two schema versions to be compared.
*)schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName has to be provided.SchemaArn or SchemaName has to be provided.}type nonrec get_schema_version_response = {created_time : created_timestamp option;The date and time the schema version was created.
*)status : schema_version_status option;The status of the schema version.
*)version_number : version_long_number option;The version number of the schema.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)data_format : data_format option;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
schema_definition : schema_definition_string option;The schema definition for the schema ID.
*)schema_version_id : schema_version_id_string option;The SchemaVersionId of the schema version.
}type nonrec get_schema_version_input = {schema_version_number : schema_version_number option;The version number of the schema.
*)schema_version_id : schema_version_id_string option;The SchemaVersionId of the schema version. This field is required for fetching by schema ID. Either this or the SchemaId wrapper has to be provided.
schema_id : schema_id option;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName and RegistryName has to be provided.SchemaArn or SchemaName and RegistryName has to be provided.}type nonrec get_schema_response = {updated_time : updated_timestamp option;The date and time the schema was updated.
*)created_time : created_timestamp option;The date and time the schema was created.
*)schema_status : schema_status option;The status of the schema.
*)next_schema_version : version_long_number option;The next version of the schema associated with the returned schema definition.
*)latest_schema_version : version_long_number option;The latest version of the schema associated with the returned schema definition.
*)schema_checkpoint : schema_checkpoint_number option;The version number of the checkpoint (the last time the compatibility mode was changed).
*)compatibility : compatibility option;The compatibility mode of the schema.
*)data_format : data_format option;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
description : description_string option;A description of schema if specified when created
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)schema_name : schema_registry_name_string option;The name of the schema.
*)registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registry.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)}type nonrec get_schema_input = {schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName and RegistryName has to be provided.SchemaArn or SchemaName and RegistryName has to be provided.}type nonrec get_schema_by_definition_response = {created_time : created_timestamp option;The date and time the schema was created.
*)status : schema_version_status option;The status of the schema version.
*)data_format : data_format option;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)schema_version_id : schema_version_id_string option;The schema ID of the schema version.
*)}type nonrec get_schema_by_definition_input = {schema_definition : schema_definition_string;The definition of the schema for which schema details are required.
*)schema_id : schema_id;This is a wrapper structure to contain schema identity fields. The structure contains:
SchemaArn or SchemaName has to be provided.SchemaArn or SchemaName has to be provided.}type nonrec get_resource_policy_response = {update_time : timestamp option;The date and time at which the policy was last updated.
*)create_time : timestamp option;The date and time at which the policy was created.
*)policy_hash : hash_string option;Contains the hash value associated with this policy.
*)policy_in_json : policy_json_string option;Contains the requested policy document, in JSON format.
*)}type nonrec get_resource_policy_request = {resource_arn : glue_resource_arn option;The ARN of the Glue resource for which to retrieve the resource policy. If not supplied, the Data Catalog resource policy is returned. Use GetResourcePolicies to view all existing resource policies. For more information see Specifying Glue Resource ARNs.
}type nonrec get_resource_policies_response_list = glue_policy listtype nonrec get_resource_policies_response = {next_token : token option;A continuation token, if the returned list does not contain the last resource policy available.
*)get_resource_policies_response_list : get_resource_policies_response_list
option;A list of the individual resource policies and the account-level resource policy.
*)}type nonrec get_registry_response = {updated_time : updated_timestamp option;The date and time the registry was updated.
*)created_time : created_timestamp option;The date and time the registry was created.
*)status : registry_status option;The status of the registry.
*)description : description_string option;A description of the registry.
*)registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registry.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)}type nonrec get_registry_input = {registry_id : registry_id;This is a wrapper structure that may contain the registry name and Amazon Resource Name (ARN).
*)}type nonrec get_plan_response = {scala_code : scala_code option;The Scala code to perform the mapping.
*)python_script : python_script option;A Python script to perform the mapping.
*)}type nonrec catalog_entry = {table_name : name_string;The name of the table in question.
*)database_name : name_string;The database in which the table metadata resides.
*)}Specifies a table definition in the Glue Data Catalog.
type nonrec catalog_entries = catalog_entry listtype nonrec additional_plan_options_map =
(generic_string * generic_string) listtype nonrec get_plan_request = {additional_plan_options_map : additional_plan_options_map option;A map to hold additional optional key-value parameters.
Currently, these key-value pairs are supported:
inferSchema — Specifies whether to set inferSchema to true or false for the default script generated by an Glue job. For example, to set inferSchema to true, pass the following key value pair:
--additional-plan-options-map '{"inferSchema":"true"}'
language : language option;The programming language of the code to perform the mapping.
*)location : location option;The parameters for the mapping.
*)sinks : catalog_entries option;The target tables.
*)source : catalog_entry;The source table.
*)mapping : mapping_list;The list of mappings from a source table to target tables.
*)}type nonrec get_partitions_response = {next_token : token option;A continuation token, if the returned list of partitions does not include the last one.
*)partitions : partition_list option;A list of requested partitions.
*)}type nonrec get_partitions_request = {audit_context : audit_context option;query_as_of_time : timestamp option;The time as of when to read the partition contents. If not set, the most recent transaction commit time will be used. Cannot be specified along with TransactionId.
transaction_id : transaction_id_string option;The transaction ID at which to read the partition contents.
*)exclude_column_schema : boolean_nullable option;When true, specifies not returning the partition column schema. Useful when you are interested only in other partition attributes such as partition values or location. This approach avoids the problem of a large response by not returning duplicate data.
*)max_results : page_size option;The maximum number of partitions to return in a single response.
*)segment : segment option;The segment of the table's partitions to scan in this request.
*)next_token : token option;A continuation token, if this is not the first call to retrieve these partitions.
*)expression : predicate_string option;An expression that filters the partitions to be returned.
The expression uses SQL syntax similar to the SQL WHERE filter clause. The SQL statement parser JSQLParser parses the expression.
Operators: The following are the operators that you can use in the Expression API call:
= Checks whether the values of the two operands are equal; if yes, then the condition becomes true.
Example: Assume 'variable a' holds 10 and 'variable b' holds 20.
(a = b) is not true.
< > Checks whether the values of two operands are equal; if the values are not equal, then the condition becomes true.
Example: (a < > b) is true.
> Checks whether the value of the left operand is greater than the value of the right operand; if yes, then the condition becomes true.
Example: (a > b) is not true.
< Checks whether the value of the left operand is less than the value of the right operand; if yes, then the condition becomes true.
Example: (a < b) is true.
>= Checks whether the value of the left operand is greater than or equal to the value of the right operand; if yes, then the condition becomes true.
Example: (a >= b) is not true.
<= Checks whether the value of the left operand is less than or equal to the value of the right operand; if yes, then the condition becomes true.
Example: (a <= b) is true.
AND, OR, IN, BETWEEN, LIKE, NOT, IS NULL Logical operators.
Supported Partition Key Types: The following are the supported partition keys.
stringdatetimestampintbigintlongtinyintsmallintdecimalIf an type is encountered that is not valid, an exception is thrown.
The following list shows the valid operators on each type. When you define a crawler, the partitionKey type is created as a STRING, to be compatible with the catalog partitions.
Sample API Call:
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_partition_response = {partition : partition option;The requested information, in the form of a Partition object.
}type nonrec get_partition_request = {audit_context : audit_context option;partition_values : value_string_list;The values that define the partition.
*)table_name : name_string;The name of the partition's table.
*)database_name : name_string;The name of the catalog database where the partition resides.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partition in question resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_partition_indexes_response = {next_token : token option;A continuation token, present if the current list segment is not the last.
*)partition_index_descriptor_list : partition_index_descriptor_list option;A list of index descriptors.
*)}type nonrec get_partition_indexes_request = {next_token : token option;A continuation token, included if this is a continuation call.
*)table_name : name_string;Specifies the name of a table for which you want to retrieve the partition indexes.
*)database_name : name_string;Specifies the name of a database from which you want to retrieve partition indexes.
*)catalog_id : catalog_id_string option;The catalog ID where the table resides.
*)}type nonrec get_materialized_view_refresh_task_run_response = {materialized_view_refresh_task_run : materialized_view_refresh_task_run option;A MaterializedViewRefreshTaskRun object representing the details of the task run.
*)}type nonrec get_materialized_view_refresh_task_run_request = {materialized_view_refresh_task_run_id : uui_dv4;The identifier for the particular materialized view refresh task run.
*)catalog_id : name_string;The ID of the Data Catalog where the table resides. If none is supplied, the account ID is used by default.
*)}type nonrec get_mapping_response = {mapping : mapping_list;A list of mappings to the specified targets.
*)}type nonrec get_mapping_request = {location : location option;Parameters for the mapping.
*)sinks : catalog_entries option;A list of target tables.
*)source : catalog_entry;Specifies the source table.
*)}type nonrec get_ml_transforms_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)transforms : transform_list;A list of machine learning transforms.
*)}type nonrec get_ml_transforms_request = {sort : transform_sort_criteria option;The sorting criteria.
*)filter : transform_filter_criteria option;The filter transformation criteria.
*)max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A paginated token to offset the results.
*)}type nonrec get_ml_transform_response = {transform_encryption : transform_encryption option;The encryption-at-rest settings of the transform that apply to accessing user data. Machine learning transforms can access user data encrypted in Amazon S3 using KMS.
*)max_retries : nullable_integer option;The maximum number of times to retry a task for this transform after a task run fails.
*)timeout : timeout option;The timeout for a task run for this transform in minutes. This is the maximum time that a task run for this transform can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when this task runs.
worker_type : worker_type option;The type of predefined worker that is allocated when this task runs. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker provides 4 vCPU, 16 GB of memory and a 64GB disk, and 1 executor per worker.G.2X worker type, each worker provides 8 vCPU, 32 GB of memory and a 128GB disk, and 1 executor per worker.max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that are allocated to task runs for this transform. You can allocate from 2 to 100 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
When the WorkerType field is set to a value other than Standard, the MaxCapacity field is set automatically and becomes read-only.
glue_version : glue_version_string option;This value determines which version of Glue this machine learning transform is compatible with. Glue 1.0 is recommended for most customers. If the value is not set, the Glue compatibility defaults to Glue 0.9. For more information, see Glue Versions in the developer guide.
*)role : role_string option;The name or Amazon Resource Name (ARN) of the IAM role with the required permissions.
*)schema : transform_schema option;The
Mapobject that represents the schema that this transform accepts. Has an upper bound of 100 columns.
*)label_count : label_count option;The number of labels available for this transform.
*)evaluation_metrics : evaluation_metrics option;The latest evaluation metrics.
*)parameters : transform_parameters option;The configuration parameters that are specific to the algorithm used.
*)input_record_tables : glue_tables option;A list of Glue table definitions used by the transform.
*)last_modified_on : timestamp option;The date and time when the transform was last modified.
*)created_on : timestamp option;The date and time when the transform was created.
*)status : transform_status_type option;The last known status of the transform (to indicate whether it can be used or not). One of "NOT_READY", "READY", or "DELETING".
*)description : description_string option;A description of the transform.
*)name : name_string option;The unique name given to the transform when it was created.
*)transform_id : hash_string option;The unique identifier of the transform, generated at the time that the transform was created.
*)}type nonrec get_ml_transform_request = {transform_id : hash_string;The unique identifier of the transform, generated at the time that the transform was created.
*)}type nonrec get_ml_task_runs_response = {next_token : pagination_token option;A pagination token, if more results are available.
*)task_runs : task_run_list option;A list of task runs that are associated with the transform.
*)}type nonrec get_ml_task_runs_request = {sort : task_run_sort_criteria option;The sorting criteria, in the TaskRunSortCriteria structure, for the task run.
filter : task_run_filter_criteria option;The filter criteria, in the TaskRunFilterCriteria structure, for the task run.
max_results : page_size option;The maximum number of results to return.
*)next_token : pagination_token option;A token for pagination of the results. The default is empty.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec get_ml_task_run_response = {execution_time : execution_time option;The amount of time (in seconds) that the task run consumed resources.
*)completed_on : timestamp option;The date and time when this task run was completed.
*)last_modified_on : timestamp option;The date and time when this task run was last modified.
*)started_on : timestamp option;The date and time when this task run started.
*)error_string : generic_string option;The error strings that are associated with the task run.
*)properties : task_run_properties option;The list of properties that are associated with the task run.
*)log_group_name : generic_string option;The names of the log groups that are associated with the task run.
*)status : task_status_type option;The status for this task run.
*)task_run_id : hash_string option;The unique run identifier associated with this run.
*)transform_id : hash_string option;The unique identifier of the task run.
*)}type nonrec get_ml_task_run_request = {task_run_id : hash_string;The unique identifier of the task run.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec get_jobs_response = {next_token : generic_string option;A continuation token, if not all job definitions have yet been returned.
*)jobs : job_list option;A list of job definitions.
*)}type nonrec get_jobs_request = {max_results : page_size option;The maximum size of the response.
*)next_token : generic_string option;A continuation token, if this is a continuation call.
*)}type nonrec get_job_runs_response = {next_token : generic_string option;A continuation token, if not all requested job runs have been returned.
*)job_runs : job_run_list option;A list of job-run metadata objects.
*)}type nonrec get_job_runs_request = {max_results : orchestration_page_size200 option;The maximum size of the response.
*)next_token : generic_string option;A continuation token, if this is a continuation call.
*)job_name : name_string;The name of the job definition for which to retrieve all job runs.
*)}type nonrec get_job_run_request = {predecessors_included : boolean_value option;True if a list of predecessor runs should be returned.
*)run_id : id_string;The ID of the job run.
*)job_name : name_string;Name of the job definition being run.
*)}type nonrec get_job_request = {job_name : name_string;The name of the job definition to retrieve.
*)}type nonrec get_job_bookmark_response = {job_bookmark_entry : job_bookmark_entry option;A structure that defines a point that a job can resume processing.
*)}type nonrec get_integration_table_properties_response = {target_table_config : target_table_config option;A structure for the target table configuration.
*)source_table_config : source_table_config option;A structure for the source table configuration.
*)table_name : string128 option;The name of the table to be replicated.
*)resource_arn : string512 option;The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
*)}type nonrec get_integration_table_properties_request = {table_name : string128;The name of the table to be replicated.
*)resource_arn : string512;The Amazon Resource Name (ARN) of the target table for which to retrieve integration table properties. Currently, this API only supports retrieving properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for retrieving integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
*)}type nonrec get_integration_resource_property_response = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_property_arn : string512 option;The resource ARN created through this create API. The format is something like arn:aws:glue:::integrationresourceproperty/*
*)resource_arn : string512 option;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec get_integration_resource_property_request = {resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec get_glue_identity_center_configuration_response = {user_background_sessions_enabled : nullable_boolean option;Indicates whether users can run background sessions when using Identity Center authentication with Glue services.
*)scopes : orchestration_string_list option;A list of Identity Center scopes that define the permissions and access levels for the Glue configuration.
*)instance_arn : identity_center_instance_arn option;The Amazon Resource Name (ARN) of the Identity Center instance associated with the Glue configuration.
*)application_arn : application_arn option;The Amazon Resource Name (ARN) of the Identity Center application associated with the Glue configuration.
*)}Response containing the Glue Identity Center configuration details.
type nonrec get_glossary_term_response = {long_description : glossary_long_description option;The long description of the glossary term.
*)short_description : glossary_short_description option;The short description of the glossary term.
*)name : glossary_term_name option;The name of the glossary term.
*)glossary_id : glossary_id option;The unique identifier of the glossary containing this term.
*)id : glossary_term_id option;The unique identifier of the glossary term.
*)}type nonrec get_glossary_term_request = {identifier : glossary_term_id;The unique identifier of the glossary term to retrieve.
*)}type nonrec get_glossary_response = {description : metadata_description option;The description of the glossary.
*)name : glossary_name option;The name of the glossary.
*)id : glossary_id option;The unique identifier of the glossary.
*)}type nonrec get_glossary_request = {identifier : glossary_id;The unique identifier of the glossary to retrieve.
*)}type nonrec get_form_type_response = {schema : form_type_schema option;The Smithy IDL schema of the form type.
*)name : form_type_name option;The name of the form type.
*)id : form_type_id option;The identifier of the form type.
*)}type nonrec get_form_type_request = {identifier : form_type_id;The identifier of the form type to retrieve.
*)}type nonrec get_entity_records_response = {next_token : next_token option;A continuation token, present if the current segment is not the last.
*)records : records option;A list of the requested objects.
*)}type nonrec connection_options = (option_key * option_value) listtype nonrec get_entity_records_request = {selected_fields : selected_fields option;List of fields that we want to fetch as part of preview data.
*)order_by : Smaws_Lib.Smithy_api.Types.string_ option;A parameter that orders the response preview data.
*)limit : limit;Limits the number of records fetched with the request.
*)filter_predicate : filter_predicate option;A filter predicate that you can apply in the query request.
*)connection_options : connection_options option;Connector options that are required to query the data.
*)data_store_api_version : api_version option;The API version of the SaaS connector.
*)next_token : next_token option;A continuation token, included if this is a continuation call.
*)entity_name : entity_name;Name of the entity that we want to query the preview data from the given connection type.
*)catalog_id : catalog_id_string option;The catalog ID of the catalog that contains the connection. This can be null, By default, the Amazon Web Services Account ID is the catalog ID.
*)connection_name : name_string option;The name of the connection that contains the connection type credentials.
*)}type nonrec dev_endpoint = {arguments : map_value option;A map of arguments used to configure the DevEndpoint.
Valid arguments are:
"--enable-glue-datacatalog": ""You can specify a version of Python support for development endpoints by using the Arguments parameter in the CreateDevEndpoint or UpdateDevEndpoint APIs. If no arguments are provided, the version defaults to Python 2.
security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this DevEndpoint.
public_keys : public_keys_list option;A list of public keys to be used by the DevEndpoints for authentication. Using this attribute is preferred over a single public key because the public keys allow you to have a different private key per client.
If you previously created an endpoint with a public key, you must remove that key to be able to set a list of public keys. Call the UpdateDevEndpoint API operation with the public key content in the deletePublicKeys attribute, and the list of new keys in the addPublicKeys attribute.
public_key : generic_string option;The public key to be used by this DevEndpoint for authentication. This attribute is provided for backward compatibility because the recommended attribute to use is public keys.
last_modified_timestamp : timestamp_value option;The point in time at which this DevEndpoint was last modified.
created_timestamp : timestamp_value option;The point in time at which this DevEndpoint was created.
*)last_update_status : generic_string option;The status of the last update.
*)failure_reason : generic_string option;The reason for a current failure in this DevEndpoint.
extra_jars_s3_path : generic_string option;The path to one or more Java .jar files in an S3 bucket that should be loaded in your DevEndpoint.
You can only use pure Java/Scala libraries with a DevEndpoint.
extra_python_libs_s3_path : generic_string option;The paths to one or more Python libraries in an Amazon S3 bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma.
You can only use pure Python libraries with a DevEndpoint. Libraries that rely on C extensions, such as the pandas Python data analysis library, are not currently supported.
vpc_id : generic_string option;The ID of the virtual private cloud (VPC) used by this DevEndpoint.
availability_zone : generic_string option;The Amazon Web Services Availability Zone where this DevEndpoint is located.
number_of_nodes : integer_value option;The number of Glue Data Processing Units (DPUs) allocated to this DevEndpoint.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated to the development endpoint.
The maximum number of workers you can define are 299 for G.1X, and 149 for G.2X.
glue_version : glue_version_string option;Glue version determines the versions of Apache Spark and Python that Glue supports. The Python version indicates the version supported for running your ETL scripts on development endpoints.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
Development endpoints that are created without specifying a Glue version default to Glue 0.9.
You can specify a version of Python support for development endpoints by using the Arguments parameter in the CreateDevEndpoint or UpdateDevEndpoint APIs. If no arguments are provided, the version defaults to Python 2.
worker_type : worker_type option;The type of predefined worker that is allocated to the development endpoint. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. We recommend this worker type for memory-intensive jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. We recommend this worker type for memory-intensive jobs.Known issue: when a development endpoint is created with the G.2X WorkerType configuration, the Spark drivers for the development endpoint will run on 4 vCPU, 16 GB of memory, and a 64 GB disk.
status : generic_string option;The current status of this DevEndpoint.
public_address : generic_string option;The public IP address used by this DevEndpoint. The PublicAddress field is present only when you create a non-virtual private cloud (VPC) DevEndpoint.
zeppelin_remote_spark_interpreter_port : integer_value option;The Apache Zeppelin port for the remote Apache Spark interpreter.
*)private_address : generic_string option;A private IP address to access the DevEndpoint within a VPC if the DevEndpoint is created within one. The PrivateAddress field is present only when you create the DevEndpoint within your VPC.
yarn_endpoint_address : generic_string option;The YARN endpoint address used by this DevEndpoint.
subnet_id : generic_string option;The subnet ID for this DevEndpoint.
security_group_ids : string_list option;A list of security group identifiers used in this DevEndpoint.
role_arn : role_arn option;The Amazon Resource Name (ARN) of the IAM role used in this DevEndpoint.
endpoint_name : generic_string option;The name of the DevEndpoint.
}A development endpoint where a developer can remotely debug extract, transform, and load (ETL) scripts.
type nonrec dev_endpoint_list = dev_endpoint listtype nonrec get_dev_endpoints_response = {next_token : generic_string option;A continuation token, if not all DevEndpoint definitions have yet been returned.
dev_endpoints : dev_endpoint_list option;A list of DevEndpoint definitions.
}type nonrec get_dev_endpoints_request = {next_token : generic_string option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum size of information to return.
*)}type nonrec get_dev_endpoint_response = {dev_endpoint : dev_endpoint option;A DevEndpoint definition.
}type nonrec get_dev_endpoint_request = {endpoint_name : generic_string;Name of the DevEndpoint to retrieve information for.
}type nonrec code_gen_node = {line_number : integer option;The line number of the node.
*)args : code_gen_node_args;Properties of the node, in the form of name-value pairs.
*)node_type : code_gen_node_type;The type of node that this is.
*)id : code_gen_identifier;A node identifier that is unique within the node's graph.
*)}Represents a node in a directed acyclic graph (DAG)
type nonrec dag_nodes = code_gen_node listtype nonrec code_gen_edge = {target_parameter : code_gen_arg_name option;The target of the edge.
*)target : code_gen_identifier;The ID of the node at which the edge ends.
*)source : code_gen_identifier;The ID of the node at which the edge starts.
*)}Represents a directional edge in a directed acyclic graph (DAG).
type nonrec dag_edges = code_gen_edge listtype nonrec get_dataflow_graph_request = {python_script : python_script option;The Python script to transform.
*)}type nonrec database = {federated_database : federated_database option;A FederatedDatabase structure that references an entity outside the Glue Data Catalog.
catalog_id : catalog_id_string option;The ID of the Data Catalog in which the database resides.
*)target_database : database_identifier option;A DatabaseIdentifier structure that describes a target database for resource linking.
create_table_default_permissions : principal_permissions_list option;Creates a set of default permissions on the table for principals. Used by Lake Formation. Not used in the normal course of Glue operations.
*)create_time : timestamp option;The time at which the metadata database was created in the catalog.
*)parameters : parameters_map option;These key-value pairs define parameters and properties of the database.
*)location_uri : ur_i option;The location of the database (for example, an HDFS path).
*)description : description_string option;A description of the database.
*)name : name_string;The name of the database. For Hive compatibility, this is folded to lowercase when it is stored.
*)}The Database object represents a logical grouping of tables that might reside in a Hive metastore or an RDBMS.
type nonrec database_list = database listtype nonrec get_databases_response = {next_token : token option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)database_list : database_list;A list of Database objects from the specified catalog.
}type nonrec database_attributes_list = database_attributes listtype nonrec get_databases_request = {attributes_to_get : database_attributes_list option;Specifies the database fields returned by the GetDatabases call. This parameter doesn’t accept an empty list. The request must include the NAME.
max_results : catalog_getter_page_size option;The maximum number of databases to return in one response.
*)next_token : token option;A continuation token, if this is a continuation call.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog from which to retrieve Databases. If none is provided, the Amazon Web Services account ID is used by default.
}type nonrec get_database_response = {database : database option;The definition of the specified database in the Data Catalog.
*)}type nonrec get_database_request = {name : name_string;The name of the database to retrieve. For Hive compatibility, this should be all lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the database resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_data_quality_ruleset_response = {data_quality_security_configuration : name_string option;The name of the security configuration created with the data quality encryption option.
*)recommendation_run_id : hash_string option;When a ruleset was created from a recommendation run, this run ID is generated to link the two together.
*)last_modified_on : timestamp option;A timestamp. The last point in time when this data quality ruleset was modified.
*)created_on : timestamp option;A timestamp. The time and date that this data quality ruleset was created.
*)target_table : data_quality_target_table option;The name and database name of the target table.
*)ruleset : data_quality_ruleset_string option;A Data Quality Definition Language (DQDL) ruleset. For more information, see the Glue developer guide.
*)description : description_string option;A description of the ruleset.
*)name : name_string option;The name of the ruleset.
*)}Returns the data quality ruleset response.
type nonrec data_quality_result_id_list = hash_string listtype nonrec get_data_quality_ruleset_evaluation_run_response = {additional_data_sources : data_source_map option;A map of reference strings to additional data sources you can specify for an evaluation run.
*)result_ids : data_quality_result_id_list option;A list of result IDs for the data quality results for the run.
*)ruleset_names : ruleset_names option;A list of ruleset names for the run. Currently, this parameter takes only one Ruleset name.
*)execution_time : execution_time option;The amount of time (in seconds) that the run consumed resources.
*)completed_on : timestamp option;The date and time when this run was completed.
*)last_modified_on : timestamp option;A timestamp. The last point in time when this data quality rule recommendation run was modified.
*)started_on : timestamp option;The date and time when this run started.
*)error_string : generic_string option;The error strings that are associated with the run.
*)status : task_status_type option;The status for this run.
*)additional_run_options : data_quality_evaluation_run_additional_run_options
option;Additional run options you can specify for an evaluation run.
*)timeout : timeout option;The timeout for a run in minutes. This is the maximum time that a run can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of G.1X workers to be used in the run. The default is 5.
role : role_string option;An IAM role supplied to encrypt the results of the run.
*)data_source : data_source option;The data source (an Glue table) associated with this evaluation run.
*)run_id : hash_string option;The unique run identifier associated with this run.
*)}type nonrec get_data_quality_ruleset_evaluation_run_request = {run_id : hash_string;The unique run identifier associated with this run.
*)}type nonrec get_data_quality_rule_recommendation_run_response = {data_quality_security_configuration : name_string option;The name of the security configuration created with the data quality encryption option.
*)created_ruleset_name : name_string option;The name of the ruleset that was created by the run.
*)recommended_ruleset : data_quality_ruleset_string option;When a start rule recommendation run completes, it creates a recommended ruleset (a set of rules). This member has those rules in Data Quality Definition Language (DQDL) format.
*)execution_time : execution_time option;The amount of time (in seconds) that the run consumed resources.
*)completed_on : timestamp option;The date and time when this run was completed.
*)last_modified_on : timestamp option;A timestamp. The last point in time when this data quality rule recommendation run was modified.
*)started_on : timestamp option;The date and time when this run started.
*)error_string : generic_string option;The error strings that are associated with the run.
*)status : task_status_type option;The status for this run.
*)timeout : timeout option;The timeout for a run in minutes. This is the maximum time that a run can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of G.1X workers to be used in the run. The default is 5.
role : role_string option;An IAM role supplied to encrypt the results of the run.
*)data_source : data_source option;The data source (an Glue table) associated with this run.
*)run_id : hash_string option;The unique run identifier associated with this run.
*)}The response for the Data Quality rule recommendation run.
type nonrec get_data_quality_rule_recommendation_run_request = {run_id : hash_string;The unique run identifier associated with this run.
*)}type nonrec evaluated_metrics_map = (name_string * nullable_double) listtype nonrec data_quality_rule_result = {labels : labels option;A map containing labels assigned to the data quality rule.
*)rule_metrics : rule_metrics_map option;A map containing metrics associated with the evaluation of the rule based on row-level results.
*)evaluated_rule : data_quality_rule_result_description option;The evaluated rule.
*)evaluated_metrics : evaluated_metrics_map option;A map of metrics associated with the evaluation of the rule.
*)result_ : data_quality_rule_result_status option;A pass or fail status for the rule.
*)evaluation_message : data_quality_rule_result_description option;An evaluation message.
*)description : data_quality_rule_result_description option;A description of the data quality rule.
*)name : name_string option;The name of the data quality rule.
*)}Describes the result of the evaluation of a data quality rule.
type nonrec data_quality_rule_results = data_quality_rule_result listtype nonrec data_quality_analyzer_result = {evaluated_metrics : evaluated_metrics_map option;A map of metrics associated with the evaluation of the analyzer.
*)evaluation_message : data_quality_rule_result_description option;An evaluation message.
*)description : data_quality_rule_result_description option;A description of the data quality analyzer.
*)name : name_string option;The name of the data quality analyzer.
*)}Describes the result of the evaluation of a data quality analyzer.
type nonrec data_quality_analyzer_results = data_quality_analyzer_result listtype nonrec data_quality_observation = {metric_based_observation : metric_based_observation option;An object of type MetricBasedObservation representing the observation that is based on evaluated data quality metrics.
description : data_quality_observation_description option;A description of the data quality observation.
*)}Describes the observation generated after evaluating the rules and analyzers.
type nonrec data_quality_observations = data_quality_observation listtype nonrec data_quality_aggregated_metrics = {total_rules_failed : nullable_double option;The total number of data quality rules that failed their evaluation criteria.
*)total_rules_passed : nullable_double option;The total number of data quality rules that passed their evaluation criteria.
*)total_rules_processed : nullable_double option;The total number of data quality rules that were evaluated.
*)total_rows_failed : nullable_double option;The total number of rows that failed one or more data quality rules.
*)total_rows_passed : nullable_double option;The total number of rows that passed all applicable data quality rules.
*)total_rows_processed : nullable_double option;The total number of rows that were processed during the data quality evaluation.
*)}A summary of metrics showing the total counts of processed rows and rules, including their pass/fail statistics based on row-level results.
type nonrec get_data_quality_result_response = {aggregated_metrics : data_quality_aggregated_metrics option;A summary of DataQualityAggregatedMetrics objects showing the total counts of processed rows and rules, including their pass/fail statistics based on row-level results.
observations : data_quality_observations option;A list of DataQualityObservation objects representing the observations generated after evaluating the rules and analyzers.
analyzer_results : data_quality_analyzer_results option;A list of DataQualityAnalyzerResult objects representing the results for each analyzer.
rule_results : data_quality_rule_results option;A list of DataQualityRuleResult objects representing the results for each rule.
ruleset_evaluation_run_id : hash_string option;The unique run ID associated with the ruleset evaluation.
*)job_run_id : hash_string option;The job run ID associated with the data quality result, if any.
*)job_name : name_string option;The job name associated with the data quality result, if any.
*)completed_on : timestamp option;The date and time when the run for this data quality result was completed.
*)started_on : timestamp option;The date and time when the run for this data quality result started.
*)evaluation_context : generic_string option;In the context of a job in Glue Studio, each node in the canvas is typically assigned some sort of name and data quality nodes will have names. In the case of multiple nodes, the evaluationContext can differentiate the nodes.
ruleset_name : name_string option;The name of the ruleset associated with the data quality result.
*)data_source : data_source option;The table associated with the data quality result, if any.
*)score : generic_bounded_double option;An aggregate data quality score. Represents the ratio of rules that passed to the total number of rules.
*)profile_id : hash_string option;The Profile ID for the data quality result.
*)result_id : hash_string option;A unique result ID for the data quality result.
*)}The response for the data quality result.
type nonrec get_data_quality_result_request = {result_id : hash_string;A unique result ID for the data quality result.
*)}type nonrec get_data_quality_model_result_response = {model : statistic_model_results option;A list of StatisticModelResult
completed_on : timestamp option;The timestamp when the data quality model training completed.
*)}type nonrec get_data_quality_model_result_request = {profile_id : hash_string;The Profile ID.
*)statistic_id : hash_string;The Statistic ID.
*)}type nonrec get_data_quality_model_response = {failure_reason : hash_string option;The training failure reason.
*)completed_on : timestamp option;The timestamp when the data quality model training completed.
*)started_on : timestamp option;The timestamp when the data quality model training started.
*)status : data_quality_model_status option;The training status of the data quality model.
*)}type nonrec get_data_quality_model_request = {profile_id : hash_string;The Profile ID.
*)statistic_id : hash_string option;The Statistic ID.
*)}type nonrec get_data_catalog_encryption_settings_response = {data_catalog_encryption_settings : data_catalog_encryption_settings option;The requested security configuration.
*)}type nonrec get_data_catalog_encryption_settings_request = {catalog_id : catalog_id_string option;The ID of the Data Catalog to retrieve the security configuration for. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_dashboard_url_response = {url : sensitive_url;The URL for the Spark monitoring dashboard.
*)}type nonrec get_dashboard_url_request = {request_origin : orchestration_name_string option;The origin of the request.
*)resource_type : glue_resource_type;The type of the resource. Valid values are SESSION and JOB.
resource_id : name_string;The unique identifier of the resource for which to retrieve the dashboard URL.
*)}type nonrec get_custom_entity_type_response = {context_words : context_words option;A list of context words if specified when you created the custom pattern. If none of these context words are found within the vicinity of the regular expression the data will not be detected as sensitive data.
*)regex_string : name_string option;A regular expression string that is used for detecting sensitive data in a custom pattern.
*)name : name_string option;The name of the custom pattern that you retrieved.
*)}type nonrec get_custom_entity_type_request = {name : name_string;The name of the custom pattern that you want to retrieve.
*)}type nonrec crawler = {lake_formation_configuration : lake_formation_configuration option;Specifies whether the crawler should use Lake Formation credentials for the crawler instead of the IAM role credentials.
*)crawler_security_configuration : crawler_security_configuration option;The name of the SecurityConfiguration structure to be used by this crawler.
configuration : crawler_configuration option;Crawler configuration information. This versioned JSON string allows users to specify aspects of a crawler's behavior. For more information, see Setting crawler configuration options.
*)version : version_id option;The version of the crawler.
*)last_crawl : last_crawl_info option;The status of the last crawl, and potentially error information if an error occurred.
*)last_updated : timestamp option;The time that the crawler was last updated.
*)creation_time : timestamp option;The time that the crawler was created.
*)crawl_elapsed_time : milliseconds_count option;If the crawler is running, contains the total time elapsed since the last crawl began.
*)schedule : schedule option;For scheduled crawlers, the schedule when the crawler runs.
*)table_prefix : table_prefix option;The prefix added to the names of tables that are created.
*)state : crawler_state option;Indicates whether the crawler is running, or whether a run is pending.
*)lineage_configuration : lineage_configuration option;A configuration that specifies whether data lineage is enabled for the crawler.
*)schema_change_policy : schema_change_policy option;The policy that specifies update and delete behaviors for the crawler.
*)recrawl_policy : recrawl_policy option;A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.
*)classifiers : classifier_name_list option;A list of UTF-8 strings that specify the custom classifiers that are associated with the crawler.
*)description : description_string option;A description of the crawler.
*)database_name : database_name option;The name of the database in which the crawler's output is stored.
*)targets : crawler_targets option;A collection of targets to crawl.
*)role : role option;The Amazon Resource Name (ARN) of an IAM role that's used to access customer resources, such as Amazon Simple Storage Service (Amazon S3) data.
*)name : name_string option;The name of the crawler.
*)}Specifies a crawler program that examines a data source and uses classifiers to try to determine its schema. If successful, the crawler records metadata concerning the data source in the Glue Data Catalog.
type nonrec crawler_list = crawler listtype nonrec get_crawlers_response = {next_token : token option;A continuation token, if the returned list has not reached the end of those defined in this customer account.
*)crawlers : crawler_list option;A list of crawler metadata.
*)}type nonrec get_crawler_response = {crawler : crawler option;The metadata for the specified crawler.
*)}type nonrec get_crawler_request = {name : name_string;The name of the crawler to retrieve metadata for.
*)}type nonrec crawler_metrics = {tables_deleted : non_negative_integer option;The number of tables deleted by this crawler.
*)tables_updated : non_negative_integer option;The number of tables updated by this crawler.
*)tables_created : non_negative_integer option;The number of tables created by this crawler.
*)median_runtime_seconds : non_negative_double option;The median duration of this crawler's runs, in seconds.
*)last_runtime_seconds : non_negative_double option;The duration of the crawler's most recent run, in seconds.
*)still_estimating : boolean_ option;True if the crawler is still estimating how long it will take to complete this run.
*)time_left_seconds : non_negative_double option;The estimated time left to complete a running crawl.
*)crawler_name : name_string option;The name of the crawler.
*)}Metrics for a specified crawler.
type nonrec crawler_metrics_list = crawler_metrics listtype nonrec get_crawler_metrics_response = {next_token : token option;A continuation token, if the returned list does not contain the last metric available.
*)crawler_metrics_list : crawler_metrics_list option;A list of metrics for the specified crawler.
*)}type nonrec get_crawler_metrics_request = {next_token : token option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum size of a list to return.
*)crawler_name_list : crawler_name_list option;A list of the names of crawlers about which to retrieve metrics.
*)}type nonrec authentication_configuration = {o_auth2_properties : o_auth2_properties option;The properties for OAuth2 authentication.
*)kms_key_arn : kms_key_arn option;The Amazon Resource Name (ARN) of the KMS key used to encrypt sensitive authentication information. This key is used to protect credentials and other sensitive data stored within the authentication configuration.
*)secret_arn : secret_arn option;The secret manager ARN to store credentials.
*)authentication_type : authentication_type option;A structure containing the authentication configuration.
*)}A structure containing the authentication configuration.
type nonrec connection = {compatible_compute_environments : compute_environment_list option;A list of compute environments compatible with the connection.
*)connection_schema_version : connection_schema_version option;The version of the connection schema for this connection. Version 2 supports properties for specific compute environments.
*)authentication_configuration : authentication_configuration option;The authentication properties of the connection.
*)last_connection_validation_time : timestamp option;A timestamp of the time this connection was last validated.
*)status_reason : long_value_string option;The reason for the connection status.
*)status : connection_status option;The status of the connection. Can be one of: READY, IN_PROGRESS, or FAILED.
last_updated_by : name_string option;The user, group, or role that last updated this connection definition.
*)last_updated_time : timestamp option;The timestamp of the last time the connection definition was updated.
*)creation_time : timestamp option;The timestamp of the time that this connection definition was created.
*)physical_connection_requirements : physical_connection_requirements option;The physical connection requirements, such as virtual private cloud (VPC) and SecurityGroup, that are needed to make this connection successfully.
python_properties : property_map option;Connection properties specific to the Python compute environment.
*)athena_properties : property_map option;Connection properties specific to the Athena compute environment.
*)spark_properties : property_map option;Connection properties specific to the Spark compute environment.
*)connection_properties : connection_properties option;These key-value pairs define parameters for the connection when using the version 1 Connection schema:
HOST - The host URI: either the fully qualified domain name (FQDN) or the IPv4 address of the database host.PORT - The port number, between 1024 and 65535, of the port on which the database host is listening for database connections.USER_NAME - The name under which to log in to the database. The value string for USER_NAME is "USERNAME".PASSWORD - A password, if one is used, for the user name.ENCRYPTED_PASSWORD - When you enable connection password protection by setting ConnectionPasswordEncryption in the Data Catalog encryption settings, this field stores the encrypted password.JDBC_DRIVER_JAR_URI - The Amazon Simple Storage Service (Amazon S3) path of the JAR file that contains the JDBC driver to use.JDBC_DRIVER_CLASS_NAME - The class name of the JDBC driver to use.JDBC_ENGINE - The name of the JDBC engine to use.JDBC_ENGINE_VERSION - The version of the JDBC engine to use.CONFIG_FILES - (Reserved for future use.)INSTANCE_ID - The instance ID to use.JDBC_CONNECTION_URL - The URL for connecting to a JDBC data source.JDBC_ENFORCE_SSL - A case-insensitive Boolean string (true, false) specifying whether Secure Sockets Layer (SSL) with hostname matching is enforced for the JDBC connection on the client. The default is false.CUSTOM_JDBC_CERT - An Amazon S3 location specifying the customer's root certificate. Glue uses this root certificate to validate the customer’s certificate when connecting to the customer database. Glue only handles X.509 certificates. The certificate provided must be DER-encoded and supplied in Base64 encoding PEM format.SKIP_CUSTOM_JDBC_CERT_VALIDATION - By default, this is false. Glue validates the Signature algorithm and Subject Public Key Algorithm for the customer certificate. The only permitted algorithms for the Signature algorithm are SHA256withRSA, SHA384withRSA or SHA512withRSA. For the Subject Public Key Algorithm, the key length must be at least 2048. You can set the value of this property to true to skip Glue’s validation of the customer certificate.CUSTOM_JDBC_CERT_STRING - A custom JDBC certificate string which is used for domain match or distinguished name match to prevent a man-in-the-middle attack. In Oracle database, this is used as the SSL_SERVER_CERT_DN; in Microsoft SQL Server, this is used as the hostNameInCertificate.CONNECTION_URL - The URL for connecting to a general (non-JDBC) data source.SECRET_ID - The secret ID used for the secret manager of credentials.CONNECTOR_URL - The connector URL for a MARKETPLACE or CUSTOM connection.CONNECTOR_TYPE - The connector type for a MARKETPLACE or CUSTOM connection.CONNECTOR_CLASS_NAME - The connector class name for a MARKETPLACE or CUSTOM connection.KAFKA_BOOTSTRAP_SERVERS - A comma-separated list of host and port pairs that are the addresses of the Apache Kafka brokers in a Kafka cluster to which a Kafka client will connect to and bootstrap itself.KAFKA_SSL_ENABLED - Whether to enable or disable SSL on an Apache Kafka connection. Default value is "true".KAFKA_CUSTOM_CERT - The Amazon S3 URL for the private CA cert file (.pem format). The default is an empty string.KAFKA_SKIP_CUSTOM_CERT_VALIDATION - Whether to skip the validation of the CA cert file or not. Glue validates for three algorithms: SHA256withRSA, SHA384withRSA and SHA512withRSA. Default value is "false".KAFKA_CLIENT_KEYSTORE - The Amazon S3 location of the client keystore file for Kafka client side authentication (Optional).KAFKA_CLIENT_KEYSTORE_PASSWORD - The password to access the provided keystore (Optional).KAFKA_CLIENT_KEY_PASSWORD - A keystore can consist of multiple keys, so this is the password to access the client key to be used with the Kafka server side key (Optional).ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD - The encrypted version of the Kafka client keystore password (if the user has the Glue encrypt passwords setting selected).ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD - The encrypted version of the Kafka client key password (if the user has the Glue encrypt passwords setting selected).KAFKA_SASL_MECHANISM - "SCRAM-SHA-512", "GSSAPI", "AWS_MSK_IAM", or "PLAIN". These are the supported SASL Mechanisms.KAFKA_SASL_PLAIN_USERNAME - A plaintext username used to authenticate with the "PLAIN" mechanism.KAFKA_SASL_PLAIN_PASSWORD - A plaintext password used to authenticate with the "PLAIN" mechanism.ENCRYPTED_KAFKA_SASL_PLAIN_PASSWORD - The encrypted version of the Kafka SASL PLAIN password (if the user has the Glue encrypt passwords setting selected).KAFKA_SASL_SCRAM_USERNAME - A plaintext username used to authenticate with the "SCRAM-SHA-512" mechanism.KAFKA_SASL_SCRAM_PASSWORD - A plaintext password used to authenticate with the "SCRAM-SHA-512" mechanism.ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD - The encrypted version of the Kafka SASL SCRAM password (if the user has the Glue encrypt passwords setting selected).KAFKA_SASL_SCRAM_SECRETS_ARN - The Amazon Resource Name of a secret in Amazon Web Services Secrets Manager.KAFKA_SASL_GSSAPI_KEYTAB - The S3 location of a Kerberos keytab file. A keytab stores long-term keys for one or more principals. For more information, see MIT Kerberos Documentation: Keytab.KAFKA_SASL_GSSAPI_KRB5_CONF - The S3 location of a Kerberos krb5.conf file. A krb5.conf stores Kerberos configuration information, such as the location of the KDC server. For more information, see MIT Kerberos Documentation: krb5.conf.KAFKA_SASL_GSSAPI_SERVICE - The Kerberos service name, as set with sasl.kerberos.service.name in your Kafka Configuration.KAFKA_SASL_GSSAPI_PRINCIPAL - The name of the Kerberos princial used by Glue. For more information, see Kafka Documentation: Configuring Kafka Brokers.ROLE_ARN - The role to be used for running queries.REGION - The Amazon Web Services Region where queries will be run.WORKGROUP_NAME - The name of an Amazon Redshift serverless workgroup or Amazon Athena workgroup in which queries will run.CLUSTER_IDENTIFIER - The cluster identifier of an Amazon Redshift cluster in which queries will run.DATABASE - The Amazon Redshift database that you are connecting to.match_criteria : match_criteria option;A list of criteria that can be used in selecting this connection.
*)connection_type : connection_type option;The type of the connection. Currently, SFTP is not supported.
*)description : description_string option;The description of the connection.
*)name : name_string option;The name of the connection definition.
*)}Defines a connection to a data source.
type nonrec connection_list = connection listtype nonrec get_connections_response = {next_token : token option;A continuation token, if the list of connections returned does not include the last of the filtered connections.
*)connection_list : connection_list option;A list of requested connection definitions.
*)}type nonrec get_connections_filter = {connection_schema_version : connection_schema_version option;Denotes if the connection was created with schema version 1 or 2.
*)connection_type : connection_type option;The type of connections to return. Currently, SFTP is not supported.
*)match_criteria : match_criteria option;A criteria string that must match the criteria recorded in the connection definition for that connection definition to be returned.
*)}Filters the connection definitions that are returned by the GetConnections API operation.
type nonrec get_connections_request = {max_results : page_size option;The maximum number of connections to return in one response.
*)next_token : token option;A continuation token, if this is a continuation call.
*)hide_password : boolean_ option;Allows you to retrieve the connection metadata without returning the password. For instance, the Glue console uses this flag to retrieve the connection, and does not display the password. Set this parameter when the caller might not have permission to use the KMS key to decrypt the password, but it does have permission to access the rest of the connection properties.
*)filter : get_connections_filter option;A filter that controls which connections are returned.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the connections reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec get_connection_response = {connection : connection option;The requested connection definition.
*)}type nonrec get_connection_request = {apply_override_for_compute_environment : compute_environment option;For connections that may be used in multiple services, specifies returning properties for the specified compute environment.
*)hide_password : boolean_ option;Allows you to retrieve the connection metadata without returning the password. For instance, the Glue console uses this flag to retrieve the connection, and does not display the password. Set this parameter when the caller might not have permission to use the KMS key to decrypt the password, but it does have permission to access the rest of the connection properties.
*)name : name_string;The name of the connection definition to retrieve.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the connection resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec execution_attempt = {error_message : description_string option;An error message associated with the last column statistics task run.
*)execution_timestamp : timestamp option;A timestamp when the last column statistics task run occurred.
*)column_statistics_task_run_id : hash_string option;A task run ID for the last column statistics task run.
*)status : execution_status option;The status of the last column statistics task run.
*)}A run attempt for a column statistics task run.
type nonrec column_statistics_task_settings = {last_execution_attempt : execution_attempt option;The last ExecutionAttempt for the column statistics task run.
setting_source : setting_source option;The source of setting the column statistics task. Possible values may be CATALOG or TABLE.
schedule_type : schedule_type option;The type of schedule for a column statistics task. Possible values may be CRON or AUTO.
security_configuration : crawler_security_configuration option;Name of the security configuration that is used to encrypt CloudWatch logs.
*)sample_size : sample_size_percentage option;The percentage of data to sample.
*)role : role option;The role used for running the column statistics.
*)catalog_i_d : catalog_id_string option;The ID of the Data Catalog in which the database resides.
*)column_name_list : column_name_list option;A list of column names for which to run statistics.
*)schedule : schedule option;A schedule for running the column statistics, specified in CRON syntax.
*)table_name : table_name option;The name of the table for which to generate column statistics.
*)database_name : database_name option;The name of the database where the table resides.
*)}The settings for a column statistics task.
type nonrec get_column_statistics_task_settings_response = {column_statistics_task_settings : column_statistics_task_settings option;A ColumnStatisticsTaskSettings object representing the settings for the column statistics task.
}type nonrec get_column_statistics_task_settings_request = {table_name : name_string;The name of the table for which to retrieve column statistics.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec column_statistics_task_run = {dpu_seconds : non_negative_double option;The calculated DPU usage in seconds for all autoscaled workers.
*)error_message : description_string option;The error message for the job.
*)end_time : timestamp option;The end time of the task.
*)start_time : timestamp option;The start time of the task.
*)last_updated : timestamp option;The last point in time when this task was modified.
*)creation_time : timestamp option;The time that this task was created.
*)status : column_statistics_state option;The status of the task run.
*)computation_type : computation_type option;The type of column statistics computation.
*)worker_type : name_string option;The type of workers being used for generating stats. The default is g.1x.
number_of_workers : positive_integer option;The number of workers used to generate column statistics. The job is preconfigured to autoscale up to 25 instances.
*)security_configuration : crawler_security_configuration option;Name of the security configuration that is used to encrypt CloudWatch logs for the column stats task run.
*)sample_size : sample_size_percentage option;The percentage of rows used to generate statistics. If none is supplied, the entire table will be used to generate stats.
*)role : role option;The IAM role that the service assumes to generate statistics.
*)catalog_i_d : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is supplied, the Amazon Web Services account ID is used by default.
*)column_name_list : column_name_list option;A list of the column names. If none is supplied, all column names for the table will be used by default.
*)table_name : table_name option;The name of the table for which column statistics is generated.
*)database_name : database_name option;The database where the table resides.
*)column_statistics_task_run_id : hash_string option;The identifier for the particular column statistics task run.
*)customer_id : account_id option;The Amazon Web Services account ID.
*)}The object that shows the details of the column stats run.
type nonrec column_statistics_task_runs_list = column_statistics_task_run listtype nonrec get_column_statistics_task_runs_response = {next_token : token option;A continuation token, if not all task runs have yet been returned.
*)column_statistics_task_runs : column_statistics_task_runs_list option;A list of column statistics task runs.
*)}type nonrec get_column_statistics_task_runs_request = {next_token : token option;A continuation token, if this is a continuation call.
*)max_results : page_size option;The maximum size of the response.
*)table_name : name_string;The name of the table.
*)database_name : database_name;The name of the database where the table resides.
*)}type nonrec get_column_statistics_task_run_response = {column_statistics_task_run : column_statistics_task_run option;A ColumnStatisticsTaskRun object representing the details of the column stats run.
}type nonrec get_column_statistics_task_run_request = {column_statistics_task_run_id : hash_string;The identifier for the particular column statistics task run.
*)}type nonrec column_statistics_list = column_statistics listtype nonrec column_error = {error : error_detail option;An error message with the reason for the failure of an operation.
*)column_name : name_string option;The name of the column that failed.
*)}Encapsulates a column name that failed and the reason for failure.
type nonrec column_errors = column_error listtype nonrec get_column_statistics_for_table_response = {errors : column_errors option;List of ColumnStatistics that failed to be retrieved.
*)column_statistics_list : column_statistics_list option;List of ColumnStatistics.
*)}type nonrec get_column_names_list = name_string listtype nonrec get_column_statistics_for_table_request = {column_names : get_column_names_list;A list of the column names.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec get_column_statistics_for_partition_response = {errors : column_errors option;Error occurred during retrieving column statistics data.
*)column_statistics_list : column_statistics_list option;List of ColumnStatistics that failed to be retrieved.
*)}type nonrec get_column_statistics_for_partition_request = {column_names : get_column_names_list;A list of the column names.
*)partition_values : value_string_list;A list of partition values identifying the partition.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec csv_classifier = {serde : csv_serde_option option;Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are OpenCSVSerDe, LazySimpleSerDe, and None. You can specify the None value when you want the crawler to do the detection.
custom_datatypes : custom_datatypes option;A list of custom datatypes including "BINARY", "BOOLEAN", "DATE", "DECIMAL", "DOUBLE", "FLOAT", "INT", "LONG", "SHORT", "STRING", "TIMESTAMP".
*)custom_datatype_configured : nullable_boolean option;Enables the custom datatype to be configured.
*)allow_single_column : nullable_boolean option;Enables the processing of files that contain only one column.
*)disable_value_trimming : nullable_boolean option;Specifies not to trim values before identifying the type of column values. The default value is true.
header : csv_header option;A list of strings representing column names.
*)contains_header : csv_header_option option;Indicates whether the CSV file contains a header.
*)quote_symbol : csv_quote_symbol option;A custom symbol to denote what combines content into a single column value. It must be different from the column delimiter.
*)delimiter : csv_column_delimiter option;A custom symbol to denote what separates each column entry in the row.
*)version : version_id option;The version of this classifier.
*)last_updated : timestamp option;The time that this classifier was last updated.
*)creation_time : timestamp option;The time that this classifier was registered.
*)name : name_string;The name of the classifier.
*)}A classifier for custom CSV content.
type nonrec classifier = {csv_classifier : csv_classifier option;A classifier for comma-separated values (CSV).
*)json_classifier : json_classifier option;A classifier for JSON content.
*)xml_classifier : xml_classifier option;A classifier for XML content.
*)grok_classifier : grok_classifier option;A classifier that uses grok.
}Classifiers are triggered during a crawl task. A classifier checks whether a given file is in a format it can handle. If it is, the classifier creates a schema in the form of a StructType object that matches that data format.
You can use the standard classifiers that Glue provides, or you can write your own classifiers to best categorize your data sources and specify the appropriate schemas to use for them. A classifier can be a grok classifier, an XML classifier, a JSON classifier, or a custom CSV classifier, as specified in one of the fields in the Classifier object.
type nonrec classifier_list = classifier listtype nonrec get_classifiers_response = {next_token : token option;A continuation token.
*)classifiers : classifier_list option;The requested list of classifier objects.
*)}type nonrec get_classifier_response = {classifier : classifier option;The requested classifier.
*)}type nonrec data_lake_access_properties_output = {catalog_type : name_string option;Specifies a federated catalog type for the native catalog resource. The currently supported type is aws:redshift.
status_message : name_string option;A message that gives more detailed information about the managed workgroup status.
*)redshift_database_name : name_string option;The default Redshift database resource name in the managed compute.
*)managed_workgroup_status : name_string option;The managed Redshift Serverless compute status.
*)managed_workgroup_name : name_string option;The managed Redshift Serverless compute name that is created for your catalog resource.
*)kms_key : resource_arn_string option;An encryption key that will be used for the staging bucket that will be created along with the catalog.
*)data_transfer_role : iam_role_arn option;A role that will be assumed by Glue for transferring data into/out of the staging bucket during a query.
*)data_lake_access : boolean_ option;Turns on or off data lake access for Apache Spark applications that access Amazon Redshift databases in the Data Catalog.
*)}The output properties of the data lake access configuration for your catalog resource in the Glue Data Catalog.
type nonrec catalog_properties_output = {custom_properties : parameters_map option;Additional key-value properties for the catalog, such as column statistics optimizations.
*)iceberg_optimization_properties : iceberg_optimization_properties_output option;An IcebergOptimizationPropertiesOutput object that specifies Iceberg table optimization settings for the catalog, including configurations for compaction, retention, and orphan file deletion operations.
data_lake_access_properties : data_lake_access_properties_output option;A DataLakeAccessProperties object with input properties to configure data lake access for your catalog resource in the Glue Data Catalog.
}Property attributes that include configuration properties for the catalog resource.
type nonrec catalog = {allow_full_table_external_data_access : allow_full_table_external_data_access_enum
option;Allows third-party engines to access data in Amazon S3 locations that are registered with Lake Formation.
*)create_database_default_permissions : principal_permissions_list option;An array of PrincipalPermissions objects. Creates a set of default permissions on the database(s) for principals. Used by Amazon Web Services Lake Formation. Not used in the normal course of Glue operations.
create_table_default_permissions : principal_permissions_list option;An array of PrincipalPermissions objects. Creates a set of default permissions on the table(s) for principals. Used by Amazon Web Services Lake Formation. Not used in the normal course of Glue operations.
catalog_properties : catalog_properties_output option;A CatalogProperties object that specifies data lake access properties and other custom properties.
federated_catalog : federated_catalog option;A FederatedCatalog object that points to an entity outside the Glue Data Catalog.
target_redshift_catalog : target_redshift_catalog option;A TargetRedshiftCatalog object that describes a target catalog for database resource linking.
update_time : timestamp option;The time at which the catalog was last updated.
*)create_time : timestamp option;The time at which the catalog was created.
*)parameters : parameters_map option;A map array of key-value pairs that define parameters and properties of the catalog.
*)description : description_string option;Description string, not more than 2048 bytes long, matching the URI address multi-line string pattern. A description of the catalog.
*)resource_arn : resource_arn_string option;The Amazon Resource Name (ARN) assigned to the catalog resource.
*)name : catalog_name_string;The name of the catalog. Cannot be the same as the account ID.
*)catalog_id : catalog_id_string option;The ID of the catalog. To grant access to the default catalog, this field should not be provided.
*)}The catalog object represents a logical grouping of databases in the Glue Data Catalog or a federated source. You can now create a Redshift-federated catalog or a catalog containing resource links to Redshift databases in another account or region.
type nonrec catalog_list = catalog listtype nonrec get_catalogs_response = {next_token : token option;A continuation token for paginating the returned list of tokens, returned if the current segment of the list is not the last.
*)catalog_list : catalog_list;An array of Catalog objects. A list of Catalog objects from the specified parent catalog.
}type nonrec get_catalogs_request = {has_databases : nullable_boolean option;When true, the response only includes catalogs that can contain databases. Some catalogs are organizational containers that hold only other catalogs, not databases. When this parameter is set to true, those container-only catalogs are excluded, and only catalogs capable of containing databases are returned. Defaults to false.
include_root : nullable_boolean option;Whether to list the default catalog in the account and region in the response. Defaults to false. When true and ParentCatalogId = NULL | Amazon Web Services Account ID, all catalogs and the default catalog are enumerated in the response.
When the ParentCatalogId is not equal to null, and this attribute is passed as false or true, an InvalidInputException is thrown.
recursive : boolean_ option;Whether to list all catalogs across the catalog hierarchy, starting from the ParentCatalogId. Defaults to false . When true, all catalog objects in the ParentCatalogID hierarchy are enumerated in the response.
max_results : page_size option;The maximum number of catalogs to return in one response.
*)next_token : token option;A continuation token, if this is a continuation call.
*)parent_catalog_id : catalog_id_string option;The ID of the parent catalog in which the catalog resides. If none is provided, the Amazon Web Services Account Number is used by default.
*)}type nonrec get_catalog_response = {catalog : catalog option;A Catalog object. The definition of the specified catalog in the Glue Data Catalog.
}type nonrec get_catalog_request = {catalog_id : catalog_id_string;The ID of the parent catalog in which the catalog resides. If none is provided, the Amazon Web Services Account Number is used by default.
*)}type nonrec catalog_import_status = {imported_by : name_string option;The name of the person who initiated the migration.
*)import_time : timestamp option;The time that the migration was started.
*)import_completed : boolean_ option;True if the migration has completed, or False otherwise.
}A structure containing migration status information.
type nonrec get_catalog_import_status_response = {import_status : catalog_import_status option;The status of the specified catalog migration.
*)}type nonrec get_catalog_import_status_request = {catalog_id : catalog_id_string option;The ID of the catalog to migrate. Currently, this should be the Amazon Web Services account ID.
*)}type nonrec blueprint_run = {role_arn : orchestration_iam_role_arn option;The role ARN. This role will be assumed by the Glue service and will be used to create the workflow and other entities of a workflow.
*)parameters : blueprint_parameters option;The blueprint parameters as a string. You will have to provide a value for each key that is required from the parameter spec that is defined in the Blueprint$ParameterSpec.
rollback_error_message : message_string option;If there are any errors while creating the entities of a workflow, we try to roll back the created entities until that point and delete them. This attribute indicates the errors seen while trying to delete the entities that are created.
*)error_message : message_string option;Indicates any errors that are seen while running the blueprint.
*)completed_on : timestamp_value option;The date and time that the blueprint run completed.
*)started_on : timestamp_value option;The date and time that the blueprint run started.
*)state : blueprint_run_state option;The state of the blueprint run. Possible values are:
workflow_name : name_string option;The name of a workflow that is created as a result of a successful blueprint run. If a blueprint run has an error, there will not be a workflow created.
*)run_id : id_string option;The run ID for this blueprint run.
*)blueprint_name : orchestration_name_string option;The name of the blueprint.
*)}The details of a blueprint run.
type nonrec blueprint_runs = blueprint_run listtype nonrec get_blueprint_runs_response = {next_token : generic_string option;A continuation token, if not all blueprint runs have been returned.
*)blueprint_runs : blueprint_runs option;Returns a list of BlueprintRun objects.
}type nonrec get_blueprint_runs_request = {max_results : page_size option;The maximum size of a list to return.
*)next_token : generic_string option;A continuation token, if this is a continuation request.
*)blueprint_name : name_string;The name of the blueprint.
*)}type nonrec get_blueprint_run_response = {blueprint_run : blueprint_run option;Returns a BlueprintRun object.
}type nonrec get_blueprint_run_request = {run_id : id_string;The run ID for the blueprint run you want to retrieve.
*)blueprint_name : orchestration_name_string;The name of the blueprint.
*)}type nonrec blueprint = {last_active_definition : last_active_definition option;When there are multiple versions of a blueprint and the latest version has some errors, this attribute indicates the last successful blueprint definition that is available with the service.
*)error_message : error_string option;An error message.
*)status : blueprint_status option;The status of the blueprint registration.
blueprint_service_location : generic_string option;Specifies a path in Amazon S3 where the blueprint is copied when you call CreateBlueprint/UpdateBlueprint to register the blueprint in Glue.
blueprint_location : generic_string option;Specifies the path in Amazon S3 where the blueprint is published.
*)parameter_spec : blueprint_parameter_spec option;A JSON string that indicates the list of parameter specifications for the blueprint.
*)last_modified_on : timestamp_value option;The date and time the blueprint was last modified.
*)created_on : timestamp_value option;The date and time the blueprint was registered.
*)description : generic512_char_string option;The description of the blueprint.
*)name : orchestration_name_string option;The name of the blueprint.
*)}The details of a blueprint.
type nonrec get_blueprint_request = {include_parameter_spec : nullable_boolean option;Specifies whether or not to include the parameter specification.
*)include_blueprint : nullable_boolean option;Specifies whether or not to include the blueprint in the response.
*)name : name_string;The name of the blueprint.
*)}type nonrec get_asset_type_response = {forms : asset_type_forms_map option;The forms that make up the asset type, keyed by form name.
*)name : asset_type_name option;The name of the asset type.
*)id : asset_type_id option;The identifier of the asset type.
*)}type nonrec get_asset_type_request = {identifier : asset_type_id;The identifier of the asset type to retrieve.
*)}type nonrec get_asset_output = {iterable_forms : iterable_form_map option;The iterable forms available on the asset, keyed by form name (for example, columns). Use the form name with ListIterableForms or BatchGetIterableForms to retrieve the form's items.
attachments : asset_form_map option;Additional attachments on the asset for more context, keyed by attachment name.
*)forms : asset_form_map option;The forms on the asset, keyed by form name.
*)glossary_terms : glossary_term_id_list option;The identifiers of the glossary terms associated with the asset.
*)asset_type_id : asset_type_id;The identifier of the asset type for this asset.
*)updated_at : updated_at option;The timestamp at which the asset was last updated.
*)created_at : created_at option;The timestamp at which the asset was created.
*)description : asset_description option;The description of the asset.
*)name : asset_name option;The name of the asset.
*)id : asset_id;The unique identifier of the asset.
*)}type nonrec get_asset_input = {identifier : asset_id;The unique identifier of the asset to retrieve.
*)}type nonrec field_filter_operators_list = field_filter_operator listtype nonrec field = {custom_properties : custom_properties option;Optional map of keys which may be returned.
*)native_data_type : Smaws_Lib.Smithy_api.Types.string_ option;The data type returned by the SaaS API, such as “picklist” or “textarea” from Salesforce.
*)parent_field : Smaws_Lib.Smithy_api.Types.string_ option;A parent field name for a nested field.
*)supported_filter_operators : field_filter_operators_list option;Indicates the support filter operators for this field.
*)supported_values : list_of_string option;A list of supported values for the field.
*)is_default_on_create : bool_ option;Indicates whether this field is populated automatically when the object is created, such as a created at timestamp.
*)is_upsertable : bool_ option;Indicates whether this field can be upserted as part of a destination write.
*)is_updateable : bool_ option;Indicates whether this field can be updated as part of a destination write.
*)is_createable : bool_ option;Indicates whether this field can be created as part of a destination write.
*)is_partitionable : bool_ option;Indicates whether a given field can be used in partitioning the query made to SaaS.
*)is_filterable : bool_ option;Indicates whether this field can used in a filter clause (WHERE clause) of a SQL statement when querying data.
is_retrievable : bool_ option;Indicates whether this field can be added in Select clause of SQL query or whether it is retrievable or not.
*)is_nullable : bool_ option;Indicates whether this field can be nullable or not.
*)is_primary_key : bool_ option;Indicates whether this field can used as a primary key for the given entity.
*)field_type : field_data_type option;The type of data in the field.
*)description : field_description option;A description of the field.
*)label : field_label option;A readable label used for the field.
*)field_name : entity_field_name option;A unique identifier for the field.
*)}The Field object has information about the different properties associated with a field in the connector.
type nonrec fields_list = field listtype nonrec federated_resource_already_exists_exception = {associated_glue_resource : glue_resource_arn option;The associated Glue resource already exists.
*)message : message_string option;The message describing the problem.
*)}A federated resource already exists.
type nonrec error_by_name = (name_string * error_detail) listtype nonrec disassociate_glossary_terms_response = {glossary_terms : glossary_term_id_list option;The remaining glossary terms associated with the asset.
*)asset_identifier : asset_id option;The unique identifier of the asset.
*)}type nonrec disassociate_glossary_terms_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)glossary_term_identifiers : glossary_term_id_list;The list of glossary term identifiers to disassociate from the asset.
*)asset_identifier : asset_id;The unique identifier of the asset to disassociate glossary terms from.
*)}type nonrec dev_endpoint_names = generic_string listtype nonrec describe_integrations_response = {marker : string128 option;A value that indicates the starting point for the next set of response records in a subsequent request.
*)integrations : integrations_list option;A list of zero-ETL integrations.
*)}type nonrec describe_integrations_request = {filters : integration_filter_list option;A list of key and values, to filter down the results. Supported keys are "Status", "IntegrationName", and "SourceArn". IntegrationName is limited to only one value.
*)max_records : integration_integer option;The total number of items to return in the output.
*)marker : string128 option;A value that indicates the starting point for the next set of response records in a subsequent request.
*)integration_identifier : string128 option;The Amazon Resource Name (ARN) for the integration.
*)}type nonrec describe_inbound_integrations_response = {marker : string128 option;A value that indicates the starting point for the next set of response records in a subsequent request.
*)inbound_integrations : inbound_integrations_list option;A list of inbound integrations.
*)}type nonrec describe_inbound_integrations_request = {target_arn : string512 option;The Amazon Resource Name (ARN) of the target resource in the integration.
*)max_records : integration_integer option;The total number of items to return in the output.
*)marker : string128 option;A token to specify where to start paginating. This is the marker from a previously truncated response.
*)integration_arn : string128 option;The Amazon Resource Name (ARN) of the integration.
*)}type nonrec describe_entity_response = {next_token : next_token option;A continuation token, present if the current segment is not the last.
*)fields : fields_list option;Describes the fields for that connector entity. This is the list of Field objects. Field is very similar to column in a database. The Field object has information about different properties associated with fields in the connector.
}type nonrec describe_entity_request = {data_store_api_version : api_version option;The version of the API used for the data store.
*)next_token : next_token option;A continuation token, included if this is a continuation call.
*)entity_name : entity_name;The name of the entity that you want to describe from the connection type.
*)catalog_id : catalog_id_string option;The catalog ID of the catalog that contains the connection. This can be null, By default, the Amazon Web Services Account ID is the catalog ID.
*)connection_name : name_string;The name of the connection that contains the connection type credentials.
*)}type nonrec auth_configuration = {custom_authentication_properties : properties_map option;A map of key-value pairs for the custom authentication properties. Each value is a a Property object.
basic_authentication_properties : properties_map option;A map of key-value pairs for the OAuth2 properties. Each value is a a Property object.
o_auth2_properties : properties_map option;A map of key-value pairs for the OAuth2 properties. Each value is a a Property object.
secret_arn : property option;The Amazon Resource Name (ARN) for the Secrets Manager.
*)authentication_type : property;The type of authentication for a connection.
*)}The authentication configuration for a connection returned by the DescribeConnectionType API.
type nonrec compute_environment_configuration = {physical_connection_properties_required : bool_ option;Indicates whether PhysicalConnectionProperties are required for the compute environment.
connection_properties_required_overrides : list_of_string;The connection properties that are required as overrides for the compute environment.
*)connection_option_name_overrides : property_name_overrides;The connection option name overrides for the compute environment.
*)connection_property_name_overrides : property_name_overrides;The connection property name overrides for the compute environment.
*)connection_options : properties_map;The parameters used as connection options for the compute environment.
*)supported_authentication_types : authentication_types;The supported authentication types for the compute environment.
*)compute_environment : compute_environment;The type of compute environment.
*)description : compute_environment_configuration_description_string;A description of the compute environment.
*)name : compute_environment_name;A name for the compute environment configuration.
*)}An object containing configuration for a compute environment (such as Spark, Python or Athena) returned by the DescribeConnectionType API.
type nonrec compute_environment_configuration_map =
(compute_environment_name * compute_environment_configuration) listtype nonrec describe_connection_type_response = {rest_configuration : rest_configuration option;HTTP request and response configuration, validation endpoint, and entity configurations for REST based data source.
*)spark_connection_properties : properties_map option;Connection properties specific to the Spark compute environment.
*)python_connection_properties : properties_map option;Connection properties specific to the Python compute environment.
*)athena_connection_properties : properties_map option;Connection properties specific to the Athena compute environment.
*)physical_connection_requirements : properties_map option;Physical requirements for a connection, such as VPC, Subnet and Security Group specifications.
*)compute_environment_configurations : compute_environment_configuration_map
option;The compute environments that are supported by the connection.
*)authentication_configuration : auth_configuration option;The type of authentication used for the connection.
*)connection_options : properties_map option;Returns properties that can be set when creating a connection in the ConnectionInput.ConnectionProperties. ConnectionOptions defines parameters that can be set in a Spark ETL script in the connection options map passed to a dataframe.
connection_properties : properties_map option;Connection properties which are common across compute environments.
*)capabilities : capabilities option;The supported authentication types, data interface types (compute environments), and data operations of the connector.
*)description : description option;A description of the connection type.
*)connection_type : name_string option;The name of the connection type.
*)}type nonrec describe_connection_type_request = {connection_type : name_string;The name of the connection type to be described.
*)}type nonrec delete_workflow_response = {name : name_string option;Name of the workflow specified in input.
*)}type nonrec delete_user_defined_function_request = {function_name : name_string;The name of the function definition to be deleted.
*)database_name : name_string;The name of the catalog database where the function is located.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the function to be deleted is located. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_usage_profile_request = {name : name_string;The name of the usage profile to delete.
*)}type nonrec delete_trigger_response = {name : name_string option;The name of the trigger that was deleted.
*)}type nonrec delete_table_version_request = {version_id : version_string;The ID of the table version to be deleted. A VersionID is a string representation of an integer. Each version is incremented by 1.
table_name : name_string;The name of the table. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The database in the catalog in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the tables reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_table_request = {transaction_id : transaction_id_string option;The transaction ID at which to delete the table contents.
*)name : name_string;The name of the table to be deleted. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The name of the catalog database in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_table_optimizer_request = {type_ : table_optimizer_type;The type of table optimizer.
*)table_name : name_string;The name of the table.
*)database_name : name_string;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string;The Catalog ID of the table.
*)}type nonrec delete_session_response = {id : name_string option;Returns the ID of the deleted session.
*)}type nonrec delete_session_request = {request_origin : orchestration_name_string option;The name of the origin of the delete session request.
*)id : name_string;The ID of the session to be deleted.
*)}type nonrec delete_security_configuration_request = {name : name_string;The name of the security configuration to delete.
*)}type nonrec delete_schema_versions_response = {schema_version_errors : schema_version_error_list option;A list of SchemaVersionErrorItem objects, each containing an error and schema version.
}type nonrec delete_schema_versions_input = {versions : versions_string;A version range may be supplied which may be of the format:
schema_id : schema_id;This is a wrapper structure that may contain the schema name and Amazon Resource Name (ARN).
*)}type nonrec delete_schema_response = {status : schema_status option;The status of the schema.
*)schema_name : schema_registry_name_string option;The name of the schema being deleted.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema being deleted.
*)}type nonrec delete_schema_input = {schema_id : schema_id;This is a wrapper structure that may contain the schema name and Amazon Resource Name (ARN).
*)}type nonrec delete_resource_policy_request = {resource_arn : glue_resource_arn option;The ARN of the Glue resource for the resource policy to be deleted.
*)policy_hash_condition : hash_string option;The hash value returned when this policy was set.
*)}type nonrec delete_registry_response = {status : registry_status option;The status of the registry. A successful operation will return the Deleting status.
registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registry being deleted.
*)registry_name : schema_registry_name_string option;The name of the registry being deleted.
*)}type nonrec delete_registry_input = {registry_id : registry_id;This is a wrapper structure that may contain the registry name and Amazon Resource Name (ARN).
*)}type nonrec delete_partition_request = {partition_values : value_string_list;The values that define the partition.
*)table_name : name_string;The name of the table that contains the partition to be deleted.
*)database_name : name_string;The name of the catalog database in which the table in question resides.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partition to be deleted resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_partition_index_request = {index_name : name_string;The name of the partition index to be deleted.
*)table_name : name_string;Specifies the name of a table from which you want to delete a partition index.
*)database_name : name_string;Specifies the name of a database from which you want to delete a partition index.
*)catalog_id : catalog_id_string option;The catalog ID where the table resides.
*)}type nonrec delete_ml_transform_response = {transform_id : hash_string option;The unique identifier of the transform that was deleted.
*)}type nonrec delete_ml_transform_request = {transform_id : hash_string;The unique identifier of the transform to delete.
*)}type nonrec delete_job_response = {job_name : name_string option;The name of the job definition that was deleted.
*)}type nonrec delete_job_request = {job_name : name_string;The name of the job definition to delete.
*)}type nonrec delete_integration_response = {data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)errors : integration_error_list option;A list of errors associated with the integration.
*)create_time : integration_timestamp;The time when the integration was created, in UTC.
*)status : integration_status;The status of the integration being deleted.
The possible statuses are:
additional_encryption_context : integration_additional_encryption_context_map
option;An optional set of non-secret key–value pairs that contains additional contextual information for encryption.
*)kms_key_id : string2048 option;The ARN of a KMS key used for encrypting the channel.
*)integration_arn : string128;The Amazon Resource Name (ARN) for the integration.
*)description : integration_description option;A description of the integration.
*)integration_name : string128;A unique name for an integration in Glue.
*)target_arn : string512;The ARN of the target for the integration.
*)source_arn : string512;The ARN of the source for the integration.
*)}type nonrec delete_integration_resource_property_request = {resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec delete_integration_request = {integration_identifier : string128;The Amazon Resource Name (ARN) for the integration.
*)}type nonrec delete_glossary_term_request = {identifier : glossary_term_id;The unique identifier of the glossary term to delete.
*)}type nonrec delete_glossary_request = {identifier : glossary_id;The unique identifier of the glossary to delete.
*)}type nonrec delete_form_type_request = {identifier : form_type_id;The identifier of the form type to delete.
*)}type nonrec delete_dev_endpoint_request = {endpoint_name : generic_string;The name of the DevEndpoint.
}type nonrec delete_database_request = {name : name_string;The name of the database to delete. For Hive compatibility, this must be all lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the database resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_data_quality_ruleset_request = {name : name_string;A name for the data quality ruleset.
*)}type nonrec delete_custom_entity_type_response = {name : name_string option;The name of the custom pattern you deleted.
*)}type nonrec delete_custom_entity_type_request = {name : name_string;The name of the custom pattern that you want to delete.
*)}type nonrec delete_connection_type_request = {connection_type : name_string;The name of the connection type to delete. Must reference an existing registered connection type.
*)}type nonrec delete_connection_request = {connection_name : name_string;The name of the connection to delete.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the connection resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_connection_name_list = name_string listtype nonrec delete_column_statistics_task_settings_request = {table_name : name_string;The name of the table for which to delete column statistics.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec delete_column_statistics_for_table_request = {column_name : name_string;The name of the column.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_column_statistics_for_partition_request = {column_name : name_string;Name of the column.
*)partition_values : value_string_list;A list of partition values identifying the partition.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec delete_blueprint_response = {name : name_string option;Returns the name of the blueprint that was deleted.
*)}type nonrec delete_blueprint_request = {name : name_string;The name of the blueprint to delete.
*)}type nonrec delete_attachment_response = {asset_identifier : asset_id option;The unique identifier of the asset.
*)}type nonrec delete_attachment_request = {attachment_name : attachment_name;The name of the attachment to delete.
*)item_identifier : item_identifier option;The identifier of the item within the iterable form. Required when iterableFormName is specified.
iterable_form_name : iterable_form_name option;The name of the iterable form. When specified along with itemIdentifier, the attachment is deleted from an item within the iterable form rather than from the asset itself.
asset_identifier : asset_id;The unique identifier of the asset from which to delete the attachment.
*)}type nonrec delete_asset_type_request = {identifier : asset_type_id;The identifier of the asset type to delete.
*)}type nonrec delete_asset_request = {identifier : asset_id;The unique identifier of the asset to delete.
*)}type nonrec data_quality_result = {aggregated_metrics : data_quality_aggregated_metrics option;A summary of DataQualityAggregatedMetrics objects showing the total counts of processed rows and rules, including their pass/fail statistics based on row-level results.
observations : data_quality_observations option;A list of DataQualityObservation objects representing the observations generated after evaluating the rules and analyzers.
analyzer_results : data_quality_analyzer_results option;A list of DataQualityAnalyzerResult objects representing the results for each analyzer.
rule_results : data_quality_rule_results option;A list of DataQualityRuleResult objects representing the results for each rule.
ruleset_evaluation_run_id : hash_string option;The unique run ID for the ruleset evaluation for this data quality result.
*)job_run_id : hash_string option;The job run ID associated with the data quality result, if any.
*)job_name : name_string option;The job name associated with the data quality result, if any.
*)completed_on : timestamp option;The date and time when this data quality run completed.
*)started_on : timestamp option;The date and time when this data quality run started.
*)evaluation_context : generic_string option;In the context of a job in Glue Studio, each node in the canvas is typically assigned some sort of name and data quality nodes will have names. In the case of multiple nodes, the evaluationContext can differentiate the nodes.
ruleset_name : name_string option;The name of the ruleset associated with the data quality result.
*)data_source : data_source option;The table associated with the data quality result, if any.
*)score : generic_bounded_double option;An aggregate data quality score. Represents the ratio of rules that passed to the total number of rules.
*)profile_id : hash_string option;The Profile ID for the data quality result.
*)result_id : hash_string option;A unique result ID for the data quality result.
*)}Describes a data quality result.
type nonrec data_quality_results_list = data_quality_result listtype nonrec data_quality_result_ids = hash_string listtype nonrec custom_entity_type_names = name_string listtype nonrec create_xml_classifier_request = {row_tag : row_tag option;The XML tag designating the element that contains each record in an XML document being parsed. This can't identify a self-closing element (closed by />). An empty row element that contains only attributes can be parsed as long as it ends with a closing tag (for example,
is okay, but
is not).
*)name : name_string;The name of the classifier.
*)classification : classification;An identifier of the data format that the classifier matches.
*)}Specifies an XML classifier for CreateClassifier to create.
type nonrec create_workflow_response = {name : name_string option;The name of the workflow which was provided as part of the request.
*)}type nonrec create_workflow_request = {max_concurrent_runs : nullable_integer option;You can use this parameter to prevent unwanted multiple updates to data, to control costs, or in some cases, to prevent exceeding the maximum number of concurrent runs of any of the component jobs. If you leave this parameter blank, there is no limit to the number of concurrent workflow runs.
*)default_run_properties : workflow_run_properties option;A collection of properties to be used as part of each execution of the workflow.
Run properties may be logged. Do not pass plaintext secrets as properties. Retrieve secrets from a Glue Connection, Amazon Web Services Secrets Manager or other secret management mechanism if you intend to use them within the workflow run.
*)description : workflow_description_string option;A description of the workflow.
*)name : name_string;The name to be assigned to the workflow. It should be unique within your account.
*)}type nonrec create_user_defined_function_request = {function_input : user_defined_function_input;A FunctionInput object that defines the function to create in the Data Catalog.
database_name : name_string;The name of the catalog database in which to create the function.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which to create the function. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec create_usage_profile_response = {name : name_string option;The name of the usage profile that was created.
*)}type nonrec create_usage_profile_request = {configuration : profile_configuration;A ProfileConfiguration object specifying the job and session values for the profile.
description : description_string option;A description of the usage profile.
*)name : name_string;The name of the usage profile.
*)}type nonrec create_trigger_request = {event_batching_condition : event_batching_condition option;Batch condition that must be met (specified number of events received or batch time window expired) before EventBridge event trigger fires.
*)start_on_creation : boolean_value option;Set to true to start SCHEDULED and CONDITIONAL triggers when created. True is not supported for ON_DEMAND triggers.
description : description_string option;A description of the new trigger.
*)actions : action_list;The actions initiated by this trigger when it fires.
*)predicate : predicate option;A predicate to specify when the new trigger should fire.
This field is required when the trigger type is CONDITIONAL.
schedule : generic_string option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
This field is required when the trigger type is SCHEDULED.
*)type_ : trigger_type;The type of the new trigger.
*)workflow_name : name_string option;The name of the workflow associated with the trigger.
*)name : name_string;The name of the trigger.
*)}type nonrec create_table_request = {open_table_format_input : open_table_format_input option;Specifies an OpenTableFormatInput structure when creating an open format table.
transaction_id : transaction_id_string option;The ID of the transaction.
*)partition_indexes : partition_index_list option;A list of partition indexes, PartitionIndex structures, to create in the table.
table_input : table_input option;The TableInput object that defines the metadata table to create in the catalog.
name : name_string option;The unique identifier for the table within the specified database that will be created in the Glue Data Catalog.
*)database_name : name_string;The catalog database in which to create the new table. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which to create the Table. If none is supplied, the Amazon Web Services account ID is used by default.
}type nonrec create_table_optimizer_request = {table_optimizer_configuration : table_optimizer_configuration;A TableOptimizerConfiguration object representing the configuration of a table optimizer.
type_ : table_optimizer_type;The type of table optimizer.
*)table_name : name_string;The name of the table.
*)database_name : name_string;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string;The Catalog ID of the table.
*)}type nonrec create_session_response = {session : session option;Returns the session object in the response.
*)}type nonrec create_session_request = {session_type : session_type option;The type of session to create.
*)request_origin : orchestration_name_string option;The origin of the request.
*)glue_version : glue_version_string option;The Glue version determines the versions of Apache Spark and Python that Glue supports. The GlueVersion must be greater than 2.0.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with the session
*)worker_type : worker_type option;The type of predefined worker that is allocated when a job runs. Accepts a value of G.1X, G.2X, G.4X, or G.8X for Spark jobs. Accepts the value Z.2X for Ray notebooks.
G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm).G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.number_of_workers : nullable_integer option;The number of workers of a defined WorkerType to use for the session.
max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that can be allocated when the job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB memory.
*)connections : connections_list option;The number of connections to use for the session.
*)default_arguments : orchestration_arguments_map option;A map array of key-value pairs. Max is 75 pairs.
*)idle_timeout : timeout option;The number of minutes when idle before session times out. Default for Spark ETL jobs is value of Timeout. Consult the documentation for other job types.
*)timeout : timeout option;The number of minutes before session times out. Default for Spark ETL jobs is 48 hours (2880 minutes). Consult the documentation for other job types.
*)command : session_command;The SessionCommand that runs the job.
role : orchestration_role_arn;The IAM Role ARN
*)description : description_string option;The description of the session.
*)id : name_string;The ID of the session request.
*)}Request to create a new session.
type nonrec create_security_configuration_response = {created_timestamp : timestamp_value option;The time at which the new security configuration was created.
*)name : name_string option;The name assigned to the new security configuration.
*)}type nonrec create_security_configuration_request = {encryption_configuration : encryption_configuration;The encryption configuration for the new security configuration.
*)name : name_string;The name for the new security configuration.
*)}type nonrec create_script_response = {scala_code : scala_code option;The Scala code generated from the DAG.
*)python_script : python_script option;The Python script generated from the DAG.
*)}type nonrec create_schema_response = {schema_version_status : schema_version_status option;The status of the first schema version created.
*)schema_version_id : schema_version_id_string option;The unique identifier of the first schema version.
*)schema_status : schema_status option;The status of the schema.
*)next_schema_version : version_long_number option;The next version of the schema associated with the returned schema definition.
*)latest_schema_version : version_long_number option;The latest version of the schema associated with the returned schema definition.
*)schema_checkpoint : schema_checkpoint_number option;The version number of the checkpoint (the last time the compatibility mode was changed).
*)compatibility : compatibility option;The schema compatibility mode.
*)data_format : data_format option;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
description : description_string option;A description of the schema if specified when created.
*)schema_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the schema.
*)schema_name : schema_registry_name_string option;The name of the schema.
*)registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the registry.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)}type nonrec create_schema_input = {schema_definition : schema_definition_string option;The schema definition using the DataFormat setting for SchemaName.
description : description_string option;An optional description of the schema. If description is not provided, there will not be any automatic default value for this.
*)compatibility : compatibility option;The compatibility mode of the schema. The possible values are:
data_format : data_format;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
schema_name : schema_registry_name_string;Name of the schema to be created of max length of 255, and may only contain letters, numbers, hyphen, underscore, dollar sign, or hash mark. No whitespace.
*)registry_id : registry_id option;This is a wrapper shape to contain the registry identity fields. If this is not provided, the default registry will be used. The ARN format for the same will be:
arn:aws:glue:us-east-2::registry/default-registry:random-5-letter-id.
*)}type nonrec create_registry_response = {description : description_string option;A description of the registry.
*)registry_name : schema_registry_name_string option;The name of the registry.
*)registry_arn : glue_resource_arn option;The Amazon Resource Name (ARN) of the newly created registry.
*)}type nonrec create_registry_input = {description : description_string option;A description of the registry. If description is not provided, there will not be any default value for this.
*)registry_name : schema_registry_name_string;Name of the registry to be created of max length of 255, and may only contain letters, numbers, hyphen, underscore, dollar sign, or hash mark. No whitespace.
*)}type nonrec create_partition_request = {partition_input : partition_input;A PartitionInput structure defining the partition to be created.
table_name : name_string;The name of the metadata table in which the partition is to be created.
*)database_name : name_string;The name of the metadata database in which the partition is to be created.
*)catalog_id : catalog_id_string option;The Amazon Web Services account ID of the catalog in which the partition is to be created.
*)}type nonrec create_partition_index_request = {partition_index : partition_index;Specifies a PartitionIndex structure to create a partition index in an existing table.
table_name : name_string;Specifies the name of a table in which you want to create a partition index.
*)database_name : name_string;Specifies the name of a database in which you want to create a partition index.
*)catalog_id : catalog_id_string option;The catalog ID where the table resides.
*)}type nonrec create_ml_transform_response = {transform_id : hash_string option;A unique identifier that is generated for the transform.
*)}type nonrec create_ml_transform_request = {transform_encryption : transform_encryption option;The encryption-at-rest settings of the transform that apply to accessing user data. Machine learning transforms can access user data encrypted in Amazon S3 using KMS.
*)max_retries : nullable_integer option;The maximum number of times to retry a task for this transform after a task run fails.
*)timeout : timeout option;The timeout of the task run for this transform in minutes. This is the maximum time that a task run for this transform can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when this task runs.
If WorkerType is set, then NumberOfWorkers is required (and vice versa).
worker_type : worker_type option;The type of predefined worker that is allocated when this task runs. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker provides 4 vCPU, 16 GB of memory and a 64GB disk, and 1 executor per worker.G.2X worker type, each worker provides 8 vCPU, 32 GB of memory and a 128GB disk, and 1 executor per worker.MaxCapacity is a mutually exclusive option with NumberOfWorkers and WorkerType.
NumberOfWorkers or WorkerType is set, then MaxCapacity cannot be set.MaxCapacity is set then neither NumberOfWorkers or WorkerType can be set.WorkerType is set, then NumberOfWorkers is required (and vice versa).MaxCapacity and NumberOfWorkers must both be at least 1.max_capacity : nullable_double option;The number of Glue data processing units (DPUs) that are allocated to task runs for this transform. You can allocate from 2 to 100 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
MaxCapacity is a mutually exclusive option with NumberOfWorkers and WorkerType.
NumberOfWorkers or WorkerType is set, then MaxCapacity cannot be set.MaxCapacity is set then neither NumberOfWorkers or WorkerType can be set.WorkerType is set, then NumberOfWorkers is required (and vice versa).MaxCapacity and NumberOfWorkers must both be at least 1.When the WorkerType field is set to a value other than Standard, the MaxCapacity field is set automatically and becomes read-only.
When the WorkerType field is set to a value other than Standard, the MaxCapacity field is set automatically and becomes read-only.
glue_version : glue_version_string option;This value determines which version of Glue this machine learning transform is compatible with. Glue 1.0 is recommended for most customers. If the value is not set, the Glue compatibility defaults to Glue 0.9. For more information, see Glue Versions in the developer guide.
*)role : role_string;The name or Amazon Resource Name (ARN) of the IAM role with the required permissions. The required permissions include both Glue service role permissions to Glue resources, and Amazon S3 permissions required by the transform.
parameters : transform_parameters;The algorithmic parameters that are specific to the transform type used. Conditionally dependent on the transform type.
*)input_record_tables : glue_tables;A list of Glue table definitions used by the transform.
*)description : description_string option;A description of the machine learning transform that is being defined. The default is an empty string.
*)name : name_string;The unique name that you give the transform when you create it.
*)}type nonrec create_json_classifier_request = {json_path : json_path;A JsonPath string defining the JSON data for the classifier to classify. Glue supports a subset of JsonPath, as described in Writing JsonPath Custom Classifiers.
name : name_string;The name of the classifier.
*)}Specifies a JSON classifier for CreateClassifier to create.
type nonrec create_job_response = {name : name_string option;The unique name that was provided for this job definition.
*)}type nonrec create_job_request = {maintenance_window : maintenance_window option;This field specifies a day of the week and hour for a maintenance window for streaming jobs. Glue periodically performs maintenance activities. During these maintenance windows, Glue will need to restart your streaming jobs.
Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.
*)source_control_details : source_control_details option;The details for a source control configuration for a job, allowing synchronization of job artifacts to or from a remote repository.
*)execution_class : execution_class option;Indicates whether the job is run with a standard or flexible execution class. The standard execution-class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.
The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.
Only jobs with Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX. The flexible execution class is available for Spark jobs.
code_gen_configuration_nodes : code_gen_configuration_nodes option;The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.
*)worker_type : worker_type option;The type of predefined worker that is allocated when a job runs. Accepts a value of G.1X, G.2X, G.4X, G.8X or G.025X for Spark jobs. Accepts the value Z.2X for Ray jobs.
G.1X worker type, each worker maps to 1 DPU (4 vCPUs, 16 GB of memory) with 94GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPUs, 32 GB of memory) with 138GB disk, and provides 1 executor per worker. We recommend this worker type for workloads such as data transforms, joins, and queries, to offers a scalable and cost effective way to run most jobs.G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs in the following Amazon Web Services Regions: US East (Ohio), US East (N. Virginia), US West (N. California), US West (Oregon), Asia Pacific (Mumbai), Asia Pacific (Seoul), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), Europe (London), Europe (Spain), Europe (Stockholm), and South America (São Paulo).G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk, and provides 1 executor per worker. We recommend this worker type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries. This worker type is available only for Glue version 3.0 or later Spark ETL jobs, in the same Amazon Web Services Regions as supported for the G.4X worker type.G.025X worker type, each worker maps to 0.25 DPU (2 vCPUs, 4 GB of memory) with 84GB disk, and provides 1 executor per worker. We recommend this worker type for low volume streaming jobs. This worker type is only available for Glue version 3.0 or later streaming jobs.Z.2X worker type, each worker maps to 2 M-DPU (8vCPUs, 64 GB of memory) with 128 GB disk, and provides up to 8 Ray workers based on the autoscaler.number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated when a job runs.
glue_version : glue_version_string option;In Spark jobs, GlueVersion determines the versions of Apache Spark and Python that Glue available in a job. The Python version indicates the version supported for jobs of type Spark.
Ray jobs should set GlueVersion to 4.0 or greater. However, the versions of Ray, Python and additional libraries available in your Ray job are determined by the Runtime parameter of the Job command.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
Jobs that are created without specifying a Glue version default to Glue 5.1.
*)notification_property : notification_property option;Specifies configuration properties of a job notification.
*)security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this job.
max_capacity : nullable_double option;For Glue version 1.0 or earlier jobs, using the standard worker type, the number of Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
For Glue version 2.0+ jobs, you cannot specify a Maximum capacity. Instead, you should specify a Worker type and the Number of workers.
Do not set MaxCapacity if using WorkerType and NumberOfWorkers.
The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job, an Apache Spark ETL job, or an Apache Spark streaming ETL job:
JobCommand.Name="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.JobCommand.Name="glueetl") or Apache Spark streaming ETL job (JobCommand.Name="gluestreaming"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.timeout : timeout option;The job timeout in minutes. This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status.
Jobs must have timeout values less than 7 days or 10080 minutes. Otherwise, the jobs will throw an exception.
When the value is left blank, the timeout is defaulted to 2,880 minutes for Glue version 4.0 and earlier, or 480 minutes for Glue version 5.0 and later.
Any existing Glue jobs that had a timeout value greater than 7 days will be defaulted to 7 days. For instance if you have specified a timeout of 20 days for a batch job, it will be stopped on the 7th day.
For streaming jobs, if you have set up a maintenance window, it will be restarted during the maintenance window after 7 days.
*)allocated_capacity : integer_value option;This parameter is deprecated. Use MaxCapacity instead.
The number of Glue data processing units (DPUs) to allocate to this Job. You can allocate a minimum of 2 DPUs; the default is 10. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory. For more information, see the Glue pricing page.
*)max_retries : max_retries option;The maximum number of times to retry this job if it fails.
*)connections : connections_list option;The connections used for this job.
*)non_overridable_arguments : generic_map option;Arguments for this job that are not overridden when providing job arguments in a job run, specified as name-value pairs.
*)default_arguments : generic_map option;The default arguments for every run of this job, specified as name-value pairs.
You can specify arguments here that your own job-execution script consumes, as well as arguments that Glue itself consumes.
Job arguments may be logged. Do not pass plaintext secrets as arguments. Retrieve secrets from a Glue Connection, Secrets Manager or other secret management mechanism if you intend to keep them within the Job.
For information about how to specify and consume your own Job arguments, see the Calling Glue APIs in Python topic in the developer guide.
For information about the arguments you can provide to this field when configuring Spark jobs, see the Special Parameters Used by Glue topic in the developer guide.
For information about the arguments you can provide to this field when configuring Ray jobs, see Using job parameters in Ray jobs in the developer guide.
*)command : job_command;The JobCommand that runs this job.
execution_property : execution_property option;An ExecutionProperty specifying the maximum number of concurrent runs allowed for this job.
role : role_string;The name or Amazon Resource Name (ARN) of the IAM role associated with this job.
*)log_uri : uri_string option;This field is reserved for future use.
*)description : description_string option;Description of the job being defined.
*)job_run_queuing_enabled : nullable_boolean option;Specifies whether job run queuing is enabled for the job runs for this job.
A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.
If this field does not match the value set in the job run, then the value from the job run field will be used.
*)job_mode : job_mode option;A mode that describes how a job was created. Valid values are:
SCRIPT - The job was created using the Glue Studio script editor.VISUAL - The job was created using the Glue Studio visual editor.NOTEBOOK - The job was created using an interactive sessions notebook.When the JobMode field is missing or null, SCRIPT is assigned as the default value.
name : name_string;The name you assign to this job definition. It must be unique in your account.
*)}type nonrec create_integration_table_properties_request = {target_table_config : target_table_config option;A structure for the target table configuration.
*)source_table_config : source_table_config option;A structure for the source table configuration. See the SourceTableConfig structure to see list of supported source properties.
table_name : string128;The name of the table to be replicated.
*)resource_arn : string512;The Amazon Resource Name (ARN) of the target table for which to create integration table properties. Currently, this API only supports creating integration table properties for target tables, and the provided ARN should be the ARN of the target table in the Glue Data Catalog. Support for creating integration table properties for source connections (using the connection ARN) is not yet implemented and will be added in a future release.
*)}type nonrec create_integration_response = {integration_config : integration_config option;The configuration settings.
*)data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)errors : integration_error_list option;A list of errors associated with the integration creation.
*)create_time : integration_timestamp;The time when the integration was created, in UTC.
*)status : integration_status;The status of the integration being created.
The possible statuses are:
additional_encryption_context : integration_additional_encryption_context_map
option;An optional set of non-secret key–value pairs that contains additional contextual information for encryption.
*)kms_key_id : string2048 option;The ARN of a KMS key used for encrypting the channel.
*)integration_arn : string128;The Amazon Resource Name (ARN) for the created integration.
*)description : integration_description option;A description of the integration.
*)integration_name : string128;A unique name for an integration in Glue.
*)target_arn : string512;The ARN of the target resource for the integration.
*)source_arn : string512;The ARN of the source resource for the integration.
*)}type nonrec create_integration_resource_property_response = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_property_arn : string512 option;The resource ARN created through this create API. The format is something like arn:aws:glue:::integrationresourceproperty/*
*)resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec create_integration_resource_property_request = {target_processing_properties : target_processing_properties option;The resource properties associated with the integration target.
*)source_processing_properties : source_processing_properties option;The resource properties associated with the integration source.
*)resource_arn : string512;The connection ARN of the source, or the database ARN of the target.
*)}type nonrec create_integration_request = {integration_config : integration_config option;The configuration settings.
*)additional_encryption_context : integration_additional_encryption_context_map
option;An optional set of non-secret key–value pairs that contains additional contextual information for encryption. This can only be provided if KMSKeyId is provided.
kms_key_id : string2048 option;The ARN of a KMS key used for encrypting the channel.
*)data_filter : string2048 option;Selects source tables for the integration using Maxwell filter syntax.
*)description : integration_description option;A description of the integration.
*)target_arn : string512;The ARN of the target resource for the integration.
*)source_arn : string512;The ARN of the source resource for the integration.
*)integration_name : string128;A unique name for an integration in Glue.
*)}type nonrec create_grok_classifier_request = {custom_patterns : custom_patterns option;Optional custom grok patterns used by this classifier.
*)grok_pattern : grok_pattern;The grok pattern used by this classifier.
*)name : name_string;The name of the new classifier.
*)classification : classification;An identifier of the data format that the classifier matches, such as Twitter, JSON, Omniture logs, Amazon CloudWatch Logs, and so on.
*)}Specifies a grok classifier for CreateClassifier to create.
type nonrec create_glue_identity_center_configuration_response = {application_arn : application_arn option;The Amazon Resource Name (ARN) of the Identity Center application that was created for the Glue configuration.
*)}Response from creating a new Glue Identity Center configuration.
type nonrec create_glue_identity_center_configuration_request = {user_background_sessions_enabled : nullable_boolean option;Specifies whether users can run background sessions when using Identity Center authentication with Glue services.
*)scopes : identity_center_scopes_list option;A list of Identity Center scopes that define the permissions and access levels for the Glue configuration.
*)instance_arn : identity_center_instance_arn;The Amazon Resource Name (ARN) of the Identity Center instance to be associated with the Glue configuration.
*)}Request to create a new Glue Identity Center configuration.
type nonrec create_glossary_term_response = {long_description : glossary_long_description option;The long description of the glossary term.
*)short_description : glossary_short_description option;The short description of the glossary term.
*)name : glossary_term_name option;The name of the glossary term.
*)glossary_id : glossary_id option;The unique identifier of the glossary containing this term.
*)id : glossary_term_id option;The unique identifier of the glossary term.
*)}type nonrec create_glossary_term_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)long_description : glossary_long_description option;A long description of the glossary term.
*)short_description : glossary_short_description option;A short description of the glossary term.
*)name : glossary_term_name;The name of the glossary term.
*)glossary_identifier : glossary_id;The unique identifier of the glossary in which to create the term.
*)}type nonrec create_glossary_response = {description : metadata_description option;The description of the glossary.
*)name : glossary_name option;The name of the glossary.
*)id : glossary_id option;The unique identifier of the glossary.
*)}type nonrec create_glossary_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)description : metadata_description option;The description of the glossary.
*)name : glossary_name;The name of the glossary.
*)}type nonrec create_dev_endpoint_response = {arguments : map_value option;The map of arguments used to configure this DevEndpoint.
Valid arguments are:
"--enable-glue-datacatalog": ""You can specify a version of Python support for development endpoints by using the Arguments parameter in the CreateDevEndpoint or UpdateDevEndpoint APIs. If no arguments are provided, the version defaults to Python 2.
created_timestamp : timestamp_value option;The point in time at which this DevEndpoint was created.
security_configuration : name_string option;The name of the SecurityConfiguration structure being used with this DevEndpoint.
failure_reason : generic_string option;The reason for a current failure in this DevEndpoint.
extra_jars_s3_path : generic_string option;Path to one or more Java .jar files in an S3 bucket that will be loaded in your DevEndpoint.
extra_python_libs_s3_path : generic_string option;The paths to one or more Python libraries in an S3 bucket that will be loaded in your DevEndpoint.
vpc_id : generic_string option;The ID of the virtual private cloud (VPC) used by this DevEndpoint.
availability_zone : generic_string option;The Amazon Web Services Availability Zone where this DevEndpoint is located.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated to the development endpoint.
glue_version : glue_version_string option;Glue version determines the versions of Apache Spark and Python that Glue supports. The Python version indicates the version supported for running your ETL scripts on development endpoints.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
*)worker_type : worker_type option;The type of predefined worker that is allocated to the development endpoint. May be a value of Standard, G.1X, or G.2X.
*)number_of_nodes : integer_value option;The number of Glue Data Processing Units (DPUs) allocated to this DevEndpoint.
*)zeppelin_remote_spark_interpreter_port : integer_value option;The Apache Zeppelin port for the remote Apache Spark interpreter.
*)yarn_endpoint_address : generic_string option;The address of the YARN endpoint used by this DevEndpoint.
role_arn : role_arn option;The Amazon Resource Name (ARN) of the role assigned to the new DevEndpoint.
subnet_id : generic_string option;The subnet ID assigned to the new DevEndpoint.
security_group_ids : string_list option;The security groups assigned to the new DevEndpoint.
status : generic_string option;The current status of the new DevEndpoint.
endpoint_name : generic_string option;The name assigned to the new DevEndpoint.
}type nonrec create_dev_endpoint_request = {arguments : map_value option;A map of arguments used to configure the DevEndpoint.
security_configuration : name_string option;The name of the SecurityConfiguration structure to be used with this DevEndpoint.
extra_jars_s3_path : generic_string option;The path to one or more Java .jar files in an S3 bucket that should be loaded in your DevEndpoint.
extra_python_libs_s3_path : generic_string option;The paths to one or more Python libraries in an Amazon S3 bucket that should be loaded in your DevEndpoint. Multiple values must be complete paths separated by a comma.
You can only use pure Python libraries with a DevEndpoint. Libraries that rely on C extensions, such as the pandas Python data analysis library, are not yet supported.
number_of_workers : nullable_integer option;The number of workers of a defined workerType that are allocated to the development endpoint.
The maximum number of workers you can define are 299 for G.1X, and 149 for G.2X.
glue_version : glue_version_string option;Glue version determines the versions of Apache Spark and Python that Glue supports. The Python version indicates the version supported for running your ETL scripts on development endpoints.
For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.
Development endpoints that are created without specifying a Glue version default to Glue 0.9.
You can specify a version of Python support for development endpoints by using the Arguments parameter in the CreateDevEndpoint or UpdateDevEndpoint APIs. If no arguments are provided, the version defaults to Python 2.
worker_type : worker_type option;The type of predefined worker that is allocated to the development endpoint. Accepts a value of Standard, G.1X, or G.2X.
Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. We recommend this worker type for memory-intensive jobs.G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. We recommend this worker type for memory-intensive jobs.Known issue: when a development endpoint is created with the G.2X WorkerType configuration, the Spark drivers for the development endpoint will run on 4 vCPU, 16 GB of memory, and a 64 GB disk.
number_of_nodes : integer_value option;The number of Glue Data Processing Units (DPUs) to allocate to this DevEndpoint.
public_keys : public_keys_list option;A list of public keys to be used by the development endpoints for authentication. The use of this attribute is preferred over a single public key because the public keys allow you to have a different private key per client.
If you previously created an endpoint with a public key, you must remove that key to be able to set a list of public keys. Call the UpdateDevEndpoint API with the public key content in the deletePublicKeys attribute, and the list of new keys in the addPublicKeys attribute.
public_key : generic_string option;The public key to be used by this DevEndpoint for authentication. This attribute is provided for backward compatibility because the recommended attribute to use is public keys.
subnet_id : generic_string option;The subnet ID for the new DevEndpoint to use.
security_group_ids : string_list option;Security group IDs for the security groups to be used by the new DevEndpoint.
role_arn : role_arn;The IAM role for the DevEndpoint.
endpoint_name : generic_string;The name to be assigned to the new DevEndpoint.
}type nonrec create_database_request = {database_input : database_input;The metadata for the database.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which to create the database. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec create_data_quality_ruleset_response = {name : name_string option;A unique name for the data quality ruleset.
*)}type nonrec create_data_quality_ruleset_request = {client_token : hash_string option;Used for idempotency and is recommended to be set to a random ID (such as a UUID) to avoid creating or starting multiple instances of the same resource.
*)data_quality_security_configuration : name_string option;The name of the security configuration created with the data quality encryption option.
*)target_table : data_quality_target_table option;A target table associated with the data quality ruleset.
*)ruleset : data_quality_ruleset_string;A Data Quality Definition Language (DQDL) ruleset. For more information, see the Glue developer guide.
*)description : description_string option;A description of the data quality ruleset.
*)name : name_string;A unique name for the data quality ruleset.
*)}A request to create a data quality ruleset.
type nonrec create_custom_entity_type_response = {name : name_string option;The name of the custom pattern you created.
*)}type nonrec create_custom_entity_type_request = {context_words : context_words option;A list of context words. If none of these context words are found within the vicinity of the regular expression the data will not be detected as sensitive data.
If no context words are passed only a regular expression is checked.
*)regex_string : name_string;A regular expression string that is used for detecting sensitive data in a custom pattern.
*)name : name_string;A name for the custom pattern that allows it to be retrieved or deleted later. This name must be unique per Amazon Web Services account.
*)}type nonrec create_csv_classifier_request = {serde : csv_serde_option option;Sets the SerDe for processing CSV in the classifier, which will be applied in the Data Catalog. Valid values are OpenCSVSerDe, LazySimpleSerDe, and None. You can specify the None value when you want the crawler to do the detection.
custom_datatypes : custom_datatypes option;Creates a list of supported custom datatypes.
*)custom_datatype_configured : nullable_boolean option;Enables the configuration of custom datatypes.
*)allow_single_column : nullable_boolean option;Enables the processing of files that contain only one column.
*)disable_value_trimming : nullable_boolean option;Specifies not to trim values before identifying the type of column values. The default value is true.
*)header : csv_header option;A list of strings representing column names.
*)contains_header : csv_header_option option;Indicates whether the CSV file contains a header.
*)quote_symbol : csv_quote_symbol option;A custom symbol to denote what combines content into a single column value. Must be different from the column delimiter.
*)delimiter : csv_column_delimiter option;A custom symbol to denote what separates each column entry in the row.
*)name : name_string;The name of the classifier.
*)}Specifies a custom CSV classifier for CreateClassifier to create.
type nonrec create_crawler_request = {crawler_security_configuration : crawler_security_configuration option;The name of the SecurityConfiguration structure to be used by this crawler.
configuration : crawler_configuration option;Crawler configuration information. This versioned JSON string allows users to specify aspects of a crawler's behavior. For more information, see Setting crawler configuration options.
*)lake_formation_configuration : lake_formation_configuration option;Specifies Lake Formation configuration settings for the crawler.
*)lineage_configuration : lineage_configuration option;Specifies data lineage configuration settings for the crawler.
*)recrawl_policy : recrawl_policy option;A policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run.
*)schema_change_policy : schema_change_policy option;The policy for the crawler's update and deletion behavior.
*)table_prefix : table_prefix option;The table prefix used for catalog tables that are created.
*)classifiers : classifier_name_list option;A list of custom classifiers that the user has registered. By default, all built-in classifiers are included in a crawl, but these custom classifiers always override the default classifiers for a given classification.
*)schedule : cron_expression option;A cron expression used to specify the schedule (see Time-Based Schedules for Jobs and Crawlers. For example, to run something every day at 12:15 UTC, you would specify: cron(15 12 * * ? *).
targets : crawler_targets;A list of collection of targets to crawl.
*)description : description_string option;A description of the new crawler.
*)database_name : database_name option;The Glue database where results are written, such as: arn:aws:daylight:us-east-1::database/sometable/*.
role : role;The IAM role or Amazon Resource Name (ARN) of an IAM role used by the new crawler to access customer resources.
*)name : name_string;Name of the new crawler.
*)}type nonrec create_connection_response = {create_connection_status : connection_status option;The status of the connection creation request. The request can take some time for certain authentication types, for example when creating an OAuth connection with token exchange over VPC.
*)}type nonrec create_connection_request = {connection_input : connection_input;A ConnectionInput object defining the connection to create.
catalog_id : catalog_id_string option;The ID of the Data Catalog in which to create the connection. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec create_column_statistics_task_settings_request = {security_configuration : name_string option;Name of the security configuration that is used to encrypt CloudWatch logs.
*)catalog_i_d : name_string option;The ID of the Data Catalog in which the database resides.
*)sample_size : sample_size_percentage option;The percentage of data to sample.
*)column_name_list : column_name_list option;A list of column names for which to run statistics.
*)schedule : cron_expression option;A schedule for running the column statistics, specified in CRON syntax.
*)role : name_string;The role used for running the column statistics.
*)table_name : name_string;The name of the table for which to generate column statistics.
*)database_name : name_string;The name of the database where the table resides.
*)}type nonrec create_classifier_request = {csv_classifier : create_csv_classifier_request option;A CsvClassifier object specifying the classifier to create.
json_classifier : create_json_classifier_request option;A JsonClassifier object specifying the classifier to create.
xml_classifier : create_xml_classifier_request option;An XMLClassifier object specifying the classifier to create.
grok_classifier : create_grok_classifier_request option;A GrokClassifier object specifying the classifier to create.
}type nonrec create_catalog_request = {catalog_input : catalog_input;A CatalogInput object that defines the metadata for the catalog.
name : catalog_name_string;The name of the catalog to create.
*)}type nonrec create_blueprint_response = {name : name_string option;Returns the name of the blueprint that was registered.
*)}type nonrec create_blueprint_request = {blueprint_location : orchestration_s3_location;Specifies a path in Amazon S3 where the blueprint is published.
*)description : generic512_char_string option;A description of the blueprint.
*)name : orchestration_name_string;The name of the blueprint.
*)}type nonrec check_schema_version_validity_response = {error : schema_validation_error option;A validation failure error message.
*)valid : is_version_valid option;Return true, if the schema is valid and false otherwise.
*)}type nonrec check_schema_version_validity_input = {schema_definition : schema_definition_string;The definition of the schema that has to be validated.
*)data_format : data_format;The data format of the schema definition. Currently AVRO, JSON and PROTOBUF are supported.
}type nonrec cancel_statement_request = {request_origin : orchestration_name_string option;The origin of the request to cancel the statement.
*)id : integer_value;The ID of the statement to be cancelled.
*)session_id : name_string;The Session ID of the statement to be cancelled.
*)}type nonrec cancel_ml_task_run_response = {status : task_status_type option;The status for this run.
*)task_run_id : hash_string option;The unique identifier for the task run.
*)transform_id : hash_string option;The unique identifier of the machine learning transform.
*)}type nonrec cancel_ml_task_run_request = {task_run_id : hash_string;A unique identifier for the task run.
*)transform_id : hash_string;The unique identifier of the machine learning transform.
*)}type nonrec cancel_data_quality_ruleset_evaluation_run_request = {run_id : hash_string;The unique run identifier associated with this run.
*)}type nonrec cancel_data_quality_rule_recommendation_run_request = {run_id : hash_string;The unique run identifier associated with this run.
*)}type nonrec blueprints = blueprint listtype nonrec batch_update_partition_failure_entry = {error_detail : error_detail option;The details about the batch update partition error.
*)partition_value_list : bounded_partition_value_list option;A list of values defining the partitions.
*)}Contains information about a batch update partition error.
type nonrec batch_update_partition_failure_list =
batch_update_partition_failure_entry listtype nonrec batch_update_partition_response = {errors : batch_update_partition_failure_list option;The errors encountered when trying to update the requested partitions. A list of BatchUpdatePartitionFailureEntry objects.
}type nonrec batch_update_partition_request_entry = {partition_input : partition_input;The structure used to update a partition.
*)partition_value_list : bounded_partition_value_list;A list of values defining the partitions.
*)}A structure that contains the values and structure used to update a partition.
type nonrec batch_update_partition_request_entry_list =
batch_update_partition_request_entry listtype nonrec batch_update_partition_request = {entries : batch_update_partition_request_entry_list;A list of up to 100 BatchUpdatePartitionRequestEntry objects to update.
table_name : name_string;The name of the metadata table in which the partition is to be updated.
*)database_name : name_string;The name of the metadata database in which the partition is to be updated.
*)catalog_id : catalog_id_string option;The ID of the catalog in which the partition is to be updated. Currently, this should be the Amazon Web Services account ID.
*)}type nonrec batch_table_optimizer = {table_optimizer : table_optimizer option;A TableOptimizer object that contains details on the configuration and last run of a table optimizer.
table_name : table_name_string option;The name of the table.
*)database_name : database_name_string option;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string option;The Catalog ID of the table.
*)}Contains details for one of the table optimizers returned by the BatchGetTableOptimizer operation.
type nonrec batch_table_optimizers = batch_table_optimizer listtype nonrec batch_stop_job_run_successful_submission = {job_run_id : id_string option;The JobRunId of the job run that was stopped.
job_name : name_string option;The name of the job definition used in the job run that was stopped.
*)}Records a successful request to stop a specified JobRun.
type nonrec batch_stop_job_run_successful_submission_list =
batch_stop_job_run_successful_submission listtype nonrec batch_stop_job_run_error = {error_detail : error_detail option;Specifies details about the error that was encountered.
*)job_run_id : id_string option;The JobRunId of the job run in question.
job_name : name_string option;The name of the job definition that is used in the job run in question.
*)}Records an error that occurred when attempting to stop a specified job run.
type nonrec batch_stop_job_run_error_list = batch_stop_job_run_error listtype nonrec batch_stop_job_run_response = {errors : batch_stop_job_run_error_list option;A list of the errors that were encountered in trying to stop JobRuns, including the JobRunId for which each error was encountered and details about the error.
successful_submissions : batch_stop_job_run_successful_submission_list option;A list of the JobRuns that were successfully submitted for stopping.
*)}type nonrec batch_stop_job_run_job_run_id_list = id_string listtype nonrec batch_stop_job_run_request = {job_run_ids : batch_stop_job_run_job_run_id_list;A list of the JobRunIds that should be stopped for that job definition.
job_name : name_string;The name of the job definition for which to stop job runs.
*)}type nonrec annotation_error = {failure_reason : description_string option;The reason why the annotation failed.
*)statistic_id : hash_string option;The Statistic ID for the failed annotation.
*)profile_id : hash_string option;The Profile ID for the failed annotation.
*)}A failed annotation.
type nonrec annotation_error_list = annotation_error listtype nonrec batch_put_data_quality_statistic_annotation_response = {failed_inclusion_annotations : annotation_error_list option;A list of AnnotationError's.
}type nonrec batch_put_data_quality_statistic_annotation_request = {client_token : hash_string option;Client Token.
*)inclusion_annotations : inclusion_annotation_list;A list of DatapointInclusionAnnotation's. The InclusionAnnotations must contain a profileId and statisticId. If there are multiple InclusionAnnotations, the list must refer to a single statisticId across multiple profileIds.
}type nonrec batch_get_workflows_response = {missing_workflows : workflow_names option;A list of names of workflows not found.
*)workflows : workflows option;A list of workflow resource metadata.
*)}type nonrec batch_get_workflows_request = {include_graph : nullable_boolean option;Specifies whether to include a graph when returning the workflow resource metadata.
*)names : workflow_names;A list of workflow names, which may be the names returned from the ListWorkflows operation.
}type nonrec batch_get_triggers_response = {triggers_not_found : trigger_name_list option;A list of names of triggers not found.
*)triggers : trigger_list option;A list of trigger definitions.
*)}type nonrec batch_get_triggers_request = {trigger_names : trigger_name_list;A list of trigger names, which may be the names returned from the ListTriggers operation.
}type nonrec batch_get_table_optimizer_error = {type_ : table_optimizer_type option;The type of table optimizer.
*)table_name : table_name_string option;The name of the table.
*)database_name : database_name_string option;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string option;The Catalog ID of the table.
*)error : error_detail option;An ErrorDetail object containing code and message details about the error.
}Contains details on one of the errors in the error list returned by the BatchGetTableOptimizer operation.
type nonrec batch_get_table_optimizer_errors =
batch_get_table_optimizer_error listtype nonrec batch_get_table_optimizer_response = {failures : batch_get_table_optimizer_errors option;A list of errors from the operation.
*)table_optimizers : batch_table_optimizers option;A list of BatchTableOptimizer objects.
}type nonrec batch_get_table_optimizer_entry = {type_ : table_optimizer_type option;The type of table optimizer.
*)table_name : table_name_string option;The name of the table.
*)database_name : database_name_string option;The name of the database in the catalog in which the table resides.
*)catalog_id : catalog_id_string option;The Catalog ID of the table.
*)}Represents a table optimizer to retrieve in the BatchGetTableOptimizer operation.
type nonrec batch_get_table_optimizer_entries =
batch_get_table_optimizer_entry listtype nonrec batch_get_table_optimizer_request = {entries : batch_get_table_optimizer_entries;A list of BatchGetTableOptimizerEntry objects specifying the table optimizers to retrieve.
}type nonrec batch_get_partition_value_list = partition_value_list listtype nonrec batch_get_partition_response = {unprocessed_keys : batch_get_partition_value_list option;A list of the partition values in the request for which partitions were not returned.
*)partitions : partition_list option;A list of the requested partitions.
*)}type nonrec batch_get_partition_request = {query_session_context : query_session_context option;audit_context : audit_context option;partitions_to_get : batch_get_partition_value_list;A list of partition values identifying the partitions to retrieve.
*)table_name : name_string;The name of the partitions' table.
*)database_name : name_string;The name of the catalog database where the partitions reside.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partitions in question reside. If none is supplied, the Amazon Web Services account ID is used by default.
*)}type nonrec batch_get_jobs_response = {jobs_not_found : job_name_list option;A list of names of jobs not found.
*)jobs : job_list option;A list of job definitions.
*)}type nonrec batch_get_jobs_request = {job_names : job_name_list;A list of job names, which might be the names returned from the ListJobs operation.
}type nonrec batch_get_iterable_forms_response = {errors : item_error_list option;The list of errors for items that could not be retrieved.
*)items : iterable_form_item_list option;The list of retrieved iterable form items.
*)}type nonrec batch_get_iterable_forms_request = {item_identifiers : item_identifier_list;The list of item identifiers to retrieve. Each identifier can be an item ID or item name.
*)iterable_form_name : iterable_form_name;The name of the iterable form to retrieve items from.
*)asset_identifier : asset_id;The unique identifier of the asset.
*)}type nonrec batch_get_dev_endpoints_response = {dev_endpoints_not_found : dev_endpoint_names option;A list of DevEndpoints not found.
dev_endpoints : dev_endpoint_list option;A list of DevEndpoint definitions.
}type nonrec batch_get_dev_endpoints_request = {dev_endpoint_names : dev_endpoint_names;The list of DevEndpoint names, which might be the names returned from the ListDevEndpoint operation.
}type nonrec batch_get_data_quality_result_response = {results_not_found : data_quality_result_ids option;A list of result IDs for which results were not found.
*)results : data_quality_results_list;A list of DataQualityResult objects representing the data quality results.
}type nonrec batch_get_data_quality_result_request = {result_ids : data_quality_result_ids;A list of unique result IDs for the data quality results.
*)}type nonrec batch_get_custom_entity_types_response = {custom_entity_types_not_found : custom_entity_type_names option;A list of the names of custom patterns that were not found.
*)custom_entity_types : custom_entity_types option;A list of CustomEntityType objects representing the custom patterns that have been created.
}type nonrec batch_get_custom_entity_types_request = {names : custom_entity_type_names;A list of names of the custom patterns that you want to retrieve.
*)}type nonrec batch_get_crawlers_response = {crawlers_not_found : crawler_name_list option;A list of names of crawlers that were not found.
*)crawlers : crawler_list option;A list of crawler definitions.
*)}type nonrec batch_get_crawlers_request = {crawler_names : crawler_name_list;A list of crawler names, which might be the names returned from the ListCrawlers operation.
}type nonrec batch_get_blueprints_response = {missing_blueprints : blueprint_names option;Returns a list of BlueprintNames that were not found.
blueprints : blueprints option;Returns a list of blueprint as a Blueprints object.
}type nonrec batch_get_blueprint_names = orchestration_name_string listtype nonrec batch_get_blueprints_request = {include_parameter_spec : nullable_boolean option;Specifies whether or not to include the parameters, as a JSON string, for the blueprint in the response.
*)include_blueprint : nullable_boolean option;Specifies whether or not to include the blueprint in the response.
*)names : batch_get_blueprint_names;A list of blueprint names.
*)}type nonrec batch_delete_table_version_response = {errors : table_version_errors option;A list of errors encountered while trying to delete the specified table versions.
*)}type nonrec batch_delete_table_version_list = version_string listtype nonrec batch_delete_table_version_request = {version_ids : batch_delete_table_version_list;A list of the IDs of versions to be deleted. A VersionId is a string representation of an integer. Each version is incremented by 1.
table_name : name_string;The name of the table. For Hive compatibility, this name is entirely lowercase.
*)database_name : name_string;The database in the catalog in which the table resides. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the tables reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec batch_delete_table_response = {errors : table_errors option;A list of errors encountered in attempting to delete the specified tables.
*)}type nonrec batch_delete_table_name_list = name_string listtype nonrec batch_delete_table_request = {transaction_id : transaction_id_string option;The transaction ID at which to delete the table contents.
*)tables_to_delete : batch_delete_table_name_list;A list of the table to delete.
*)database_name : name_string;The name of the catalog database in which the tables to delete reside. For Hive compatibility, this name is entirely lowercase.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the table resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec batch_delete_partition_value_list = partition_value_list listtype nonrec batch_delete_partition_response = {errors : partition_errors option;The errors encountered when trying to delete the requested partitions.
*)}type nonrec batch_delete_partition_request = {partitions_to_delete : batch_delete_partition_value_list;A list of PartitionInput structures that define the partitions to be deleted.
table_name : name_string;The name of the table that contains the partitions to be deleted.
*)database_name : name_string;The name of the catalog database in which the table in question resides.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog where the partition to be deleted resides. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec batch_delete_connection_response = {errors : error_by_name option;A map of the names of connections that were not successfully deleted to error details.
*)succeeded : name_string_list option;A list of names of the connection definitions that were successfully deleted.
*)}type nonrec batch_delete_connection_request = {connection_name_list : delete_connection_name_list;A list of names of the connections to delete.
*)catalog_id : catalog_id_string option;The ID of the Data Catalog in which the connections reside. If none is provided, the Amazon Web Services account ID is used by default.
*)}type nonrec batch_create_partition_response = {errors : partition_errors option;The errors encountered when trying to create the requested partitions.
*)}type nonrec batch_create_partition_request = {partition_input_list : partition_input_list;A list of PartitionInput structures that define the partitions to be created.
table_name : name_string;The name of the metadata table in which the partition is to be created.
*)database_name : name_string;The name of the metadata database in which the partition is to be created.
*)catalog_id : catalog_id_string option;The ID of the catalog in which the partition is to be created. Currently, this should be the Amazon Web Services account ID.
*)}type nonrec associate_glossary_terms_response = {glossary_terms : glossary_term_id_list option;The glossary terms now associated with the asset.
*)asset_identifier : asset_id option;The unique identifier of the asset.
*)}type nonrec associate_glossary_terms_request = {client_token : hash_string option;A unique, case-sensitive identifier that you provide to ensure the idempotency of the request.
*)glossary_term_identifiers : glossary_term_id_list;The list of glossary term identifiers to associate with the asset.
*)asset_identifier : asset_id;The unique identifier of the asset to associate glossary terms with.
*)}