From 8e7094c87fa7ad049e6ea600a5f90f1c97d683bd Mon Sep 17 00:00:00 2001 From: arcadia-devtools Date: Wed, 1 Jun 2022 15:35:19 +0300 Subject: intermediate changes ref:b1e863e3aa12b888b40dd5ec36411ca050a25bd8 --- contrib/python/boto3/py3/.dist-info/METADATA | 4 +- contrib/python/boto3/py3/boto3/__init__.py | 2 +- contrib/python/botocore/py3/.dist-info/METADATA | 2 +- contrib/python/botocore/py3/botocore/__init__.py | 2 +- .../botocore/data/glue/2017-03-31/service-2.json | 2784 +++++++++++++++++++- .../botocore/data/kms/2014-11-01/service-2.json | 36 +- 6 files changed, 2769 insertions(+), 61 deletions(-) (limited to 'contrib/python') diff --git a/contrib/python/boto3/py3/.dist-info/METADATA b/contrib/python/boto3/py3/.dist-info/METADATA index c717d1a3a67..171470a83f7 100644 --- a/contrib/python/boto3/py3/.dist-info/METADATA +++ b/contrib/python/boto3/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: boto3 -Version: 1.23.1 +Version: 1.23.2 Summary: The AWS SDK for Python Home-page: https://github.com/boto/boto3 Author: Amazon Web Services @@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.10 Requires-Python: >= 3.6 License-File: LICENSE License-File: NOTICE -Requires-Dist: botocore (<1.27.0,>=1.26.1) +Requires-Dist: botocore (<1.27.0,>=1.26.2) Requires-Dist: jmespath (<2.0.0,>=0.7.1) Requires-Dist: s3transfer (<0.6.0,>=0.5.0) Provides-Extra: crt diff --git a/contrib/python/boto3/py3/boto3/__init__.py b/contrib/python/boto3/py3/boto3/__init__.py index 81dcf3bf832..3fc7194a33f 100644 --- a/contrib/python/boto3/py3/boto3/__init__.py +++ b/contrib/python/boto3/py3/boto3/__init__.py @@ -17,7 +17,7 @@ from boto3.compat import _warn_deprecated_python from boto3.session import Session __author__ = 'Amazon Web Services' -__version__ = '1.23.1' +__version__ = '1.23.2' # The default Boto3 session; autoloaded when needed. diff --git a/contrib/python/botocore/py3/.dist-info/METADATA b/contrib/python/botocore/py3/.dist-info/METADATA index 5d5e49ee9cf..31c9168b7af 100644 --- a/contrib/python/botocore/py3/.dist-info/METADATA +++ b/contrib/python/botocore/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: botocore -Version: 1.26.1 +Version: 1.26.2 Summary: Low-level, data-driven core of boto 3. Home-page: https://github.com/boto/botocore Author: Amazon Web Services diff --git a/contrib/python/botocore/py3/botocore/__init__.py b/contrib/python/botocore/py3/botocore/__init__.py index 17135c4b802..804b4a4cbfa 100644 --- a/contrib/python/botocore/py3/botocore/__init__.py +++ b/contrib/python/botocore/py3/botocore/__init__.py @@ -16,7 +16,7 @@ import logging import os import re -__version__ = '1.26.1' +__version__ = '1.26.2' class NullHandler(logging.Handler): diff --git a/contrib/python/botocore/py3/botocore/data/glue/2017-03-31/service-2.json b/contrib/python/botocore/py3/botocore/data/glue/2017-03-31/service-2.json index 07fdb63cbe4..a1ff3c44409 100644 --- a/contrib/python/botocore/py3/botocore/data/glue/2017-03-31/service-2.json +++ b/contrib/python/botocore/py3/botocore/data/glue/2017-03-31/service-2.json @@ -3086,11 +3086,88 @@ "type":"list", "member":{"shape":"Action"} }, + "AdditionalOptions":{ + "type":"map", + "key":{"shape":"EnclosedInStringProperty"}, + "value":{"shape":"EnclosedInStringProperty"} + }, "AdditionalPlanOptionsMap":{ "type":"map", "key":{"shape":"GenericString"}, "value":{"shape":"GenericString"} }, + "AggFunction":{ + "type":"string", + "enum":[ + "avg", + "countDistinct", + "count", + "first", + "last", + "kurtosis", + "max", + "min", + "skewness", + "stddev_samp", + "stddev_pop", + "sum", + "sumDistinct", + "var_samp", + "var_pop" + ] + }, + "Aggregate":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Groups", + "Aggs" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

Specifies the fields and rows to use as inputs for the aggregate transform.

" + }, + "Groups":{ + "shape":"GlueStudioPathList", + "documentation":"

Specifies the fields to group by.

" + }, + "Aggs":{ + "shape":"AggregateOperations", + "documentation":"

Specifies the aggregate functions to be performed on specified fields.

" + } + }, + "documentation":"

Specifies a transform that groups rows by chosen fields and computes the aggregated value by specified function.

" + }, + "AggregateOperation":{ + "type":"structure", + "required":[ + "Column", + "AggFunc" + ], + "members":{ + "Column":{ + "shape":"EnclosedInStringProperties", + "documentation":"

Specifies the column on the data set on which the aggregation function will be applied.

" + }, + "AggFunc":{ + "shape":"AggFunction", + "documentation":"

Specifies the aggregation function to apply.

Possible aggregation functions include: avg countDistinct, count, first, last, kurtosis, max, min, skewness, stddev_samp, stddev_pop, sum, sumDistinct, var_samp, var_pop

" + } + }, + "documentation":"

Specifies the set of parameters needed to perform aggregation in the aggregate transform.

" + }, + "AggregateOperations":{ + "type":"list", + "member":{"shape":"AggregateOperation"}, + "max":30, + "min":1 + }, "AlreadyExistsException":{ "type":"structure", "members":{ @@ -3102,6 +3179,70 @@ "documentation":"

A resource to be created or added already exists.

", "exception":true }, + "ApplyMapping":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Mapping" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Mapping":{ + "shape":"Mappings", + "documentation":"

Specifies the mapping of data property keys in the data source to data property keys in the data target.

" + } + }, + "documentation":"

Specifies a transform that maps data property keys in the data source to data property keys in the data target. You can rename keys, modify the data types for keys, and choose which keys to drop from the dataset.

" + }, + "AthenaConnectorSource":{ + "type":"structure", + "required":[ + "Name", + "ConnectionName", + "ConnectorName", + "ConnectionType", + "SchemaName" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection that is associated with the connector.

" + }, + "ConnectorName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of a connector that assists with accessing the data store in Glue Studio.

" + }, + "ConnectionType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of connection, such as marketplace.athena or custom.athena, designating a connection to an Amazon Athena data store.

" + }, + "ConnectionTable":{ + "shape":"EnclosedInStringPropertyWithQuote", + "documentation":"

The name of the table in the data source.

" + }, + "SchemaName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the Cloudwatch log group to read from. For example, /aws-glue/jobs/output.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the custom Athena source.

" + } + }, + "documentation":"

Specifies a connector to an Amazon Athena data source.

" + }, "AttemptCount":{"type":"integer"}, "AuditColumnNamesList":{ "type":"list", @@ -3162,6 +3303,34 @@ "type":"list", "member":{"shape":"BackfillError"} }, + "BasicCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of your data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database that contains the table you want to use as the target. This database must already exist in the Data Catalog.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The table that defines the schema of your output data. This table must already exist in the Data Catalog.

" + } + }, + "documentation":"

Specifies a target that uses a Glue Data Catalog table.

" + }, "BatchCreatePartitionRequest":{ "type":"structure", "required":[ @@ -3936,6 +4105,35 @@ "max":100, "min":0 }, + "BoxedBoolean":{ + "type":"boolean", + "box":true + }, + "BoxedDoubleFraction":{ + "type":"double", + "box":true, + "max":1, + "min":0 + }, + "BoxedLong":{ + "type":"long", + "box":true + }, + "BoxedNonNegativeInt":{ + "type":"integer", + "box":true, + "min":0 + }, + "BoxedNonNegativeLong":{ + "type":"long", + "box":true, + "min":0 + }, + "BoxedPositiveInt":{ + "type":"integer", + "box":true, + "min":0 + }, "CancelMLTaskRunRequest":{ "type":"structure", "required":[ @@ -4055,6 +4253,124 @@ }, "documentation":"

A structure containing migration status information.

" }, + "CatalogKafkaSource":{ + "type":"structure", + "required":[ + "Name", + "Table", + "Database" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "WindowSize":{ + "shape":"BoxedPositiveInt", + "documentation":"

The amount of time to spend processing each micro batch.

", + "box":true + }, + "DetectSchema":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to automatically determine the schema from the incoming data.

", + "box":true + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "StreamingOptions":{ + "shape":"KafkaStreamingSourceOptions", + "documentation":"

Specifies the streaming options.

" + }, + "DataPreviewOptions":{ + "shape":"StreamingDataPreviewOptions", + "documentation":"

Specifies options related to data preview for viewing a sample of your data.

" + } + }, + "documentation":"

Specifies an Apache Kafka data store in the Data Catalog.

" + }, + "CatalogKinesisSource":{ + "type":"structure", + "required":[ + "Name", + "Table", + "Database" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "WindowSize":{ + "shape":"BoxedPositiveInt", + "documentation":"

The amount of time to spend processing each micro batch.

" + }, + "DetectSchema":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to automatically determine the schema from the incoming data.

", + "box":true + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "StreamingOptions":{ + "shape":"KinesisStreamingSourceOptions", + "documentation":"

Additional options for the Kinesis streaming data source.

" + }, + "DataPreviewOptions":{ + "shape":"StreamingDataPreviewOptions", + "documentation":"

Additional options for data preview.

" + } + }, + "documentation":"

Specifies a Kinesis data source in the Glue Data Catalog.

" + }, + "CatalogSchemaChangePolicy":{ + "type":"structure", + "members":{ + "EnableUpdateCatalog":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to use the specified update behavior when the crawler finds a changed schema.

" + }, + "UpdateBehavior":{ + "shape":"UpdateCatalogBehavior", + "documentation":"

The update behavior when the crawler finds a changed schema.

" + } + }, + "documentation":"

A policy that specifies update behavior for the crawler.

" + }, + "CatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a data store in the Glue Data Catalog.

" + }, "CatalogTablesList":{ "type":"list", "member":{"shape":"NameString"}, @@ -4170,6 +4486,212 @@ }, "CodeGenArgName":{"type":"string"}, "CodeGenArgValue":{"type":"string"}, + "CodeGenConfigurationNode":{ + "type":"structure", + "members":{ + "AthenaConnectorSource":{ + "shape":"AthenaConnectorSource", + "documentation":"

Specifies a connector to an Amazon Athena data source.

" + }, + "JDBCConnectorSource":{ + "shape":"JDBCConnectorSource", + "documentation":"

Specifies a connector to a JDBC data source.

" + }, + "SparkConnectorSource":{ + "shape":"SparkConnectorSource", + "documentation":"

Specifies a connector to an Apache Spark data source.

" + }, + "CatalogSource":{ + "shape":"CatalogSource", + "documentation":"

Specifies a data store in the Glue Data Catalog.

" + }, + "RedshiftSource":{ + "shape":"RedshiftSource", + "documentation":"

Specifies an Amazon Redshift data store.

" + }, + "S3CatalogSource":{ + "shape":"S3CatalogSource", + "documentation":"

Specifies an Amazon S3 data store in the Glue Data Catalog.

" + }, + "S3CsvSource":{ + "shape":"S3CsvSource", + "documentation":"

Specifies a command-separated value (CSV) data store stored in Amazon S3.

" + }, + "S3JsonSource":{ + "shape":"S3JsonSource", + "documentation":"

Specifies a JSON data store stored in Amazon S3.

" + }, + "S3ParquetSource":{ + "shape":"S3ParquetSource", + "documentation":"

Specifies an Apache Parquet data store stored in Amazon S3.

" + }, + "RelationalCatalogSource":{"shape":"RelationalCatalogSource"}, + "DynamoDBCatalogSource":{"shape":"DynamoDBCatalogSource"}, + "JDBCConnectorTarget":{ + "shape":"JDBCConnectorTarget", + "documentation":"

Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.

" + }, + "SparkConnectorTarget":{ + "shape":"SparkConnectorTarget", + "documentation":"

Specifies a target that uses an Apache Spark connector.

" + }, + "CatalogTarget":{ + "shape":"BasicCatalogTarget", + "documentation":"

Specifies a target that uses a Glue Data Catalog table.

" + }, + "RedshiftTarget":{ + "shape":"RedshiftTarget", + "documentation":"

Specifies a target that uses Amazon Redshift.

" + }, + "S3CatalogTarget":{ + "shape":"S3CatalogTarget", + "documentation":"

Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.

" + }, + "S3GlueParquetTarget":{ + "shape":"S3GlueParquetTarget", + "documentation":"

Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.

" + }, + "S3DirectTarget":{ + "shape":"S3DirectTarget", + "documentation":"

Specifies a data target that writes to Amazon S3.

" + }, + "ApplyMapping":{ + "shape":"ApplyMapping", + "documentation":"

Specifies a transform that maps data property keys in the data source to data property keys in the data target. You can rename keys, modify the data types for keys, and choose which keys to drop from the dataset.

" + }, + "SelectFields":{ + "shape":"SelectFields", + "documentation":"

Specifies a transform that chooses the data property keys that you want to keep.

" + }, + "DropFields":{ + "shape":"DropFields", + "documentation":"

Specifies a transform that chooses the data property keys that you want to drop.

" + }, + "RenameField":{ + "shape":"RenameField", + "documentation":"

Specifies a transform that renames a single data property key.

" + }, + "Spigot":{ + "shape":"Spigot", + "documentation":"

Specifies a transform that writes samples of the data to an Amazon S3 bucket.

" + }, + "Join":{ + "shape":"Join", + "documentation":"

Specifies a transform that joins two datasets into one dataset using a comparison phrase on the specified data property keys. You can use inner, outer, left, right, left semi, and left anti joins.

" + }, + "SplitFields":{ + "shape":"SplitFields", + "documentation":"

Specifies a transform that splits data property keys into two DynamicFrames. The output is a collection of DynamicFrames: one with selected data property keys, and one with the remaining data property keys.

" + }, + "SelectFromCollection":{ + "shape":"SelectFromCollection", + "documentation":"

Specifies a transform that chooses one DynamicFrame from a collection of DynamicFrames. The output is the selected DynamicFrame

" + }, + "FillMissingValues":{ + "shape":"FillMissingValues", + "documentation":"

Specifies a transform that locates records in the dataset that have missing values and adds a new field with a value determined by imputation. The input data set is used to train the machine learning model that determines what the missing value should be.

" + }, + "Filter":{ + "shape":"Filter", + "documentation":"

Specifies a transform that splits a dataset into two, based on a filter condition.

" + }, + "CustomCode":{ + "shape":"CustomCode", + "documentation":"

Specifies a transform that uses custom code you provide to perform the data transformation. The output is a collection of DynamicFrames.

" + }, + "SparkSQL":{ + "shape":"SparkSQL", + "documentation":"

Specifies a transform where you enter a SQL query using Spark SQL syntax to transform the data. The output is a single DynamicFrame.

" + }, + "DirectKinesisSource":{ + "shape":"DirectKinesisSource", + "documentation":"

Specifies a direct Amazon Kinesis data source.

" + }, + "DirectKafkaSource":{ + "shape":"DirectKafkaSource", + "documentation":"

Specifies an Apache Kafka data store.

" + }, + "CatalogKinesisSource":{ + "shape":"CatalogKinesisSource", + "documentation":"

Specifies a Kinesis data source in the Glue Data Catalog.

" + }, + "CatalogKafkaSource":{ + "shape":"CatalogKafkaSource", + "documentation":"

Specifies an Apache Kafka data store in the Data Catalog.

" + }, + "DropNullFields":{ + "shape":"DropNullFields", + "documentation":"

Specifies a transform that removes columns from the dataset if all values in the column are 'null'. By default, Glue Studio will recognize null objects, but some values such as empty strings, strings that are \"null\", -1 integers or other placeholders such as zeros, are not automatically recognized as nulls.

" + }, + "Merge":{ + "shape":"Merge", + "documentation":"

Specifies a transform that merges a DynamicFrame with a staging DynamicFrame based on the specified primary keys to identify records. Duplicate records (records with the same primary keys) are not de-duplicated.

" + }, + "Union":{ + "shape":"Union", + "documentation":"

Specifies a transform that combines the rows from two or more datasets into a single result.

" + }, + "PIIDetection":{ + "shape":"PIIDetection", + "documentation":"

Specifies a transform that identifies, removes or masks PII data.

" + }, + "Aggregate":{ + "shape":"Aggregate", + "documentation":"

Specifies a transform that groups rows by chosen fields and computes the aggregated value by specified function.

" + }, + "DropDuplicates":{ + "shape":"DropDuplicates", + "documentation":"

Specifies a transform that removes rows of repeating data from a data set.

" + }, + "GovernedCatalogTarget":{ + "shape":"GovernedCatalogTarget", + "documentation":"

Specifies a data target that writes to a goverened catalog.

" + }, + "GovernedCatalogSource":{ + "shape":"GovernedCatalogSource", + "documentation":"

Specifies a data source in a goverened Data Catalog.

" + }, + "MicrosoftSQLServerCatalogSource":{ + "shape":"MicrosoftSQLServerCatalogSource", + "documentation":"

Specifies a Microsoft SQL server data source in the Glue Data Catalog.

" + }, + "MySQLCatalogSource":{ + "shape":"MySQLCatalogSource", + "documentation":"

Specifies a MySQL data source in the Glue Data Catalog.

" + }, + "OracleSQLCatalogSource":{ + "shape":"OracleSQLCatalogSource", + "documentation":"

Specifies an Oracle data source in the Glue Data Catalog.

" + }, + "PostgreSQLCatalogSource":{ + "shape":"PostgreSQLCatalogSource", + "documentation":"

Specifies a PostgresSQL data source in the Glue Data Catalog.

" + }, + "MicrosoftSQLServerCatalogTarget":{ + "shape":"MicrosoftSQLServerCatalogTarget", + "documentation":"

Specifies a target that uses Microsoft SQL.

" + }, + "MySQLCatalogTarget":{ + "shape":"MySQLCatalogTarget", + "documentation":"

Specifies a target that uses MySQL.

" + }, + "OracleSQLCatalogTarget":{ + "shape":"OracleSQLCatalogTarget", + "documentation":"

Specifies a target that uses Oracle SQL.

" + }, + "PostgreSQLCatalogTarget":{ + "shape":"PostgreSQLCatalogTarget", + "documentation":"

Specifies a target that uses Postgres SQL.

" + } + }, + "documentation":"

CodeGenConfigurationNode enumerates all valid Node types. One and only one of its member variables can be populated.

" + }, + "CodeGenConfigurationNodes":{ + "type":"map", + "key":{"shape":"NodeId"}, + "value":{"shape":"CodeGenConfigurationNode"}, + "sensitive":true + }, "CodeGenEdge":{ "type":"structure", "required":[ @@ -4477,6 +4999,13 @@ "FULL_ALL" ] }, + "CompressionType":{ + "type":"string", + "enum":[ + "gzip", + "bzip2" + ] + }, "ConcurrentModificationException":{ "type":"structure", "members":{ @@ -5517,6 +6046,10 @@ "WorkerType":{ "shape":"WorkerType", "documentation":"

The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, or G.2X.

" + }, + "CodeGenConfigurationNodes":{ + "shape":"CodeGenConfigurationNodes", + "documentation":"

The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.

" } } }, @@ -6207,6 +6740,38 @@ "min":1, "pattern":"[^\\r\\n]" }, + "CustomCode":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Code", + "ClassName" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"ManyInputs", + "documentation":"

The data inputs identified by their node names.

" + }, + "Code":{ + "shape":"ExtendedString", + "documentation":"

The custom code that is used to perform the data transformation.

" + }, + "ClassName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name defined for the custom code node class.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the custom code transform.

" + } + }, + "documentation":"

Specifies a transform that uses custom code you provide to perform the data transformation. The output is a collection of DynamicFrames.

" + }, "CustomEntityType":{ "type":"structure", "required":[ @@ -6379,6 +6944,24 @@ "member":{"shape":"Database"} }, "DatabaseName":{"type":"string"}, + "Datatype":{ + "type":"structure", + "required":[ + "Id", + "Label" + ], + "members":{ + "Id":{ + "shape":"GenericLimitedString", + "documentation":"

The datatype of the value.

" + }, + "Label":{ + "shape":"GenericLimitedString", + "documentation":"

A label assigned to the datatype.

" + } + }, + "documentation":"

A structure representing the datatype of the value.

" + }, "DateColumnStatisticsData":{ "type":"structure", "required":[ @@ -7180,21 +7763,101 @@ "max":25, "min":1 }, - "Double":{"type":"double"}, - "DoubleColumnStatisticsData":{ + "DirectKafkaSource":{ "type":"structure", - "required":[ - "NumberOfNulls", - "NumberOfDistinctValues" - ], + "required":["Name"], "members":{ - "MinimumValue":{ - "shape":"Double", - "documentation":"

The lowest value in the column.

" + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" }, - "MaximumValue":{ - "shape":"Double", - "documentation":"

The highest value in the column.

" + "StreamingOptions":{ + "shape":"KafkaStreamingSourceOptions", + "documentation":"

Specifies the streaming options.

" + }, + "WindowSize":{ + "shape":"BoxedPositiveInt", + "documentation":"

The amount of time to spend processing each micro batch.

", + "box":true + }, + "DetectSchema":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to automatically determine the schema from the incoming data.

", + "box":true + }, + "DataPreviewOptions":{ + "shape":"StreamingDataPreviewOptions", + "documentation":"

Specifies options related to data preview for viewing a sample of your data.

" + } + }, + "documentation":"

Specifies an Apache Kafka data store.

" + }, + "DirectKinesisSource":{ + "type":"structure", + "required":["Name"], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "WindowSize":{ + "shape":"BoxedPositiveInt", + "documentation":"

The amount of time to spend processing each micro batch.

", + "box":true + }, + "DetectSchema":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to automatically determine the schema from the incoming data.

", + "box":true + }, + "StreamingOptions":{ + "shape":"KinesisStreamingSourceOptions", + "documentation":"

Additional options for the Kinesis streaming data source.

" + }, + "DataPreviewOptions":{ + "shape":"StreamingDataPreviewOptions", + "documentation":"

Additional options for data preview.

" + } + }, + "documentation":"

Specifies a direct Amazon Kinesis data source.

" + }, + "DirectSchemaChangePolicy":{ + "type":"structure", + "members":{ + "EnableUpdateCatalog":{ + "shape":"BoxedBoolean", + "documentation":"

Whether to use the specified update behavior when the crawler finds a changed schema.

" + }, + "UpdateBehavior":{ + "shape":"UpdateCatalogBehavior", + "documentation":"

The update behavior when the crawler finds a changed schema.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies the table in the database that the schema change policy applies to.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies the database that the schema change policy applies to.

" + } + }, + "documentation":"

A policy that specifies update behavior for the crawler.

" + }, + "Double":{"type":"double"}, + "DoubleColumnStatisticsData":{ + "type":"structure", + "required":[ + "NumberOfNulls", + "NumberOfDistinctValues" + ], + "members":{ + "MinimumValue":{ + "shape":"Double", + "documentation":"

The lowest value in the column.

" + }, + "MaximumValue":{ + "shape":"Double", + "documentation":"

The highest value in the column.

" }, "NumberOfNulls":{ "shape":"NonNegativeLong", @@ -7208,6 +7871,100 @@ "documentation":"

Defines column statistics supported for floating-point number data columns.

" }, "DoubleValue":{"type":"double"}, + "DropDuplicates":{ + "type":"structure", + "required":[ + "Name", + "Inputs" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Columns":{ + "shape":"LimitedPathList", + "documentation":"

The name of the columns to be merged or removed if repeating.

" + } + }, + "documentation":"

Specifies a transform that removes rows of repeating data from a data set.

" + }, + "DropFields":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Paths" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Paths":{ + "shape":"GlueStudioPathList", + "documentation":"

A JSON path to a variable in the data structure.

" + } + }, + "documentation":"

Specifies a transform that chooses the data property keys that you want to drop.

" + }, + "DropNullFields":{ + "type":"structure", + "required":[ + "Name", + "Inputs" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "NullCheckBoxList":{ + "shape":"NullCheckBoxList", + "documentation":"

A structure that represents whether certain values are recognized as null values for removal.

" + }, + "NullTextList":{ + "shape":"NullValueFields", + "documentation":"

A structure that specifies a list of NullValueField structures that represent a custom null value such as zero or other value being used as a null placeholder unique to the dataset.

The DropNullFields transform removes custom null values only if both the value of the null placeholder and the datatype match the data.

" + } + }, + "documentation":"

Specifies a transform that removes columns from the dataset if all values in the column are 'null'. By default, Glue Studio will recognize null objects, but some values such as empty strings, strings that are \"null\", -1 integers or other placeholders such as zeros, are not automatically recognized as nulls.

" + }, + "DynamoDBCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a DynamoDB data source in the Glue Data Catalog.

" + }, "DynamoDBTarget":{ "type":"structure", "members":{ @@ -7255,6 +8012,22 @@ "FALSE" ] }, + "EnclosedInStringProperties":{ + "type":"list", + "member":{"shape":"EnclosedInStringProperty"} + }, + "EnclosedInStringPropertiesMinOne":{ + "type":"list", + "member":{"shape":"EnclosedInStringProperty"} + }, + "EnclosedInStringProperty":{ + "type":"string", + "pattern":"([\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF]|[^\\S\\r\\n\"'])*" + }, + "EnclosedInStringPropertyWithQuote":{ + "type":"string", + "pattern":"([\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF]|[^\\S\\r\\n])*" + }, "EncryptionAtRest":{ "type":"structure", "required":["CatalogEncryptionMode"], @@ -7395,13 +8168,146 @@ }, "documentation":"

Specifies configuration properties for an exporting labels task run.

" }, + "ExtendedString":{ + "type":"string", + "pattern":"[\\s\\S]*" + }, "FieldType":{"type":"string"}, + "FillMissingValues":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "ImputedPath" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "ImputedPath":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A JSON path to a variable in the data structure for the dataset that is imputed.

" + }, + "FilledPath":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A JSON path to a variable in the data structure for the dataset that is filled.

" + } + }, + "documentation":"

Specifies a transform that locates records in the dataset that have missing values and adds a new field with a value determined by imputation. The input data set is used to train the machine learning model that determines what the missing value should be.

" + }, + "Filter":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "LogicalOperator", + "Filters" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "LogicalOperator":{ + "shape":"FilterLogicalOperator", + "documentation":"

The operator used to filter rows by comparing the key value to a specified value.

" + }, + "Filters":{ + "shape":"FilterExpressions", + "documentation":"

Specifies a filter expression.

" + } + }, + "documentation":"

Specifies a transform that splits a dataset into two, based on a filter condition.

" + }, + "FilterExpression":{ + "type":"structure", + "required":[ + "Operation", + "Values" + ], + "members":{ + "Operation":{ + "shape":"FilterOperation", + "documentation":"

The type of operation to perform in the expression.

" + }, + "Negated":{ + "shape":"BoxedBoolean", + "documentation":"

Whether the expression is to be negated.

" + }, + "Values":{ + "shape":"FilterValues", + "documentation":"

A list of filter values.

" + } + }, + "documentation":"

Specifies a filter expression.

" + }, + "FilterExpressions":{ + "type":"list", + "member":{"shape":"FilterExpression"} + }, + "FilterLogicalOperator":{ + "type":"string", + "enum":[ + "AND", + "OR" + ] + }, + "FilterOperation":{ + "type":"string", + "enum":[ + "EQ", + "LT", + "GT", + "LTE", + "GTE", + "REGEX", + "ISNULL" + ] + }, "FilterString":{ "type":"string", "max":2048, "min":0, "pattern":"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\t]*" }, + "FilterValue":{ + "type":"structure", + "required":[ + "Type", + "Value" + ], + "members":{ + "Type":{ + "shape":"FilterValueType", + "documentation":"

The type of filter value.

" + }, + "Value":{ + "shape":"EnclosedInStringProperties", + "documentation":"

The value to be associated.

" + } + }, + "documentation":"

Represents a single entry in the list of values for a FilterExpression.

" + }, + "FilterValueType":{ + "type":"string", + "enum":[ + "COLUMNEXTRACTED", + "CONSTANT" + ] + }, + "FilterValues":{ + "type":"list", + "member":{"shape":"FilterValue"} + }, "FindMatchesMetrics":{ "type":"structure", "members":{ @@ -7488,6 +8394,10 @@ "max":1.0, "min":0.0 }, + "GenericLimitedString":{ + "type":"string", + "pattern":"[A-Za-z0-9_-]*" + }, "GenericMap":{ "type":"map", "key":{"shape":"GenericString"}, @@ -9538,12 +10448,70 @@ }, "documentation":"

A structure for returning a resource policy.

" }, + "GlueRecordType":{ + "type":"string", + "enum":[ + "DATE", + "STRING", + "TIMESTAMP", + "INT", + "FLOAT", + "LONG", + "BIGDECIMAL", + "BYTE", + "SHORT", + "DOUBLE" + ] + }, "GlueResourceArn":{ "type":"string", "max":10240, "min":1, "pattern":"arn:(aws|aws-us-gov|aws-cn):glue:.*" }, + "GlueSchema":{ + "type":"structure", + "members":{ + "Columns":{ + "shape":"GlueStudioSchemaColumnList", + "documentation":"

Specifies the column definitions that make up a Glue schema.

" + } + }, + "documentation":"

Specifies a user-defined schema when a schema cannot be determined by AWS Glue.

" + }, + "GlueSchemas":{ + "type":"list", + "member":{"shape":"GlueSchema"} + }, + "GlueStudioColumnNameString":{ + "type":"string", + "max":1024, + "min":0, + "pattern":"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\t]*" + }, + "GlueStudioPathList":{ + "type":"list", + "member":{"shape":"EnclosedInStringProperties"} + }, + "GlueStudioSchemaColumn":{ + "type":"structure", + "required":["Name"], + "members":{ + "Name":{ + "shape":"GlueStudioColumnNameString", + "documentation":"

The name of the column in the Glue Studio schema.

" + }, + "Type":{ + "shape":"ColumnTypeString", + "documentation":"

The hive type for this column in the Glue Studio schema.

" + } + }, + "documentation":"

Specifies a single column in a Glue schema definition.

" + }, + "GlueStudioSchemaColumnList":{ + "type":"list", + "member":{"shape":"GlueStudioSchemaColumn"} + }, "GlueTable":{ "type":"structure", "required":[ @@ -9582,6 +10550,73 @@ "min":1, "pattern":"^\\w+\\.\\w+$" }, + "GovernedCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database table to read from.

" + }, + "PartitionPredicate":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted. Set to \"\" – empty by default.

" + }, + "AdditionalOptions":{ + "shape":"S3SourceAdditionalOptions", + "documentation":"

Specifies additional connection options.

" + } + }, + "documentation":"

Specifies the data store in the governed Glue Data Catalog.

" + }, + "GovernedCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Table", + "Database" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "PartitionKeys":{ + "shape":"GlueStudioPathList", + "documentation":"

Specifies native partitioning using a sequence of keys.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "SchemaChangePolicy":{ + "shape":"CatalogSchemaChangePolicy", + "documentation":"

A policy that specifies update behavior for the governed catalog.

" + } + }, + "documentation":"

Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.

" + }, "GrokClassifier":{ "type":"structure", "required":[ @@ -9749,23 +10784,200 @@ "exception":true }, "IsVersionValid":{"type":"boolean"}, - "JdbcTarget":{ + "JDBCConnectorOptions":{ "type":"structure", "members":{ - "ConnectionName":{ - "shape":"ConnectionName", - "documentation":"

The name of the connection to use to connect to the JDBC target.

" + "FilterPredicate":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Extra condition clause to filter data from source. For example:

BillingCity='Mountain View'

When using a query instead of a table name, you should validate that the query works with the specified filterPredicate.

" }, - "Path":{ - "shape":"Path", - "documentation":"

The path of the JDBC target.

" + "PartitionColumn":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of an integer column that is used for partitioning. This option works only when it's included with lowerBound, upperBound, and numPartitions. This option works the same way as in the Spark SQL JDBC reader.

" }, - "Exclusions":{ - "shape":"PathList", - "documentation":"

A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.

" + "LowerBound":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The minimum value of partitionColumn that is used to decide partition stride.

" + }, + "UpperBound":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The maximum value of partitionColumn that is used to decide partition stride.

" + }, + "NumPartitions":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The number of partitions. This value, along with lowerBound (inclusive) and upperBound (exclusive), form partition strides for generated WHERE clause expressions that are used to split the partitionColumn.

" + }, + "JobBookmarkKeys":{ + "shape":"EnclosedInStringProperties", + "documentation":"

The name of the job bookmark keys on which to sort.

" + }, + "JobBookmarkKeysSortOrder":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies an ascending or descending sort order.

" + }, + "DataTypeMapping":{ + "shape":"JDBCDataTypeMapping", + "documentation":"

Custom data type mapping that builds a mapping from a JDBC data type to an Glue data type. For example, the option \"dataTypeMapping\":{\"FLOAT\":\"STRING\"} maps data fields of JDBC type FLOAT into the Java String type by calling the ResultSet.getString() method of the driver, and uses it to build the Glue record. The ResultSet object is implemented by each driver, so the behavior is specific to the driver you use. Refer to the documentation for your JDBC driver to understand how the driver performs the conversions.

" } }, - "documentation":"

Specifies a JDBC data store to crawl.

" + "documentation":"

Additional connection options for the connector.

" + }, + "JDBCConnectorSource":{ + "type":"structure", + "required":[ + "Name", + "ConnectionName", + "ConnectorName", + "ConnectionType" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection that is associated with the connector.

" + }, + "ConnectorName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of a connector that assists with accessing the data store in Glue Studio.

" + }, + "ConnectionType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of connection, such as marketplace.jdbc or custom.jdbc, designating a connection to a JDBC data store.

" + }, + "AdditionalOptions":{ + "shape":"JDBCConnectorOptions", + "documentation":"

Additional connection options for the connector.

" + }, + "ConnectionTable":{ + "shape":"EnclosedInStringPropertyWithQuote", + "documentation":"

The name of the table in the data source.

" + }, + "Query":{ + "shape":"SqlQuery", + "documentation":"

The table or SQL query to get the data from. You can specify either ConnectionTable or query, but not both.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the custom JDBC source.

" + } + }, + "documentation":"

Specifies a connector to a JDBC data source.

" + }, + "JDBCConnectorTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "ConnectionName", + "ConnectionTable", + "ConnectorName", + "ConnectionType" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection that is associated with the connector.

" + }, + "ConnectionTable":{ + "shape":"EnclosedInStringPropertyWithQuote", + "documentation":"

The name of the table in the data target.

" + }, + "ConnectorName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of a connector that will be used.

" + }, + "ConnectionType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of connection, such as marketplace.jdbc or custom.jdbc, designating a connection to a JDBC data target.

" + }, + "AdditionalOptions":{ + "shape":"AdditionalOptions", + "documentation":"

Additional connection options for the connector.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the JDBC target.

" + } + }, + "documentation":"

Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.

" + }, + "JDBCDataType":{ + "type":"string", + "enum":[ + "ARRAY", + "BIGINT", + "BINARY", + "BIT", + "BLOB", + "BOOLEAN", + "CHAR", + "CLOB", + "DATALINK", + "DATE", + "DECIMAL", + "DISTINCT", + "DOUBLE", + "FLOAT", + "INTEGER", + "JAVA_OBJECT", + "LONGNVARCHAR", + "LONGVARBINARY", + "LONGVARCHAR", + "NCHAR", + "NCLOB", + "NULL", + "NUMERIC", + "NVARCHAR", + "OTHER", + "REAL", + "REF", + "REF_CURSOR", + "ROWID", + "SMALLINT", + "SQLXML", + "STRUCT", + "TIME", + "TIME_WITH_TIMEZONE", + "TIMESTAMP", + "TIMESTAMP_WITH_TIMEZONE", + "TINYINT", + "VARBINARY", + "VARCHAR" + ] + }, + "JDBCDataTypeMapping":{ + "type":"map", + "key":{"shape":"JDBCDataType"}, + "value":{"shape":"GlueRecordType"} + }, + "JdbcTarget":{ + "type":"structure", + "members":{ + "ConnectionName":{ + "shape":"ConnectionName", + "documentation":"

The name of the connection to use to connect to the JDBC target.

" + }, + "Path":{ + "shape":"Path", + "documentation":"

The path of the JDBC target.

" + }, + "Exclusions":{ + "shape":"PathList", + "documentation":"

A list of glob patterns used to exclude from the crawl. For more information, see Catalog Tables with a Crawler.

" + } + }, + "documentation":"

Specifies a JDBC data store to crawl.

" }, "JdbcTargetList":{ "type":"list", @@ -9855,6 +11067,10 @@ "GlueVersion":{ "shape":"GlueVersionString", "documentation":"

Glue version determines the versions of Apache Spark and Python that Glue supports. The Python version indicates the version supported for jobs of type Spark.

For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.

Jobs that are created without specifying a Glue version default to Glue 0.9.

" + }, + "CodeGenConfigurationNodes":{ + "shape":"CodeGenConfigurationNodes", + "documentation":"

The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.

" } }, "documentation":"

Specifies a job definition.

" @@ -10139,10 +11355,77 @@ "GlueVersion":{ "shape":"GlueVersionString", "documentation":"

Glue version determines the versions of Apache Spark and Python that Glue supports. The Python version indicates the version supported for jobs of type Spark.

For more information about the available Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.

" + }, + "CodeGenConfigurationNodes":{ + "shape":"CodeGenConfigurationNodes", + "documentation":"

The representation of a directed acyclic graph on which both the Glue Studio visual component and Glue Studio code generation is based.

" } }, "documentation":"

Specifies information used to update an existing job definition. The previous job definition is completely overwritten by this information.

" }, + "Join":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "JoinType", + "Columns" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"TwoInputs", + "documentation":"

The data inputs identified by their node names.

" + }, + "JoinType":{ + "shape":"JoinType", + "documentation":"

Specifies the type of join to be performed on the datasets.

" + }, + "Columns":{ + "shape":"JoinColumns", + "documentation":"

A list of the two columns to be joined.

" + } + }, + "documentation":"

Specifies a transform that joins two datasets into one dataset using a comparison phrase on the specified data property keys. You can use inner, outer, left, right, left semi, and left anti joins.

" + }, + "JoinColumn":{ + "type":"structure", + "required":[ + "From", + "Keys" + ], + "members":{ + "From":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The column to be joined.

" + }, + "Keys":{ + "shape":"GlueStudioPathList", + "documentation":"

The key of the column to be joined.

" + } + }, + "documentation":"

Specifies a column to be joined.

" + }, + "JoinColumns":{ + "type":"list", + "member":{"shape":"JoinColumn"}, + "max":2, + "min":2 + }, + "JoinType":{ + "type":"string", + "enum":[ + "equijoin", + "left", + "right", + "outer", + "leftsemi", + "leftanti" + ] + }, "JsonClassifier":{ "type":"structure", "required":[ @@ -10175,6 +11458,72 @@ }, "JsonPath":{"type":"string"}, "JsonValue":{"type":"string"}, + "KafkaStreamingSourceOptions":{ + "type":"structure", + "members":{ + "BootstrapServers":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A list of bootstrap server URLs, for example, as b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094. This option must be specified in the API call or defined in the table metadata in the Data Catalog.

" + }, + "SecurityProtocol":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The protocol used to communicate with brokers. The possible values are \"SSL\" or \"PLAINTEXT\".

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection.

" + }, + "TopicName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The topic name as specified in Apache Kafka. You must specify at least one of \"topicName\", \"assign\" or \"subscribePattern\".

" + }, + "Assign":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The specific TopicPartitions to consume. You must specify at least one of \"topicName\", \"assign\" or \"subscribePattern\".

" + }, + "SubscribePattern":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A Java regex string that identifies the topic list to subscribe to. You must specify at least one of \"topicName\", \"assign\" or \"subscribePattern\".

" + }, + "Classification":{ + "shape":"EnclosedInStringProperty", + "documentation":"

An optional classification.

" + }, + "Delimiter":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies the delimiter character.

" + }, + "StartingOffsets":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The starting position in the Kafka topic to read data from. The possible values are \"earliest\" or \"latest\". The default value is \"latest\".

" + }, + "EndingOffsets":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The end point when a batch query is ended. Possible values are either \"latest\" or a JSON string that specifies an ending offset for each TopicPartition.

" + }, + "PollTimeoutMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The timeout in milliseconds to poll data from Kafka in Spark job executors. The default value is 512.

" + }, + "NumRetries":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

The number of times to retry before failing to fetch Kafka offsets. The default value is 3.

" + }, + "RetryIntervalMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The time in milliseconds to wait before retrying to fetch Kafka offsets. The default value is 10.

" + }, + "MaxOffsetsPerTrigger":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The rate limit on the maximum number of offsets that are processed per trigger interval. The specified total number of offsets is proportionally split across topicPartitions of different volumes. The default value is null, which means that the consumer reads all offsets until the known latest offset.

" + }, + "MinPartitions":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

The desired minimum number of partitions to read from Kafka. The default value is null, which means that the number of spark partitions is equal to the number of Kafka partitions.

" + } + }, + "documentation":"

Additional options for streaming.

" + }, "KeyList":{ "type":"list", "member":{"shape":"NameString"}, @@ -10209,6 +11558,84 @@ "min":1, "pattern":"[\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\t]*" }, + "KinesisStreamingSourceOptions":{ + "type":"structure", + "members":{ + "EndpointUrl":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The URL of the Kinesis endpoint.

" + }, + "StreamName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the Kinesis data stream.

" + }, + "Classification":{ + "shape":"EnclosedInStringProperty", + "documentation":"

An optional classification.

" + }, + "Delimiter":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies the delimiter character.

" + }, + "StartingPosition":{ + "shape":"StartingPosition", + "documentation":"

The starting position in the Kinesis data stream to read data from. The possible values are \"latest\", \"trim_horizon\", or \"earliest\". The default value is \"latest\".

" + }, + "MaxFetchTimeInMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The maximum time spent in the job executor to fetch a record from the Kinesis data stream per shard, specified in milliseconds (ms). The default value is 1000.

" + }, + "MaxFetchRecordsPerShard":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The maximum number of records to fetch per shard in the Kinesis data stream. The default value is 100000.

" + }, + "MaxRecordPerRead":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The maximum number of records to fetch from the Kinesis data stream in each getRecords operation. The default value is 10000.

" + }, + "AddIdleTimeBetweenReads":{ + "shape":"BoxedBoolean", + "documentation":"

Adds a time delay between two consecutive getRecords operations. The default value is \"False\". This option is only configurable for Glue version 2.0 and above.

" + }, + "IdleTimeBetweenReadsInMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The minimum time delay between two consecutive getRecords operations, specified in ms. The default value is 1000. This option is only configurable for Glue version 2.0 and above.

" + }, + "DescribeShardInterval":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The minimum time interval between two ListShards API calls for your script to consider resharding. The default value is 1s.

" + }, + "NumRetries":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

The maximum number of retries for Kinesis Data Streams API requests. The default value is 3.

" + }, + "RetryIntervalMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The cool-off time period (specified in ms) before retrying the Kinesis Data Streams API call. The default value is 1000.

" + }, + "MaxRetryIntervalMs":{ + "shape":"BoxedNonNegativeLong", + "documentation":"

The maximum cool-off time period (specified in ms) between two retries of a Kinesis Data Streams API call. The default value is 10000.

" + }, + "AvoidEmptyBatches":{ + "shape":"BoxedBoolean", + "documentation":"

Avoids creating an empty microbatch job by checking for unread data in the Kinesis data stream before the batch is started. The default value is \"False\".

" + }, + "StreamArn":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The Amazon Resource Name (ARN) of the Kinesis data stream.

" + }, + "RoleArn":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The Amazon Resource Name (ARN) of the role to assume using AWS Security Token Service (AWS STS). This role must have permissions for describe or read record operations for the Kinesis data stream. You must use this parameter when accessing a data stream in a different account. Used in conjunction with \"awsSTSSessionName\".

" + }, + "RoleSessionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

An identifier for the session assuming the role using AWS STS. You must use this parameter when accessing a data stream in a different account. Used in conjunction with \"awsSTSRoleARN\".

" + } + }, + "documentation":"

Additional options for the Amazon Kinesis streaming data source.

" + }, "KmsKeyArn":{ "type":"string", "pattern":"arn:aws:kms:.*" @@ -10310,6 +11737,14 @@ ] }, "LatestSchemaVersionBoolean":{"type":"boolean"}, + "LimitedPathList":{ + "type":"list", + "member":{"shape":"LimitedStringList"} + }, + "LimitedStringList":{ + "type":"list", + "member":{"shape":"GenericLimitedString"} + }, "LineageConfiguration":{ "type":"structure", "members":{ @@ -10913,6 +12348,11 @@ "SSE-KMS" ] }, + "ManyInputs":{ + "type":"list", + "member":{"shape":"NodeId"}, + "min":1 + }, "MapValue":{ "type":"map", "key":{"shape":"GenericString"}, @@ -10920,6 +12360,36 @@ "max":100, "min":0 }, + "Mapping":{ + "type":"structure", + "members":{ + "ToKey":{ + "shape":"EnclosedInStringProperty", + "documentation":"

After the apply mapping, what the name of the column should be. Can be the same as FromPath.

" + }, + "FromPath":{ + "shape":"EnclosedInStringProperties", + "documentation":"

The table or column to be modified.

" + }, + "FromType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of the data to be modified.

" + }, + "ToType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The data type that the data is to be modified to.

" + }, + "Dropped":{ + "shape":"BoxedBoolean", + "documentation":"

If true, then the column is removed.

" + }, + "Children":{ + "shape":"Mappings", + "documentation":"

Only applicable to nested data structures. If you want to change the parent structure, but also one of its children, you can fill out this data strucutre. It is also Mapping, but its FromPath will be the parent's FromPath plus the FromPath from this structure.

For the children part, suppose you have the structure:

{ \"FromPath\": \"OuterStructure\", \"ToKey\": \"OuterStructure\", \"ToType\": \"Struct\", \"Dropped\": false, \"Chidlren\": [{ \"FromPath\": \"inner\", \"ToKey\": \"inner\", \"ToType\": \"Double\", \"Dropped\": false, }] }

You can specify a Mapping that looks like:

{ \"FromPath\": \"OuterStructure\", \"ToKey\": \"OuterStructure\", \"ToType\": \"Struct\", \"Dropped\": false, \"Chidlren\": [{ \"FromPath\": \"inner\", \"ToKey\": \"inner\", \"ToType\": \"Double\", \"Dropped\": false, }] }

" + } + }, + "documentation":"

Specifies the mapping of data property keys.

" + }, "MappingEntry":{ "type":"structure", "members":{ @@ -10954,6 +12424,16 @@ "type":"list", "member":{"shape":"MappingEntry"} }, + "Mappings":{ + "type":"list", + "member":{"shape":"Mapping"} + }, + "MaskValue":{ + "type":"string", + "max":256, + "min":0, + "pattern":"[*A-Za-z0-9_-]*" + }, "MatchCriteria":{ "type":"list", "member":{"shape":"NameString"}, @@ -10968,6 +12448,34 @@ "min":1 }, "MaxRetries":{"type":"integer"}, + "Merge":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Source", + "PrimaryKeys" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"TwoInputs", + "documentation":"

The data inputs identified by their node names.

" + }, + "Source":{ + "shape":"NodeId", + "documentation":"

The source DynamicFrame that will be merged with a staging DynamicFrame.

" + }, + "PrimaryKeys":{ + "shape":"GlueStudioPathList", + "documentation":"

The list of primary key fields to match records from the source and staging dynamic frames.

" + } + }, + "documentation":"

Specifies a transform that merges a DynamicFrame with a staging DynamicFrame based on the specified primary keys to identify records. Duplicate records (records with the same primary keys) are not de-duplicated.

" + }, "MessagePrefix":{ "type":"string", "max":255, @@ -11028,6 +12536,57 @@ "min":1, "pattern":"[a-zA-Z0-9+-=._./@]+" }, + "MicrosoftSQLServerCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a Microsoft SQL server data source in the Glue Data Catalog.

" + }, + "MicrosoftSQLServerCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + } + }, + "documentation":"

Specifies a target that uses Microsoft SQL.

" + }, "MillisecondsCount":{"type":"long"}, "MongoDBTarget":{ "type":"structure", @@ -11051,6 +12610,57 @@ "type":"list", "member":{"shape":"MongoDBTarget"} }, + "MySQLCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a MySQL data source in the Glue Data Catalog.

" + }, + "MySQLCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + } + }, + "documentation":"

Specifies a target that uses MySQL.

" + }, "NameString":{ "type":"string", "max":255, @@ -11102,6 +12712,10 @@ }, "documentation":"

A node represents an Glue component (trigger, crawler, or job) on a workflow graph.

" }, + "NodeId":{ + "type":"string", + "pattern":"[A-Za-z0-9_-]*" + }, "NodeIdList":{ "type":"list", "member":{"shape":"NameString"} @@ -11110,6 +12724,10 @@ "type":"list", "member":{"shape":"Node"} }, + "NodeName":{ + "type":"string", + "pattern":"([\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF]|[^\\r\\n])*" + }, "NodeType":{ "type":"string", "enum":[ @@ -11122,6 +12740,10 @@ "type":"double", "min":0.0 }, + "NonNegativeInt":{ + "type":"integer", + "min":0 + }, "NonNegativeInteger":{ "type":"integer", "min":0 @@ -11145,6 +12767,48 @@ "box":true, "min":1 }, + "NullCheckBoxList":{ + "type":"structure", + "members":{ + "IsEmpty":{ + "shape":"BoxedBoolean", + "documentation":"

Specifies that an empty string is considered as a null value.

" + }, + "IsNullString":{ + "shape":"BoxedBoolean", + "documentation":"

Specifies that a value spelling out the word 'null' is considered as a null value.

" + }, + "IsNegOne":{ + "shape":"BoxedBoolean", + "documentation":"

Specifies that an integer value of -1 is considered as a null value.

" + } + }, + "documentation":"

Represents whether certain values are recognized as null values for removal.

" + }, + "NullValueField":{ + "type":"structure", + "required":[ + "Value", + "Datatype" + ], + "members":{ + "Value":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The value of the null placeholder.

" + }, + "Datatype":{ + "shape":"Datatype", + "documentation":"

The datatype of the value.

" + } + }, + "documentation":"

Represents a custom null value such as a zeros or other value being used as a null placeholder unique to the dataset.

" + }, + "NullValueFields":{ + "type":"list", + "member":{"shape":"NullValueField"}, + "max":50, + "min":0 + }, "NullableBoolean":{ "type":"boolean", "box":true @@ -11157,6 +12821,12 @@ "type":"integer", "box":true }, + "OneInput":{ + "type":"list", + "member":{"shape":"NodeId"}, + "max":1, + "min":1 + }, "OperationTimeoutException":{ "type":"structure", "members":{ @@ -11168,6 +12838,57 @@ "documentation":"

The operation timed out.

", "exception":true }, + "OracleSQLCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies an Oracle data source in the Glue Data Catalog.

" + }, + "OracleSQLCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + } + }, + "documentation":"

Specifies a target that uses Oracle SQL.

" + }, "OrchestrationArgumentsMap":{ "type":"map", "key":{"shape":"OrchestrationNameString"}, @@ -11257,6 +12978,50 @@ }, "documentation":"

A structure containing other metadata for a schema version belonging to the same metadata key.

" }, + "PIIDetection":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "PiiType", + "EntityTypesToDetect" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The node ID inputs to the transform.

" + }, + "PiiType":{ + "shape":"PiiType", + "documentation":"

Indicates the type of PIIDetection transform.

" + }, + "EntityTypesToDetect":{ + "shape":"EnclosedInStringProperties", + "documentation":"

Indicates the types of entities the PIIDetection transform will identify as PII data.

PII type entities include: PERSON_NAME, DATE, USA_SNN, EMAIL, USA_ITIN, USA_PASSPORT_NUMBER, PHONE_NUMBER, BANK_ACCOUNT, IP_ADDRESS, MAC_ADDRESS, USA_CPT_CODE, USA_HCPCS_CODE, USA_NATIONAL_DRUG_CODE, USA_MEDICARE_BENEFICIARY_IDENTIFIER, USA_HEALTH_INSURANCE_CLAIM_NUMBER,CREDIT_CARD,USA_NATIONAL_PROVIDER_IDENTIFIER,USA_DEA_NUMBER,USA_DRIVING_LICENSE

" + }, + "OutputColumnName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Indicates the output column name that will contain any entity type detected in that row.

" + }, + "SampleFraction":{ + "shape":"BoxedDoubleFraction", + "documentation":"

Indicates the fraction of the data to sample when scanning for PII entities.

" + }, + "ThresholdFraction":{ + "shape":"BoxedDoubleFraction", + "documentation":"

Indicates the fraction of the data that must be met in order for a column to be identified as PII data.

" + }, + "MaskValue":{ + "shape":"MaskValue", + "documentation":"

Indicates the value that will replace the detected entity.

" + } + }, + "documentation":"

Specifies a transform that identifies, removes or masks PII data.

" + }, "PageSize":{ "type":"integer", "box":true, @@ -11273,6 +13038,16 @@ "type":"string", "max":512000 }, + "ParquetCompressionType":{ + "type":"string", + "enum":[ + "snappy", + "lzo", + "gzip", + "uncompressed", + "none" + ] + }, "Partition":{ "type":"structure", "members":{ @@ -11504,10 +13279,80 @@ }, "documentation":"

Specifies the physical requirements for a connection.

" }, + "PiiType":{ + "type":"string", + "enum":[ + "RowAudit", + "RowMasking", + "ColumnAudit", + "ColumnMasking" + ] + }, "PolicyJsonString":{ "type":"string", "min":2 }, + "PollingTime":{ + "type":"long", + "box":true, + "min":10 + }, + "PositiveLong":{ + "type":"long", + "box":true, + "min":1 + }, + "PostgreSQLCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a PostgresSQL data source in the Glue Data Catalog.

" + }, + "PostgreSQLCatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + } + }, + "documentation":"

Specifies a target that uses Postgres SQL.

" + }, "Predecessor":{ "type":"structure", "members":{ @@ -11572,6 +13417,12 @@ "GROUP" ] }, + "Prob":{ + "type":"double", + "box":true, + "max":1, + "min":0 + }, "PropertyPredicate":{ "type":"structure", "members":{ @@ -11791,6 +13642,15 @@ } } }, + "QuoteChar":{ + "type":"string", + "enum":[ + "quote", + "quillemet", + "single_quote", + "disabled" + ] + }, "RecordsCount":{ "type":"long", "box":true @@ -11813,6 +13673,77 @@ }, "documentation":"

When crawling an Amazon S3 data source after the first crawl is complete, specifies whether to crawl the entire dataset again or to crawl only folders that were added since the last crawler run. For more information, see Incremental Crawls in Glue in the developer guide.

" }, + "RedshiftSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the Amazon Redshift data store.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database table to read from.

" + }, + "RedshiftTmpDir":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The Amazon S3 path where temporary data can be staged when copying out of the database.

" + }, + "TmpDirIAMRole":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The IAM role with permissions.

" + } + }, + "documentation":"

Specifies an Amazon Redshift data store.

" + }, + "RedshiftTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + }, + "RedshiftTmpDir":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The Amazon S3 path where temporary data can be staged when copying out of the database.

" + }, + "TmpDirIAMRole":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The IAM role with permissions.

" + }, + "UpsertRedshiftOptions":{ + "shape":"UpsertRedshiftTargetOptions", + "documentation":"

The set of options to configure an upsert operation when writing to a Redshift target.

" + } + }, + "documentation":"

Specifies a target that uses Amazon Redshift.

" + }, "RegisterSchemaVersionInput":{ "type":"structure", "required":[ @@ -11902,6 +13833,29 @@ "DELETING" ] }, + "RelationalCatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to read from.

" + } + }, + "documentation":"

Specifies a Relational database data source in the Glue Data Catalog.

" + }, "RemoveSchemaVersionMetadataInput":{ "type":"structure", "required":["MetadataKeyValue"], @@ -11961,6 +13915,34 @@ } } }, + "RenameField":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "SourcePath", + "TargetPath" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "SourcePath":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A JSON path to a variable in the data structure for the source data.

" + }, + "TargetPath":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A JSON path to a variable in the data structure for the target data.

" + } + }, + "documentation":"

Specifies a transform that renames a single data property key.

" + }, "ReplaceBoolean":{"type":"boolean"}, "ResetJobBookmarkRequest":{ "type":"structure", @@ -12092,28 +14074,245 @@ "Code" ], "members":{ - "SessionId":{ - "shape":"NameString", - "documentation":"

The Session Id of the statement to be run.

" + "SessionId":{ + "shape":"NameString", + "documentation":"

The Session Id of the statement to be run.

" + }, + "Code":{ + "shape":"OrchestrationStatementCodeString", + "documentation":"

The statement code to be run.

" + }, + "RequestOrigin":{ + "shape":"OrchestrationNameString", + "documentation":"

The origin of the request.

" + } + } + }, + "RunStatementResponse":{ + "type":"structure", + "members":{ + "Id":{ + "shape":"IntegerValue", + "documentation":"

Returns the Id of the statement that was run.

" + } + } + }, + "S3CatalogSource":{ + "type":"structure", + "required":[ + "Name", + "Database", + "Table" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database to read from.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The database table to read from.

" + }, + "PartitionPredicate":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Partitions satisfying this predicate are deleted. Files within the retention period in these partitions are not deleted. Set to \"\" – empty by default.

" + }, + "AdditionalOptions":{ + "shape":"S3SourceAdditionalOptions", + "documentation":"

Specifies additional connection options.

" + } + }, + "documentation":"

Specifies an Amazon S3 data store in the Glue Data Catalog.

" + }, + "S3CatalogTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Table", + "Database" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "PartitionKeys":{ + "shape":"GlueStudioPathList", + "documentation":"

Specifies native partitioning using a sequence of keys.

" + }, + "Table":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the table in the database to write to.

" + }, + "Database":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the database to write to.

" + }, + "SchemaChangePolicy":{ + "shape":"CatalogSchemaChangePolicy", + "documentation":"

A policy that specifies update behavior for the crawler.

" + } + }, + "documentation":"

Specifies a data target that writes to Amazon S3 using the Glue Data Catalog.

" + }, + "S3CsvSource":{ + "type":"structure", + "required":[ + "Name", + "Paths", + "Separator", + "QuoteChar" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Paths":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A list of the Amazon S3 paths to read from.

" + }, + "CompressionType":{ + "shape":"CompressionType", + "documentation":"

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are \"gzip\" and \"bzip\").

" + }, + "Exclusions":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A string containing a JSON list of Unix-style glob patterns to exclude. For example, \"[\\\"**.pdf\\\"]\" excludes all PDF files.

" + }, + "GroupSize":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, \"groupFiles\" must be set to \"inPartition\" for this to take effect.

" + }, + "GroupFiles":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to \"inPartition\". To disable grouping when there are more than 50,000 files, set this parameter to \"none\".

" + }, + "Recurse":{ + "shape":"BoxedBoolean", + "documentation":"

If set to true, recursively reads files in all subdirectories under the specified paths.

" + }, + "MaxBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.

" + }, + "MaxFilesInBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.

" + }, + "AdditionalOptions":{ + "shape":"S3DirectSourceAdditionalOptions", + "documentation":"

Specifies additional connection options.

" + }, + "Separator":{ + "shape":"Separator", + "documentation":"

Specifies the delimiter character. The default is a comma: \",\", but any other character can be specified.

" + }, + "Escaper":{ + "shape":"EnclosedInStringPropertyWithQuote", + "documentation":"

Specifies a character to use for escaping. This option is used only when reading CSV files. The default value is none. If enabled, the character which immediately follows is used as-is, except for a small set of well-known escapes (\\n, \\r, \\t, and \\0).

" + }, + "QuoteChar":{ + "shape":"QuoteChar", + "documentation":"

Specifies the character to use for quoting. The default is a double quote: '\"'. Set this to -1 to turn off quoting entirely.

" + }, + "Multiline":{ + "shape":"BoxedBoolean", + "documentation":"

A Boolean value that specifies whether a single record can span multiple lines. This can occur when a field contains a quoted new-line character. You must set this option to True if any record spans multiple lines. The default value is False, which allows for more aggressive file-splitting during parsing.

" + }, + "WithHeader":{ + "shape":"BoxedBoolean", + "documentation":"

A Boolean value that specifies whether to treat the first line as a header. The default value is False.

" + }, + "WriteHeader":{ + "shape":"BoxedBoolean", + "documentation":"

A Boolean value that specifies whether to write the header to output. The default value is True.

" + }, + "SkipFirst":{ + "shape":"BoxedBoolean", + "documentation":"

A Boolean value that specifies whether to skip the first data line. The default value is False.

" + }, + "OptimizePerformance":{ + "shape":"BooleanValue", + "documentation":"

A Boolean value that specifies whether to use the advanced SIMD CSV reader along with Apache Arrow based columnar memory formats. Only available in Glue version 3.0.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the S3 CSV source.

" + } + }, + "documentation":"

Specifies a command-separated value (CSV) data store stored in Amazon S3.

" + }, + "S3DirectSourceAdditionalOptions":{ + "type":"structure", + "members":{ + "BoundedSize":{ + "shape":"BoxedLong", + "documentation":"

Sets the upper limit for the target size of the dataset in bytes that will be processed.

" }, - "Code":{ - "shape":"OrchestrationStatementCodeString", - "documentation":"

The statement code to be run.

" + "BoundedFiles":{ + "shape":"BoxedLong", + "documentation":"

Sets the upper limit for the target number of files that will be processed.

" }, - "RequestOrigin":{ - "shape":"OrchestrationNameString", - "documentation":"

The origin of the request.

" + "EnableSamplePath":{ + "shape":"BoxedBoolean", + "documentation":"

Sets option to enable a sample path.

" + }, + "SamplePath":{ + "shape":"EnclosedInStringProperty", + "documentation":"

If enabled, specifies the sample path.

" } - } + }, + "documentation":"

Specifies additional connection options for the Amazon S3 data store.

" }, - "RunStatementResponse":{ + "S3DirectTarget":{ "type":"structure", + "required":[ + "Name", + "Inputs", + "Path", + "Format" + ], "members":{ - "Id":{ - "shape":"IntegerValue", - "documentation":"

Returns the Id of the statement that was run.

" + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "PartitionKeys":{ + "shape":"GlueStudioPathList", + "documentation":"

Specifies native partitioning using a sequence of keys.

" + }, + "Path":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A single Amazon S3 path to write to.

" + }, + "Compression":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are \"gzip\" and \"bzip\").

" + }, + "Format":{ + "shape":"TargetFormat", + "documentation":"

Specifies the data output format for the target.

" + }, + "SchemaChangePolicy":{ + "shape":"DirectSchemaChangePolicy", + "documentation":"

A policy that specifies update behavior for the crawler.

" } - } + }, + "documentation":"

Specifies a data target that writes to Amazon S3.

" }, "S3Encryption":{ "type":"structure", @@ -12141,6 +14340,171 @@ "SSE-S3" ] }, + "S3GlueParquetTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Path" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "PartitionKeys":{ + "shape":"GlueStudioPathList", + "documentation":"

Specifies native partitioning using a sequence of keys.

" + }, + "Path":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A single Amazon S3 path to write to.

" + }, + "Compression":{ + "shape":"ParquetCompressionType", + "documentation":"

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are \"gzip\" and \"bzip\").

" + }, + "SchemaChangePolicy":{ + "shape":"DirectSchemaChangePolicy", + "documentation":"

A policy that specifies update behavior for the crawler.

" + } + }, + "documentation":"

Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.

" + }, + "S3JsonSource":{ + "type":"structure", + "required":[ + "Name", + "Paths" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Paths":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A list of the Amazon S3 paths to read from.

" + }, + "CompressionType":{ + "shape":"CompressionType", + "documentation":"

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are \"gzip\" and \"bzip\").

" + }, + "Exclusions":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A string containing a JSON list of Unix-style glob patterns to exclude. For example, \"[\\\"**.pdf\\\"]\" excludes all PDF files.

" + }, + "GroupSize":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, \"groupFiles\" must be set to \"inPartition\" for this to take effect.

" + }, + "GroupFiles":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to \"inPartition\". To disable grouping when there are more than 50,000 files, set this parameter to \"none\".

" + }, + "Recurse":{ + "shape":"BoxedBoolean", + "documentation":"

If set to true, recursively reads files in all subdirectories under the specified paths.

" + }, + "MaxBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.

" + }, + "MaxFilesInBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.

" + }, + "AdditionalOptions":{ + "shape":"S3DirectSourceAdditionalOptions", + "documentation":"

Specifies additional connection options.

" + }, + "JsonPath":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A JsonPath string defining the JSON data.

" + }, + "Multiline":{ + "shape":"BoxedBoolean", + "documentation":"

A Boolean value that specifies whether a single record can span multiple lines. This can occur when a field contains a quoted new-line character. You must set this option to True if any record spans multiple lines. The default value is False, which allows for more aggressive file-splitting during parsing.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the S3 JSON source.

" + } + }, + "documentation":"

Specifies a JSON data store stored in Amazon S3.

" + }, + "S3ParquetSource":{ + "type":"structure", + "required":[ + "Name", + "Paths" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data store.

" + }, + "Paths":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A list of the Amazon S3 paths to read from.

" + }, + "CompressionType":{ + "shape":"ParquetCompressionType", + "documentation":"

Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are \"gzip\" and \"bzip\").

" + }, + "Exclusions":{ + "shape":"EnclosedInStringProperties", + "documentation":"

A string containing a JSON list of Unix-style glob patterns to exclude. For example, \"[\\\"**.pdf\\\"]\" excludes all PDF files.

" + }, + "GroupSize":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The target group size in bytes. The default is computed based on the input data size and the size of your cluster. When there are fewer than 50,000 input files, \"groupFiles\" must be set to \"inPartition\" for this to take effect.

" + }, + "GroupFiles":{ + "shape":"EnclosedInStringProperty", + "documentation":"

Grouping files is turned on by default when the input contains more than 50,000 files. To turn on grouping with fewer than 50,000 files, set this parameter to \"inPartition\". To disable grouping when there are more than 50,000 files, set this parameter to \"none\".

" + }, + "Recurse":{ + "shape":"BoxedBoolean", + "documentation":"

If set to true, recursively reads files in all subdirectories under the specified paths.

" + }, + "MaxBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option controls the duration in milliseconds after which the s3 listing is likely to be consistent. Files with modification timestamps falling within the last maxBand milliseconds are tracked specially when using JobBookmarks to account for Amazon S3 eventual consistency. Most users don't need to set this option. The default is 900000 milliseconds, or 15 minutes.

" + }, + "MaxFilesInBand":{ + "shape":"BoxedNonNegativeInt", + "documentation":"

This option specifies the maximum number of files to save from the last maxBand seconds. If this number is exceeded, extra files are skipped and only processed in the next job run.

" + }, + "AdditionalOptions":{ + "shape":"S3DirectSourceAdditionalOptions", + "documentation":"

Specifies additional connection options.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the S3 Parquet source.

" + } + }, + "documentation":"

Specifies an Apache Parquet data store stored in Amazon S3.

" + }, + "S3SourceAdditionalOptions":{ + "type":"structure", + "members":{ + "BoundedSize":{ + "shape":"BoxedLong", + "documentation":"

Sets the upper limit for the target size of the dataset in bytes that will be processed.

" + }, + "BoundedFiles":{ + "shape":"BoxedLong", + "documentation":"

Sets the upper limit for the target number of files that will be processed.

" + } + }, + "documentation":"

Specifies additional connection options for the Amazon S3 data store.

" + }, "S3Target":{ "type":"structure", "members":{ @@ -12553,6 +14917,62 @@ }, "documentation":"

Defines a non-overlapping region of a table's partitions, allowing multiple requests to be run in parallel.

" }, + "SelectFields":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Paths" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Paths":{ + "shape":"GlueStudioPathList", + "documentation":"

A JSON path to a variable in the data structure.

" + } + }, + "documentation":"

Specifies a transform that chooses the data property keys that you want to keep.

" + }, + "SelectFromCollection":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Index" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Index":{ + "shape":"NonNegativeInt", + "documentation":"

The index for the DynamicFrame to be selected.

" + } + }, + "documentation":"

Specifies a transform that chooses one DynamicFrame from a collection of DynamicFrames. The output is the selected DynamicFrame

" + }, + "Separator":{ + "type":"string", + "enum":[ + "comma", + "ctrla", + "pipe", + "semicolon", + "tab" + ] + }, "SerDeInfo":{ "type":"structure", "members":{ @@ -12714,6 +15134,195 @@ "ASCENDING" ] }, + "SparkConnectorSource":{ + "type":"structure", + "required":[ + "Name", + "ConnectionName", + "ConnectorName", + "ConnectionType" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data source.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection that is associated with the connector.

" + }, + "ConnectorName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of a connector that assists with accessing the data store in Glue Studio.

" + }, + "ConnectionType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of connection, such as marketplace.spark or custom.spark, designating a connection to an Apache Spark data store.

" + }, + "AdditionalOptions":{ + "shape":"AdditionalOptions", + "documentation":"

Additional connection options for the connector.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies data schema for the custom spark source.

" + } + }, + "documentation":"

Specifies a connector to an Apache Spark data source.

" + }, + "SparkConnectorTarget":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "ConnectionName", + "ConnectorName", + "ConnectionType" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the data target.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The nodes that are inputs to the data target.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of a connection for an Apache Spark connector.

" + }, + "ConnectorName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of an Apache Spark connector.

" + }, + "ConnectionType":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The type of connection, such as marketplace.spark or custom.spark, designating a connection to an Apache Spark data store.

" + }, + "AdditionalOptions":{ + "shape":"AdditionalOptions", + "documentation":"

Additional connection options for the connector.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the custom spark target.

" + } + }, + "documentation":"

Specifies a target that uses an Apache Spark connector.

" + }, + "SparkSQL":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "SqlQuery", + "SqlAliases" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"ManyInputs", + "documentation":"

The data inputs identified by their node names. You can associate a table name with each input node to use in the SQL query. The name you choose must meet the Spark SQL naming restrictions.

" + }, + "SqlQuery":{ + "shape":"SqlQuery", + "documentation":"

A SQL query that must use Spark SQL syntax and return a single data set.

" + }, + "SqlAliases":{ + "shape":"SqlAliases", + "documentation":"

A list of aliases. An alias allows you to specify what name to use in the SQL for a given input. For example, you have a datasource named \"MyDataSource\". If you specify From as MyDataSource, and Alias as SqlName, then in your SQL you can do:

select * from SqlName

and that gets data from MyDataSource.

" + }, + "OutputSchemas":{ + "shape":"GlueSchemas", + "documentation":"

Specifies the data schema for the SparkSQL transform.

" + } + }, + "documentation":"

Specifies a transform where you enter a SQL query using Spark SQL syntax to transform the data. The output is a single DynamicFrame.

" + }, + "Spigot":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Path" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Path":{ + "shape":"EnclosedInStringProperty", + "documentation":"

A path in Amazon S3 where the transform will write a subset of records from the dataset to a JSON file in an Amazon S3 bucket.

" + }, + "Topk":{ + "shape":"Topk", + "documentation":"

Specifies a number of records to write starting from the beginning of the dataset.

" + }, + "Prob":{ + "shape":"Prob", + "documentation":"

The probability (a decimal value with a maximum value of 1) of picking any given record. A value of 1 indicates that each row read from the dataset should be included in the sample output.

" + } + }, + "documentation":"

Specifies a transform that writes samples of the data to an Amazon S3 bucket.

" + }, + "SplitFields":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "Paths" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"OneInput", + "documentation":"

The data inputs identified by their node names.

" + }, + "Paths":{ + "shape":"GlueStudioPathList", + "documentation":"

A JSON path to a variable in the data structure.

" + } + }, + "documentation":"

Specifies a transform that splits data property keys into two DynamicFrames. The output is a collection of DynamicFrames: one with selected data property keys, and one with the remaining data property keys.

" + }, + "SqlAlias":{ + "type":"structure", + "required":[ + "From", + "Alias" + ], + "members":{ + "From":{ + "shape":"NodeId", + "documentation":"

A table, or a column in a table.

" + }, + "Alias":{ + "shape":"EnclosedInStringPropertyWithQuote", + "documentation":"

A temporary name given to a table, or a column in a table.

" + } + }, + "documentation":"

Represents a single entry in the list of values for SqlAliases.

" + }, + "SqlAliases":{ + "type":"list", + "member":{"shape":"SqlAlias"} + }, + "SqlQuery":{ + "type":"string", + "pattern":"([\\u0020-\\uD7FF\\uE000-\\uFFFD\\uD800\\uDC00-\\uDBFF\\uDFFF\\s])*" + }, "StartBlueprintRunRequest":{ "type":"structure", "required":[ @@ -12988,6 +15597,14 @@ }, "documentation":"

The batch condition that started the workflow run. Either the number of events in the batch size arrived, in which case the BatchSize member is non-zero, or the batch window expired, in which case the BatchWindow member is non-zero.

" }, + "StartingPosition":{ + "type":"string", + "enum":[ + "latest", + "trim_horizon", + "earliest" + ] + }, "Statement":{ "type":"structure", "members":{ @@ -13230,6 +15847,20 @@ }, "documentation":"

Describes the physical storage of table data.

" }, + "StreamingDataPreviewOptions":{ + "type":"structure", + "members":{ + "PollingTime":{ + "shape":"PollingTime", + "documentation":"

The polling time in milliseconds.

" + }, + "RecordPollingLimit":{ + "shape":"PositiveLong", + "documentation":"

The limit to the number of records polled.

" + } + }, + "documentation":"

Specifies options related to data preview for viewing a sample of your data.

" + }, "StringColumnStatisticsData":{ "type":"structure", "required":[ @@ -13536,6 +16167,16 @@ "max":50, "min":0 }, + "TargetFormat":{ + "type":"string", + "enum":[ + "json", + "csv", + "avro", + "orc", + "parquet" + ] + }, "TaskRun":{ "type":"structure", "members":{ @@ -13690,6 +16331,12 @@ "Timestamp":{"type":"timestamp"}, "TimestampValue":{"type":"timestamp"}, "Token":{"type":"string"}, + "Topk":{ + "type":"integer", + "box":true, + "max":100, + "min":0 + }, "TotalSegmentsInteger":{ "type":"integer", "max":10, @@ -13941,6 +16588,12 @@ }, "documentation":"

A structure used to provide information used to update a trigger. This object updates the previous trigger definition by overwriting it completely.

" }, + "TwoInputs":{ + "type":"list", + "member":{"shape":"NodeId"}, + "max":2, + "min":2 + }, "TypeString":{ "type":"string", "max":20000, @@ -13965,6 +16618,36 @@ "type":"list", "member":{"shape":"UnfilteredPartition"} }, + "Union":{ + "type":"structure", + "required":[ + "Name", + "Inputs", + "UnionType" + ], + "members":{ + "Name":{ + "shape":"NodeName", + "documentation":"

The name of the transform node.

" + }, + "Inputs":{ + "shape":"TwoInputs", + "documentation":"

The node ID inputs to the transform.

" + }, + "UnionType":{ + "shape":"UnionType", + "documentation":"

Indicates the type of Union transform.

Specify ALL to join all rows from data sources to the resulting DynamicFrame. The resulting union does not remove duplicate rows.

Specify DISTINCT to remove duplicate rows in the resulting DynamicFrame.

" + } + }, + "documentation":"

Specifies a transform that combines the rows from two or more datasets into a single result.

" + }, + "UnionType":{ + "type":"string", + "enum":[ + "ALL", + "DISTINCT" + ] + }, "UntagResourceRequest":{ "type":"structure", "required":[ @@ -14024,6 +16707,13 @@ } } }, + "UpdateCatalogBehavior":{ + "type":"string", + "enum":[ + "UPDATE_IN_DATABASE", + "LOG" + ] + }, "UpdateClassifierRequest":{ "type":"structure", "members":{ @@ -14715,6 +17405,24 @@ "documentation":"

Specifies an XML classifier to be updated.

" }, "UpdatedTimestamp":{"type":"string"}, + "UpsertRedshiftTargetOptions":{ + "type":"structure", + "members":{ + "TableLocation":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The physical location of the Redshift table.

" + }, + "ConnectionName":{ + "shape":"EnclosedInStringProperty", + "documentation":"

The name of the connection to use to write to Redshift.

" + }, + "UpsertKeys":{ + "shape":"EnclosedInStringPropertiesMinOne", + "documentation":"

The keys used to determine whether to perform an update or insert.

" + } + }, + "documentation":"

The options to configure an upsert operation when writing to a Redshift target .

" + }, "UriString":{"type":"string"}, "UserDefinedFunction":{ "type":"structure", diff --git a/contrib/python/botocore/py3/botocore/data/kms/2014-11-01/service-2.json b/contrib/python/botocore/py3/botocore/data/kms/2014-11-01/service-2.json index 3aea4e68a9f..b2e95ef1603 100644 --- a/contrib/python/botocore/py3/botocore/data/kms/2014-11-01/service-2.json +++ b/contrib/python/botocore/py3/botocore/data/kms/2014-11-01/service-2.json @@ -124,7 +124,7 @@ {"shape":"CustomKeyStoreInvalidStateException"}, {"shape":"CloudHsmClusterInvalidConfigurationException"} ], - "documentation":"

Creates a unique customer managed KMS key in your Amazon Web Services account and Region.

In addition to the required parameters, you can use the optional parameters to specify a key policy, description, tags, and other useful elements for any key type.

KMS is replacing the term customer master key (CMK) with KMS key and KMS key. The concept has not changed. To prevent breaking changes, KMS is keeping some variations of this term.

To create different types of KMS keys, use the following guidance:

Symmetric encryption KMS key

To create a symmetric encryption KMS key, you aren't required to specify any parameters. The default value for KeySpec, SYMMETRIC_DEFAULT, and the default value for KeyUsage, ENCRYPT_DECRYPT, create a symmetric encryption KMS key.

If you need a key for basic encryption and decryption or you are creating a KMS key to protect your resources in an Amazon Web Services service, create a symmetric encryption KMS key. The key material in a symmetric encryption key never leaves KMS unencrypted. You can use a symmetric encryption KMS key to encrypt and decrypt data up to 4,096 bytes, but they are typically used to generate data keys and data keys pairs. For details, see GenerateDataKey and GenerateDataKeyPair.

Asymmetric KMS keys

To create an asymmetric KMS key, use the KeySpec parameter to specify the type of key material in the KMS key. Then, use the KeyUsage parameter to determine whether the KMS key will be used to encrypt and decrypt or sign and verify. You can't change these properties after the KMS key is created.

Asymmetric KMS keys contain an RSA key pair or an Elliptic Curve (ECC) key pair. The private key in an asymmetric KMS key never leaves AWS KMS unencrypted. However, you can use the GetPublicKey operation to download the public key so it can be used outside of AWS KMS. KMS keys with RSA key pairs can be used to encrypt or decrypt data or sign and verify messages (but not both). KMS keys with ECC key pairs can be used only to sign and verify messages. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

HMAC KMS key

To create an HMAC KMS key, set the KeySpec parameter to a key spec value for HMAC KMS keys. Then set the KeyUsage parameter to GENERATE_VERIFY_MAC. You must set the key usage even though GENERATE_VERIFY_MAC is the only valid key usage value for HMAC KMS keys. You can't change these properties after the KMS key is created.

HMAC KMS keys are symmetric keys that never leave KMS unencrypted. You can use HMAC keys to generate (GenerateMac) and verify (VerifyMac) HMAC codes for messages up to 4096 bytes.

HMAC KMS keys are not supported in all Amazon Web Services Regions. If you try to create an HMAC KMS key in an Amazon Web Services Region in which HMAC keys are not supported, the CreateKey operation returns an UnsupportedOperationException. For a list of Regions in which HMAC KMS keys are supported, see HMAC keys in KMS in the Key Management Service Developer Guide.

Multi-Region primary keys
Imported key material

To create a multi-Region primary key in the local Amazon Web Services Region, use the MultiRegion parameter with a value of True. To create a multi-Region replica key, that is, a KMS key with the same key ID and key material as a primary key, but in a different Amazon Web Services Region, use the ReplicateKey operation. To change a replica key to a primary key, and its primary key to a replica key, use the UpdatePrimaryRegion operation.

You can create multi-Region KMS keys for all supported KMS key types: symmetric encryption KMS keys, HMAC KMS keys, asymmetric encryption KMS keys, and asymmetric signing KMS keys. You can also create multi-Region keys with imported key material. However, you can't create multi-Region keys in a custom key store.

This operation supports multi-Region keys, an KMS feature that lets you create multiple interoperable KMS keys in different Amazon Web Services Regions. Because these KMS keys have the same key ID, key material, and other metadata, you can use them interchangeably to encrypt data in one Amazon Web Services Region and decrypt it in a different Amazon Web Services Region without re-encrypting the data or making a cross-Region call. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

To import your own key material, begin by creating a symmetric encryption KMS key with no key material. To do this, use the Origin parameter of CreateKey with a value of EXTERNAL. Next, use GetParametersForImport operation to get a public key and import token, and use the public key to encrypt your key material. Then, use ImportKeyMaterial with your import token to import the key material. For step-by-step instructions, see Importing Key Material in the Key Management Service Developer Guide .

This feature supports only symmetric encryption KMS keys, including multi-Region symmetric encryption KMS keys. You cannot import key material into any other type of KMS key.

To create a multi-Region primary key with imported key material, use the Origin parameter of CreateKey with a value of EXTERNAL and the MultiRegion parameter with a value of True. To create replicas of the multi-Region primary key, use the ReplicateKey operation. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

Custom key store

To create a symmetric encryption KMS key in a custom key store, use the CustomKeyStoreId parameter to specify the custom key store. You must also use the Origin parameter with a value of AWS_CLOUDHSM. The CloudHSM cluster that is associated with the custom key store must have at least two active HSMs in different Availability Zones in the Amazon Web Services Region.

Custom key stores support only symmetric encryption KMS keys. You cannot create an HMAC KMS key or an asymmetric KMS key in a custom key store. For information about custom key stores in KMS see Custom key stores in KMS in the Key Management Service Developer Guide .

Cross-account use: No. You cannot use this operation to create a KMS key in a different Amazon Web Services account.

Required permissions: kms:CreateKey (IAM policy). To use the Tags parameter, kms:TagResource (IAM policy). For examples and information about related permissions, see Allow a user to create KMS keys in the Key Management Service Developer Guide.

Related operations:

" + "documentation":"

Creates a unique customer managed KMS key in your Amazon Web Services account and Region.

In addition to the required parameters, you can use the optional parameters to specify a key policy, description, tags, and other useful elements for any key type.

KMS is replacing the term customer master key (CMK) with KMS key and KMS key. The concept has not changed. To prevent breaking changes, KMS is keeping some variations of this term.

To create different types of KMS keys, use the following guidance:

Symmetric encryption KMS key

To create a symmetric encryption KMS key, you aren't required to specify any parameters. The default value for KeySpec, SYMMETRIC_DEFAULT, and the default value for KeyUsage, ENCRYPT_DECRYPT, create a symmetric encryption KMS key.

If you need a key for basic encryption and decryption or you are creating a KMS key to protect your resources in an Amazon Web Services service, create a symmetric encryption KMS key. The key material in a symmetric encryption key never leaves KMS unencrypted. You can use a symmetric encryption KMS key to encrypt and decrypt data up to 4,096 bytes, but they are typically used to generate data keys and data keys pairs. For details, see GenerateDataKey and GenerateDataKeyPair.

Asymmetric KMS keys

To create an asymmetric KMS key, use the KeySpec parameter to specify the type of key material in the KMS key. Then, use the KeyUsage parameter to determine whether the KMS key will be used to encrypt and decrypt or sign and verify. You can't change these properties after the KMS key is created.

Asymmetric KMS keys contain an RSA key pair or an Elliptic Curve (ECC) key pair. The private key in an asymmetric KMS key never leaves KMS unencrypted. However, you can use the GetPublicKey operation to download the public key so it can be used outside of KMS. KMS keys with RSA key pairs can be used to encrypt or decrypt data or sign and verify messages (but not both). KMS keys with ECC key pairs can be used only to sign and verify messages. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

HMAC KMS key

To create an HMAC KMS key, set the KeySpec parameter to a key spec value for HMAC KMS keys. Then set the KeyUsage parameter to GENERATE_VERIFY_MAC. You must set the key usage even though GENERATE_VERIFY_MAC is the only valid key usage value for HMAC KMS keys. You can't change these properties after the KMS key is created.

HMAC KMS keys are symmetric keys that never leave KMS unencrypted. You can use HMAC keys to generate (GenerateMac) and verify (VerifyMac) HMAC codes for messages up to 4096 bytes.

HMAC KMS keys are not supported in all Amazon Web Services Regions. If you try to create an HMAC KMS key in an Amazon Web Services Region in which HMAC keys are not supported, the CreateKey operation returns an UnsupportedOperationException. For a list of Regions in which HMAC KMS keys are supported, see HMAC keys in KMS in the Key Management Service Developer Guide.

Multi-Region primary keys
Imported key material

To create a multi-Region primary key in the local Amazon Web Services Region, use the MultiRegion parameter with a value of True. To create a multi-Region replica key, that is, a KMS key with the same key ID and key material as a primary key, but in a different Amazon Web Services Region, use the ReplicateKey operation. To change a replica key to a primary key, and its primary key to a replica key, use the UpdatePrimaryRegion operation.

You can create multi-Region KMS keys for all supported KMS key types: symmetric encryption KMS keys, HMAC KMS keys, asymmetric encryption KMS keys, and asymmetric signing KMS keys. You can also create multi-Region keys with imported key material. However, you can't create multi-Region keys in a custom key store.

This operation supports multi-Region keys, an KMS feature that lets you create multiple interoperable KMS keys in different Amazon Web Services Regions. Because these KMS keys have the same key ID, key material, and other metadata, you can use them interchangeably to encrypt data in one Amazon Web Services Region and decrypt it in a different Amazon Web Services Region without re-encrypting the data or making a cross-Region call. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

To import your own key material, begin by creating a symmetric encryption KMS key with no key material. To do this, use the Origin parameter of CreateKey with a value of EXTERNAL. Next, use GetParametersForImport operation to get a public key and import token, and use the public key to encrypt your key material. Then, use ImportKeyMaterial with your import token to import the key material. For step-by-step instructions, see Importing Key Material in the Key Management Service Developer Guide .

This feature supports only symmetric encryption KMS keys, including multi-Region symmetric encryption KMS keys. You cannot import key material into any other type of KMS key.

To create a multi-Region primary key with imported key material, use the Origin parameter of CreateKey with a value of EXTERNAL and the MultiRegion parameter with a value of True. To create replicas of the multi-Region primary key, use the ReplicateKey operation. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

Custom key store

To create a symmetric encryption KMS key in a custom key store, use the CustomKeyStoreId parameter to specify the custom key store. You must also use the Origin parameter with a value of AWS_CLOUDHSM. The CloudHSM cluster that is associated with the custom key store must have at least two active HSMs in different Availability Zones in the Amazon Web Services Region.

Custom key stores support only symmetric encryption KMS keys. You cannot create an HMAC KMS key or an asymmetric KMS key in a custom key store. For information about custom key stores in KMS see Custom key stores in KMS in the Key Management Service Developer Guide .

Cross-account use: No. You cannot use this operation to create a KMS key in a different Amazon Web Services account.

Required permissions: kms:CreateKey (IAM policy). To use the Tags parameter, kms:TagResource (IAM policy). For examples and information about related permissions, see Allow a user to create KMS keys in the Key Management Service Developer Guide.

Related operations:

" }, "Decrypt":{ "name":"Decrypt", @@ -146,7 +146,7 @@ {"shape":"KMSInternalException"}, {"shape":"KMSInvalidStateException"} ], - "documentation":"

Decrypts ciphertext that was encrypted by a KMS key using any of the following operations:

You can use this operation to decrypt ciphertext that was encrypted under a symmetric encryption KMS key or an asymmetric encryption KMS key. When the KMS key is asymmetric, you must specify the KMS key and the encryption algorithm that was used to encrypt the ciphertext. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

The Decrypt operation also decrypts ciphertext that was encrypted outside of KMS by the public key in an KMS asymmetric KMS key. However, it cannot decrypt symmetric ciphertext produced by other libraries, such as the Amazon Web Services Encryption SDK or Amazon S3 client-side encryption. These libraries return a ciphertext format that is incompatible with KMS.

If the ciphertext was encrypted under a symmetric encryption KMS key, the KeyId parameter is optional. KMS can get this information from metadata that it adds to the symmetric ciphertext blob. This feature adds durability to your implementation by ensuring that authorized users can decrypt ciphertext decades after it was encrypted, even if they've lost track of the key ID. However, specifying the KMS key is always recommended as a best practice. When you use the KeyId parameter to specify a KMS key, KMS only uses the KMS key you specify. If the ciphertext was encrypted under a different KMS key, the Decrypt operation fails. This practice ensures that you use the KMS key that you intend.

Whenever possible, use key policies to give users permission to call the Decrypt operation on a particular KMS key, instead of using IAM policies. Otherwise, you might create an IAM user policy that gives the user Decrypt permission on all KMS keys. This user could decrypt ciphertext that was encrypted by KMS keys in other accounts if the key policy for the cross-account KMS key permits it. If you must use an IAM policy for Decrypt permissions, limit the user to particular KMS keys or particular trusted accounts. For details, see Best practices for IAM policies in the Key Management Service Developer Guide.

Applications in Amazon Web Services Nitro Enclaves can call this operation by using the Amazon Web Services Nitro Enclaves Development Kit. For information about the supporting parameters, see How Amazon Web Services Nitro Enclaves use KMS in the Key Management Service Developer Guide.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:Decrypt (key policy)

Related operations:

" + "documentation":"

Decrypts ciphertext that was encrypted by a KMS key using any of the following operations:

You can use this operation to decrypt ciphertext that was encrypted under a symmetric encryption KMS key or an asymmetric encryption KMS key. When the KMS key is asymmetric, you must specify the KMS key and the encryption algorithm that was used to encrypt the ciphertext. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

The Decrypt operation also decrypts ciphertext that was encrypted outside of KMS by the public key in an KMS asymmetric KMS key. However, it cannot decrypt ciphertext produced by other libraries, such as the Amazon Web Services Encryption SDK or Amazon S3 client-side encryption. These libraries return a ciphertext format that is incompatible with KMS.

If the ciphertext was encrypted under a symmetric encryption KMS key, the KeyId parameter is optional. KMS can get this information from metadata that it adds to the symmetric ciphertext blob. This feature adds durability to your implementation by ensuring that authorized users can decrypt ciphertext decades after it was encrypted, even if they've lost track of the key ID. However, specifying the KMS key is always recommended as a best practice. When you use the KeyId parameter to specify a KMS key, KMS only uses the KMS key you specify. If the ciphertext was encrypted under a different KMS key, the Decrypt operation fails. This practice ensures that you use the KMS key that you intend.

Whenever possible, use key policies to give users permission to call the Decrypt operation on a particular KMS key, instead of using IAM policies. Otherwise, you might create an IAM user policy that gives the user Decrypt permission on all KMS keys. This user could decrypt ciphertext that was encrypted by KMS keys in other accounts if the key policy for the cross-account KMS key permits it. If you must use an IAM policy for Decrypt permissions, limit the user to particular KMS keys or particular trusted accounts. For details, see Best practices for IAM policies in the Key Management Service Developer Guide.

Applications in Amazon Web Services Nitro Enclaves can call this operation by using the Amazon Web Services Nitro Enclaves Development Kit. For information about the supporting parameters, see How Amazon Web Services Nitro Enclaves use KMS in the Key Management Service Developer Guide.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:Decrypt (key policy)

Related operations:

" }, "DeleteAlias":{ "name":"DeleteAlias", @@ -259,7 +259,7 @@ {"shape":"KMSInvalidStateException"}, {"shape":"UnsupportedOperationException"} ], - "documentation":"

Disables automatic rotation of the key material for the specified symmetric encryption KMS key.

You cannot enable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: No. You cannot perform this operation on a KMS key in a different Amazon Web Services account.

Required permissions: kms:DisableKeyRotation (key policy)

Related operations:

" + "documentation":"

Disables automatic rotation of the key material of the specified symmetric encryption KMS key.

Automatic key rotation is supported only on symmetric encryption KMS keys. You cannot enable or disable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. The key rotation status of these KMS keys is always false. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

You can enable (EnableKeyRotation) and disable automatic rotation of the key material in customer managed KMS keys. Key material rotation of Amazon Web Services managed KMS keys is not configurable. KMS always rotates the key material for every year. Rotation of Amazon Web Services owned KMS keys varies.

In May 2022, KMS changed the rotation schedule for Amazon Web Services managed keys from every three years to every year. For details, see EnableKeyRotation.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: No. You cannot perform this operation on a KMS key in a different Amazon Web Services account.

Required permissions: kms:DisableKeyRotation (key policy)

Related operations:

" }, "DisconnectCustomKeyStore":{ "name":"DisconnectCustomKeyStore", @@ -309,7 +309,7 @@ {"shape":"KMSInvalidStateException"}, {"shape":"UnsupportedOperationException"} ], - "documentation":"

Enables automatic rotation of the key material for the specified symmetric encryption KMS key.

You cannot enable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: No. You cannot perform this operation on a KMS key in a different Amazon Web Services account.

Required permissions: kms:EnableKeyRotation (key policy)

Related operations:

" + "documentation":"

Enables automatic rotation of the key material of the specified symmetric encryption KMS key.

When you enable automatic rotation of acustomer managed KMS key, KMS rotates the key material of the KMS key one year (approximately 365 days) from the enable date and every year thereafter. You can monitor rotation of the key material for your KMS keys in CloudTrail and Amazon CloudWatch. To disable rotation of the key material in a customer managed KMS key, use the DisableKeyRotation operation.

Automatic key rotation is supported only on symmetric encryption KMS keys. You cannot enable or disable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. The key rotation status of these KMS keys is always false. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

You cannot enable or disable automatic rotation Amazon Web Services managed KMS keys. KMS always rotates the key material of Amazon Web Services managed keys every year. Rotation of Amazon Web Services owned KMS keys varies.

In May 2022, KMS changed the rotation schedule for Amazon Web Services managed keys from every three years (approximately 1,095 days) to every year (approximately 365 days).

New Amazon Web Services managed keys are automatically rotated one year after they are created, and approximately every year thereafter.

Existing Amazon Web Services managed keys are automatically rotated one year after their most recent rotation, and every year thereafter.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: No. You cannot perform this operation on a KMS key in a different Amazon Web Services account.

Required permissions: kms:EnableKeyRotation (key policy)

Related operations:

" }, "Encrypt":{ "name":"Encrypt", @@ -411,7 +411,7 @@ {"shape":"KMSInternalException"}, {"shape":"KMSInvalidStateException"} ], - "documentation":"

Returns a unique symmetric data key for use outside of KMS. This operation returns a data key that is encrypted under a symmetric encryption KMS key that you specify. The bytes in the key are random; they are not related to the caller or to the KMS key.

GenerateDataKeyWithoutPlaintext is identical to the GenerateDataKey operation except that it does not return a plaintext copy of the data key.

This operation is useful for systems that need to encrypt data at some point, but not immediately. When you need to encrypt the data, you call the Decrypt operation on the encrypted copy of the key. It's also useful in distributed systems with different levels of trust. For example, you might store encrypted data in containers. One component of your system creates new containers and stores an encrypted data key with each container. Then, a different component puts the data into the containers. That component first decrypts the data key, uses the plaintext data key to encrypt data, puts the encrypted data into the container, and then destroys the plaintext data key. In this system, the component that creates the containers never sees the plaintext data key.

To request an asymmetric data key pair, use the GenerateDataKeyPair or GenerateDataKeyPairWithoutPlaintext operations.

To generate a data key, you must specify the symmetric encryption KMS key that is used to encrypt the data key. You cannot use an asymmetric KMS key or a key in a custom key store to generate a data key. To get the type of your KMS key, use the DescribeKey operation.

If the operation succeeds, you will find the encrypted copy of the data key in the CiphertextBlob field.

You can use an optional encryption context to add additional security to the encryption operation. If you specify an EncryptionContext, you must specify the same encryption context (a case-sensitive exact match) when decrypting the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. For more information, see Encryption Context in the Key Management Service Developer Guide.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:GenerateDataKeyWithoutPlaintext (key policy)

Related operations:

" + "documentation":"

Returns a unique symmetric data key for use outside of KMS. This operation returns a data key that is encrypted under a symmetric encryption KMS key that you specify. The bytes in the key are random; they are not related to the caller or to the KMS key.

GenerateDataKeyWithoutPlaintext is identical to the GenerateDataKey operation except that it does not return a plaintext copy of the data key.

This operation is useful for systems that need to encrypt data at some point, but not immediately. When you need to encrypt the data, you call the Decrypt operation on the encrypted copy of the key.

It's also useful in distributed systems with different levels of trust. For example, you might store encrypted data in containers. One component of your system creates new containers and stores an encrypted data key with each container. Then, a different component puts the data into the containers. That component first decrypts the data key, uses the plaintext data key to encrypt data, puts the encrypted data into the container, and then destroys the plaintext data key. In this system, the component that creates the containers never sees the plaintext data key.

To request an asymmetric data key pair, use the GenerateDataKeyPair or GenerateDataKeyPairWithoutPlaintext operations.

To generate a data key, you must specify the symmetric encryption KMS key that is used to encrypt the data key. You cannot use an asymmetric KMS key or a key in a custom key store to generate a data key. To get the type of your KMS key, use the DescribeKey operation.

If the operation succeeds, you will find the encrypted copy of the data key in the CiphertextBlob field.

You can use an optional encryption context to add additional security to the encryption operation. If you specify an EncryptionContext, you must specify the same encryption context (a case-sensitive exact match) when decrypting the encrypted data key. Otherwise, the request to decrypt fails with an InvalidCiphertextException. For more information, see Encryption Context in the Key Management Service Developer Guide.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:GenerateDataKeyWithoutPlaintext (key policy)

Related operations:

" }, "GenerateMac":{ "name":"GenerateMac", @@ -430,7 +430,7 @@ {"shape":"KMSInternalException"}, {"shape":"KMSInvalidStateException"} ], - "documentation":"

Generates a hash-based message authentication code (HMAC) for a message using an HMAC KMS key and a MAC algorithm that the key supports. The MAC algorithm computes the HMAC for the message and the key as described in RFC 2104.

You can use the HMAC that this operation generates with the VerifyMac operation to demonstrate that the original message has not changed. Also, because a secret key is used to create the hash, you can verify that the party that generated the hash has the required secret key. This operation is part of KMS support for HMAC KMS keys. For details, see HMAC keys in KMS in the Key Management Service Developer Guide .

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:GenerateMac (key policy)

Related operations: VerifyMac

" + "documentation":"

Generates a hash-based message authentication code (HMAC) for a message using an HMAC KMS key and a MAC algorithm that the key supports. The MAC algorithm computes the HMAC for the message and the key as described in RFC 2104.

You can use the HMAC that this operation generates with the VerifyMac operation to demonstrate that the original message has not changed. Also, because a secret key is used to create the hash, you can verify that the party that generated the hash has the required secret key. This operation is part of KMS support for HMAC KMS keys. For details, see HMAC keys in KMS in the Key Management Service Developer Guide .

Best practices recommend that you limit the time during which any signing mechanism, including an HMAC, is effective. This deters an attack where the actor uses a signed message to establish validity repeatedly or long after the message is superseded. HMAC tags do not include a timestamp, but you can include a timestamp in the token or message to help you detect when its time to refresh the HMAC.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:GenerateMac (key policy)

Related operations: VerifyMac

" }, "GenerateRandom":{ "name":"GenerateRandom", @@ -481,7 +481,7 @@ {"shape":"KMSInvalidStateException"}, {"shape":"UnsupportedOperationException"} ], - "documentation":"

Gets a Boolean value that indicates whether automatic rotation of the key material is enabled for the specified KMS key.

You cannot enable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key. The key rotation status for these KMS keys is always false.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation on a KMS key in a different Amazon Web Services account, specify the key ARN in the value of the KeyId parameter.

Required permissions: kms:GetKeyRotationStatus (key policy)

Related operations:

" + "documentation":"

Gets a Boolean value that indicates whether automatic rotation of the key material is enabled for the specified KMS key.

When you enable automatic rotation for customer managed KMS keys, KMS rotates the key material of the KMS key one year (approximately 365 days) from the enable date and every year thereafter. You can monitor rotation of the key material for your KMS keys in CloudTrail and Amazon CloudWatch.

Automatic key rotation is supported only on symmetric encryption KMS keys. You cannot enable or disable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. The key rotation status of these KMS keys is always false. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key..

You can enable (EnableKeyRotation) and disable automatic rotation (DisableKeyRotation) of the key material in customer managed KMS keys. Key material rotation of Amazon Web Services managed KMS keys is not configurable. KMS always rotates the key material in Amazon Web Services managed KMS keys every year. The key rotation status for Amazon Web Services managed KMS keys is always true.

In May 2022, KMS changed the rotation schedule for Amazon Web Services managed keys from every three years to every year. For details, see EnableKeyRotation.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation on a KMS key in a different Amazon Web Services account, specify the key ARN in the value of the KeyId parameter.

Required permissions: kms:GetKeyRotationStatus (key policy)

Related operations:

" }, "GetParametersForImport":{ "name":"GetParametersForImport", @@ -779,7 +779,7 @@ {"shape":"KMSInternalException"}, {"shape":"KMSInvalidStateException"} ], - "documentation":"

Creates a digital signature for a message or message digest by using the private key in an asymmetric signing KMS key. To verify the signature, use the Verify operation, or use the public key in the same asymmetric KMS key outside of KMS. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

Digital signatures are generated and verified by using asymmetric key pair, such as an RSA or ECC pair that is represented by an asymmetric KMS key. The key owner (or an authorized user) uses their private key to sign a message. Anyone with the public key can verify that the message was signed with that particular private key and that the message hasn't changed since it was signed.

To use the Sign operation, provide the following information:

When signing a message, be sure to record the KMS key and the signing algorithm. This information is required to verify the signature.

To verify the signature that this operation generates, use the Verify operation. Or use the GetPublicKey operation to download the public key and then use the public key to verify the signature outside of KMS.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:Sign (key policy)

Related operations: Verify

" + "documentation":"

Creates a digital signature for a message or message digest by using the private key in an asymmetric signing KMS key. To verify the signature, use the Verify operation, or use the public key in the same asymmetric KMS key outside of KMS. For information about asymmetric KMS keys, see Asymmetric KMS keys in the Key Management Service Developer Guide.

Digital signatures are generated and verified by using asymmetric key pair, such as an RSA or ECC pair that is represented by an asymmetric KMS key. The key owner (or an authorized user) uses their private key to sign a message. Anyone with the public key can verify that the message was signed with that particular private key and that the message hasn't changed since it was signed.

To use the Sign operation, provide the following information:

When signing a message, be sure to record the KMS key and the signing algorithm. This information is required to verify the signature.

Best practices recommend that you limit the time during which any signature is effective. This deters an attack where the actor uses a signed message to establish validity repeatedly or long after the message is superseded. Signatures do not include a timestamp, but you can include a timestamp in the signed message to help you detect when its time to refresh the signature.

To verify the signature that this operation generates, use the Verify operation. Or use the GetPublicKey operation to download the public key and then use the public key to verify the signature outside of KMS.

The KMS key that you use for this operation must be in a compatible key state. For details, see Key states of KMS keys in the Key Management Service Developer Guide.

Cross-account use: Yes. To perform this operation with a KMS key in a different Amazon Web Services account, specify the key ARN or alias ARN in the value of the KeyId parameter.

Required permissions: kms:Sign (key policy)

Related operations: Verify

" }, "TagResource":{ "name":"TagResource", @@ -1202,7 +1202,7 @@ "members":{ "Policy":{ "shape":"PolicyType", - "documentation":"

The key policy to attach to the KMS key.

If you provide a key policy, it must meet the following criteria:

If you do not provide a key policy, KMS attaches a default key policy to the KMS key. For more information, see Default Key Policy in the Key Management Service Developer Guide.

The key policy size quota is 32 kilobytes (32768 bytes).

For help writing and formatting a JSON policy document, see the IAM JSON Policy Reference in the Identity and Access Management User Guide .

" + "documentation":"

The key policy to attach to the KMS key. If you do not specify a key policy, KMS attaches a default key policy to the KMS key. For more information, see Default key policy in the Key Management Service Developer Guide.

If you provide a key policy, it must meet the following criteria:

A key policy document must conform to the following rules.

For help writing and formatting a JSON policy document, see the IAM JSON Policy Reference in the Identity and Access Management User Guide .

" }, "Description":{ "shape":"DescriptionType", @@ -1220,7 +1220,7 @@ }, "KeySpec":{ "shape":"KeySpec", - "documentation":"

Specifies the type of KMS key to create. The default value, SYMMETRIC_DEFAULT, creates a KMS key with a 256-bit symmetric key for encryption and decryption. For help choosing a key spec for your KMS key, see Choosing a KMS key type in the Key Management Service Developer Guide .

The KeySpec determines whether the KMS key contains a symmetric key or an asymmetric key pair. It also determines the algorithms that the KMS key supports. You can't change the KeySpec after the KMS key is created. To further restrict the algorithms that can be used with the KMS key, use a condition key in its key policy or IAM policy. For more information, see kms:EncryptionAlgorithm, kms:MacAlgorithm or kms:Signing Algorithm in the Key Management Service Developer Guide .

Amazon Web Services services that are integrated with KMS use symmetric encryption KMS keys to protect your data. These services do not support asymmetric KMS keys or HMAC KMS keys.

KMS supports the following key specs for KMS keys:

" + "documentation":"

Specifies the type of KMS key to create. The default value, SYMMETRIC_DEFAULT, creates a KMS key with a 256-bit symmetric key for encryption and decryption. For help choosing a key spec for your KMS key, see Choosing a KMS key type in the Key Management Service Developer Guide .

The KeySpec determines whether the KMS key contains a symmetric key or an asymmetric key pair. It also determines the cryptographic algorithms that the KMS key supports. You can't change the KeySpec after the KMS key is created. To further restrict the algorithms that can be used with the KMS key, use a condition key in its key policy or IAM policy. For more information, see kms:EncryptionAlgorithm, kms:MacAlgorithm or kms:Signing Algorithm in the Key Management Service Developer Guide .

Amazon Web Services services that are integrated with KMS use symmetric encryption KMS keys to protect your data. These services do not support asymmetric KMS keys or HMAC KMS keys.

KMS supports the following key specs for KMS keys:

" }, "Origin":{ "shape":"OriginType", @@ -1240,7 +1240,7 @@ }, "MultiRegion":{ "shape":"NullableBooleanType", - "documentation":"

Creates a multi-Region primary key that you can replicate into other Amazon Web Services Regions. You cannot change this value after you create the KMS key.

For a multi-Region key, set this parameter to True. For a single-Region KMS key, omit this parameter or set it to False. The default value is False.

This operation supports multi-Region keys, an KMS feature that lets you create multiple interoperable KMS keys in different Amazon Web Services Regions. Because these KMS keys have the same key ID, key material, and other metadata, you can use them interchangeably to encrypt data in one Amazon Web Services Region and decrypt it in a different Amazon Web Services Region without re-encrypting the data or making a cross-Region call. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

This value creates a primary key, not a replica. To create a replica key, use the ReplicateKey operation.

You can create a symmetric or asymmetric multi-Region key, and you can create a multi-Region key with imported key material. However, you cannot create a multi-Region key in a custom key store.

" + "documentation":"

Creates a multi-Region primary key that you can replicate into other Amazon Web Services Regions. You cannot change this value after you create the KMS key.

For a multi-Region key, set this parameter to True. For a single-Region KMS key, omit this parameter or set it to False. The default value is False.

This operation supports multi-Region keys, an KMS feature that lets you create multiple interoperable KMS keys in different Amazon Web Services Regions. Because these KMS keys have the same key ID, key material, and other metadata, you can use them interchangeably to encrypt data in one Amazon Web Services Region and decrypt it in a different Amazon Web Services Region without re-encrypting the data or making a cross-Region call. For more information about multi-Region keys, see Multi-Region keys in KMS in the Key Management Service Developer Guide.

This value creates a primary key, not a replica. To create a replica key, use the ReplicateKey operation.

You can create a multi-Region version of a symmetric encryption KMS key, an HMAC KMS key, an asymmetric KMS key, or a KMS key with imported key material. However, you cannot create a multi-Region key in a custom key store.

" } } }, @@ -1584,7 +1584,7 @@ "members":{ "KeyId":{ "shape":"KeyIdType", - "documentation":"

Identifies a symmetric encryption KMS key. You cannot enable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

Specify the key ID or key ARN of the KMS key.

For example:

To get the key ID and key ARN for a KMS key, use ListKeys or DescribeKey.

" + "documentation":"

Identifies a symmetric encryption KMS key. You cannot enable or disable automatic rotation of asymmetric KMS keys, HMAC KMS keys, KMS keys with imported key material, or KMS keys in a custom key store. The key rotation status of these KMS keys is always false. To enable or disable automatic rotation of a set of related multi-Region keys, set the property on the primary key.

Specify the key ID or key ARN of the KMS key.

For example:

To get the key ID and key ARN for a KMS key, use ListKeys or DescribeKey.

" } } }, @@ -1732,7 +1732,7 @@ }, "KeyId":{ "shape":"KeyIdType", - "documentation":"

Specifies the symmetric encryption KMS key that encrypts the private key in the data key pair. You cannot specify an asymmetric KMS key or a KMS key in a custom key store. To get the type and origin of your KMS key, use the DescribeKey operation.

To specify a KMS key, use its key ID, key ARN, alias name, or alias ARN. When using an alias name, prefix it with \"alias/\". To specify a KMS key in a different Amazon Web Services account, you must use the key ARN or alias ARN.

For example:

To get the key ID and key ARN for a KMS key, use ListKeys or DescribeKey. To get the alias name and alias ARN, use ListAliases.

" + "documentation":"

Specifies the symmetric encryption KMS key that encrypts the private key in the data key pair. You cannot specify an asymmetric KMS key or a KMS key in a custom key store. To get the type and origin of your KMS key, use the DescribeKey operation.

To specify a KMS key, use its key ID, key ARN, alias name, or alias ARN. When using an alias name, prefix it with \"alias/\". To specify a KMS key in a different Amazon Web Services account, you must use the key ARN or alias ARN.

For example:

To get the key ID and key ARN for a KMS key, use ListKeys or DescribeKey. To get the alias name and alias ARN, use ListAliases.

" }, "KeyPairSpec":{ "shape":"DataKeyPairSpec", @@ -2196,7 +2196,7 @@ "members":{ "message":{"shape":"ErrorMessageType"} }, - "documentation":"

The request was rejected because the specified KMS key cannot decrypt the data. The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request must identify the same KMS key that was used to encrypt the ciphertext.

", + "documentation":"

The request was rejected because the specified KMS key cannot decrypt the data. The KeyId in a Decrypt request and the SourceKeyId in a ReEncrypt request must identify the same KMS key that was used to encrypt the ciphertext.

", "exception":true }, "IncorrectKeyMaterialException":{ @@ -2859,7 +2859,7 @@ }, "Policy":{ "shape":"PolicyType", - "documentation":"

The key policy to attach to the KMS key.

The key policy must meet the following criteria:

The key policy cannot exceed 32 kilobytes (32768 bytes). For more information, see Resource Quotas in the Key Management Service Developer Guide.

" + "documentation":"

The key policy to attach to the KMS key.

The key policy must meet the following criteria:

A key policy document must conform to the following rules.

" }, "BypassPolicyLockoutSafetyCheck":{ "shape":"BooleanType", @@ -2956,7 +2956,7 @@ }, "Policy":{ "shape":"PolicyType", - "documentation":"

The key policy to attach to the KMS key. This parameter is optional. If you do not provide a key policy, KMS attaches the default key policy to the KMS key.

The key policy is not a shared property of multi-Region keys. You can specify the same key policy or a different key policy for each key in a set of related multi-Region keys. KMS does not synchronize this property.

If you provide a key policy, it must meet the following criteria:

" + "documentation":"

The key policy to attach to the KMS key. This parameter is optional. If you do not provide a key policy, KMS attaches the default key policy to the KMS key.

The key policy is not a shared property of multi-Region keys. You can specify the same key policy or a different key policy for each key in a set of related multi-Region keys. KMS does not synchronize this property.

If you provide a key policy, it must meet the following criteria:

A key policy document must conform to the following rules.

" }, "BypassPolicyLockoutSafetyCheck":{ "shape":"BooleanType", @@ -3033,7 +3033,7 @@ }, "PendingWindowInDays":{ "shape":"PendingWindowInDaysType", - "documentation":"

The waiting period, specified in number of days. After the waiting period ends, KMS deletes the KMS key.

If the KMS key is a multi-Region primary key with replicas, the waiting period begins when the last of its replica keys is deleted. Otherwise, the waiting period begins immediately.

This value is optional. If you include a value, it must be between 7 and 30, inclusive. If you do not include a value, it defaults to 30.

" + "documentation":"

The waiting period, specified in number of days. After the waiting period ends, KMS deletes the KMS key.

If the KMS key is a multi-Region primary key with replica keys, the waiting period begins when the last of its replica keys is deleted. Otherwise, the waiting period begins immediately.

This value is optional. If you include a value, it must be between 7 and 30, inclusive. If you do not include a value, it defaults to 30.

" } } }, @@ -3397,5 +3397,5 @@ "enum":["RSA_2048"] } }, - "documentation":"Key Management Service

Key Management Service (KMS) is an encryption and key management web service. This guide describes the KMS operations that you can call programmatically. For general information about KMS, see the Key Management Service Developer Guide .

KMS is replacing the term customer master key (CMK) with KMS key and KMS key. The concept has not changed. To prevent breaking changes, KMS is keeping some variations of this term.

Amazon Web Services provides SDKs that consist of libraries and sample code for various programming languages and platforms (Java, Ruby, .Net, macOS, Android, etc.). The SDKs provide a convenient way to create programmatic access to KMS and other Amazon Web Services services. For example, the SDKs take care of tasks such as signing requests (see below), managing errors, and retrying requests automatically. For more information about the Amazon Web Services SDKs, including how to download and install them, see Tools for Amazon Web Services.

We recommend that you use the Amazon Web Services SDKs to make programmatic API calls to KMS.

If you need to use FIPS 140-2 validated cryptographic modules when communicating with Amazon Web Services, use the FIPS endpoint in your preferred Amazon Web Services Region. For more information about the available FIPS endpoints, see Service endpoints in the Key Management Service topic of the Amazon Web Services General Reference.

Clients must support TLS (Transport Layer Security) 1.0. We recommend TLS 1.2. Clients must also support cipher suites with Perfect Forward Secrecy (PFS) such as Ephemeral Diffie-Hellman (DHE) or Elliptic Curve Ephemeral Diffie-Hellman (ECDHE). Most modern systems such as Java 7 and later support these modes.

Signing Requests

Requests must be signed by using an access key ID and a secret access key. We strongly recommend that you do not use your Amazon Web Services account (root) access key ID and secret key for everyday work with KMS. Instead, use the access key ID and secret access key for an IAM user. You can also use the Amazon Web Services Security Token Service to generate temporary security credentials that you can use to sign requests.

All KMS operations require Signature Version 4.

Logging API Requests

KMS supports CloudTrail, a service that logs Amazon Web Services API calls and related events for your Amazon Web Services account and delivers them to an Amazon S3 bucket that you specify. By using the information collected by CloudTrail, you can determine what requests were made to KMS, who made the request, when it was made, and so on. To learn more about CloudTrail, including how to turn it on and find your log files, see the CloudTrail User Guide.

Additional Resources

For more information about credentials and request signing, see the following:

Commonly Used API Operations

Of the API operations discussed in this guide, the following will prove the most useful for most applications. You will likely perform operations other than these, such as creating keys and assigning policies, by using the console.

" + "documentation":"Key Management Service

Key Management Service (KMS) is an encryption and key management web service. This guide describes the KMS operations that you can call programmatically. For general information about KMS, see the Key Management Service Developer Guide .

KMS is replacing the term customer master key (CMK) with KMS key and KMS key. The concept has not changed. To prevent breaking changes, KMS is keeping some variations of this term.

Amazon Web Services provides SDKs that consist of libraries and sample code for various programming languages and platforms (Java, Ruby, .Net, macOS, Android, etc.). The SDKs provide a convenient way to create programmatic access to KMS and other Amazon Web Services services. For example, the SDKs take care of tasks such as signing requests (see below), managing errors, and retrying requests automatically. For more information about the Amazon Web Services SDKs, including how to download and install them, see Tools for Amazon Web Services.

We recommend that you use the Amazon Web Services SDKs to make programmatic API calls to KMS.

If you need to use FIPS 140-2 validated cryptographic modules when communicating with Amazon Web Services, use the FIPS endpoint in your preferred Amazon Web Services Region. For more information about the available FIPS endpoints, see Service endpoints in the Key Management Service topic of the Amazon Web Services General Reference.

All KMS API calls must be signed and be transmitted using Transport Layer Security (TLS). KMS recommends you always use the latest supported TLS version. Clients must also support cipher suites with Perfect Forward Secrecy (PFS) such as Ephemeral Diffie-Hellman (DHE) or Elliptic Curve Ephemeral Diffie-Hellman (ECDHE). Most modern systems such as Java 7 and later support these modes.

Signing Requests

Requests must be signed by using an access key ID and a secret access key. We strongly recommend that you do not use your Amazon Web Services account (root) access key ID and secret key for everyday work with KMS. Instead, use the access key ID and secret access key for an IAM user. You can also use the Amazon Web Services Security Token Service to generate temporary security credentials that you can use to sign requests.

All KMS operations require Signature Version 4.

Logging API Requests

KMS supports CloudTrail, a service that logs Amazon Web Services API calls and related events for your Amazon Web Services account and delivers them to an Amazon S3 bucket that you specify. By using the information collected by CloudTrail, you can determine what requests were made to KMS, who made the request, when it was made, and so on. To learn more about CloudTrail, including how to turn it on and find your log files, see the CloudTrail User Guide.

Additional Resources

For more information about credentials and request signing, see the following:

Commonly Used API Operations

Of the API operations discussed in this guide, the following will prove the most useful for most applications. You will likely perform operations other than these, such as creating keys and assigning policies, by using the console.

" } -- cgit v1.3