From 8a9d97bb79076ed648ff4337b43dee1bd2ed807b Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 20 Nov 2024 12:59:49 -0800 Subject: [PATCH 1/3] chore: Update release policy, fix linter (#733) --- SUPPORT_POLICY.rst | 4 ++++ .../internal/mpl/tampering_mpl_materials.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/SUPPORT_POLICY.rst b/SUPPORT_POLICY.rst index 76cd580d4..1e20c300f 100644 --- a/SUPPORT_POLICY.rst +++ b/SUPPORT_POLICY.rst @@ -30,6 +30,10 @@ This table describes the current support status of each major version of the AWS - - * - 3.x + - General Availability + - Maintenance + - 2025-05-20 + * - 4.x - General Availability - - diff --git a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py index 00a0eb252..4caf85635 100644 --- a/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py +++ b/test_vector_handlers/src/awses_test_vectors/internal/mpl/tampering_mpl_materials.py @@ -40,7 +40,7 @@ class HalfSigningCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFromMPL): wrapped_default_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManagerFromMPL)) - def __init__(self, master_key_provider): + def __init__(self, master_key_provider): # pylint: disable=super-init-not-called """Create a new CMM that wraps a the given CMM.""" mpl = AwsCryptographicMaterialProviders(MaterialProvidersConfig()) mpl_cmm = mpl.create_default_cryptographic_materials_manager( @@ -89,7 +89,7 @@ class HalfSigningEncryptionMaterialsFromMPL(EncryptionMaterialsFromMPL): _underlying_materials: EncryptionMaterialsFromMPL - def __init__(self, underlying_materials): + def __init__(self, underlying_materials): # pylint: disable=super-init-not-called """Create a HalfSigningEncryptionMaterialsFromMPL wrapper around underlying_materials. """ @@ -159,7 +159,7 @@ class ProviderInfoChangingCryptoMaterialsManagerFromMPL(CryptoMaterialsManagerFr wrapped_cmm = attr.ib(validator=attr.validators.instance_of(CryptoMaterialsManager)) new_provider_info = attr.ib(validator=attr.validators.instance_of(six.string_types)) - def __init__(self, materials_manager, new_provider_info): + def __init__(self, materials_manager, new_provider_info): # pylint: disable=super-init-not-called """Create a new CMM that wraps a the given CMM.""" self.wrapped_cmm = materials_manager self.new_provider_info = new_provider_info From 899af22c25fdd5d896817e40514b5ea0e1537e90 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Thu, 21 Nov 2024 10:54:02 -0800 Subject: [PATCH 2/3] chore(CI): Build test vectors from source (#724) --- codebuild/py311/awses_local_mpl.yml | 2 ++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 2 ++ .../decrypt_golden_manifest_with_keyrings.yml | 2 ++ ...decrypt_golden_manifest_with_masterkey.yml | 2 +- .../py311/decrypt_keyrings_with_keyrings.yml | 2 ++ .../py311/decrypt_masterkey_with_keyrings.yml | 2 ++ .../decrypt_net_401_vectors_keyrings.yml | 2 ++ codebuild/py311/encrypt_keyrings.yml | 2 ++ .../generate_decrypt_vectors_keyrings.yml | 2 ++ codebuild/py312/awses_local_mpl.yml | 2 ++ .../decrypt_dafny_esdk_vectors_keyrings.yml | 2 ++ .../decrypt_golden_manifest_with_keyrings.yml | 2 ++ ...decrypt_golden_manifest_with_masterkey.yml | 2 +- .../py312/decrypt_hkeyring_with_keyrings.yml | 2 ++ .../py312/decrypt_hkeyring_with_masterkey.yml | 2 +- .../py312/decrypt_keyrings_with_keyrings.yml | 2 ++ .../py312/decrypt_masterkey_with_keyrings.yml | 2 ++ .../decrypt_net_401_vectors_keyrings.yml | 2 ++ codebuild/py312/encrypt_keyrings.yml | 2 ++ .../generate_decrypt_vectors_keyrings.yml | 2 ++ .../generate_hkeyring_decrypt_vectors.yml | 2 ++ test_vector_handlers/requirements_mpl.txt | 2 +- .../scripts/install_mpl_test_vector_runner.sh | 24 +++++++++++++++++++ test_vector_handlers/tox.ini | 3 +++ 24 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 test_vector_handlers/scripts/install_mpl_test_vector_runner.sh diff --git a/codebuild/py311/awses_local_mpl.yml b/codebuild/py311/awses_local_mpl.yml index 859931aa3..c6a975df8 100644 --- a/codebuild/py311/awses_local_mpl.yml +++ b/codebuild/py311/awses_local_mpl.yml @@ -21,6 +21,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - tox diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml index d69ce9370..1eadf2958 100644 --- a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml @@ -22,6 +22,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | diff --git a/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml index 154863bcc..c65816d69 100644 --- a/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml +++ b/codebuild/py311/decrypt_golden_manifest_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download "golden manifest" - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip - unzip python-2.3.0.zip -d python-2.3.0 diff --git a/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml index 4ae4bb280..3b71759d3 100644 --- a/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml +++ b/codebuild/py311/decrypt_golden_manifest_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py311-full_decrypt-mpl" + TOXENV: "py311-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py311/decrypt_keyrings_with_keyrings.yml b/codebuild/py311/decrypt_keyrings_with_keyrings.yml index fec275d48..f202c9c69 100644 --- a/codebuild/py311/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py311/decrypt_keyrings_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_keyrings.zip 311_keyrings.zip - unzip 311_keyrings.zip diff --git a/codebuild/py311/decrypt_masterkey_with_keyrings.yml b/codebuild/py311/decrypt_masterkey_with_keyrings.yml index 8543077bd..1542b1acb 100644 --- a/codebuild/py311/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py311/decrypt_masterkey_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/311_masterkey.zip 311_masterkey.zip - unzip 311_masterkey.zip diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index 1a23f0917..b5e7d7233 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -19,6 +19,8 @@ phases: python: 3.11 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py311/encrypt_keyrings.yml b/codebuild/py311/encrypt_keyrings.yml index 8b7cb94b4..8e315ca9f 100644 --- a/codebuild/py311/encrypt_keyrings.yml +++ b/codebuild/py311/encrypt_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py311/generate_decrypt_vectors_keyrings.yml b/codebuild/py311/generate_decrypt_vectors_keyrings.yml index 179ec0f12..0c9fc8333 100644 --- a/codebuild/py311/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py311/generate_decrypt_vectors_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.11 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/awses_local_mpl.yml b/codebuild/py312/awses_local_mpl.yml index c92265541..ea4d4f229 100644 --- a/codebuild/py312/awses_local_mpl.yml +++ b/codebuild/py312/awses_local_mpl.yml @@ -23,6 +23,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - cd /root/.pyenv/plugins/python-build/../.. && git pull && cd - - pyenv install --skip-existing 3.12.0 - pyenv local 3.12.0 diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml index 810d16b74..2a8439253 100644 --- a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml @@ -22,6 +22,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch test vectors from Dafny ESDK's most recent run # (Assuming the first result is most recent; seems to be correct...) - | diff --git a/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml b/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml index 54bb10c33..c0442a10b 100644 --- a/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml +++ b/codebuild/py312/decrypt_golden_manifest_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download "golden manifest" - curl -L -o python-2.3.0.zip https://github.com/awslabs/aws-encryption-sdk-test-vectors/raw/master/vectors/awses-decrypt/python-2.3.0.zip - unzip python-2.3.0.zip -d python-2.3.0 diff --git a/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml b/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml index df204d4c5..0e2ecb287 100644 --- a/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml +++ b/codebuild/py312/decrypt_golden_manifest_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-full_decrypt-mpl" + TOXENV: "py312-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml index 5bcd26738..844f4c464 100644 --- a/codebuild/py312/decrypt_hkeyring_with_keyrings.yml +++ b/codebuild/py312/decrypt_hkeyring_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors # This manifest has coverage for both HKeyring and required encryption context CMM - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_hkeyring_reccmm_manifest.zip 312_hkeyring_reccmm_manifest.zip diff --git a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml index be67235d7..04def1a98 100644 --- a/codebuild/py312/decrypt_hkeyring_with_masterkey.yml +++ b/codebuild/py312/decrypt_hkeyring_with_masterkey.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TOXENV: "py312-full_decrypt-mpl" + TOXENV: "py312-full_decrypt" AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID: >- arn:aws:kms:us-west-2:658956600833:key/b35311ef1-d8dc-4780-9f5a-55776cbb2f7f AWS_ENCRYPTION_SDK_PYTHON_INTEGRATION_TEST_AWS_KMS_KEY_ID_2: >- diff --git a/codebuild/py312/decrypt_keyrings_with_keyrings.yml b/codebuild/py312/decrypt_keyrings_with_keyrings.yml index 3ab7058f9..27f9f1b5e 100644 --- a/codebuild/py312/decrypt_keyrings_with_keyrings.yml +++ b/codebuild/py312/decrypt_keyrings_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_keyrings.zip 312_keyrings.zip - unzip 312_keyrings.zip diff --git a/codebuild/py312/decrypt_masterkey_with_keyrings.yml b/codebuild/py312/decrypt_masterkey_with_keyrings.yml index 21f646370..e22bd2ace 100644 --- a/codebuild/py312/decrypt_masterkey_with_keyrings.yml +++ b/codebuild/py312/decrypt_masterkey_with_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Download previously generated vectors - aws s3 cp s3://generated-vectors-artifacts-bucket/$CODEBUILD_RESOLVED_SOURCE_VERSION/312_masterkey.zip 312_masterkey.zip - unzip 312_masterkey.zip diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml index aec3916e5..3156e06ab 100644 --- a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml @@ -19,6 +19,8 @@ phases: python: 3.12 pre_build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip diff --git a/codebuild/py312/encrypt_keyrings.yml b/codebuild/py312/encrypt_keyrings.yml index 56a389e6f..db12d6c9d 100644 --- a/codebuild/py312/encrypt_keyrings.yml +++ b/codebuild/py312/encrypt_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/generate_decrypt_vectors_keyrings.yml b/codebuild/py312/generate_decrypt_vectors_keyrings.yml index ae79b86ce..1760333f0 100644 --- a/codebuild/py312/generate_decrypt_vectors_keyrings.yml +++ b/codebuild/py312/generate_decrypt_vectors_keyrings.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers - | diff --git a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml index b0a755360..15b4f485e 100644 --- a/codebuild/py312/generate_hkeyring_decrypt_vectors.yml +++ b/codebuild/py312/generate_hkeyring_decrypt_vectors.yml @@ -18,6 +18,8 @@ phases: python: 3.12 build: commands: + # Build Python MPL TestVector runner from source + - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh - pip install "tox < 4.0" - cd test_vector_handlers/test/aws-crypto-tools-test-vector-framework # Checkout WIP branch with manifest containing HKeyring and required EC CMM test cases diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt index 6995b3187..f1a309033 100644 --- a/test_vector_handlers/requirements_mpl.txt +++ b/test_vector_handlers/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptography-internal-mpl-testvectors @ git+https://github.com/aws/aws-cryptographic-material-providers-library.git@python-main-with-dafny-code#subdirectory=TestVectorsAwsCryptographicMaterialProviders/runtimes/python \ No newline at end of file +aws-cryptographic-material-providers==1.7.4 diff --git a/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh new file mode 100644 index 000000000..b99226e66 --- /dev/null +++ b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh @@ -0,0 +1,24 @@ +# Builds the Python MPL TestVector runner from source. +# This package is used by the ESDK-Python test vectors for testing with the MPL. + +# This script is intended to be used by ESDK-Python's integration tests. +# You may need or want to make local changes to get this work on your machine. + +# Change to the directory of the script +cd "$(dirname "$0")" + +# Get MPL version specified in requirements.txt +export mplVersion=$(grep 'aws-cryptographic-material-providers==' ../requirements_mpl.txt | sed -E 's/.*==(.+)/\1/') + +# Clone MPL repo to get test vectors runner source code and the Dafny version to use +git clone --branch v$mplVersion --recurse-submodules https://github.com/aws/aws-cryptographic-material-providers-library.git + +# Download Dafny to build the test vector runner; get Dafny version from ESDK's project.properties file +export dafnyVersion=$(grep '^dafnyVersion=' aws-cryptographic-material-providers-library/project.properties | cut -d '=' -f 2) +curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip -L -o dafny.zip +unzip -qq dafny.zip && rm dafny.zip +export PATH="$PWD/dafny:$PATH" + +# Build MPL test vector runner from source +cd aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/ +make transpile_python diff --git a/test_vector_handlers/tox.ini b/test_vector_handlers/tox.ini index cdb1137fb..7fe77ff50 100644 --- a/test_vector_handlers/tox.ini +++ b/test_vector_handlers/tox.ini @@ -51,6 +51,9 @@ deps = -rtest/requirements.txt # Install the MPL requirements if the `-mpl` suffix is present mpl: -rrequirements_mpl.txt + # This is the filepath used in ESDK-Python's integration tests. + # You may need to change this to run this project locally. + mpl: scripts/aws-cryptographic-material-providers-library/TestVectorsAwsCryptographicMaterialProviders/runtimes/python .. commands = awses_local: {[testenv:base-command]commands} From 18352146239976a93dfd26955b5b695e5a62ab68 Mon Sep 17 00:00:00 2001 From: Lucas McDonald Date: Wed, 26 Mar 2025 15:00:25 -0700 Subject: [PATCH 3/3] fix: Improve header serialization (#747) --- CHANGELOG.rst | 20 +++ README.rst | 4 +- .../py310/decrypt_dafny_esdk_vectors.yml | 8 +- codebuild/py310/decrypt_net_401_vectors.yml | 2 +- .../decrypt_dafny_esdk_vectors_keyrings.yml | 8 +- .../decrypt_dafny_esdk_vectors_masterkey.yml | 8 +- .../decrypt_net_401_vectors_keyrings.yml | 2 +- .../decrypt_net_401_vectors_masterkey.yml | 2 +- .../decrypt_dafny_esdk_vectors_keyrings.yml | 8 +- .../decrypt_dafny_esdk_vectors_masterkey.yml | 8 +- codebuild/py312/decrypt_hkeyring_with_net.yml | 6 +- .../decrypt_net_401_vectors_keyrings.yml | 2 +- .../decrypt_net_401_vectors_masterkey.yml | 2 +- codebuild/py38/decrypt_dafny_esdk_vectors.yml | 8 +- codebuild/py38/decrypt_net_401_vectors.yml | 2 +- codebuild/py39/decrypt_dafny_esdk_vectors.yml | 8 +- codebuild/py39/decrypt_net_401_vectors.yml | 2 +- performance_tests/README.rst | 2 +- performance_tests/requirements_mpl.txt | 2 +- requirements_mpl.txt | 2 +- setup.py | 5 +- src/aws_encryption_sdk/identifiers.py | 2 +- .../internal/formatting/deserialize.py | 11 ++ .../internal/formatting/serialize.py | 26 +++- test/unit/test_deserialize.py | 10 ++ test/unit/test_serialize.py | 146 ++++++++++++++++++ test/unit/test_values.py | 62 ++++++++ test_vector_handlers/requirements_mpl.txt | 2 +- .../scripts/install_mpl_test_vector_runner.sh | 4 +- 29 files changed, 317 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 56cba75a2..c9ba4ef56 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,24 @@ Changelog ********* +4.0.1 -- 2025-03-26 +=================== + +Fixes +----------- +* fix: Improve header serialization + `#747 `_ + + ESDK-Python <4.0.1 would truncate non-ASCII key provider IDs it wrote to message headers. + If a Raw or Custom MasterKeyProvider or Keyring supplied a non-ASCII key provider ID / key namespace, + ESDK-Python would truncate the the key provider ID it wrote to the message's header. + The message can be decrypted by replacing the truncated provider ID with the expected provider ID in decryption code. + Contact AWS for any questions about this approach. + +Maintenance +----------- +* deps: Extend supported `MPL`_ versions to include v1.10.0 + 4.0.0 -- 2024-10-29 =================== @@ -23,6 +41,7 @@ Breaking Changes However, messages that are constructed with the required EC CMM are not backward compatible with ESDK <4.0.0, as no version of ESDK <4.0.0 supports reading messages encrypted with the required EC CMM. A message that is encrypted with the required EC CMM from the MPL must be decrypted with a CMM from the MPL. + For more information on using the required EC CMM, see `AWS Documentation `_. Fixes ----------- @@ -425,6 +444,7 @@ Minor =================== * Initial public release +.. _MPL: https://github.com/aws/aws-cryptographic-material-providers-library .. _breaking changes in attrs 17.1.0: https://attrs.readthedocs.io/en/stable/changelog.html .. _tox: https://tox.readthedocs.io/en/latest/ .. _pylint: https://www.pylint.org/ diff --git a/README.rst b/README.rst index b968ae13e..ebc71fa85 100644 --- a/README.rst +++ b/README.rst @@ -42,7 +42,7 @@ Required Prerequisites Recommended Prerequisites ========================= -* aws-cryptographic-material-providers: == 1.7.4 +* aws-cryptographic-material-providers: == 1.10.0 * Requires Python 3.11+. Installation @@ -246,7 +246,7 @@ sharing entries in that cache across threads needs to be done carefully **Important:** Components from the `AWS Cryptographic Material Providers Library (MPL)`_ have separate thread safety considerations. -For more information, see the note on thread safety in that project's README (TODO-MPL: link) +For more information, see the note on thread safety in that project's `README `_. .. _AWS Encryption SDK: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/introduction.html diff --git a/codebuild/py310/decrypt_dafny_esdk_vectors.yml b/codebuild/py310/decrypt_dafny_esdk_vectors.yml index 505f3157c..019a9adf6 100644 --- a/codebuild/py310/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py310/decrypt_dafny_esdk_vectors.yml @@ -28,7 +28,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -36,7 +36,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -44,9 +44,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py310/decrypt_net_401_vectors.yml b/codebuild/py310/decrypt_net_401_vectors.yml index 82ac642d9..0d81b349e 100644 --- a/codebuild/py310/decrypt_net_401_vectors.yml +++ b/codebuild/py310/decrypt_net_401_vectors.yml @@ -21,7 +21,7 @@ phases: commands: # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml index 1eadf2958..384f24fed 100644 --- a/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_keyrings.yml @@ -30,7 +30,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -38,7 +38,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -46,9 +46,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml index 6106906b5..1d42953d9 100644 --- a/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml +++ b/codebuild/py311/decrypt_dafny_esdk_vectors_masterkey.yml @@ -28,7 +28,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -36,7 +36,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -44,9 +44,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml index b5e7d7233..0cae98c4d 100644 --- a/codebuild/py311/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py311/decrypt_net_401_vectors_keyrings.yml @@ -23,7 +23,7 @@ phases: - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py311/decrypt_net_401_vectors_masterkey.yml b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml index f6f0482e7..157e732f8 100644 --- a/codebuild/py311/decrypt_net_401_vectors_masterkey.yml +++ b/codebuild/py311/decrypt_net_401_vectors_masterkey.yml @@ -31,7 +31,7 @@ phases: # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml index 2a8439253..e20277d94 100644 --- a/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_keyrings.yml @@ -30,7 +30,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -38,7 +38,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -46,9 +46,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml index b375651c5..50948e31c 100644 --- a/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml +++ b/codebuild/py312/decrypt_dafny_esdk_vectors_masterkey.yml @@ -28,7 +28,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -36,7 +36,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -44,9 +44,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py312/decrypt_hkeyring_with_net.yml b/codebuild/py312/decrypt_hkeyring_with_net.yml index 1a1ab1827..e1816282b 100644 --- a/codebuild/py312/decrypt_hkeyring_with_net.yml +++ b/codebuild/py312/decrypt_hkeyring_with_net.yml @@ -25,15 +25,15 @@ phases: - export DAFNY_AWS_ESDK_TEST_VECTOR_MANIFEST_PATH="${PWD}/312_hkeyring_reccmm_manifest/manifest.json" # Clone SDK-Dafny repo to get test vectors runner source code and the Dafny version to use - - git clone --recurse-submodules https://github.com/aws/aws-encryption-sdk-dafny.git + - git clone --recurse-submodules https://github.com/aws/aws-encryption-sdk.git # Download Dafny to build the test vector runner; get Dafny version from ESDK's project.properties file - - export dafnyVersion=$(grep '^dafnyVersion=' aws-encryption-sdk-dafny/AwsEncryptionSDK/project.properties | cut -d '=' -f 2) + - export dafnyVersion=$(grep '^dafnyVersion=' aws-encryption-sdk/project.properties | cut -d '=' -f 2) - curl https://github.com/dafny-lang/dafny/releases/download/v$dafnyVersion/dafny-$dafnyVersion-x64-ubuntu-20.04.zip -L -o dafny.zip - unzip -qq dafny.zip && rm dafny.zip - export PATH="$PWD/dafny:$PATH" # Build MPL test vector runner from source - - cd aws-encryption-sdk-dafny/mpl/TestVectorsAwsCryptographicMaterialProviders/ + - cd aws-encryption-sdk/mpl/TestVectorsAwsCryptographicMaterialProviders/ - make transpile_net # Change ESDK TestVectors project to reference the published .NET ESDK diff --git a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml index 3156e06ab..281023f53 100644 --- a/codebuild/py312/decrypt_net_401_vectors_keyrings.yml +++ b/codebuild/py312/decrypt_net_401_vectors_keyrings.yml @@ -23,7 +23,7 @@ phases: - sh test_vector_handlers/scripts/install_mpl_test_vector_runner.sh # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py312/decrypt_net_401_vectors_masterkey.yml b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml index 5d1ef9d94..81daba522 100644 --- a/codebuild/py312/decrypt_net_401_vectors_masterkey.yml +++ b/codebuild/py312/decrypt_net_401_vectors_masterkey.yml @@ -31,7 +31,7 @@ phases: # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py38/decrypt_dafny_esdk_vectors.yml b/codebuild/py38/decrypt_dafny_esdk_vectors.yml index 968a74690..3bc966126 100644 --- a/codebuild/py38/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py38/decrypt_dafny_esdk_vectors.yml @@ -28,7 +28,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -36,7 +36,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -44,9 +44,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py38/decrypt_net_401_vectors.yml b/codebuild/py38/decrypt_net_401_vectors.yml index 298711975..91163f14d 100644 --- a/codebuild/py38/decrypt_net_401_vectors.yml +++ b/codebuild/py38/decrypt_net_401_vectors.yml @@ -21,7 +21,7 @@ phases: commands: # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/codebuild/py39/decrypt_dafny_esdk_vectors.yml b/codebuild/py39/decrypt_dafny_esdk_vectors.yml index ddb50db1c..a22c4d079 100644 --- a/codebuild/py39/decrypt_dafny_esdk_vectors.yml +++ b/codebuild/py39/decrypt_dafny_esdk_vectors.yml @@ -28,7 +28,7 @@ phases: MOST_RECENT_RUN_ID=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token ${GITHUB_TOKEN}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs?branch=mainline&status=completed&page=1&exclude_pull_requests=true" \ | jq 'first(.workflow_runs[] | select(.name=="Daily CI") | .id)') - | echo "DEBUG: Fetching artifact from run $MOST_RECENT_RUN_ID" @@ -36,7 +36,7 @@ phases: MOST_RECENT_RUN_DOWNLOAD_URL=$(curl -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/aws/aws-encryption-sdk-dafny/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-latest_vector_artifact" \ + "https://api.github.com/repos/aws/aws-encryption-sdk/actions/runs/$MOST_RECENT_RUN_ID/artifacts?name=ubuntu-22.04_vector_artifact" \ | jq '.artifacts[0].archive_download_url') - | echo "DEBUG: Fetching artifact at $MOST_RECENT_RUN_DOWNLOAD_URL" @@ -44,9 +44,9 @@ phases: curl -L -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_TOKEN" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-latest_test_vector_artifact.zip + $(echo $MOST_RECENT_RUN_DOWNLOAD_URL | tr -d '"') -o ubuntu-22.04_vector_artifact.zip # This unzips to `net41.zip`. - - unzip ubuntu-latest_test_vector_artifact + - unzip ubuntu-22.04_vector_artifact # This unzips to `net41/`. - unzip net41.zip -d net41 build: diff --git a/codebuild/py39/decrypt_net_401_vectors.yml b/codebuild/py39/decrypt_net_401_vectors.yml index 635abc95b..d8a9ff453 100644 --- a/codebuild/py39/decrypt_net_401_vectors.yml +++ b/codebuild/py39/decrypt_net_401_vectors.yml @@ -21,7 +21,7 @@ phases: commands: # Fetch ESDK .NET v4.0.1 Test Vectors - VECTOR_ZIP=$CODEBUILD_SRC_DIR/v4-Net-4.0.1.zip - - VECTORS_URL=https://github.com/aws/aws-encryption-sdk-dafny/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip + - VECTORS_URL=https://github.com/aws/aws-encryption-sdk/raw/mainline/AwsEncryptionSDK/runtimes/net/TestVectorsNative/TestVectors/resources/v4-Net-4.0.1.zip - curl -s --output $VECTOR_ZIP --location $VECTORS_URL - UNZIPPED_VECTORS_DIR=$CODEBUILD_SRC_DIR/test_vector_handlers/net_401_vectors - unzip $VECTOR_ZIP -d $UNZIPPED_VECTORS_DIR diff --git a/performance_tests/README.rst b/performance_tests/README.rst index ee3dd797b..1e23548dd 100644 --- a/performance_tests/README.rst +++ b/performance_tests/README.rst @@ -49,7 +49,7 @@ Required Prerequisites Recommended Prerequisites ========================= -* aws-cryptographic-material-providers: == 1.7.4 +* aws-cryptographic-material-providers: == 1.10.0 * Requires Python 3.11+. ***** diff --git a/performance_tests/requirements_mpl.txt b/performance_tests/requirements_mpl.txt index 1615fff0d..be77ea2da 100644 --- a/performance_tests/requirements_mpl.txt +++ b/performance_tests/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptographic-material-providers==1.7.4 \ No newline at end of file +aws-cryptographic-material-providers>=1.7.4,<=1.10.0 \ No newline at end of file diff --git a/requirements_mpl.txt b/requirements_mpl.txt index f1a309033..8912641d0 100644 --- a/requirements_mpl.txt +++ b/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptographic-material-providers==1.7.4 +aws-cryptographic-material-providers>=1.7.4,<=1.10.0 diff --git a/setup.py b/setup.py index ab5ac71a3..587495fd4 100644 --- a/setup.py +++ b/setup.py @@ -39,11 +39,8 @@ def get_requirements(): keywords="aws-encryption-sdk aws kms encryption", license="Apache License 2.0", install_requires=get_requirements(), - # pylint: disable=fixme - # TODO-MPL: Point at PyPI once MPL is released. - # This blocks releasing ESDK-Python MPL integration. extras_require={ - "MPL": ["aws-cryptographic-material-providers==1.7.4"], + "MPL": ["aws-cryptographic-material-providers>=1.7.4,<=1.10.0"], }, classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/src/aws_encryption_sdk/identifiers.py b/src/aws_encryption_sdk/identifiers.py index 3e5a9940b..8a5a2c8a1 100644 --- a/src/aws_encryption_sdk/identifiers.py +++ b/src/aws_encryption_sdk/identifiers.py @@ -17,7 +17,7 @@ # We only actually need these imports when running the mypy checks pass -__version__ = "4.0.0" +__version__ = "4.0.1" USER_AGENT_SUFFIX = "AwsEncryptionSdkPython/{}".format(__version__) diff --git a/src/aws_encryption_sdk/internal/formatting/deserialize.py b/src/aws_encryption_sdk/internal/formatting/deserialize.py index 4511ebb69..c90dc9124 100644 --- a/src/aws_encryption_sdk/internal/formatting/deserialize.py +++ b/src/aws_encryption_sdk/internal/formatting/deserialize.py @@ -145,6 +145,17 @@ def deserialize_encrypted_data_keys(stream, max_encrypted_data_keys=None): (key_provider_information,) = unpack_values(">{}s".format(key_provider_information_length), stream) (encrypted_data_key_length,) = unpack_values(">H", stream) encrypted_data_key = stream.read(encrypted_data_key_length) + # ESDK-Python <4.0.1 incorrectly computed the key provider length for non-ASCII key provider IDs. + # The length in the header was computed as the length of the key provider ID as a string instead of + # the length of the key provider ID as UTF-8 bytes. + # If a non-ASCII key provider ID were supplied, the key provider ID's UTF-8 bytes written to the header + # would be truncated, and attempting to decrypt the message would result in a deserialization error. + # That error would be raised when calling `to_str(key_provider_identifier)` below. + # An impacted message can be decrypted by replacing the truncated provider ID with the expected provider ID + # in decryption code. + # Contact AWS for any questions about this approach. + # ESDK-Python >=4.0.1 corrects the serialization logic and writes the correct length and expected bytes + # to the message header. encrypted_data_keys.add( EncryptedDataKey( key_provider=MasterKeyInfo( diff --git a/src/aws_encryption_sdk/internal/formatting/serialize.py b/src/aws_encryption_sdk/internal/formatting/serialize.py index c9679a08d..27e98c399 100644 --- a/src/aws_encryption_sdk/internal/formatting/serialize.py +++ b/src/aws_encryption_sdk/internal/formatting/serialize.py @@ -35,16 +35,30 @@ def serialize_encrypted_data_key(encrypted_data_key): "H" # encrypted data key length "{enc_data_key_len}s" # encrypted data key ) + # ESDK-Python <4.0.1 incorrectly computed len_key_provider_id_bytes for non-ASCII key provider IDs. + # len_key_provider_id_bytes was computed as the length of the key provider ID as a string instead of + # the length of the key provider ID as UTF-8 bytes. + # If a non-ASCII key provider ID were supplied, the key provider ID as UTF-8 bytes written to the header + # would be truncated, and attempting to decrypt the message would result in a deserialization error. + # The message can be decrypted by replacing the truncated provider ID with the expected provider ID + # in decryption code. + # Contact AWS for any questions about this approach. + # ESDK-Python >=4.0.1 corrects the serialization logic and writes the correct length and expected bytes + # to the message header. + key_provider_id_bytes = to_bytes(encrypted_data_key.key_provider.provider_id) + len_key_provider_id_bytes = len(key_provider_id_bytes) + key_info_bytes = to_bytes(encrypted_data_key.key_provider.key_info) + len_key_info_bytes = len(key_info_bytes) return struct.pack( encrypted_data_key_format.format( - provider_id_len=len(encrypted_data_key.key_provider.provider_id), - provider_info_len=len(encrypted_data_key.key_provider.key_info), + provider_id_len=len_key_provider_id_bytes, + provider_info_len=len_key_info_bytes, enc_data_key_len=len(encrypted_data_key.encrypted_data_key), ), - len(encrypted_data_key.key_provider.provider_id), - to_bytes(encrypted_data_key.key_provider.provider_id), - len(encrypted_data_key.key_provider.key_info), - to_bytes(encrypted_data_key.key_provider.key_info), + len_key_provider_id_bytes, + key_provider_id_bytes, + len_key_info_bytes, + key_info_bytes, len(encrypted_data_key.encrypted_data_key), encrypted_data_key.encrypted_data_key, ) diff --git a/test/unit/test_deserialize.py b/test/unit/test_deserialize.py index fc77634c2..f0273c428 100644 --- a/test/unit/test_deserialize.py +++ b/test/unit/test_deserialize.py @@ -265,6 +265,16 @@ def test_deserialize_body_frame_final(self): assert test_body == VALUES["deserialized_body_final_frame_single"] assert test_final + def test_GIVEN_final_frame_content_length_equals_header_frame_length_WHEN_deserialize_header_THEN_no_error(self): + """Validate that the deserialize_body_frame function + behaves as expected for a valid final body frame + where the final frame length equals the header frame length. + """ + stream = io.BytesIO(VALUES["serialized_final_frame_512_length"]) + aws_encryption_sdk.internal.formatting.deserialize.deserialize_frame( + stream=stream, header=VALUES["deserialized_header_frame_512_frame"] + ) + def test_deserialize_body_frame_final_invalid_final_frame_length(self): """Validate that the deserialize_body_frame function behaves as expected for a valid final body frame. diff --git a/test/unit/test_serialize.py b/test/unit/test_serialize.py index 4bcd703d8..1df4d1bd7 100644 --- a/test/unit/test_serialize.py +++ b/test/unit/test_serialize.py @@ -1,9 +1,13 @@ # Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 """Unit test suite for aws_encryption_sdk.internal.formatting.serialize""" +import io +import struct + import pytest from mock import MagicMock, patch, sentinel +import aws_encryption_sdk.internal.formatting.deserialize import aws_encryption_sdk.internal.formatting.serialize from aws_encryption_sdk.exceptions import SerializationError from aws_encryption_sdk.identifiers import ContentAADString, SerializationVersion @@ -15,6 +19,8 @@ pytestmark = [pytest.mark.unit, pytest.mark.local] +provider_input_strings = ["", "abc", "𐀂", "abc𐀂", "𐀂abc", "秘密代码", "abc秘密代码", "秘密代码abc", "秘密代码abc𐀂", "𐀂abc秘密代码123𐀂"] + @pytest.mark.parametrize( "sequence_number, error_message", @@ -80,6 +86,146 @@ def apply_fixtures(self): self.mock_encrypt_patcher.stop() self.mock_valid_frame_length_patcher.stop() + @pytest.mark.parametrize("provider_id", provider_input_strings) + @pytest.mark.parametrize("provider_info", provider_input_strings) + def test_GIVEN_valid_encrypted_data_key_WHEN_serialize_encrypted_data_key_THEN_deserialize_equals_input( + self, + provider_id, + provider_info, + ): + # Given: Some valid encrypted data key + key_provider = MasterKeyInfo(provider_id=provider_id, key_info=provider_info) + encrypted_data_key = EncryptedDataKey( + key_provider=key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + # When: serialize_encrypted_data_key + serialized_edk = aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key + ) + + # Then: Can deserialize the value + serialized_edks = bytes() + # Hardcode to have only 1 EDK + serialized_edks += struct.pack(">H", 1) + serialized_edks += serialized_edk + # Deserialization must not raise exception + deserialized = aws_encryption_sdk.internal.formatting.deserialize.deserialize_encrypted_data_keys( + stream=io.BytesIO(serialized_edks) + ) + assert deserialized == {encrypted_data_key} + assert len(deserialized) == 1 + deserialized_edk = list(deserialized)[0] + assert deserialized_edk.key_provider == encrypted_data_key.key_provider + assert deserialized_edk.key_provider.provider_id == encrypted_data_key.key_provider.provider_id + assert deserialized_edk.key_provider.key_info == encrypted_data_key.key_provider.key_info + assert deserialized_edk.encrypted_data_key == encrypted_data_key.encrypted_data_key + + @pytest.mark.parametrize("edk_1_provider_id", provider_input_strings) + @pytest.mark.parametrize("edk_1_provider_info", provider_input_strings) + @pytest.mark.parametrize("edk_2_provider_id", provider_input_strings) + @pytest.mark.parametrize("edk_2_provider_info", provider_input_strings) + def test_GIVEN_two_distinct_valid_encrypted_data_keys_WHEN_serialize_encrypted_data_keys_THEN_deserialize_equals_inputs( # noqa pylint: disable=line-too-long + self, + edk_1_provider_id, + edk_1_provider_info, + edk_2_provider_id, + edk_2_provider_info, + ): + # pylint: disable=too-many-locals + # Given: Two distinct valid encrypted data keys + edk_1_key_provider = MasterKeyInfo(provider_id=edk_1_provider_id, key_info=edk_1_provider_info) + encrypted_data_key_1 = EncryptedDataKey( + key_provider=edk_1_key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + edk_2_key_provider = MasterKeyInfo(provider_id=edk_2_provider_id, key_info=edk_2_provider_info) + encrypted_data_key_2 = EncryptedDataKey( + key_provider=edk_2_key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + # Must be distinct + if encrypted_data_key_1 == encrypted_data_key_2: + return + + # When: serialize_encrypted_data_key + serialized_edk_1 = aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key_1 + ) + serialized_edk_2 = aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key_2 + ) + + # Then: Can deserialize the value + serialized_edks = bytes() + # Hardcode to have only 2 EDKs + serialized_edks += struct.pack(">H", 2) + serialized_edks += serialized_edk_1 + serialized_edks += serialized_edk_2 + # Deserialization must not raise exception + deserialized = aws_encryption_sdk.internal.formatting.deserialize.deserialize_encrypted_data_keys( + stream=io.BytesIO(serialized_edks) + ) + assert deserialized == {encrypted_data_key_1, encrypted_data_key_2} + assert len(deserialized) == 2 + deserialized_edk_list = list(deserialized) + + deserialized_edk_some = deserialized_edk_list[0] + deserialized_edk_other = deserialized_edk_list[1] + + assert ( + (deserialized_edk_some == encrypted_data_key_1 and deserialized_edk_other == encrypted_data_key_2) + or (deserialized_edk_some == encrypted_data_key_2 and deserialized_edk_other == encrypted_data_key_1) + ) + + def test_GIVEN_invalid_encrypted_data_key_WHEN_serialize_THEN_raises_UnicodeEncodeError( + self, + ): + # Given: Some invalid encrypted data key + + # This is invalid because "\ud800\udc02" cannot be encoded to UTF-8. + # This value MUST be able to be encoded to UTF-8, or serialization will fail. + invalid_provider_string = "\ud800\udc02" + + # Then: raises UnicodeEncodeError + with pytest.raises(UnicodeEncodeError): + key_provider = MasterKeyInfo(provider_id=invalid_provider_string, key_info=invalid_provider_string) + + encrypted_data_key = EncryptedDataKey( + key_provider=key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + # When: serialize_encrypted_data_key + aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key + ) + + # Then: raises UnicodeEncodeError + with pytest.raises(UnicodeEncodeError): + key_provider = MasterKeyInfo(provider_id=invalid_provider_string, key_info="abc") + + encrypted_data_key = EncryptedDataKey( + key_provider=key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + # When: serialize_encrypted_data_key + aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key + ) + + # Then: raises UnicodeEncodeError + with pytest.raises(UnicodeEncodeError): + key_provider = MasterKeyInfo(provider_id="abc", key_info=invalid_provider_string) + + encrypted_data_key = EncryptedDataKey( + key_provider=key_provider, encrypted_data_key=VALUES["encrypted_data_key"] + ) + + # When: serialize_encrypted_data_key + aws_encryption_sdk.internal.formatting.serialize.serialize_encrypted_data_key( + encrypted_data_key=encrypted_data_key + ) + def test_serialize_header_v1(self): """Validate that the _serialize_header function behaves as expected. diff --git a/test/unit/test_values.py b/test/unit/test_values.py index 9ae2df354..6e5883127 100644 --- a/test/unit/test_values.py +++ b/test/unit/test_values.py @@ -230,6 +230,49 @@ def array_byte(source): VALUES["final_frame_base"].tag, ] ) +# This is a valid frame from a ESDK-.NET-encrypted message. +# ESDK Python versions before v4.0.0 would raise a SerializationError when deserializing this frame +# because its frame length (512; the b"\x00\x00\x02\x00" string) +# equals the configured frame length. +# In other ESDK implementations, the final frame length would never equal the frame length +# because they would append an empty final frame. +# Both are valid implementations of the ESDK specification, +# and the ESDK-Python must support this case. +VALUES["serialized_final_frame_512_length"] = b"".join( + [ + b"\xff\xff\xff\xff", + b"\x00\x00\x00\x14", + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14', + b"\x00\x00\x02\x00", + b'''\x87r+k7 \xc7\xc3\xbf)T.8,}\xc5a.H]\x16/08k2 + )\xb5QB\xccP\xc2\xc6\xeanf\x06Z7\xbb\xcd\x87L\xa6 + ~~\xdc\xab~\x0e\xf6\x05\n\xa9\x94X[\xb8En?x$\x11 + \x10\x84g0i\xeai\xf9\x8c\xe6}\xc3\xa1Gig\xbdA\x1an + \x1b\x9d\xf1\rW\xc8\xad|\x04hSt\x10\xc7\x0e\'\x8f + \xe8\x94\x9d\xdb\x82\xdb"\x95\xbc\xf5\xc5\xd0\xddQ + \xba\xaa\xbf6\x1e\xd8\xffB\xed\xee\xda1\x15\xf6=x + \xe14\xe7\xf5\xb7t\x10\x11\xa4!,!\xfa\xc7\xf1\t\xf7 + \xc3X?eI\xcdk\xf3\xb5\x80b\xdd;*\xe9\x9c\xd5\x83[\xc4c + \xe4[mA\x87\xd9\x94g\xd6\\<\xd1\xff\xcc<\xef\xe2\xbc\xda> + \xda|\xa1L\xd1\xf4u\x07Y\x13\xa3\xd4\x15\x1fS\x98\x00^ + \x1d^\xcdu\x17\xc8.\xfb\x9d\xaaU\xbf\x8f\xa96%YPX\xe6 + \xf5\\\x141\xe5\xdd\x9a,\xc7d\xca\xffQ\x02:\xd87s:\x9a + \xdf\xd5\'\xf0!\x13\xafuU\xf7w\x15\xbd\xecS \xf2h\xa4 + \xdd\xfb9\xbb\xb3\xd7?3\xc0\xeed\x0e\x17\x1b\xccN\xf9)s + \xd1\x97\x84\xb6\xce5\xca\x9b\xde\xa9\x0e$>x\xd9\x9cD= + \xd5\xa3\xa1qb#\x8c\xc1\x81Nv\x8dA0\'{~\x1c\xf1?\n\x7fAX\x9f, + \xe1\xe6d\xc5\xed\x9e\xa9o\x1bpp\xac\x1b\x03P\xd8\xae\xd6\xf6 + \xaca;N\xd6C\x08\x99!\x0bU8\x85(g\xe6\x8fD\xf7\x19\xb0]4 + \x19hB\x15\xa7\xee\xd8\xc0\xe9D\x850\xb6\x05\xd1\xa3`%\xcb + \xfb\x88&"\xdfnm\xa6\xf1X\xc4\x84\x1c\xc3\xe8]\x05mh$\xff]= + \xab\xa2p\x8e\x82:U\xef\xf3\x86X\xe16\x1f\xc7\x7f\x8dv\x1a + \xe4\r5\x8a\xea\x90\xb2\x1cA(\x9b\xedyT0\xd4h\tJ\xa4<\x07C9 + \xa3a]\x7f\x17Ak\x1d\xb9gA\x04\xbaq\xe5(y-\xc4!\x87\xa83 + \xdd\xf3\xea\xa7\x12X\xb6l\x98\xdf,\xc8\xe6\x9f7\xb0\xcd + \xb3\x9a\xf4\xe7a"H\xd9L\xd7.\x0f\x7f1W''', + b'XK#8\xb3\xab\x07\x11\x94\xf7\xac\xea\xd0g\x9b#', + ] +) VALUES["serialized_final_frame_bad_length"] = b"".join( [ b"\xff\xff\xff\xff", @@ -361,6 +404,25 @@ def array_byte(source): header_iv_length=Algorithm.AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384.iv_len, frame_length=2 ** 16, ) +VALUES["deserialized_header_frame_512_frame"] = MessageHeader( + version=SerializationVersion.V1, + type=ObjectType.CUSTOMER_AE_DATA, + algorithm=Algorithm.AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384, + message_id=VALUES["message_id"], + encryption_context=VALUES["updated_encryption_context"], + encrypted_data_keys=set( + [ + EncryptedDataKey( + key_provider=VALUES["data_keys"][0].key_provider, + encrypted_data_key=VALUES["data_keys"][0].encrypted_data_key, + ) + ] + ), + content_type=ContentType.FRAMED_DATA, + content_aad_length=0, + header_iv_length=Algorithm.AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384.iv_len, + frame_length=512, +) VALUES["deserialized_header_small_frame"] = MessageHeader( version=SerializationVersion.V1, type=ObjectType.CUSTOMER_AE_DATA, diff --git a/test_vector_handlers/requirements_mpl.txt b/test_vector_handlers/requirements_mpl.txt index f1a309033..8912641d0 100644 --- a/test_vector_handlers/requirements_mpl.txt +++ b/test_vector_handlers/requirements_mpl.txt @@ -1 +1 @@ -aws-cryptographic-material-providers==1.7.4 +aws-cryptographic-material-providers>=1.7.4,<=1.10.0 diff --git a/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh index b99226e66..aecff1a26 100644 --- a/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh +++ b/test_vector_handlers/scripts/install_mpl_test_vector_runner.sh @@ -7,8 +7,8 @@ # Change to the directory of the script cd "$(dirname "$0")" -# Get MPL version specified in requirements.txt -export mplVersion=$(grep 'aws-cryptographic-material-providers==' ../requirements_mpl.txt | sed -E 's/.*==(.+)/\1/') +# Get highest MPL version specified in requirements.txt +export mplVersion=$(grep 'aws-cryptographic-material-providers' ../requirements_mpl.txt | tr ',' '\n' | grep '<=' | sed -E 's/[^0-9]*//') # Clone MPL repo to get test vectors runner source code and the Dafny version to use git clone --branch v$mplVersion --recurse-submodules https://github.com/aws/aws-cryptographic-material-providers-library.git