diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index 9971c63d55..d71ce0a595 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -155,7 +155,7 @@ action_executions_output_ttl = 7 collection_interval = 600 [keyvalue] -# Location of the symmetric encryption key for encrypting values in kvstore. This key should be in JSON and should've been generated using keyczar. +# Location of the symmetric encryption key for encrypting values in kvstore. This key should be in JSON and should've been generated using st2-generate-symmetric-crypto-key tool. encryption_key_path = # Allow encryption of values in key value stored qualified as "secret". enable_encryption = True diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index ef4b222b65..bfdde2c57b 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -21,7 +21,6 @@ import six import mock -import unittest2 from oslo_config import cfg from python_runner import python_runner @@ -200,7 +199,6 @@ def test_simple_action_no_status_backward_compatibility(self): self.assertTrue(output is not None) self.assertEqual(output['result'], [1, 2]) - @unittest2.skipIf(six.PY3, 'keyczar doesn\'t work under Python 3') def test_simple_action_config_value_provided_overriden_in_datastore(self): pack = 'dummy_pack_5' user = 'joe' diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 0a6b7f7c5c..c89806badc 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -34,6 +34,7 @@ stevedore==1.28.0 paramiko==2.4.1 networkx==1.11 python-keyczar==0.716 +cryptography==2.2.2 retrying==1.3.3 # Note: We use latest version of virtualenv which uses pip 9.0 virtualenv==15.1.0 diff --git a/requirements.txt b/requirements.txt index 312856a703..4d96a90ddf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ RandomWords apscheduler==3.5.1 argcomplete bcrypt +cryptography==2.2.2 eventlet==0.23.0 flex==6.13.1 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper @@ -37,7 +38,6 @@ python-dateutil python-editor==1.0.3 python-gnupg==0.4.2 python-json-logger -python-keyczar==0.716 python-statsd==2.1.0 pytz==2018.4 pyyaml<4.0,>=3.12 diff --git a/st2common/bin/st2-generate-symmetric-crypto-key b/st2common/bin/st2-generate-symmetric-crypto-key index 3ea2147148..ca7c46f65d 100755 --- a/st2common/bin/st2-generate-symmetric-crypto-key +++ b/st2common/bin/st2-generate-symmetric-crypto-key @@ -1,14 +1,31 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import argparse import os import sys -from keyczar.keys import AesKey +from st2common.util.crypto import AESKey def main(key_path, force=False): + key_path = os.path.abspath(key_path) base_path = os.path.dirname(key_path) + if not os.access(base_path, os.W_OK): print('ERROR: You do not have sufficient permissions to write to path: %s.' % key_path) print('Try setting up permissions correctly and then run this tool.') @@ -23,9 +40,11 @@ def main(key_path, force=False): print('WARNING: Rewriting existing key with new key!') + # Explicitly chose large key size + aes_key = AESKey.generate(key_size=256) + with open(key_path, 'w') as key_file: - k = AesKey.Generate() - key_file.write(str(k)) + key_file.write(aes_key.to_json()) key_file.flush() msg = ('Key written to %s. ' % key_path + 'Secure the permissions so only StackStorm API ' + diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index 8827e5f347..a5b3a9e2e6 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -13,7 +13,7 @@ oslo.config paramiko pyyaml pymongo -python-keyczar +cryptography requests retrying semver diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 57bd87a659..9f524ba2b8 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -1,5 +1,6 @@ # Don't edit this file. It's generated automatically! apscheduler==3.5.1 +cryptography==2.2.2 eventlet==0.23.0 flex==6.13.1 greenlet==0.4.13 @@ -15,7 +16,6 @@ paramiko==2.4.1 prometheus_client==0.1.1 pymongo==3.6.1 python-dateutil -python-keyczar==0.716 python-statsd==2.1.0 pyyaml<4.0,>=3.12 requests[security]<2.15,>=2.14.1 diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 05cd73fd53..d9f51c538d 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -273,7 +273,8 @@ def register_opts(ignore_errors=False): cfg.StrOpt( 'encryption_key_path', default='', help='Location of the symmetric encryption key for encrypting values in kvstore. ' - 'This key should be in JSON and should\'ve been generated using keyczar.') + 'This key should be in JSON and should\'ve been generated using ' + 'st2-generate-symmetric-crypto-key tool.') ] do_register_opts(keyvalue_opts, group='keyvalue') diff --git a/st2common/st2common/util/crypto.py b/st2common/st2common/util/crypto.py index 700ce1b518..e6b0ecfefd 100644 --- a/st2common/st2common/util/crypto.py +++ b/st2common/st2common/util/crypto.py @@ -13,40 +13,299 @@ # See the License for the specific language governing permissions and # limitations under the License. +""" +Module for handling symmetric encryption and decryption of short text values (mostly used for +encrypted datastore values aka secrets). + +NOTE: In the past, this module used and relied on keyczar, but since keyczar doesn't support +Python 3, we moved to cryptography library. + +symmetric_encrypt and symmetric_decrypt functions except values as returned by the AESKey.Encrypt() +and AESKey.Decrypt() methods in keyczar. Those functions follow the same approach (AES in CBC mode +with SHA1 HMAC signature) as keyczar methods, but they use and rely on primitives and methods from +the cryptography library. + +This was done to make the keyczar -> cryptography migration fully backward compatible. + +Eventually, we should move to Fernet (https://cryptography.io/en/latest/fernet/) recipe for +symmetric encryption / decryption, because it offers more robustness and safer defaults (SHA256 +instead of SHA1, etc.). +""" + from __future__ import absolute_import + +import os +import json import binascii +import base64 + +from hashlib import sha1 + +import six + +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers import algorithms +from cryptography.hazmat.primitives.ciphers import modes +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import hmac +from cryptography.hazmat.backends import default_backend __all__ = [ + 'KEYCZAR_HEADER_SIZE', + 'KEYCZAR_AES_BLOCK_SIZE', + 'KEYCZAR_HLEN', + 'read_crypto_key', + 'symmetric_encrypt', - 'symmetric_decrypt' + 'symmetric_decrypt', + + 'cryptography_symmetric_encrypt', + 'cryptography_symmetric_decrypt', + + # NOTE: Keyczar functions are here for testing reasons - they are only used by tests + 'keyczar_symmetric_encrypt', + 'keyczar_symmetric_decrypt', + + 'AESKey' ] +# Keyczar related constants +KEYCZAR_HEADER_SIZE = 5 +KEYCZAR_AES_BLOCK_SIZE = 16 +KEYCZAR_HLEN = sha1().digest_size + +# Minimum key size which can be used for symmetric crypto +MINIMUM_AES_KEY_SIZE = 128 -def read_crypto_key(key_path, key_type=None): +DEFAULT_AES_KEY_SIZE = 256 + +assert DEFAULT_AES_KEY_SIZE >= MINIMUM_AES_KEY_SIZE + + +class AESKey(object): + """ + Class representing AES key object. """ - Return the crypto key given a path to key file and the key type. - :param key_path: Absolute path to file containing crypto key. - :type key_path: ``str`` + aes_key_string = None + hmac_key_string = None + hmac_key_size = None + mode = None + size = None - :param key_type: Type of crypto key. - :type key_type: :class:`keyczar.keys.KeyType` + def __init__(self, aes_key_string, hmac_key_string, hmac_key_size, mode='CBC', + size=DEFAULT_AES_KEY_SIZE): + if mode not in ['CBC']: + raise ValueError('Unsupported mode: %s' % (mode)) - :rtype: ``str`` + if size < MINIMUM_AES_KEY_SIZE: + raise ValueError('Unsafe key size: %s' % (size)) + + self.aes_key_string = aes_key_string + self.hmac_key_string = hmac_key_string + self.hmac_key_size = int(hmac_key_size) + self.mode = mode.upper() + self.size = int(size) + + # We also store bytes version of the key since bytes are needed by encrypt and decrypt + # methods + self.hmac_key_bytes = Base64WSDecode(self.hmac_key_string) + self.aes_key_bytes = Base64WSDecode(self.aes_key_string) + + @classmethod + def generate(self, key_size=DEFAULT_AES_KEY_SIZE): + """ + Generate a new AES key with the corresponding HMAC key. + + :rtype: :class:`AESKey` + """ + if key_size < MINIMUM_AES_KEY_SIZE: + raise ValueError('Unsafe key size: %s' % (key_size)) + + aes_key_bytes = os.urandom(int(key_size / 8)) + aes_key_string = Base64WSEncode(aes_key_bytes) + + hmac_key_bytes = os.urandom(int(key_size / 8)) + hmac_key_string = Base64WSEncode(hmac_key_bytes) + + return AESKey(aes_key_string=aes_key_string, hmac_key_string=hmac_key_string, + hmac_key_size=key_size, mode='CBC', size=key_size) + + def to_json(self): + """ + Return JSON representation of this key which is fully compatible with keyczar JSON key + file format. + + :rtype: ``str`` + """ + data = { + 'hmacKey': { + 'hmacKeyString': self.hmac_key_string, + 'size': self.hmac_key_size + }, + 'aesKeyString': self.aes_key_string, + 'mode': self.mode.upper(), + 'size': int(self.size) + } + return json.dumps(data) + + def __repr__(self): + return ('' % (self.hmac_key_size, self.mode, + self.size)) + + +def read_crypto_key(key_path): + """ + Read crypto key from keyczar JSON key file format and return parsed AESKey object. + + :param key_path: Absolute path to file containing crypto key in Keyczar JSON format. + :type key_path: ``str`` + + :rtype: :class:`AESKey` """ + with open(key_path, 'r') as fp: + content = fp.read() - # Late import to avoid very expensive in-direct import (~1 second) when this function - # is not called / used - from keyczar.keys import AesKey - key_type = key_type or AesKey + content = json.loads(content) - with open(key_path) as key_file: - key = key_type.Read(key_file.read()) - return key + try: + aes_key = AESKey(aes_key_string=content['aesKeyString'], + hmac_key_string=content['hmacKey']['hmacKeyString'], + hmac_key_size=content['hmacKey']['size'], + mode=content['mode'].upper(), + size=content['size']) + except KeyError as e: + msg = 'Invalid or malformed key file "%s": %s' % (key_path, str(e)) + raise KeyError(msg) + + return aes_key def symmetric_encrypt(encrypt_key, plaintext): + return cryptography_symmetric_encrypt(encrypt_key=encrypt_key, plaintext=plaintext) + + +def symmetric_decrypt(decrypt_key, ciphertext): + return cryptography_symmetric_decrypt(decrypt_key=decrypt_key, ciphertext=ciphertext) + + +def cryptography_symmetric_encrypt(encrypt_key, plaintext): + """ + Encrypt the provided plaintext using AES encryption. + + NOTE 1: This function return a string which is fully compatible with Keyczar.Encrypt() method. + + NOTE 2: This function is loosely based on keyczar AESKey.Encrypt() (Apache 2.0 license). + + The final encrypted string value consists of: + + [message bytes][HMAC signature bytes for the message] where message consists of + [keyczar header plaintext][IV bytes][ciphertext bytes] + + NOTE: Header itself is unused, but it's added so the format is compatible with keyczar format. + + """ + assert isinstance(encrypt_key, AESKey), 'encrypt_key needs to be AESKey class instance' + assert isinstance(plaintext, (six.text_type, six.string_types, six.binary_type)), \ + 'plaintext needs to either be a string/unicode or bytes' + + aes_key_bytes = encrypt_key.aes_key_bytes + hmac_key_bytes = encrypt_key.hmac_key_bytes + + assert isinstance(aes_key_bytes, six.binary_type) + assert isinstance(hmac_key_bytes, six.binary_type) + + # Pad data + data = pkcs5_pad(plaintext) + + # Generate IV + iv_bytes = os.urandom(KEYCZAR_AES_BLOCK_SIZE) + + backend = default_backend() + cipher = Cipher(algorithms.AES(aes_key_bytes), modes.CBC(iv_bytes), backend=backend) + encryptor = cipher.encryptor() + + # NOTE: We don't care about actual Keyczar header value, we only care about the length (5 + # bytes) so we simply add 5 0's + header_bytes = b'00000' + + if isinstance(data, (six.text_type, six.string_types)): + # Convert data to bytes + data = data.encode('utf-8') + + ciphertext_bytes = encryptor.update(data) + encryptor.finalize() + msg_bytes = header_bytes + iv_bytes + ciphertext_bytes + + # Generate HMAC signature for the message (header + IV + ciphertext) + h = hmac.HMAC(hmac_key_bytes, hashes.SHA1(), backend=backend) + h.update(msg_bytes) + sig_bytes = h.finalize() + + result = msg_bytes + sig_bytes + + # Convert resulting byte string to hex notation ASCII string + result = binascii.hexlify(result).upper() + + return result + + +def cryptography_symmetric_decrypt(decrypt_key, ciphertext): + """ + Decrypt the provided ciphertext which has been encrypted using symmetric_encrypt() method (it + assumes input is in hex notation as returned by binascii.hexlify). + + NOTE 1: This function assumes ciphertext has been encrypted using symmetric AES crypto from + keyczar library. Underneath it uses crypto primitives from cryptography library which is Python + 3 compatible. + + NOTE 2: This function is loosely based on keyczar AESKey.Decrypt() (Apache 2.0 license). + """ + assert isinstance(decrypt_key, AESKey), 'decrypt_key needs to be AESKey class instance' + assert isinstance(ciphertext, (six.text_type, six.string_types, six.binary_type)), \ + 'ciphertext needs to either be a string/unicode or bytes' + + aes_key_bytes = decrypt_key.aes_key_bytes + hmac_key_bytes = decrypt_key.hmac_key_bytes + + assert isinstance(aes_key_bytes, six.binary_type) + assert isinstance(hmac_key_bytes, six.binary_type) + + # Convert from hex notation ASCII string to bytes + ciphertext = binascii.unhexlify(ciphertext) + + data_bytes = ciphertext[KEYCZAR_HEADER_SIZE:] # remove header + + # Verify ciphertext contains IV + HMAC signature + if len(data_bytes) < (KEYCZAR_AES_BLOCK_SIZE + KEYCZAR_HLEN): + raise ValueError('Invalid or malformed ciphertext (too short)') + + iv_bytes = data_bytes[:KEYCZAR_AES_BLOCK_SIZE] # first block is IV + ciphertext_bytes = data_bytes[KEYCZAR_AES_BLOCK_SIZE:-KEYCZAR_HLEN] # strip IV and signature + signature_bytes = data_bytes[-KEYCZAR_HLEN:] # last 20 bytes are signature + + # Verify HMAC signature + backend = default_backend() + h = hmac.HMAC(hmac_key_bytes, hashes.SHA1(), backend=backend) + h.update(ciphertext[:-KEYCZAR_HLEN]) + h.verify(signature_bytes) + + # Decrypt ciphertext + cipher = Cipher(algorithms.AES(aes_key_bytes), modes.CBC(iv_bytes), backend=backend) + + decryptor = cipher.decryptor() + decrypted = decryptor.update(ciphertext_bytes) + decryptor.finalize() + + # Unpad + decrypted = pkcs5_unpad(decrypted) + return decrypted + +### +# NOTE: Those methods below are deprecated and only used for testing purposes +## + + +def keyczar_symmetric_encrypt(encrypt_key, plaintext): """ Encrypt the given message using the encrypt_key. Returns a UTF-8 str ready to be stored in database. Note that we convert the hex notation @@ -57,28 +316,128 @@ def symmetric_encrypt(encrypt_key, plaintext): 'Initialization Vector' per run and the IV is part of the output. :param encrypt_key: Symmetric AES key to use for encryption. - :type encrypt_key: :class:`keyczar.keys.AesKey` + :type encrypt_key: :class:`AESKey` :param plaintext: Plaintext / message to be encrypted. :type plaintext: ``str`` :rtype: ``str`` """ + from keyczar.keys import AesKey as KeyczarAesKey + from keyczar.keys import HmacKey as KeyczarHmacKey + from keyczar.keyinfo import GetMode + + encrypt_key = KeyczarAesKey(encrypt_key.aes_key_string, + KeyczarHmacKey(encrypt_key.hmac_key_string, + encrypt_key.hmac_key_size), + encrypt_key.size, + GetMode(encrypt_key.mode)) + return binascii.hexlify(encrypt_key.Encrypt(plaintext)).upper() -def symmetric_decrypt(decrypt_key, ciphertext): +def keyczar_symmetric_decrypt(decrypt_key, ciphertext): """ Decrypt the given crypto text into plain text. Returns the original string input. Note that we first convert the string to hex notation and then decrypt. This is reverse of the encrypt operation. :param decrypt_key: Symmetric AES key to use for decryption. - :type decrypt_key: :class:`keyczar.keys.AesKey` + :type decrypt_key: :class:`keyczar.keys.AESKey` :param crypto: Crypto text to be decrypted. :type crypto: ``str`` :rtype: ``str`` """ + from keyczar.keys import AesKey as KeyczarAesKey + from keyczar.keys import HmacKey as KeyczarHmacKey + from keyczar.keyinfo import GetMode + + decrypt_key = KeyczarAesKey(decrypt_key.aes_key_string, + KeyczarHmacKey(decrypt_key.hmac_key_string, + decrypt_key.hmac_key_size), + decrypt_key.size, + GetMode(decrypt_key.mode)) + return decrypt_key.Decrypt(binascii.unhexlify(ciphertext)) + + +def pkcs5_pad(data): + """ + Pad data using PKCS5 + """ + pad = KEYCZAR_AES_BLOCK_SIZE - len(data) % KEYCZAR_AES_BLOCK_SIZE + data = data + pad * chr(pad) + return data + + +def pkcs5_unpad(data): + """ + Unpad data padded using PKCS5. + """ + if isinstance(data, six.binary_type): + # Make sure we are operating with a string type + data = data.decode('utf-8') + + pad = ord(data[-1]) + data = data[:-pad] + return data + + +def Base64WSEncode(s): + """ + Return Base64 web safe encoding of s. Suppress padding characters (=). + + Uses URL-safe alphabet: - replaces +, _ replaces /. Will convert s of type + unicode to string type first. + + @param s: string to encode as Base64 + @type s: string + + @return: Base64 representation of s. + @rtype: string + + NOTE: Taken from keyczar (Apache 2.0 license) + """ + if isinstance(s, six.text_type): + # Make sure input string is always converted to bytes (if not already) + s = s.encode('utf-8') + + return base64.urlsafe_b64encode(s).decode('utf-8').replace("=", "") + + +def Base64WSDecode(s): + """ + Return decoded version of given Base64 string. Ignore whitespace. + + Uses URL-safe alphabet: - replaces +, _ replaces /. Will convert s of type + unicode to string type first. + + @param s: Base64 string to decode + @type s: string + + @return: original string that was encoded as Base64 + @rtype: string + + @raise Base64DecodingError: If length of string (ignoring whitespace) is one + more than a multiple of four. + + NOTE: Taken from keyczar (Apache 2.0 license) + """ + s = ''.join(s.splitlines()) + s = str(s.replace(" ", "")) # kill whitespace, make string (not unicode) + d = len(s) % 4 + + if d == 1: + raise ValueError('Base64 decoding errors') + elif d == 2: + s += "==" + elif d == 3: + s += "=" + + try: + return base64.urlsafe_b64decode(s) + except TypeError as e: + # Decoding raises TypeError if s contains invalid characters. + raise ValueError('Base64 decoding error: %s' % (str(e))) diff --git a/st2common/tests/unit/test_crypto_utils.py b/st2common/tests/unit/test_crypto_utils.py index b623d59130..54ed22d9b3 100644 --- a/st2common/tests/unit/test_crypto_utils.py +++ b/st2common/tests/unit/test_crypto_utils.py @@ -14,12 +14,36 @@ # limitations under the License. from __future__ import absolute_import -from keyczar.keys import AesKey -from unittest2 import TestCase +import os + +import six +import json +import binascii -import st2common.util.crypto as crypto_utils +import unittest2 +from unittest2 import TestCase from six.moves import range +from cryptography.exceptions import InvalidSignature + +from st2common.util.crypto import KEYCZAR_HEADER_SIZE +from st2common.util.crypto import AESKey +from st2common.util.crypto import read_crypto_key +from st2common.util.crypto import symmetric_encrypt +from st2common.util.crypto import symmetric_decrypt +from st2common.util.crypto import keyczar_symmetric_decrypt +from st2common.util.crypto import keyczar_symmetric_encrypt +from st2common.util.crypto import cryptography_symmetric_encrypt +from st2common.util.crypto import cryptography_symmetric_decrypt + +from st2tests.fixturesloader import get_fixtures_base_path + +__all__ = [ + 'CryptoUtilsTestCase', + 'CryptoUtilsKeyczarCompatibilityTestCase' +] + +KEY_FIXTURES_PATH = os.path.join(get_fixtures_base_path(), 'keyczar_keys/') class CryptoUtilsTestCase(TestCase): @@ -27,12 +51,12 @@ class CryptoUtilsTestCase(TestCase): @classmethod def setUpClass(cls): super(CryptoUtilsTestCase, cls).setUpClass() - CryptoUtilsTestCase.test_crypto_key = AesKey.Generate() + CryptoUtilsTestCase.test_crypto_key = AESKey.generate() def test_symmetric_encrypt_decrypt(self): original = 'secret' - crypto = crypto_utils.symmetric_encrypt(CryptoUtilsTestCase.test_crypto_key, original) - plain = crypto_utils.symmetric_decrypt(CryptoUtilsTestCase.test_crypto_key, crypto) + crypto = symmetric_encrypt(CryptoUtilsTestCase.test_crypto_key, original) + plain = symmetric_decrypt(CryptoUtilsTestCase.test_crypto_key, crypto) self.assertEqual(plain, original) def test_encrypt_output_is_diff_due_to_diff_IV(self): @@ -40,7 +64,188 @@ def test_encrypt_output_is_diff_due_to_diff_IV(self): cryptos = set() for _ in range(0, 10000): - crypto = crypto_utils.symmetric_encrypt(CryptoUtilsTestCase.test_crypto_key, - original) + crypto = symmetric_encrypt(CryptoUtilsTestCase.test_crypto_key, original) self.assertTrue(crypto not in cryptos) cryptos.add(crypto) + + def test_decrypt_ciphertext_is_too_short(self): + aes_key = AESKey.generate() + plaintext = 'hello world ponies 1' + encrypted = cryptography_symmetric_encrypt(aes_key, plaintext) + + # Verify original non manipulated value can be decrypted + decrypted = cryptography_symmetric_decrypt(aes_key, encrypted) + self.assertEqual(decrypted, plaintext) + + # Corrupt / shortern the encrypted data + encrypted_malformed = binascii.unhexlify(encrypted) + header = encrypted_malformed[:KEYCZAR_HEADER_SIZE] + encrypted_malformed = encrypted_malformed[KEYCZAR_HEADER_SIZE:] + + # Remove 40 bytes from ciphertext bytes + encrypted_malformed = encrypted_malformed[40:] + + # Add back header + encrypted_malformed = header + encrypted_malformed + encrypted_malformed = binascii.hexlify(encrypted_malformed) + + # Verify corrupted value results in an excpetion + expected_msg = 'Invalid or malformed ciphertext' + self.assertRaisesRegexp(ValueError, expected_msg, cryptography_symmetric_decrypt, + aes_key, encrypted_malformed) + + def test_exception_is_thrown_on_invalid_hmac_signature(self): + aes_key = AESKey.generate() + plaintext = 'hello world ponies 2' + encrypted = cryptography_symmetric_encrypt(aes_key, plaintext) + + # Verify original non manipulated value can be decrypted + decrypted = cryptography_symmetric_decrypt(aes_key, encrypted) + self.assertEqual(decrypted, plaintext) + + # Corrupt the HMAC signature (last part is the HMAC signature) + encrypted_malformed = binascii.unhexlify(encrypted) + encrypted_malformed = encrypted_malformed[:-3] + encrypted_malformed += b'abc' + encrypted_malformed = binascii.hexlify(encrypted_malformed) + + # Verify corrupted value results in an excpetion + expected_msg = 'Signature did not match digest' + self.assertRaisesRegexp(InvalidSignature, expected_msg, cryptography_symmetric_decrypt, + aes_key, encrypted_malformed) + + +class CryptoUtilsKeyczarCompatibilityTestCase(TestCase): + """ + Tests which verify that new cryptography based symmetric_encrypt and symmetric_decrypt are + fully compatible with keyczar output format and also return keyczar based format. + """ + + def test_aes_key_class(self): + # 1. Unsupported mode + expected_msg = 'Unsupported mode: EBC' + self.assertRaisesRegexp(ValueError, expected_msg, AESKey, aes_key_string='a', + hmac_key_string='b', hmac_key_size=128, mode='EBC') + + # 2. AES key is too small + expected_msg = 'Unsafe key size: 64' + self.assertRaisesRegexp(ValueError, expected_msg, AESKey, aes_key_string='a', + hmac_key_string='b', hmac_key_size=128, mode='CBC', size=64) + + def test_loading_keys_from_keyczar_formatted_key_files(self): + key_path = os.path.join(KEY_FIXTURES_PATH, 'one.json') + aes_key = read_crypto_key(key_path=key_path) + + self.assertEqual(aes_key.hmac_key_string, 'lgI9YdOKlIOtPQFdgB0B6zr0AZ6L2QJuFQg4gTu2dxc') + self.assertEqual(aes_key.hmac_key_size, 256) + + self.assertEqual(aes_key.aes_key_string, 'vKmBE2YeQ9ATyovel7NDjdnbvOMcoU5uPtUVxWxWm58') + self.assertEqual(aes_key.mode, 'CBC') + self.assertEqual(aes_key.size, 256) + + key_path = os.path.join(KEY_FIXTURES_PATH, 'two.json') + aes_key = read_crypto_key(key_path=key_path) + + self.assertEqual(aes_key.hmac_key_string, '92ok9S5extxphADmUhObPSD5wugey8eTffoJ2CEg_2s') + self.assertEqual(aes_key.hmac_key_size, 256) + + self.assertEqual(aes_key.aes_key_string, 'fU9hT9pm-b9hu3VyQACLXe2Z7xnaJMZrXiTltyLUzgs') + self.assertEqual(aes_key.mode, 'CBC') + self.assertEqual(aes_key.size, 256) + + key_path = os.path.join(KEY_FIXTURES_PATH, 'five.json') + aes_key = read_crypto_key(key_path=key_path) + + self.assertEqual(aes_key.hmac_key_string, 'GCX2uMfOzp1JXYgqH8piEE4_mJOPXydH_fRHPDw9bkM') + self.assertEqual(aes_key.hmac_key_size, 256) + + self.assertEqual(aes_key.aes_key_string, 'EeBcUcbH14tL0w_fF5siEw') + self.assertEqual(aes_key.mode, 'CBC') + self.assertEqual(aes_key.size, 128) + + def test_key_generation_file_format_is_fully_keyczar_compatible(self): + # Verify that the code can read and correctly parse keyczar formatted key files + aes_key = AESKey.generate() + key_json = aes_key.to_json() + json_parsed = json.loads(key_json) + + expected = { + 'hmacKey': { + 'hmacKeyString': aes_key.hmac_key_string, + 'size': aes_key.hmac_key_size + }, + 'aesKeyString': aes_key.aes_key_string, + 'mode': aes_key.mode, + 'size': aes_key.size + } + + self.assertEqual(json_parsed, expected) + + def test_symmetric_encrypt_decrypt_cryptography(self): + key = AESKey.generate() + plaintexts = [ + 'a b c', + 'ab', + 'hello foo', + 'hell', + 'bar5' + 'hello hello bar bar hello', + 'a', + '', + 'c' + ] + + for plaintext in plaintexts: + encrypted = cryptography_symmetric_encrypt(key, plaintext) + decrypted = cryptography_symmetric_decrypt(key, encrypted) + + self.assertEqual(decrypted, plaintext) + + @unittest2.skipIf(six.PY3, 'keyczar doesn\'t work under Python 3') + def test_symmetric_encrypt_decrypt_roundtrips_1(self): + encrypt_keys = [ + AESKey.generate(), + AESKey.generate(), + AESKey.generate(), + AESKey.generate() + ] + + # Verify all keys are unique + aes_key_strings = set() + hmac_key_strings = set() + + for key in encrypt_keys: + aes_key_strings.add(key.aes_key_string) + hmac_key_strings.add(key.hmac_key_string) + + self.assertEqual(len(aes_key_strings), 4) + self.assertEqual(len(hmac_key_strings), 4) + + plaintext = 'hello world test dummy 8 9 5 1 bar2' + + # Verify that round trips work and that cryptography based primitives are fully compatible + # with keyczar format + + count = 0 + for key in encrypt_keys: + data_enc_keyczar = keyczar_symmetric_encrypt(key, plaintext) + data_enc_cryptography = cryptography_symmetric_encrypt(key, plaintext) + + self.assertNotEqual(data_enc_keyczar, data_enc_cryptography) + + data_dec_keyczar_keyczar = keyczar_symmetric_decrypt(key, data_enc_keyczar) + data_dec_keyczar_cryptography = keyczar_symmetric_decrypt(key, data_enc_cryptography) + + self.assertEqual(data_dec_keyczar_keyczar, plaintext) + self.assertEqual(data_dec_keyczar_cryptography, plaintext) + + data_dec_cryptography_cryptography = cryptography_symmetric_decrypt(key, + data_enc_cryptography) + data_dec_cryptography_keyczar = cryptography_symmetric_decrypt(key, data_enc_keyczar) + + self.assertEqual(data_dec_cryptography_cryptography, plaintext) + self.assertEqual(data_dec_cryptography_keyczar, plaintext) + + count += 1 + + self.assertEqual(count, 4) diff --git a/st2tests/st2tests/fixtures/keyczar_keys/five.json b/st2tests/st2tests/fixtures/keyczar_keys/five.json new file mode 100644 index 0000000000..78312ed22c --- /dev/null +++ b/st2tests/st2tests/fixtures/keyczar_keys/five.json @@ -0,0 +1 @@ +{"hmacKey": {"hmacKeyString": "GCX2uMfOzp1JXYgqH8piEE4_mJOPXydH_fRHPDw9bkM", "size": 256}, "aesKeyString": "EeBcUcbH14tL0w_fF5siEw", "mode": "CBC", "size": 128} \ No newline at end of file diff --git a/st2tests/st2tests/fixtures/keyczar_keys/four.json b/st2tests/st2tests/fixtures/keyczar_keys/four.json new file mode 100644 index 0000000000..d07f20c429 --- /dev/null +++ b/st2tests/st2tests/fixtures/keyczar_keys/four.json @@ -0,0 +1 @@ +{"hmacKey": {"hmacKeyString": "EuQw_LBAiRd8hmr7Vorb-ZVVDMY_XJcRQEo2PzCrLJI", "size": 256}, "size": 256, "aesKeyString": "pzHrshLtPPBvgn7E2aJOnN_Br1YY5tsagMeUy3PhoOU", "mode": "CBC"} \ No newline at end of file diff --git a/st2tests/st2tests/fixtures/keyczar_keys/one.json b/st2tests/st2tests/fixtures/keyczar_keys/one.json new file mode 100644 index 0000000000..eca91e5bfc --- /dev/null +++ b/st2tests/st2tests/fixtures/keyczar_keys/one.json @@ -0,0 +1 @@ +{"hmacKey": {"hmacKeyString": "lgI9YdOKlIOtPQFdgB0B6zr0AZ6L2QJuFQg4gTu2dxc", "size": 256}, "size": 256, "aesKeyString": "vKmBE2YeQ9ATyovel7NDjdnbvOMcoU5uPtUVxWxWm58", "mode": "CBC"} \ No newline at end of file diff --git a/st2tests/st2tests/fixtures/keyczar_keys/three.json b/st2tests/st2tests/fixtures/keyczar_keys/three.json new file mode 100644 index 0000000000..7ec72e797d --- /dev/null +++ b/st2tests/st2tests/fixtures/keyczar_keys/three.json @@ -0,0 +1 @@ +{"hmacKey": {"hmacKeyString": "7139G4CocJhKGH7_M1iyvvuZtJirmgg4hmDaSODf-EA", "size": 256}, "size": 256, "aesKeyString": "oNAIFtoFq9OL-YIHgRqyuu3zVr9PeZsFq5kaW7-LVv0", "mode": "CBC"} \ No newline at end of file diff --git a/st2tests/st2tests/fixtures/keyczar_keys/two.json b/st2tests/st2tests/fixtures/keyczar_keys/two.json new file mode 100644 index 0000000000..336d029a49 --- /dev/null +++ b/st2tests/st2tests/fixtures/keyczar_keys/two.json @@ -0,0 +1 @@ +{"hmacKey": {"hmacKeyString": "92ok9S5extxphADmUhObPSD5wugey8eTffoJ2CEg_2s", "size": 256}, "size": 256, "aesKeyString": "fU9hT9pm-b9hu3VyQACLXe2Z7xnaJMZrXiTltyLUzgs", "mode": "CBC"} \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index 0d063e8205..2cacc80bd4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,3 +25,8 @@ psutil==5.4.5 webtest==2.0.25 rstcheck>=3.3.0,<3.4 pyrabbit +# Since StackStorm v2.8.0 we now use cryptography instead of keyczar, but we still have some tests +# which utilize keyczar and ensure new cryptography code is fully compatible with keyczar code +# (those tests only run under Python 2.7 since keyczar doesn't support Python 3.x). +# See https://github.com/StackStorm/st2/pull/4165 +python-keyczar diff --git a/tox.ini b/tox.ini index 6a3def8e9e..e18256d096 100644 --- a/tox.ini +++ b/tox.ini @@ -39,9 +39,9 @@ commands = nosetests --with-timer --rednose -sv st2client/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/unit/ nosetests --with-timer --rednose -sv st2reactor/tests/integration/ --ignore-files=test_garbage_collector.* - nosetests --with-timer --rednose -sv --ignore-files=test_kvps.* st2api/tests/unit/controllers/v1/ + nosetests --with-timer --rednose -sv st2api/tests/unit/controllers/v1/ nosetests --with-timer --rednose -sv --ignore-files=test_validator_mistral.* st2api/tests/unit/controllers/exp/ - nosetests --with-timer --rednose -sv --ignore-files=test_jinja_render_crypto_filters.* --ignore-files=test_config_loader.* --ignore-files=test_crypto_utils.* --ignore-files=test_db.* --ignore-files=test_logging.* st2common/tests/unit/ + nosetests --with-timer --rednose -sv st2common/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/action_chain_runner/tests/unit/ contrib/runners/action_chain_runner/tests/integration/ nosetests --with-timer --rednose -sv contrib/runners/cloudslang_runner/tests/unit/ nosetests --with-timer --rednose -sv contrib/runners/inquirer_runner/tests/unit/