From c931a620a1acc3e1f3a58dde36b85d15ccbd5be2 Mon Sep 17 00:00:00 2001 From: Daniel Coelho Date: Fri, 11 Jun 2021 12:10:50 -0700 Subject: [PATCH] External Provider Integration Template App --- .../README.md | 12 +++ .../kms_plugin_app/app.py | 60 +++++++++++++ .../kms_plugin_app/constants.py | 33 +++++++ .../kms_plugin_app/custom.py | 61 +++++++++++++ .../kms_plugin_app/custom2.py | 90 +++++++++++++++++++ .../kms_plugin_app/model/json_objects.py | 31 +++++++ .../kms_plugin_app/requirements.txt | 4 + .../kms_plugin_app/spec.yaml | 15 ++++ .../kms_plugin_app/utils.py | 39 ++++++++ 9 files changed, 345 insertions(+) create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/README.md create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/app.py create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/constants.py create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom.py create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom2.py create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/model/json_objects.py create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/requirements.txt create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/spec.yaml create mode 100644 samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/utils.py diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/README.md b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/README.md new file mode 100644 index 00000000..c739cb9d --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/README.md @@ -0,0 +1,12 @@ +# SQL Server BDC Encryption at Rest + +This folder contains the AppDeploy template application for integration with external key providers (such as HSMs, HashiCorp Vault, etc). + +The folder kms_plugin_app should be downloaded as a zip file for usage in respect of the instructions at [External Key Providers](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-external-provider). + +To learn more about how Key Versions are used on SQL Server Big Data Clusters see the following article: [Key Versions](https://docs.microsoft.com/sql/big-data-cluster/big-data-cluster-key-versions) + +For information on configuring and using the Encryption at Rest feature see the following guides: +* [Encryption at rest concepts and configuration guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-concepts-and-configuration) +* [SQL Server Big Data Clusters HDFS Encryption Zones usage guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-hdfs-encryption-zones) +* [SQL Server Big Data Clusters transparent data encryption (TDE) at rest usage guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-sql-server-tde) diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/app.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/app.py new file mode 100644 index 00000000..72d14dd8 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/app.py @@ -0,0 +1,60 @@ +# This is a script for running the HSM interaction service using PKCS11 +import json +import sys +# Append the current application path to sys path to be able to resolve local modules. +# +sys.path.append('.') +sys.path.append('./model') +from constants import ConfigurationConstants, Operations +import utils +from json_objects import EncryptDecryptRequest +import custom2 + +def handler(operation, payload, pin, key_attributes, version): + """ + Entry point for the application. + """ + if (payload != None and len(payload) > 0): + # The payload is base64 URL encoded and needs to be decoded first + # + json_request_payload = utils.urlsafe_base64decode_as_str(payload) + + json_key_attributes_dict = json.loads(utils.urlsafe_base64decode_as_str(key_attributes)) + + if operation == Operations.OPERATION_ENCRYPT: + encrypt_decrypt_dict = json.loads(json_request_payload) + request = EncryptDecryptRequest(**encrypt_decrypt_dict) + response = wrap_key(request, json_key_attributes_dict, pin, version) + elif operation == Operations.OPERATION_DECRYPT: + encrypt_decrypt_dict = json.loads(json_request_payload) + request = EncryptDecryptRequest(**encrypt_decrypt_dict) + response = unwrap_key(request, json_key_attributes_dict, pin, version) + elif operation == Operations.OPERATION_GET_KEY: + response = get_key(json_key_attributes_dict, pin, version) + else: + # Throw exception on unsupported operation. + # + raise Exception('Unsupported operation ' + operation) + + # The response should be a base64 url encoded JSON expected by the control plane. + # + serialized_json_response = json.dumps(response.__dict__).encode("utf-8") + return utils.urlsafe_b64encode_as_str(serialized_json_response) + +def get_key(json_key_attributes_dict, pin, version): + """ + Call in to the custom key store module to get the key. + """ + return custom2.get_key(json_key_attributes_dict, pin, version) + +def wrap_key(request, json_key_attributes_dict, pin, version): + """ + Call in to the custom key store module to encrypt. + """ + return custom2.encrypt(request, json_key_attributes_dict, pin, version) + +def unwrap_key(request, json_key_attributes_dict, pin, version): + """ + Call in to the custom key store module to decrypt. + """ + return custom2.decrypt(request, json_key_attributes_dict, pin, version) diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/constants.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/constants.py new file mode 100644 index 00000000..c398b2f5 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/constants.py @@ -0,0 +1,33 @@ +# Definition of all the constants +# +class ConfigurationConstants(object): + """ + File where the application configurations are available + """ + CONFIG_HSM_SETTINGS_FILE = "configuration.ini" + + """ + Configuration section name where environment variables + to be loaded before application execution are defined. + """ + CONFIG_SECTION_ENVIRONMENT_VARIABLE = "EnvironmentVariables" + + CONFIG_SECTION_PKCS11_CONFIGURATION = "PKCS11Configuration" + + CONFIG_KEY_PKCS11_MODULE_PATH = "PKCS11_MODULE_PATH" + +class Operations(object): + """ + Operations supported by the application. These are the operations + that the Big Data Cluster control plane will invoke. + """ + OPERATION_ENCRYPT='encrypt' + OPERATION_DECRYPT='decrypt' + OPERATION_GET_KEY='getKey' + +class CryptoConstants(object): + """ + General constants for cryptography + """ + KTY_RSA="RSA" + WRAP_RSA_OAEP="RSA-OAEP" diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom.py new file mode 100644 index 00000000..3751fa16 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom.py @@ -0,0 +1,61 @@ +# Placeholder for adding logic specific to application +# and backend key store. +# +import os +import json +from Crypto.Cipher import PKCS1_OAEP +from Crypto.PublicKey import RSA +from Crypto.Hash import SHA1 +import sys +# Append the current application path to sys path to be able to resolve local modules. +# +sys.path.append('.') +sys.path.append('./model') +from constants import ConfigurationConstants, Operations, CryptoConstants +import utils +from json_objects import EncryptDecryptRequest, JsonWebKeyResponse, EncryptDecryptResponse + + +def decrypt(request, json_key_attributes_dict, pin, version): + """ + This method will be called by the application entry point + for decrypting the payload. + request.value has the plaintext payload + request.alg contains the padding algorithm for encryption. + """ + key_name = json_key_attributes_dict["keyname"] + file_name = '{}.pem'.format(key_name) + # Decode the base64 url to get the bytes. + with open(file_name, 'r') as key_file: + key = RSA.import_key(key_file.read()) + if request.alg == CryptoConstants.WRAP_RSA_OAEP: + cipher_algo = PKCS1_OAEP.new(key, hashAlgo = SHA1) + plain_text = cipher_algo.decrypt(request.value) + response = EncryptDecryptResponse(plain_text) + return response + + +def encrypt(request, json_key_attributes_dict, pin, version): + """ + This method will be called by the application entry point + for encrypting the payload. + request.value has the plaintext payload + request.alg contains the padding algorithm for encryption. + """ + key_name = json_key_attributes_dict["keyname"] + file_name = '{}.pem'.format(key_name) + with open(file_name, 'r') as key_file: + key = RSA.import_key(key_file.read()) + if request.alg == CryptoConstants.WRAP_RSA_OAEP: + cipher_algo = PKCS1_OAEP.new(key) + cipher_text = cipher_algo.encrypt(request.value) + response = EncryptDecryptResponse(cipher_text) + return response + +def get_key(json_key_attributes_dict, pin, version): + key_name = json_key_attributes_dict["keyname"] + file_name = '{}.pem'.format(key_name) + with open(file_name, 'r') as key_file: + key = RSA.import_key(key_file.read()) + jwk = JsonWebKeyResponse(key.n, key.e) + return jwk diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom2.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom2.py new file mode 100644 index 00000000..b1fb47fb --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom2.py @@ -0,0 +1,90 @@ +# Placeholder for adding logic specific to application +# and backend key store. +# +import os +import json +from Crypto.Cipher import PKCS1_OAEP +from Crypto.PublicKey import RSA +from Crypto.Hash import SHA1 +import sys +import hvac + +# Append the current application path to sys path to be able to resolve local modules. +# +sys.path.append('.') +sys.path.append('./model') +from constants import ConfigurationConstants, Operations, CryptoConstants +import utils +from json_objects import EncryptDecryptRequest, JsonWebKeyResponse, EncryptDecryptResponse + +def decrypt(request, json_key_attributes_dict, pin, version): + """ + This method will be called by the application entry point + for decrypting the payload. + request.value has the plaintext payload + request.alg contains the padding algorithm for encryption. + """ + key_path = json_key_attributes_dict["keypath"] + vault_url = json_key_attributes_dict["vaulturl"] + key_name = json_key_attributes_dict["keyname"] + hvac_client = hvac.Client( + url=vault_url, + token=pin + ) + read_response = hvac_client.secrets.kv.read_secret_version(path=key_path) + rsa_key_pem = read_response['data']['data'][key_name] + + key = RSA.import_key(rsa_key_pem) + if request.alg == CryptoConstants.WRAP_RSA_OAEP: + cipher_algo = PKCS1_OAEP.new(key, hashAlgo = SHA1) + plain_text = cipher_algo.decrypt(request.value) + response = EncryptDecryptResponse(plain_text) + return response + + +def encrypt(request, json_key_attributes_dict, pin, version): + """ + This method will be called by the application entry point + for encrypting the payload. + request.value has the plaintext payload + request.alg contains the padding algorithm for encryption. + """ + key_path = json_key_attributes_dict["keypath"] + vault_url = json_key_attributes_dict["vaulturl"] + key_name = json_key_attributes_dict["keyname"] + hvac_client = hvac.Client( + url=vault_url, + token=pin + ) + hvac_client = hvac.Client( + url=vault_url, + token=pin + ) + read_response = hvac_client.secrets.kv.read_secret_version(path=key_path) + rsa_key_pem = read_response['data']['data'][key_name] + + key = RSA.import_key(rsa_key_pem) + if request.alg == CryptoConstants.WRAP_RSA_OAEP: + cipher_algo = PKCS1_OAEP.new(key) + cipher_text = cipher_algo.encrypt(request.value) + response = EncryptDecryptResponse(cipher_text) + return response + +def get_key(json_key_attributes_dict, pin, version): + key_path = json_key_attributes_dict["keypath"] + vault_url = json_key_attributes_dict["vaulturl"] + key_name = json_key_attributes_dict["keyname"] + hvac_client = hvac.Client( + url=vault_url, + token=pin + ) + hvac_client = hvac.Client( + url=vault_url, + token=pin + ) + read_response = hvac_client.secrets.kv.read_secret_version(path=key_path) + rsa_key_pem = read_response['data']['data'][key_name] + + key = RSA.import_key(rsa_key_pem) + jwk = JsonWebKeyResponse(key.n, key.e) + return jwk diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/model/json_objects.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/model/json_objects.py new file mode 100644 index 00000000..a8d22e07 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/model/json_objects.py @@ -0,0 +1,31 @@ +# Contains the JSON objects for the application. +# +import sys +import os +sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) +import utils +from constants import CryptoConstants + +class EncryptDecryptRequest(object): + """ + Represents the encryption and decryption request + """ + def __init__(self, value, alg): + self.value = utils.urlsafe_base64decode(value) + self.alg = alg + +class EncryptDecryptResponse(object): + """ + Represents the encryption and decryption response + """ + def __init__(self, value): + self.value = utils.urlsafe_b64encode_as_str(value) + +class JsonWebKeyResponse(object): + """ + Represents the getKey operation response + """ + def __init__(self, modulus, exponent): + self.n = utils.urlsafe_b64encode_as_str(utils._int_to_bytes(modulus)) + self.e = utils.urlsafe_b64encode_as_str(utils._int_to_bytes(exponent)) + self.kty = CryptoConstants.KTY_RSA diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/requirements.txt b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/requirements.txt new file mode 100644 index 00000000..f12d87ff --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/requirements.txt @@ -0,0 +1,4 @@ +pycrypto==2.6.1 +pycryptodome==3.10.1 +cryptography==3.2.1 +hvac==0.10.11 diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/spec.yaml b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/spec.yaml new file mode 100644 index 00000000..062e5fc7 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/spec.yaml @@ -0,0 +1,15 @@ +name: encryption +version: v1 +runtime: kms-plugin-python +src: ./app.py +entrypoint: handler +replicas: 1 +poolsize: 1 +inputs: + operation: str + payload: str + pin: str + key_attributes: str + version: str +output: + result: str diff --git a/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/utils.py b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/utils.py new file mode 100644 index 00000000..2f586038 --- /dev/null +++ b/samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/utils.py @@ -0,0 +1,39 @@ +# Definition of all the constants +import codecs +import base64 + +def urlsafe_base64decode(value): + """ + urlsafe_b64decode without padding + """ + # Python requires padding even for base64 URL encoded + # strings. Compute the required padding and append. + pad = '=' * (4 - (len(value) % 3)) + return base64.urlsafe_b64decode(value + pad) + +def urlsafe_base64decode_as_str(value): + """ + urlsafe_b64decode without padding. Returns result as UTF-8 encoded string. + """ + return urlsafe_base64decode(value).decode("utf-8") + +def urlsafe_b64encode_as_str(value): + """ + base64 url safe encoding which returns result as UTF-8 encoded string + """ + return base64.urlsafe_b64encode(value).decode("utf-8") + +def _int_to_bytes(i): + """ + Converts the given int to the big-endian bytes + """ + h = hex(i) + if len(h) > 1 and h[0:2] == "0x": + h = h[2:] + + # need to strip L in python 2.x + h = h.strip("L") + + if len(h) % 2: + h = "0" + h + return codecs.decode(h, "hex")