Merge pull request #942 from DaniBunny/bdc-external-key-provider

External Provider Integration Template App

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/README.md

@@ -0,0 +1,12 @@
+# SQL Server BDC Encryption at Rest
+
+This folder contains the AppDeploy template application for integration with external key providers (such as HSMs, HashiCorp Vault, etc).
+
+The folder kms_plugin_app should be downloaded as a zip file for usage in respect of the instructions at [External Key Providers](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-external-provider).
+
+To learn more about how Key Versions are used on SQL Server Big Data Clusters see the following article: [Key Versions](https://docs.microsoft.com/sql/big-data-cluster/big-data-cluster-key-versions)
+
+For information on configuring and using the Encryption at Rest feature see the following guides:
+* [Encryption at rest concepts and configuration guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-concepts-and-configuration)
+* [SQL Server Big Data Clusters HDFS Encryption Zones usage guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-hdfs-encryption-zones)
+* [SQL Server Big Data Clusters transparent data encryption (TDE) at rest usage guide](https://docs.microsoft.com/sql/big-data-cluster/encryption-at-rest-sql-server-tde)

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/app.py

@@ -0,0 +1,60 @@
+# This is a script for running the HSM interaction service using PKCS11
+import json
+import sys
+# Append the current application path to sys path to be able to resolve local modules.
+#
+sys.path.append('.')
+sys.path.append('./model')
+from constants import ConfigurationConstants, Operations
+import utils
+from json_objects import EncryptDecryptRequest
+import custom2
+
+def handler(operation, payload, pin, key_attributes, version):
+    """
+    Entry point for the application.
+    """
+    if (payload != None and len(payload) > 0):
+        # The payload is base64 URL encoded and needs to be decoded first
+        #
+        json_request_payload = utils.urlsafe_base64decode_as_str(payload)
+
+    json_key_attributes_dict = json.loads(utils.urlsafe_base64decode_as_str(key_attributes))
+
+    if operation == Operations.OPERATION_ENCRYPT:
+        encrypt_decrypt_dict = json.loads(json_request_payload)
+        request = EncryptDecryptRequest(**encrypt_decrypt_dict)
+        response = wrap_key(request, json_key_attributes_dict, pin, version)
+    elif operation == Operations.OPERATION_DECRYPT:
+        encrypt_decrypt_dict = json.loads(json_request_payload)
+        request = EncryptDecryptRequest(**encrypt_decrypt_dict)
+        response = unwrap_key(request, json_key_attributes_dict, pin, version)
+    elif operation == Operations.OPERATION_GET_KEY:
+        response = get_key(json_key_attributes_dict, pin, version)
+    else:
+        # Throw exception on unsupported operation.
+        #
+        raise Exception('Unsupported operation ' + operation)
+    
+    # The response should be a base64 url encoded JSON expected by the control plane.
+    #
+    serialized_json_response = json.dumps(response.__dict__).encode("utf-8")
+    return utils.urlsafe_b64encode_as_str(serialized_json_response)
+
+def get_key(json_key_attributes_dict, pin, version):
+    """
+    Call in to the custom key store module to get the key.
+    """
+    return custom2.get_key(json_key_attributes_dict, pin, version)
+
+def wrap_key(request, json_key_attributes_dict, pin, version):
+    """
+    Call in to the custom key store module to encrypt.
+    """
+    return custom2.encrypt(request, json_key_attributes_dict, pin, version)
+
+def unwrap_key(request, json_key_attributes_dict, pin, version):
+    """
+    Call in to the custom key store module to decrypt.
+    """
+    return custom2.decrypt(request, json_key_attributes_dict, pin, version)

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/constants.py

@@ -0,0 +1,33 @@
+# Definition of all the constants
+#
+class ConfigurationConstants(object):
+    """
+    File where the application configurations are available
+    """
+    CONFIG_HSM_SETTINGS_FILE = "configuration.ini"
+
+    """
+    Configuration section name where environment variables 
+    to be loaded before application execution are defined.
+    """
+    CONFIG_SECTION_ENVIRONMENT_VARIABLE = "EnvironmentVariables"
+
+    CONFIG_SECTION_PKCS11_CONFIGURATION = "PKCS11Configuration"
+
+    CONFIG_KEY_PKCS11_MODULE_PATH = "PKCS11_MODULE_PATH"
+
+class Operations(object):
+    """
+    Operations supported by the application. These are the operations 
+    that the Big Data Cluster control plane will invoke.
+    """
+    OPERATION_ENCRYPT='encrypt'
+    OPERATION_DECRYPT='decrypt'
+    OPERATION_GET_KEY='getKey'
+
+class CryptoConstants(object):
+    """
+    General constants for cryptography
+    """
+    KTY_RSA="RSA"
+    WRAP_RSA_OAEP="RSA-OAEP"

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom.py

@@ -0,0 +1,61 @@
+# Placeholder for adding logic specific to application
+# and backend key store.
+#
+import os
+import json
+from Crypto.Cipher import PKCS1_OAEP
+from Crypto.PublicKey import RSA
+from Crypto.Hash import SHA1
+import sys
+# Append the current application path to sys path to be able to resolve local modules.
+#
+sys.path.append('.')
+sys.path.append('./model')
+from constants import ConfigurationConstants, Operations, CryptoConstants
+import utils
+from json_objects import EncryptDecryptRequest, JsonWebKeyResponse, EncryptDecryptResponse
+
+
+def decrypt(request, json_key_attributes_dict, pin, version):
+    """
+    This method will be called by the application entry point
+    for decrypting the payload.
+    request.value has the plaintext payload
+    request.alg contains the padding algorithm for encryption.
+    """
+    key_name = json_key_attributes_dict["keyname"]
+    file_name = '{}.pem'.format(key_name)
+    # Decode the base64 url to get the bytes.
+    with open(file_name, 'r') as key_file:
+        key = RSA.import_key(key_file.read())
+        if request.alg == CryptoConstants.WRAP_RSA_OAEP:
+            cipher_algo = PKCS1_OAEP.new(key, hashAlgo = SHA1)
+            plain_text = cipher_algo.decrypt(request.value)
+            response = EncryptDecryptResponse(plain_text)
+            return response
+
+
+def encrypt(request, json_key_attributes_dict, pin, version):
+    """
+    This method will be called by the application entry point
+    for encrypting the payload.
+    request.value has the plaintext payload
+    request.alg contains the padding algorithm for encryption.
+    """
+    key_name = json_key_attributes_dict["keyname"]
+    file_name = '{}.pem'.format(key_name)
+    with open(file_name, 'r') as key_file:
+        key = RSA.import_key(key_file.read())
+        if request.alg == CryptoConstants.WRAP_RSA_OAEP:
+            cipher_algo = PKCS1_OAEP.new(key)
+            cipher_text = cipher_algo.encrypt(request.value)
+            response = EncryptDecryptResponse(cipher_text)
+            return response
+
+def get_key(json_key_attributes_dict, pin, version):
+    key_name = json_key_attributes_dict["keyname"]
+    file_name = '{}.pem'.format(key_name)
+    with open(file_name, 'r') as key_file:
+        key = RSA.import_key(key_file.read())
+        jwk = JsonWebKeyResponse(key.n, key.e)
+        return jwk

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/custom2.py

@@ -0,0 +1,90 @@
+# Placeholder for adding logic specific to application
+# and backend key store.
+#
+import os
+import json
+from Crypto.Cipher import PKCS1_OAEP
+from Crypto.PublicKey import RSA
+from Crypto.Hash import SHA1
+import sys
+import hvac
+
+# Append the current application path to sys path to be able to resolve local modules.
+#
+sys.path.append('.')
+sys.path.append('./model')
+from constants import ConfigurationConstants, Operations, CryptoConstants
+import utils
+from json_objects import EncryptDecryptRequest, JsonWebKeyResponse, EncryptDecryptResponse
+
+def decrypt(request, json_key_attributes_dict, pin, version):
+    """
+    This method will be called by the application entry point
+    for decrypting the payload.
+    request.value has the plaintext payload
+    request.alg contains the padding algorithm for encryption.
+    """
+    key_path = json_key_attributes_dict["keypath"]
+    vault_url = json_key_attributes_dict["vaulturl"]
+    key_name = json_key_attributes_dict["keyname"]
+    hvac_client = hvac.Client(
+        url=vault_url,
+        token=pin
+    )
+    read_response = hvac_client.secrets.kv.read_secret_version(path=key_path)
+    rsa_key_pem = read_response['data']['data'][key_name]
+
+    key = RSA.import_key(rsa_key_pem)
+    if request.alg == CryptoConstants.WRAP_RSA_OAEP:
+        cipher_algo = PKCS1_OAEP.new(key, hashAlgo = SHA1)
+        plain_text = cipher_algo.decrypt(request.value)
+        response = EncryptDecryptResponse(plain_text)
+        return response
+
+
+def encrypt(request, json_key_attributes_dict, pin, version):
+    """
+    This method will be called by the application entry point
+    for encrypting the payload.
+    request.value has the plaintext payload
+    request.alg contains the padding algorithm for encryption.
+    """
+    key_path = json_key_attributes_dict["keypath"]
+    vault_url = json_key_attributes_dict["vaulturl"]
+    key_name = json_key_attributes_dict["keyname"]
+    hvac_client = hvac.Client(
+        url=vault_url,
+        token=pin
+    )
+    hvac_client = hvac.Client(
+        url=vault_url,
+        token=pin
+    )
+    read_response = hvac_client.secrets.kv.read_secret_version(path=key_path)
+    rsa_key_pem = read_response['data']['data'][key_name]
+
+    key = RSA.import_key(rsa_key_pem)
+    if request.alg == CryptoConstants.WRAP_RSA_OAEP:
+        cipher_algo = PKCS1_OAEP.new(key)
+        cipher_text = cipher_algo.encrypt(request.value)
+        response = EncryptDecryptResponse(cipher_text)
+        return response
+
+def get_key(json_key_attributes_dict, pin, version):
+    key_path = json_key_attributes_dict["keypath"]
+    vault_url = json_key_attributes_dict["vaulturl"]
+    key_name = json_key_attributes_dict["keyname"]
+    hvac_client = hvac.Client(
+        url=vault_url,
+        token=pin
+    )
+    hvac_client = hvac.Client(
+        url=vault_url,
+        token=pin
+    )
+    read_response = hvac_client.secrets.kv.read_secret_version(path=key_path)
+    rsa_key_pem = read_response['data']['data'][key_name]
+
+    key = RSA.import_key(rsa_key_pem)
+    jwk = JsonWebKeyResponse(key.n, key.e)
+    return jwk

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/model/json_objects.py

@@ -0,0 +1,31 @@
+# Contains the JSON objects for the application.
+#
+import sys
+import os
+sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
+import utils
+from constants import CryptoConstants
+
+class EncryptDecryptRequest(object):
+    """
+    Represents the encryption and decryption request
+    """
+    def __init__(self, value, alg):
+        self.value = utils.urlsafe_base64decode(value)
+        self.alg = alg
+
+class EncryptDecryptResponse(object):
+    """ 
+    Represents the encryption and decryption response
+    """
+    def __init__(self, value):
+        self.value = utils.urlsafe_b64encode_as_str(value)
+
+class JsonWebKeyResponse(object):
+    """
+    Represents the getKey operation response
+    """
+    def __init__(self, modulus, exponent):
+        self.n = utils.urlsafe_b64encode_as_str(utils._int_to_bytes(modulus))
+        self.e = utils.urlsafe_b64encode_as_str(utils._int_to_bytes(exponent))
+        self.kty = CryptoConstants.KTY_RSA

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/requirements.txt

@@ -0,0 +1,4 @@
+pycrypto==2.6.1
+pycryptodome==3.10.1
+cryptography==3.2.1
+hvac==0.10.11

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/spec.yaml

@@ -0,0 +1,15 @@
+name: encryption
+version: v1
+runtime: kms-plugin-python
+src: ./app.py
+entrypoint: handler
+replicas: 1
+poolsize: 1
+inputs:
+  operation: str
+  payload: str
+  pin: str
+  key_attributes: str
+  version: str
+output:
+  result: str

samples/features/sql-big-data-cluster/security/encryption-at-rest-external-key-provider/kms_plugin_app/utils.py

@@ -0,0 +1,39 @@
+# Definition of all the constants
+import codecs
+import base64
+
+def urlsafe_base64decode(value):
+    """
+    urlsafe_b64decode without padding
+    """
+    # Python requires padding even for base64 URL encoded
+    # strings. Compute the required padding and append.
+    pad = '=' * (4 - (len(value) % 3))
+    return base64.urlsafe_b64decode(value + pad)
+
+def urlsafe_base64decode_as_str(value):
+    """
+    urlsafe_b64decode without padding. Returns result as UTF-8 encoded string.
+    """
+    return urlsafe_base64decode(value).decode("utf-8")
+
+def urlsafe_b64encode_as_str(value):
+    """
+    base64 url safe encoding which returns result as UTF-8 encoded string
+    """
+    return base64.urlsafe_b64encode(value).decode("utf-8")
+
+def _int_to_bytes(i):
+    """
+    Converts the given int to the big-endian bytes
+    """
+    h = hex(i)
+    if len(h) > 1 and h[0:2] == "0x":
+        h = h[2:]
+
+    # need to strip L in python 2.x
+    h = h.strip("L")
+
+    if len(h) % 2:
+        h = "0" + h
+    return codecs.decode(h, "hex")