This commit is contained in:
Umachandar Jayachandran
2019-07-23 18:14:16 -07:00
6 changed files with 571 additions and 33 deletions
@@ -16,9 +16,9 @@ Using this sample Python script, you will deploy a Kubernetes cluster in Azure u
python -m pip install requests
python -m pip install requests --upgrade
```
- Install mssqlctl CLI latest version using . Run the command below using elevated priviledges (sudo or admin cmd window):
- Install latest version of the cluster management tool **azdata** (previously named mssqlctl) using below command. Run the command below using elevated priviledges (sudo or admin cmd window):
```
pip3 install -r https://private-repo.microsoft.com/python/ctp3.1/mssqlctl/requirements.txt
pip3 install -r https://aka.ms/azdata
```
1. Login into your Azure account. Run this command:
```
@@ -1,7 +1,7 @@
#
# Prerequisites:
#
# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), mssqlctl CLI (pip3 install -r https://private-repo.microsoft.com/python/ctp3.1/mssqlctl/requirements.txt)
# Azure CLI (https://docs.microsoft.com/en-us/cli/azure/install-azure-cli), python3 (https://www.python.org/downloads), azdata CLI (pip3 install -r https://aka.ms/azdata)
#
# Run `az login` at least once BEFORE running this script
#
@@ -38,21 +38,24 @@ AKS_NODE_COUNT=input("Provide number of worker nodes for AKS cluster - Press ENT
CLUSTER_NAME=input("Provide name of AKS cluster and SQL big data cluster - Press ENTER for using `sqlbigdata`:").strip() or "sqlbigdata"
#This password will be use for Controller user, Knox user and SQL Server Master SA accounts
#
CONTROLLER_USERNAME=input("Provide username to be used for Controller user - Press ENTER for using `admin`:").strip() or "admin"
PASSWORD = getpass.getpass("Provide password to be used for Controller user, Knox user and SQL Server Master SA accounts - Press ENTER for using `MySQLBigData2019`").strip() or "MySQLBigData2019"
#docker registry details
DOCKER_REGISTRY="private-repo.microsoft.com"
DOCKER_REPOSITORY="mssql-private-preview"
DOCKER_IMAGE_TAG="ctp3.1"
# Docker registry details
# Use this only if you are using a private registry different than mcr. If so, make sure you are also setting the environment variables for DOCKER_USERNAME and DOCKER_PASSWORD
# DOCKER_REGISTRY="<your private registry>"
# DOCKER_REPOSITORY="<your private repository>"
# DOCKER_IMAGE_TAG="<your Docker image tag>"
print ('Setting environment variables')
os.environ['MSSQL_SA_PASSWORD'] = PASSWORD
os.environ['CONTROLLER_USERNAME'] = CONTROLLER_USERNAME
os.environ['CONTROLLER_PASSWORD'] = PASSWORD
os.environ['KNOX_PASSWORD'] = PASSWORD
os.environ['DOCKER_USERNAME']=DOCKER_USERNAME
os.environ['DOCKER_PASSWORD']=DOCKER_PASSWORD
# Use this only if you are using a private registry different than mcr. If so, you must set the environment variables for DOCKER_USERNAME and DOCKER_PASSWORD
# os.environ['DOCKER_USERNAME']=DOCKER_USERNAME
# os.environ['DOCKER_PASSWORD']=DOCKER_PASSWORD
os.environ['ACCEPT_EULA']="Yes"
print ("Set azure context to subcription: "+SUBSCRIPTION_ID)
@@ -71,29 +74,30 @@ command = "az aks get-credentials --overwrite-existing --name "+CLUSTER_NAME+" -
executeCmd (command)
print("Creating SQL Big Data cluster:" +CLUSTER_NAME)
command="mssqlctl bdc config init --source aks-dev-test --target custom --force"
command="azdata bdc config init --source aks-dev-test --target custom --force"
executeCmd (command)
command="mssqlctl bdc config section set -c custom -j ""metadata.name=" + CLUSTER_NAME + ""
command="azdata bdc config section set -c custom -j ""metadata.name=" + CLUSTER_NAME + ""
executeCmd (command)
command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + ""
# Use this only if you are using a private registry different than default Micrososft registry (mcr).
# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.registry=" + DOCKER_REGISTRY + ""
# executeCmd (command)
# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + ""
# executeCmd (command)
# command="azdata bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + ""
# executeCmd (command)
command="azdata bdc create -c custom --accept-eula yes"
executeCmd (command)
command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.repository=" + DOCKER_REPOSITORY + ""
executeCmd (command)
command="mssqlctl bdc config section set -c custom -j ""$.spec.controlPlane.spec.docker.imageTag=" + DOCKER_IMAGE_TAG + ""
executeCmd (command)
command="mssqlctl bdc create -c custom --accept-eula yes"
executeCmd (command)
command="mssqlctl login --cluster-name " + CLUSTER_NAME
command="azdata login --cluster-name " + CLUSTER_NAME
executeCmd (command)
print("")
print("SQL Server big data cluster endpoints: ")
command="mssqlctl bdc endpoint list -o table"
command="azdata bdc endpoint list -o table"
executeCmd(command)
@@ -24,7 +24,7 @@ python push-bdc-images-to-custom-private-repo.py
>```
When prompted, provide your input for:
- Docker registry, repository and credentials to access Microsoft private registry where the images will be pulled from (source)
- Docker registry, repository to access Microsoft registry where the images will be pulled from (source). Press enter to use default mcr registry.
- Docker registry, repository and credentials to access your private registry where the images will be pushed to (target)
## Deploy with from your private repository
@@ -13,10 +13,11 @@ def execute_cmd (cmd):
if (stderr is not None):
raise Exception(stderr)
SOURCE_DOCKER_REGISTRY = input("Provide Docker registry source - press ENTER for using `private-repo.microsoft.com`:") or "private-repo.microsoft.com"
SOURCE_DOCKER_REPOSITORY = input("Provide Docker repository source - press ENTER for using `mssql-private-preview`:") or "mssql-private-preview"
SOURCE_DOCKER_USERNAME = input("Provide Docker username for the source registry:")
SOURCE_DOCKER_PASSWORD=getpass.getpass("Provide Docker password for the source registry:")
SOURCE_DOCKER_REGISTRY = input("Provide Docker registry source - press ENTER for using `mcr.microsoft.com`:") or "mcr.microsoft.com"
SOURCE_DOCKER_REPOSITORY = input("Provide Docker repository source - press ENTER for using `mssql/bdc`:") or "mssql/bdc"
# Use this only if your source is a private Docker registry
# SOURCE_DOCKER_USERNAME = input("Provide Docker username for the source registry:")
# SOURCE_DOCKER_PASSWORD=getpass.getpass("Provide Docker password for the source registry:")
SOURCE_DOCKER_TAG = input("Provide Docker tag for the images at the source: ") or "latest"
TARGET_DOCKER_REGISTRY = input("Provide Docker registry target:")
@@ -43,12 +44,14 @@ images = [ 'mssql-appdeploy-init',
'mssql-service-proxy',
'mssql-app-service-proxy',
'mssql-ssis-app-runtime',
'mssql-monitor-telegraf']
'mssql-monitor-telegraf',
'mssql-security-support']
print("Execute docker login to source registry: " + SOURCE_DOCKER_REGISTRY)
cmd = "docker login " + SOURCE_DOCKER_REGISTRY + " -u " + SOURCE_DOCKER_USERNAME + " -p " + SOURCE_DOCKER_PASSWORD
execute_cmd(cmd)
print("")
# Use this only if your source is a private Docker registry
# print("Execute docker login to source registry: " + SOURCE_DOCKER_REGISTRY)
# cmd = "docker login " + SOURCE_DOCKER_REGISTRY + " -u " + SOURCE_DOCKER_USERNAME + " -p " + SOURCE_DOCKER_PASSWORD
# execute_cmd(cmd)
# print("")
print("Pulling images from source repository: " + SOURCE_DOCKER_REGISTRY + "/" + SOURCE_DOCKER_REPOSITORY)
@@ -0,0 +1,86 @@
# Delegate subnet for Managed Instance deployment
Script that validates and prepares virtual network and subnet for Managed Instance creation to comply with [networking requirements](https://docs.microsoft.com/azure/sql-database/sql-database-managed-instance-vnet-configuration#requirements).
### Contents
[About this sample](#about-this-sample)<br/>
[Before you begin](#before-you-begin)<br/>
[Run this sample](#run-this-sample)<br/>
[Sample details](#sample-details)<br/>
[Disclaimers](#disclaimers)<br/>
[Related links](#related-links)<br/>
<a name=about-this-sample></a>
## About this sample
- **Applies to:** Azure SQL Database
- **Key features:** Managed Instance
- **Workload:** n/a
- **Programming Language:** PowerShell
- **Authors:** Srdan Bozovic
- **Update history:** n/a
<a name=before-you-begin></a>
## Before you begin
To run this sample, you need the following prerequisites.
**Software prerequisites:**
1. PowerShell 5.1 or PowerShell Core 6.0
2. Azure PowerShell Az module
**Azure prerequisites:**
1. Permission to manage Azure virtual network
<a name=run-this-sample></a>
## Run this sample
Run the script below from either Windows or Azure Cloud Shell
```powershell
$scriptUrlBase = 'https://raw.githubusercontent.com/Microsoft/sql-server-samples/master/samples/manage/azure-sql-db-managed-instance/delegate-subnet'
$parameters = @{
subscriptionId = '<subscriptionId>'
resourceGroupName = '<resourceGroupName>'
virtualNetworkName = '<virtualNetworkName>'
subnetName = '<subnetName>'
}
Invoke-Command -ScriptBlock ([Scriptblock]::Create((iwr ($scriptUrlBase+'/delegateSubnet.ps1?t='+ [DateTime]::Now.Ticks)).Content)) -ArgumentList $parameters
```
<a name=sample-details></a>
## Sample details
This sample shows how to prepare Azure virtual network and subnet for Managed Instance deployment using PowerShell
This is done in three simple steps:
- Validate - Selected virtual netwok and subnet are validated for Managed Instance networking requirements
- Confirm - User is shown a set of changes that need to be made to prepare subnet for Managed Instance deployment and asked for consent
- Prepare - Virtual network and subnet are configured properly
<a name=disclaimers></a>
## Disclaimers
The scripts and this guide are copyright Microsoft Corporations and are provided as samples. They are not part of any Azure service and are not covered by any SLA or other Azure-related agreements. They are provided as-is with no warranties express or implied. Microsoft takes no responsibility for the use of the scripts or the accuracy of this document. Familiarize yourself with the scripts before using them.
<a name=related-links></a>
## Related Links
<!-- Links to more articles. Remember to delete "en-us" from the link path. -->
For more information, see these articles:
- [What is a Managed Instance?](https://docs.microsoft.com/azure/sql-database/sql-database-managed-instance)
- [Configure a VNet for Azure SQL Database Managed Instance](https://docs.microsoft.com/azure/sql-database/sql-database-managed-instance-vnet-configuration)
@@ -0,0 +1,445 @@
$parameters = $args[0]
$subscriptionId = $parameters['subscriptionId']
$resourceGroupName = $parameters['resourceGroupName']
$virtualNetworkName = $parameters['virtualNetworkName']
$subnetName = $parameters['subnetName']
$force = $parameters['force']
$NSnetworkModels = "Microsoft.Azure.Commands.Network.Models"
$NScollections = "System.Collections.Generic"
function VerifyPSVersion {
Write-Host "Verifying PowerShell version."
if ($PSVersionTable.PSEdition -eq "Desktop") {
if (($PSVersionTable.PSVersion.Major -ge 6) -or
(($PSVersionTable.PSVersion.Major -eq 5) -and ($PSVersionTable.PSVersion.Minor -ge 1))) {
Write-Host "PowerShell version verified." -ForegroundColor Green
}
else {
Write-Host "You need to install PowerShell version 5.1 or heigher." -ForegroundColor Red
Break;
}
}
else {
if ($PSVersionTable.PSVersion.Major -ge 6) {
Write-Host "PowerShell version verified." -ForegroundColor Green
}
else {
Write-Host "You need to install PowerShell version 6.0 or heigher." -ForegroundColor Red
Break;
}
}
}
function EnsureAzModule {
Write-Host "Checking if Az module is imported."
$module = Get-Module Az
If ($null -eq $module) {
try {
Import-Module Az -ErrorAction Stop
Write-Host "Module Az imported." -ForegroundColor Green
}
catch {
Install-Module Az -AllowClobber
Write-Host "Module Az installed." -ForegroundColor Green
}
} else {
Write-Host "Module Az imported." -ForegroundColor Green
}
}
function EnsureLogin () {
$context = Get-AzContext
If ($null -eq $context.Subscription) {
Write-Host "Sign-in..."
If ($null -eq (Connect-AzAccount -ErrorAction SilentlyContinue -ErrorVariable Errors)) {
Write-Host ("Sign-in failed: {0}" -f $Errors[0].Exception.Message) -ForegroundColor Red
Break
}
}
Write-Host "Sign-in successful." -ForegroundColor Green
}
function SelectSubscriptionId {
param (
$subscriptionId
)
Write-Host "Selecting subscription '$subscriptionId'."
$context = Get-AzContext
If($context.Subscription.Id -ne $subscriptionId)
{
Try
{
Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop | Out-null
}
Catch
{
Write-Host "Subscription selection failed: $_" -ForegroundColor Red
Break
}
}
Write-Host "Subscription selected." -ForegroundColor Green
}
function LoadVirtualNetwork {
param (
$resourceGroupName,
$virtualNetworkName
)
Write-Host("Loading virtual network '{0}' in resource group '{1}'." -f $virtualNetworkName, $resourceGroupName)
$virtualNetwork = Get-AzVirtualNetwork -ResourceGroupName $resourceGroupName -Name $virtualNetworkName -ErrorAction SilentlyContinue -WarningAction SilentlyContinue
If($null -ne $virtualNetwork.Id)
{
Write-Host "Virtual network loaded." -ForegroundColor Green
return $virtualNetwork
}
else
{
Write-Host "Virtual network not found." -ForegroundColor Red
Break
}
}
function LoadVirtualNetworkSubnet {
param (
$virtualNetwork,
$subnetName
)
Write-Host("Loading subnet '{0}'." -f $subnetName)
$subnets = $virtualNetwork.Subnets.Name
If($true -eq $subnets.Contains($subnetName))
{
$subnetIndex = $subnets.IndexOf($subnetName)
$subnet = $virtualNetwork.Subnets[$subnetIndex]
Write-Host "Subnet loaded." -ForegroundColor Green
return $subnet
}
else
{
Write-Host "Subnet not found." -ForegroundColor Red
Break
}
}
function VerifySubnet {
param (
$subnet
)
Write-Host("Verifying subnet '{0}'." -f $subnet.Name)
If($subnet.AddressPrefix.Split('/')[1] -le 28)
{
Write-Host "Passed Validation - Subnet is of enough size." -ForegroundColor Green
}
Else
{
Write-Host "Failed Validation - Minimum supported subnet size is /28." -ForegroundColor Red
Break
}
If(
($subnet.IpConfigurations.Count -eq 0) -and
(
($subnet.ResourceNavigationLinks.Count -eq 0) -or
($subnet.ResourceNavigationLinks[0].LinkedResourceType -eq 'Microsoft.Sql/virtualClusters')
)
)
{
Write-Host "Passed Validation - There are no conflicting resources inside the subnet." -ForegroundColor Green
}
Else
{
Write-Host "Failed Validation - Subnet is already in use." -ForegroundColor Red
Break
}
}
function VerifyDelegation {
param (
$subnet
)
$result = @{
isDelegatedToManagedInstance = $false;
isDelegated = $false;
success = $false;
}
$delegation = Get-AzDelegation -Subnet $subnet
If($delegation -ne $null)
{
$result['isDelegated'] = $true
$result['isDelegatedToManagedInstance'] = $delegation.ServiceName -eq "Microsoft.Sql/managedInstances"
}
$result['success'] = -not $result['isDelegated'] -or $result['isDelegatedToManagedInstance']
return $result
}
function LoadNetworkSecurityGroup {
param (
$subnet
)
Write-Host("Loading Network security group for subnet '{0}'." -f $subnet.Name)
If(
$null -ne $subnet.NetworkSecurityGroup
)
{
$nsgSegments = ($subnet.NetworkSecurityGroup.Id).Split("/", [System.StringSplitOptions]::RemoveEmptyEntries)
$nsgName = $nsgSegments[-1].Trim()
$nsgResourceGroup = $nsgSegments[3].Trim()
$networkSecurityGroup = Get-AzNetworkSecurityGroup -ResourceGroupName $nsgResourceGroup -Name $nsgName
Write-Host "Network security group security group loaded." -ForegroundColor Green
return $networkSecurityGroup
}
Else
{
return $null
}
}
function HasNSG {
param (
$subnet
)
$nsg = LoadNetworkSecurityGroup $subnet
return $nsg -ne $null
}
function LoadRouteTable {
param (
$subnet
)
Write-Host("Loading Route table for subnet '{0}'." -f $subnet.Name)
If(
$null -ne $subnet.RouteTable
)
{
$rtSegments = ($subnet.RouteTable.Id).Split("/", [System.StringSplitOptions]::RemoveEmptyEntries)
$rtName = $rtSegments[-1].Trim()
$rtResourceGroup = $rtSegments[3].Trim()
$routeTable = Get-AzRouteTable -ResourceGroupName $rtResourceGroup -Name $rtName
Write-Host "Route table loaded." -ForegroundColor Green
return $routeTable
}
return $null
}
function HasRouteTable {
param (
$subnet
)
$routeTable = LoadRouteTable $subnet
return $routeTable -ne $null
}
function CreateNSG
{
param(
$virtualNetwork,
$subnet
)
Write-Host "Creating Network security group."
$networkSecurityGroupName = "nsgManagedInstance" + (Get-Random -Maximum 1000)
$securityRules = New-Object "$NScollections.List``1[$NSnetworkModels.PSSecurityRule]"
$rule = New-AzNetworkSecurityRuleConfig `
-Name prepare-allow_tds_inbound `
-Description "Allow access to data" `
-Direction Inbound -Priority 1000 -Access Allow -Protocol Tcp `
-SourceAddressPrefix VirtualNetwork -DestinationAddressPrefix $subnet.AddressPrefix `
-SourcePortRange * -DestinationPortRange @("1433","11000-11999")
$securityRules.Add($rule)
$rule = New-AzNetworkSecurityRuleConfig `
-Name prepare-deny_all_inbound `
-Description "Deny all other inbound traffic" `
-Direction Inbound -Priority 4096 -Access Deny -Protocol * `
-SourceAddressPrefix * -DestinationAddressPrefix * `
-SourcePortRange * -DestinationPortRange *
$securityRules.Add($rule)
$rule = New-AzNetworkSecurityRuleConfig `
-Name prepare-deny_all_outbound `
-Description "Deny all other outbound traffic" `
-Direction Outbound -Priority 4096 -Access Deny -Protocol * `
-SourceAddressPrefix * -DestinationAddressPrefix * `
-SourcePortRange * -DestinationPortRange *
$securityRules.Add($rule)
Try
{
$networkSecurityGroup = New-AzNetworkSecurityGroup -Name $networkSecurityGroupName -ResourceGroupName $virtualNetwork.ResourceGroupName -Location $virtualNetwork.Location -SecurityRules $securityRules
}
Catch
{
Write-Host "Failed: $_" -ForegroundColor Red
}
Write-Host "Associating Network security group."
$subnet.NetworkSecurityGroup = $networkSecurityGroup
}
function CreateRouteTable
{
param(
$virtualNetwork,
$subnet
)
Write-Host "Creating Route table."
$routeTableName = "rtManagedInstance" + (Get-Random -Maximum 1000)
Try
{
$routeTable = New-AzRouteTable -Name $routeTableName -ResourceGroupName $virtualNetwork.ResourceGroupName -Location $virtualNetwork.Location
}
Catch
{
Write-Host "Failed: $_" -ForegroundColor Red
}
Write-Host "Associating Route table."
$subnet.RouteTable = $routeTable
}
function DelegateSubnet
{
param(
$subnet
)
Write-Host "Creating Subnet Delegation for Managed Instance."
$subnet.Delegations = New-Object "$NScollections.List``1[$NSnetworkModels.PSDelegation]"
$delegationName = "dgManagedInstance" + (Get-Random -Maximum 1000)
Try
{
$delegation = New-AzDelegation -Name $delegationName -ServiceName "Microsoft.Sql/managedInstances"
}
Catch
{
Write-Host "Failed: $_" -ForegroundColor Red
}
Write-Host "Associating Subnet Delegation for Managed Instance."
$subnet.Delegations.Add($delegation)
}
function SetVirtualNetwork
{
param($virtualNetwork)
Write-Host "Applying changes to the virtual network."
Try
{
Set-AzVirtualNetwork -VirtualNetwork $virtualNetwork -ErrorAction Stop -WarningAction SilentlyContinue | Out-Null
}
Catch
{
Write-Host "Failed: $_" -ForegroundColor Red
}
}
VerifyPSVersion
EnsureAzModule
EnsureLogin
SelectSubscriptionId -subscriptionId $subscriptionId
$virtualNetwork = LoadVirtualNetwork -resourceGroupName $resourceGroupName -virtualNetworkName $virtualNetworkName
$subnet = LoadVirtualNetworkSubnet -virtualNetwork $virtualNetwork -subnetName $subnetName
Write-Host
$delegationVerificationResult = VerifyDelegation $subnet
If($delegationVerificationResult['success'])
{
VerifySubnet $subnet
$hasNsg = HasNSG $subnet
$hasRouteTable = HasRouteTable $subnet
$isValid = $delegationVerificationResult['isDelegatedToManagedInstance'] -and $hasNsg -and $hasRouteTable
If($isValid -ne $true)
{
Write-Host
Write-Host("---------- To delegate the virtual network subnet for Managed Instance this script will: --------------- ") -ForegroundColor Yellow
Write-Host
If(-not $hasNsg)
{
Write-Host "Create Network security group and associate it to the subnet." -ForegroundColor Yellow
}
If(-not $hasRouteTable)
{
Write-Host "Create Route table and associate it to the subnet." -ForegroundColor Yellow
}
If(-not $delegationVerificationResult['isDelegatedToManagedInstance'])
{
Write-Host "Delegate subnet to Managed Instance." -ForegroundColor Yellow
}
Write-Host
Write-Host("-------------------------------------------------------------------------------------------------------- ") -ForegroundColor Yellow
Write-Host
$applyChanges = $force
If($applyChanges -ne $true)
{
$reply = Read-Host -Prompt "Do you want to make these changes? [y/n]"
$applyChanges = $reply -match "[yY]"
Write-Host
}
If ($applyChanges)
{
If(-not $hasNsg)
{
CreateNSG $virtualNetwork $subnet
}
If(-not $hasRouteTable)
{
CreateRouteTable $virtualNetwork $subnet
}
If(-not $delegationVerificationResult['isDelegatedToManagedInstance'])
{
DelegateSubnet $subnet
}
SetVirtualNetwork $virtualNetwork
Write-Host
Write-Host "Subnet delegated to the Managed Instance." -ForegroundColor Green
Write-Host "https://portal.azure.com/#create/Microsoft.SQLManagedInstance"
}
Else
{
Write-Host
Write-Host "Subnet delegation canceled." -ForegroundColor Yellow
}
}
Else
{
Write-Host "Subnet is already delegated to the Managed Instance." -ForegroundColor Green
Write-Host "https://portal.azure.com/#create/Microsoft.SQLManagedInstance"
}
}
Else
{
Write-Host
Write-Host "Subnet is already delegated to other service." -ForegroundColor Red
}