Iteration on AutoExport Commit

This is an interated version that contains changes to AutoExport.ps1 and
changes to README.md responding to psongms's comments.
This commit is contained in:
tgrieger
2016-10-03 11:33:35 -07:00
parent 630619a7ab
commit d88531ecb7
2 changed files with 129 additions and 120 deletions
@@ -17,8 +17,8 @@ Add-Type -TypeDefinition @"
# The database and server pairs that will be exported.
$databaseServerPairs =
@([pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE1"},
[pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE2"},
[pscustomobject]@{serverName="SAMPLESERVER2";databaseName="SAMPLEDATABASE3"});
[pscustomobject]@{serverName="SAMPLESERVER1";databaseName="SAMPLEDATABASE2"},
[pscustomobject]@{serverName="SAMPLESERVER2";databaseName="SAMPLEDATABASE3"});
$serverCred = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL1';
$serverCred2 = Get-AutomationPSCredential -Name 'NAMEOFSERVERCREDENTIAL2';
@@ -31,132 +31,143 @@ $retryLimit = 5;
# The number of minutes you want to wait for an operation to finish before you fail.
$waitInMinutes = 30;
$storageKeyVariableName = "STORAGEKEYVARIABLENAME";
$storageAccountName = "STORAGEACCOUNTNAME";
$automationCertificateName = "CERTIFICATENAME";
$subId = "00000000-0000-0000-0000-000000000000";
$subName = "SUBSCRIPTIONNAME";
function LogMessage($message)
{
$timestamp = Get-Date -format "yyyy-MM-dd_HH:mm.ss";
echo ($timestamp + " " + $message)
}
# This function takes the database and server names and creates a database object to use for the export.
function CreateDatabaseObject($databaseName, $serverName)
{
# Create the new object.
# Create the new object.
$dbObj = New-Object System.Object;
# Add the DatabaseName property and set it.
$dbObj | Add-Member -type NoteProperty -name DatabaseName -Value $databaseName;
# Add a unique time at the end of DatabaseCopyName so that we have a unique database name every time.
# Add the DatabaseName property and set it.
$dbObj | Add-Member -type NoteProperty -name DatabaseName -Value $databaseName;
# Add a unique time at the end of DatabaseCopyName so that we have a unique database name every time.
$currentTime = Get-Date -format "_yyyy-MM-dd_HH:mm.ss";
$dbCopyName = $databaseName + $currentTime;
# Add the DatabaseCopyName property and set it.
# Add the DatabaseCopyName property and set it.
$dbObj | Add-Member -type NoteProperty -name DatabaseCopyName -Value $dbCopyName;
# Add the ServerName property and set it.
$dbObj | Add-Member -type NoteProperty -name ServerName -Value $serverName;
# Add the Export property and set it to $null for now. This will be used to look up the export after it has been started.
$dbObj | Add-Member -type NoteProperty -name Export -Value $null;
# Add the DatabaseState property and set it to ToCopy so that the "state machine" knows to start the copy of the database.
# Add the ServerName property and set it.
$dbObj | Add-Member -type NoteProperty -name ServerName -Value $serverName;
# Add the Export property and set it to $null for now. This will be used to look up the export after it has been started.
$dbObj | Add-Member -type NoteProperty -name Export -Value $null;
# Add the DatabaseState property and set it to ToCopy so that the "state machine" knows to start the copy of the database.
$dbObj | Add-Member -type NoteProperty -name DatabaseState -Value ([DatabaseState]::ToCopy);
# Add the RetryCount property and set it to 0. This will be used to count the number of time we retry each failable operation.
$dbObj | Add-Member -type NoteProperty -name RetryCount -Value 0;
# Add the OperationStartTime property and set it to $null for now. This will be used when an operation starts to correcly do timeouts.
$dbObj | Add-Member -type NoteProperty -name OperationStartTime -Value $null;
# Add the RetryCount property and set it to 0. This will be used to count the number of time we retry each failable operation.
$dbObj | Add-Member -type NoteProperty -name RetryCount -Value 0;
# Add the OperationStartTime property and set it to $null for now. This will be used when an operation starts to correcly do timeouts.
$dbObj | Add-Member -type NoteProperty -name OperationStartTime -Value $null;
# Return the newly created object.
return $dbObj;
# Return the newly created object.
return $dbObj;
}
# This function starts the copy of the database. If there is an error, we set the state to ToDrop. Otherwise, we set the state to Copying.
function StartCopy($dbObj)
{
# Start the copy of the database.
# Start the copy of the database.
Start-AzureSqlDatabaseCopy -ServerName $dbObj.ServerName -DatabaseName $dbObj.DatabaseName -PartnerDatabase $dbObj.DatabaseCopyName;
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
if(-not $? -and $global:retryLimit -ile $dbObj.RetryCount)
{
echo ("Error occurred while starting copy of " + $dbObj.DatabaseName + ". It will not be copied. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
LogMessage ("Error occurred while starting copy of " + $dbObj.DatabaseName + ". It will not be copied. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
# Set state to ToDrop in case something does get copied.
$dbsCopying[$i].DatabaseState = ([DatabaseState]::ToDrop);
# Return so we don't execute the rest of the function.
# Return so we don't execute the rest of the function.
return;
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
LogMessage ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
echo ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
# Set the state of the database object to Copying.
$dbObj.DatabaseState = ([DatabaseState]::Copying);
echo ("Copying " + $dbObj.DatabaseName);
$dbObj.OperationStartTime = Get-Date;
LogMessage ("Copying " + $dbObj.DatabaseName + " to " + $dbObj.DatabaseCopyName);
$dbObj.OperationStartTime = Get-Date;
}
# This function checks the progress of the copy. If there is an error, we set the state to ToDrop. Otherwise, we set the state to ToExport.
function CheckCopy($dbObj)
{
# Get the status of the database copy.
$check = Get-AzureSqlDatabaseCopy -ServerName $dbObj.ServerName -DatabaseName $dbObj.DatabaseName;
$currentTime = Get-Date;
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
$check = Get-AzureSqlDatabaseCopy -ServerName $dbObj.ServerName -DatabaseName $dbObj.DatabaseName -PartnerDatabase $dbObj.DatabaseCopyName;
$currentTime = Get-Date;
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
if((-not $? -and $global:retryLimit -ile $dbObj.RetryCount) -or ($currentTime - $dbObj.OperationStartTime).TotalMinutes -gt $global:waitInMinutes)
{
echo ("Error occurred during copy of " + $dbObj.DatabaseName + ". It will not be exported. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
LogMessage ("Error occurred during copy of " + $dbObj.DatabaseName + ". It will not be exported. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
# Set state to ToDrop in case something did get copied.
$dbsCopying[$i].DatabaseState = ([DatabaseState]::ToDrop);
# Return so we don't execute the rest of the function.
# Return so we don't execute the rest of the function.
return;
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
echo ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
# Get the percent complete from the status to check if the database copy is done.
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
LogMessage ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
# Get the percent complete from the status to check if the database copy is done.
$i = $check.PercentComplete
# $i will be $null when the copy is complete.
if($i -eq $null)
{
# The copy is complete so set the state to ToExport.
# The copy is complete so set the state to ToExport.
$dbObj.DatabaseState = ([DatabaseState]::ToExport);
$dbObj.RetryCount = 0;
$dbObj.RetryCount = 0;
}
}
# This function starts the export. If there is an error, we se tthe state o ToDrop. Otherwise, we set the state to Exporting.
# This function starts the export. If there is an error, we set the state to ToDrop. Otherwise, we set the state to Exporting.
function StartExport($dbObj)
{
# Setup the server connection that the storage account is on.
$serverManageUrl = "https://autoexportserver.database.windows.net";
# Get the current time to use as a unique identifier for the blob name.
# Get the current time to use as a unique identifier for the blob name.
$currentTime = Get-Date -format "_yyyy-MM-dd_HH:mm.ss";
$blobName = $dbObj.DatabaseName + "_ExportBlob" + $currentTime;
# Use the stored credential to create a server credential to use to login to the server.
# Use the stored credential to create a server credential to use to login to the server.
$servercredential = $global:serverCredentialsDictionary[$dbObj.ServerName];
# Set up a SQL connection context to use when exporting.
# Set up a SQL connection context to use when exporting.
$ctx = New-AzureSqlDatabaseServerContext -ServerName $dbObj.ServerName -Credential $servercredential;
# Get the storage key to setup the storage context.
$storageKey = Get-AutomationVariable -Name "STORAGEKEYVARIABLENAME";
# Get the storage context.
$stgctx = New-AzureStorageContext -StorageAccountName "STORAGEACCOUNTNAME" -StorageAccountKey $storageKey;
# Start the export. If there is an error, stop the export and set the state to ToDrop.
# Get the storage key to setup the storage context.
$storageKey = Get-AutomationVariable -Name $global:storageKeyVariableName;
# Get the storage context.
$stgctx = New-AzureStorageContext -StorageAccountName $global:storageAccountName -StorageAccountKey $storageKey;
# Start the export. If there is an error, stop the export and set the state to ToDrop.
$dbObj.Export = Start-AzureSqlDatabaseExport -SqlConnectionContext $ctx -StorageContext $stgctx -StorageContainerName autoexportcontainer -DatabaseName $dbObj.DatabaseCopyName -BlobName $blobName;
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
# $? is true if the last command succeeded and false if the last command failed. If it is false, go to the ToDrop state.
if (-not $? -and $global:retryLimit -ile $dbObj.RetryCount)
{
echo ("Error occurred while starting export of " + $dbObj.DatabaseName + ". It will not be exported. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
LogMessage ("Error occurred while starting export of " + $dbObj.DatabaseName + ". It will not be exported. Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
# Set state to ToDrop so that we drop the copied database since there was an error exporting it.
$dbsToExport[$i].DatabaseState = ([DatabaseState]::ToDrop);
# Return so we don't execute the rest of the function.
# Return so we don't execute the rest of the function.
return
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
echo ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
LogMessage ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
# Set the state to Exporting.
$dbObj.DatabaseState = ([DatabaseState]::Exporting);
echo ("Exporting " + $dbObj.DatabaseCopyName);
$dbObj.OperationStartTime = Get-Date;
LogMessage ("Exporting " + $dbObj.DatabaseCopyName);
$dbObj.OperationStartTime = Get-Date;
}
# This function monitors the export progress.
@@ -164,36 +175,36 @@ function CheckExport($dbObj)
{
# Get the progress of the database's export.
$check = Get-AzureSqlDatabaseImportExportStatus -Request $dbObj.Export;
$currentTime = Get-Date;
$currentTime = Get-Date;
# The export is complete when Status is "Completed". Wait for that to happen.
if($check.Status -eq "Completed")
{
# The export id one, set the state to ToDrop because it was successful.
# The export id one, set the state to ToDrop because it was successful.
$dbObj.DatabaseState = ([DatabaseState]::ToDrop);
$dbObj.RetryCount = 0;
$dbObj.RetryCount = 0;
}
elseif($check.Status -eq "Failed" -and $dbObj.RetryCount -lt $global:retryLimit)
{
# If the status is "Failed" and we have more retries left, try to export the database copy again.
echo ("The last export failed on database " + $dbObj.DatabaseName + ", going back to ToExport state to try again");
echo $check
$dbObj.DatabaseState = ([DatabaseState]::ToExport);
$dbObj.RetryCount++;
return;
}
elseif($global:retryLimit -ile $dbObj.RetryCount -or ($currentTime - $dbObj.OperationStartTime).TotalMinutes -gt $global:waitInMinutes)
{
echo ("Error occurred while exporting " + $dbObj.DatabaseName + ". Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
# The export id one, set the state to ToDrop either because it failed.
elseif($check.Status -eq "Failed" -and $dbObj.RetryCount -lt $global:retryLimit)
{
# If the status is "Failed" and we have more retries left, try to export the database copy again.
LogMessage ("The last export failed on database " + $dbObj.DatabaseName + ", going back to ToExport state to try again");
LogMessage $check
$dbObj.DatabaseState = ([DatabaseState]::ToExport);
$dbObj.RetryCount++;
return;
}
elseif($global:retryLimit -ile $dbObj.RetryCount -or ($currentTime - $dbObj.OperationStartTime).TotalMinutes -gt $global:waitInMinutes)
{
LogMessage ("Error occurred while exporting " + $dbObj.DatabaseName + ". Deleting the database copy named " + $dbObj.DatabaseCopyName + ".");
# The export id one, set the state to ToDrop either because it failed.
$dbObj.DatabaseState = ([DatabaseState]::ToDrop);
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
echo ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
}
elseif(-not $?)
{
# We failed but we haven't hit the retry limit yet so increment RetryCount and return so we try again.
LogMessageLogMessage ("Retrying with database " + $dbObj.DatabaseName);
$dbObj.RetryCount++;
return;
}
}
# This function runs the command to drop the database and sets the state to Finished.
@@ -203,51 +214,51 @@ function StartDrop($dbObj)
Remove-AzureSqlDatabase -ServerName $dbObj.ServerName -DatabaseName $dbObj.DatabaseCopyName -Force;
# Set the state to Finished so it gets removed from the array.
$dbObj.DatabaseState = ([DatabaseState]::Finished);
echo ($dbObj.DatabaseCopyName + " dropped")
LogMessage ($dbObj.DatabaseCopyName + " dropped")
}
# Runs the "State Machine" so that different databases can progress independently.
function ExportProcess
{
# Get all database objects in the ToCopy state and start the database copy.
# Get all database objects in the ToCopy state and start the database copy.
$dbsToCopy = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToCopy);
for($i = 0; $i -lt $dbsToCopy.Count; $i++)
{
echo $dbsToCopy[$i];
LogMessage $dbsToCopy[$i];
StartCopy($dbsToCopy[$i]);
}
# Get all database objects in the Copying state and check on their copy progress.
# Get all database objects in the Copying state and check on their copy progress.
$dbsCopying = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::Copying);
for($i = 0; $i -lt $dbsCopying.Count; $i++)
{
CheckCopy($dbsCopying[$i]);
}
# Get all database objects in the ToExport state and start their export.
# Get all database objects in the ToExport state and start their export.
$dbsToExport = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToExport);
for($i = 0; $i -lt $dbsToExport.Count; $i++)
{
echo $dbsToExport[$i];
LogMessage $dbsToExport[$i];
StartExport($dbsToExport[$i]);
}
# Get all database objects in the Exporting state and check on their export progress.
# Get all database objects in the Exporting state and check on their export progress.
$dbsExporting = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::Exporting);
for($i = 0; $i -lt $dbsExporting.Count; $i++)
{
CheckExport($dbsExporting[$i]);
}
# Get all database objects in the ToDrop state and start their drop.
# Get all database objects in the ToDrop state and start their drop.
$dbsToDrop = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::ToDrop);
for($i = 0; $i -lt $dbsToDrop.Count; $i++)
{
echo $dbsToDrop[$i];
LogMessage $dbsToDrop[$i];
StartDrop($dbsToDrop[$i]);
}
# Get all database objects in the Finished state and remove them from the array.
# Get all database objects in the Finished state and remove them from the array.
$dbsFinished = $global:dbs | Where-Object DatabaseState -eq ([DatabaseState]::Finished);
for($i = 0; $i -lt $dbsFinished.Count; $i++)
{
@@ -256,25 +267,23 @@ function ExportProcess
}
# Get the certificate to authenticate the subscription
$cert = Get-AutomationCertificate -Name "CERTIFICATENAME";
$cert = Get-AutomationCertificate -Name $global:automationCertificateName;
# Set the subscription to use
$subID = "00000000-0000-0000-0000-000000000000";
$subName = "SUBSCRIPTIONNAME";
Set-AzureSubscription -SubscriptionName $subName -Certificate $cert -SubscriptionId $subID;
Select-AzureSubscription -Current $subName;
Set-AzureSubscription -SubscriptionName $global:subName -Certificate $cert -SubscriptionId $global:subID;
Select-AzureSubscription -Current $global:subName;
$currentIndex = 0;
for($currentRun = 0; $currentRun -lt ([math]::Ceiling($databaseServerPairs.Length/$batchingLimit)); $currentRun++)
{
# Loop through all the databses in the $databaseServerPairs array and add corresponding database objects into the array.
for($currentIndex; $currentIndex -lt $global:databaseServerPairs.Length -and $currentIndex -lt ($currentRun*$batchingLimit + $batchingLimit); $currentIndex++)
{
$global:dbs.Add((CreateDatabaseObject $global:databaseServerPairs[$currentIndex].DatabaseName $global:databaseServerPairs[$currentIndex].ServerName))
}
# Loop through all the databses in the $databaseServerPairs array and add corresponding database objects into the array.
for($currentIndex; $currentIndex -lt $global:databaseServerPairs.Length -and $currentIndex -lt ($currentRun*$batchingLimit + $batchingLimit); $currentIndex++)
{
$global:dbs.Add((CreateDatabaseObject $global:databaseServerPairs[$currentIndex].DatabaseName $global:databaseServerPairs[$currentIndex].ServerName))
}
# Continually call ExportProcess until all of the database objects have been removed from the array.
while($global:dbs.Count -gt 0)
{
ExportProcess
}
# Continually call ExportProcess until all of the database objects have been removed from the array.
while($global:dbs.Count -gt 0)
{
ExportProcess
}
}
@@ -42,9 +42,9 @@ Provides the scripts and lists the steps to set up automatically exporting your
- $batchingLimit: This tells the script how many databases can be worked on at the same time (basically, the maximum number of database copies that there will be at once).
- $retryLimit: This tells the script how many times it can retry an operation.
- $waitTimeInMinutes: This tells the script how long it can wait for an operation to complete before it fails.
- -Name for Get-AzureAutomationVariable: This is the AutomationAccount you created the StorageKey variable under (probably the same one you are running the RunBook under) and -Name is the name of the variable.
- -StorageAccountName for New-AzureStorageContext: This is the name of the storage account you are exporting to.
- -Name for Get-AutomationCertificate: This is the name of the certificate you setup to authenticate with Azure.
- $storageKeyVariableName: This is the AutomationAccount you created the StorageKey variable under (probably the same one you are running the RunBook under) and -Name is the name of the variable.
- $storageAccountName: This is the name of the storage account you are exporting to.
- $automationCertificateName for Get-AutomationCertificate: This is the name of the certificate you setup to authenticate with Azure.
- $subId: The ID of the subscription you are using. This will be used to tell Azure Automation which subscription to use.
- $subName: The name of the subscription you are using. This will be used to tell Azure Automation which subscription to use.
2. In AutoExportBlobRetention, here are the values that need to be modified: