mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Merge remote-tracking branch 'refs/remotes/Microsoft/master'
This commit is contained in:
Jovan Popovic
+8
-5
@@ -1,4 +1,5 @@
|
||||
/********************************************************************************
|
||||
|
||||
--/********************************************************************************
|
||||
* Note: You can export file and create format file using bcp out command:
|
||||
*
|
||||
*>bcp "SELECT Name, Color, Price, Size, Quantity, Data, Tags FROM Product" queryout product.dat -d ProductCatalog -T
|
||||
@@ -46,7 +47,8 @@ WITH ( TYPE = BLOB_STORAGE,
|
||||
|
||||
DROP TABLE IF EXISTS Product;
|
||||
GO
|
||||
|
||||
--Create a permanent table. A temp table currently is not supported for BULK INSERT, although it will will work
|
||||
--with OPENROWSET
|
||||
CREATE TABLE dbo.Product(
|
||||
Name nvarchar(50) NOT NULL,
|
||||
Color nvarchar(15) NULL,
|
||||
@@ -54,8 +56,8 @@ CREATE TABLE dbo.Product(
|
||||
Size nvarchar(5) NULL,
|
||||
Quantity int NULL,
|
||||
Data nvarchar(4000) NULL,
|
||||
Tags nvarchar(4000) NULL,
|
||||
INDEX cci CLUSTERED COLUMNSTORE
|
||||
Tags nvarchar(4000) NULL
|
||||
--,INDEX cci CLUSTERED COLUMNSTORE
|
||||
)
|
||||
GO
|
||||
|
||||
@@ -69,6 +71,7 @@ FROM 'product.csv'
|
||||
WITH ( DATA_SOURCE = 'MyAzureBlobStorage',
|
||||
FORMAT='CSV', CODEPAGE = 65001, --UTF-8 encoding
|
||||
FIRSTROW=2,
|
||||
ROWTERMINATOR = '0x0a',
|
||||
TABLOCK);
|
||||
|
||||
-- 2.2. INSERT file exported using bcp.exe into Product table
|
||||
@@ -93,4 +96,4 @@ FROM OPENROWSET(BULK 'product.bcp',
|
||||
DATA_SOURCE = 'MyAzureBlobStorage',
|
||||
FORMATFILE='data/product.fmt',
|
||||
FORMATFILE_DATA_SOURCE = 'MyAzureBlobStorage') as data
|
||||
GROUP BY Color;
|
||||
GROUP BY Color;
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"resourcegroup": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"subscriptionId": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"workspace": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"workspaceapiversion": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
}
|
||||
},
|
||||
"resources": [{
|
||||
"apiVersion": "[parameters('workspaceapiversion')]",
|
||||
"name": "[parameters('workspace')]",
|
||||
"type": "Microsoft.OperationalInsights/workspaces",
|
||||
"location": "[parameters('location')]",
|
||||
"id": "[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'))]",
|
||||
"resources": [{
|
||||
"apiVersion": "2015-11-01-preview",
|
||||
"name": "Data Sync",
|
||||
"type": "views",
|
||||
"location": "[parameters('location')]",
|
||||
"id": "[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'),'/views/Data Sync')]",
|
||||
"dependson": [
|
||||
"[Concat('/subscriptions/', parameters('subscriptionId'), '/resourceGroups/', parameters('resourcegroup'), '/providers/Microsoft.OperationalInsights/workspaces/', parameters('workspace'))]"
|
||||
],
|
||||
"properties": {
|
||||
"Id": "Data Sync",
|
||||
"Name": "Data Sync",
|
||||
"Author": "jognanay@microsoft.com",
|
||||
"Source": "Local",
|
||||
"Version": 1,
|
||||
"Dashboard": [{
|
||||
"Id": "SingleQueryDonutBuilderBladeV1",
|
||||
"Type": "Blade",
|
||||
"Version": 0,
|
||||
"Configuration": {
|
||||
"General": {
|
||||
"title": "All Sync Groups",
|
||||
"newGroup": false,
|
||||
"icon": "",
|
||||
"useIcon": false
|
||||
},
|
||||
"Header": {
|
||||
"Title": "Error and Warnings",
|
||||
"Subtitle": ""
|
||||
},
|
||||
"Donut": {
|
||||
"Query": " LogLevel_s !=Success Type=DataSyncLog_CL TimeStamp_t>NOW-1DAY |measure count() by SyncGroupName_s",
|
||||
"CenterLegend": {
|
||||
"Text": "Total",
|
||||
"Operation": "Sum",
|
||||
"ArcsToSelect": []
|
||||
},
|
||||
"Options": {
|
||||
"colors": [
|
||||
"#00188f",
|
||||
"#0072c6",
|
||||
"#00bcf2"
|
||||
],
|
||||
"valueColorMapping": []
|
||||
}
|
||||
},
|
||||
"List": {
|
||||
"Query": " LogLevel_s !=Success Type=DataSyncLog_CL TimeStamp_t>NOW-1DAY | measure count() by SyncGroupName_s",
|
||||
"HideGraph": false,
|
||||
"enableSparklines": false,
|
||||
"operation": "Summary",
|
||||
"ColumnsTitle": {
|
||||
"Name": "Sync Group",
|
||||
"Value": "Error and Warning Count"
|
||||
},
|
||||
"Color": "#0072c6",
|
||||
"thresholds": {
|
||||
"isEnabled": true,
|
||||
"values": [{
|
||||
"name": "Norefaultmal",
|
||||
"threshold": "Default",
|
||||
"color": "#009e49",
|
||||
"isDefault": true
|
||||
},
|
||||
{
|
||||
"name": "Good",
|
||||
"threshold": "0",
|
||||
"color": "#fcd116",
|
||||
"isDefault": false
|
||||
},
|
||||
{
|
||||
"name": "Error",
|
||||
"threshold": "1",
|
||||
"color": "#ba141a",
|
||||
"isDefault": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"NameDSVSeparator": "",
|
||||
"NavigationQuery": "{selected item}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Id": "SingleQueryDonutBuilderBladeV1",
|
||||
"Type": "Blade",
|
||||
"Version": 0,
|
||||
"Configuration": {
|
||||
"General": {
|
||||
"title": "Sample Sync Group 1",
|
||||
"newGroup": false,
|
||||
"icon": "",
|
||||
"useIcon": false
|
||||
},
|
||||
"Header": {
|
||||
"Title": "Contoso Sync",
|
||||
"Subtitle": ""
|
||||
},
|
||||
"Donut": {
|
||||
"Query": "SyncGroupName_s=ContosoSync TimeStamp_t>NOW-1DAY| measure count() by LogLevel_s",
|
||||
"CenterLegend": {
|
||||
"Text": "",
|
||||
"Operation": "Sum",
|
||||
"ArcsToSelect": []
|
||||
},
|
||||
"Options": {
|
||||
"colors": [
|
||||
"#00188f",
|
||||
"#0072c6",
|
||||
"#00bcf2"
|
||||
],
|
||||
"valueColorMapping": []
|
||||
}
|
||||
},
|
||||
"List": {
|
||||
"Query": "LogLevel_s!=Success TimeStamp_t>NOW-1DAY SyncGroupName_s=ContosoSync | select Details_s",
|
||||
"HideGraph": false,
|
||||
"enableSparklines": false,
|
||||
"ColumnsTitle": {
|
||||
"Name": "Recent Errors and Warnings",
|
||||
"Value": ""
|
||||
},
|
||||
"Color": "#0072c6",
|
||||
"operation": "Summary",
|
||||
"thresholds": {
|
||||
"isEnabled": false,
|
||||
"values": [{
|
||||
"name": "Normal",
|
||||
"threshold": "Default",
|
||||
"color": "#009e49",
|
||||
"isDefault": true
|
||||
},
|
||||
{
|
||||
"name": "Warning",
|
||||
"threshold": "60",
|
||||
"color": "#fcd116",
|
||||
"isDefault": false
|
||||
},
|
||||
{
|
||||
"name": "Error",
|
||||
"threshold": "90",
|
||||
"color": "#ba141a",
|
||||
"isDefault": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"NameDSVSeparator": "",
|
||||
"NavigationQuery": "{selected item}"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Id": "SingleQueryDonutBuilderBladeV1",
|
||||
"Type": "Blade",
|
||||
"Version": 0,
|
||||
"Configuration": {
|
||||
"General": {
|
||||
"title": "Sample Sync Group 2",
|
||||
"newGroup": false,
|
||||
"icon": "",
|
||||
"useIcon": false
|
||||
},
|
||||
"Header": {
|
||||
"Title": "Contsoso Sync 2",
|
||||
"Subtitle": ""
|
||||
},
|
||||
"Donut": {
|
||||
"Query": "SyncGroupName_s=ContosoSync2 TimeStamp_t>NOW-1DAY | measure count() by LogLevel_s",
|
||||
"CenterLegend": {
|
||||
"Text": "Total",
|
||||
"Operation": "Sum",
|
||||
"ArcsToSelect": []
|
||||
},
|
||||
"Options": {
|
||||
"colors": [
|
||||
"#00188f",
|
||||
"#0072c6",
|
||||
"#00bcf2"
|
||||
],
|
||||
"valueColorMapping": []
|
||||
}
|
||||
},
|
||||
"List": {
|
||||
"Query": "LogLevel_s=Error TimeStamp_t>NOW-1DAY SyncGroupName_s=ContosoSync2 |select Details_s",
|
||||
"HideGraph": false,
|
||||
"enableSparklines": false,
|
||||
"ColumnsTitle": {
|
||||
"Name": "Recent Errors and Warnings",
|
||||
"Value": ""
|
||||
},
|
||||
"Color": "#0072c6",
|
||||
"operation": "Summary",
|
||||
"thresholds": {
|
||||
"isEnabled": false,
|
||||
"values": [{
|
||||
"name": "Normal",
|
||||
"threshold": "Default",
|
||||
"color": "#009e49",
|
||||
"isDefault": true
|
||||
},
|
||||
{
|
||||
"name": "Warning",
|
||||
"threshold": "60",
|
||||
"color": "#fcd116",
|
||||
"isDefault": false
|
||||
},
|
||||
{
|
||||
"name": "Error",
|
||||
"threshold": "90",
|
||||
"color": "#ba141a",
|
||||
"isDefault": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"NameDSVSeparator": "",
|
||||
"NavigationQuery": "{selected item}"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"Filters": [],
|
||||
"OverviewTile": {
|
||||
"Id": "SingleQueryDonutBuilderTileV1",
|
||||
"Type": "OverviewTile",
|
||||
"Version": 0,
|
||||
"Configuration": {
|
||||
"Donut": {
|
||||
"Query": "Type=DataSyncLog_CL |measure count() by LogLevel_s",
|
||||
"CenterLegend": {
|
||||
"Text": "Total",
|
||||
"Operation": "Sum",
|
||||
"ArcsToSelect": []
|
||||
},
|
||||
"Options": {
|
||||
"colors": [
|
||||
"#00188f",
|
||||
"#0072c6",
|
||||
"#00bcf2"
|
||||
],
|
||||
"valueColorMapping": []
|
||||
}
|
||||
},
|
||||
"Advanced": {
|
||||
"DataFlowVerification": {
|
||||
"Enabled": false,
|
||||
"Query": "*",
|
||||
"Message": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
}]
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
#Data Sync OMS Integration Runbook
|
||||
|
||||
#To use this script Change all the strings below to reflect your information.
|
||||
|
||||
#Information for Sync Group 1
|
||||
#If you want to use all the sync groups in your subscription keep the $DS_xxxx fields empty.
|
||||
#If you want to use all sync groups in a Resource Group define the $DS_ResourceGroupName.
|
||||
#If you want to use all sync groups in a Server define $DS_ResourceGroupName and $DS_ServerName.
|
||||
#If you want to use all sync groups in a Database define $DS_ResourceGroupName, $DS_ServerName and $DS_DatabaseName.
|
||||
#If you want to use a specific sync group define $DS_ResourceGroupName, $DS_ServerName, $DS_DatabaseName and $DS_SyncGroupName.
|
||||
|
||||
$SubscriptionId = "SubscriptionId"
|
||||
$DS_ResourceGroupName = ""
|
||||
$DS_ServerName = ""
|
||||
$DS_DatabaseName = ""
|
||||
$DS_SyncGroupName = ""
|
||||
|
||||
$AC_ResourceGroupName = "ResourceGroupName"
|
||||
$AC_AccountName = "AutomationAccountName"
|
||||
$AC_LastUpdatedTimeVariableName = "DataSyncLogLastUpdatedTime"
|
||||
|
||||
# Replace with your OMS Workspace ID
|
||||
$CustomerId = "OMSCustomerID"
|
||||
|
||||
# Replace with your OMS Primary Key
|
||||
$SharedKey = "SharedKey"
|
||||
|
||||
# Specify the name of the record type that you'll be creating
|
||||
$LogType = "DataSyncLog"
|
||||
|
||||
# Specify a field with the created time for the records
|
||||
$TimeStampField = "DateValue"
|
||||
|
||||
#Specify the interval of how often you want to send data to oms
|
||||
#You can use -hours, -minutes or -days, use a negative number
|
||||
#$interval = New-TimeSpan -hours -1
|
||||
|
||||
|
||||
#Connect to Azure
|
||||
$connectionName = "AzureRunAsConnection"
|
||||
try
|
||||
{
|
||||
# Get the connection "AzureRunAsConnection "
|
||||
$servicePrincipalConnection=Get-AutomationConnection -Name $connectionName
|
||||
|
||||
"Logging in to Azure..."
|
||||
Add-AzureRmAccount `
|
||||
-ServicePrincipal `
|
||||
-TenantId $servicePrincipalConnection.TenantId `
|
||||
-ApplicationId $servicePrincipalConnection.ApplicationId `
|
||||
-CertificateThumbprint $servicePrincipalConnection.CertificateThumbprint
|
||||
}
|
||||
catch {
|
||||
if (!$servicePrincipalConnection)
|
||||
{
|
||||
$ErrorMessage = "Connection $connectionName not found."
|
||||
throw $ErrorMessage
|
||||
} else{
|
||||
Write-Error -Message $_.Exception
|
||||
throw $_.Exception
|
||||
}
|
||||
}
|
||||
|
||||
# Create the function to create the authorization signature
|
||||
Function Build-Signature ($customerId, $sharedKey, $date, $contentLength, $method, $contentType, $resource)
|
||||
{
|
||||
$xHeaders = "x-ms-date:" + $date
|
||||
$stringToHash = $method + "`n" + $contentLength + "`n" + $contentType + "`n" + $xHeaders + "`n" + $resource
|
||||
|
||||
$bytesToHash = [Text.Encoding]::UTF8.GetBytes($stringToHash)
|
||||
$keyBytes = [Convert]::FromBase64String($sharedKey)
|
||||
|
||||
$sha256 = New-Object System.Security.Cryptography.HMACSHA256
|
||||
$sha256.Key = $keyBytes
|
||||
$calculatedHash = $sha256.ComputeHash($bytesToHash)
|
||||
$encodedHash = [Convert]::ToBase64String($calculatedHash)
|
||||
$authorization = 'SharedKey {0}:{1}' -f $customerId,$encodedHash
|
||||
return $authorization
|
||||
}
|
||||
|
||||
|
||||
# Create the function to create and post the request
|
||||
Function Post-OMSData($customerId, $sharedKey, $body, $logType)
|
||||
{
|
||||
$method = "POST"
|
||||
$contentType = "application/json"
|
||||
$resource = "/api/logs"
|
||||
$rfc1123date = [DateTime]::UtcNow.ToString("r")
|
||||
$contentLength = $body.Length
|
||||
$signature = Build-Signature `
|
||||
-customerId $customerId `
|
||||
-sharedKey $sharedKey `
|
||||
-date $rfc1123date `
|
||||
-contentLength $contentLength `
|
||||
-fileName $fileName `
|
||||
-method $method `
|
||||
-contentType $contentType `
|
||||
-resource $resource
|
||||
$uri = "https://" + $customerId + ".ods.opinsights.azure.com" + $resource + "?api-version=2016-04-01"
|
||||
|
||||
$headers = @{
|
||||
"Authorization" = $signature;
|
||||
"Log-Type" = $logType;
|
||||
"x-ms-date" = $rfc1123date;
|
||||
"time-generated-field" = $TimeStampField;
|
||||
}
|
||||
|
||||
$response = Invoke-WebRequest -Uri $uri -Method $method -ContentType $contentType -Headers $headers -Body $body -UseBasicParsing
|
||||
return $response.StatusCode
|
||||
|
||||
}
|
||||
|
||||
|
||||
#Get Log Data
|
||||
select-azurermsubscription -SubscriptionId $SubscriptionId
|
||||
$endtime =[System.DateTime]::UtcNow
|
||||
$StartTime = Get-AzureRmAutomationVariable -ResourceGroupName $AC_ResourceGroupName `
|
||||
–AutomationAccountName $AC_AccountName `
|
||||
-Name $AC_LastUpdatedTimeVariableName | Select -ExpandProperty Value
|
||||
|
||||
|
||||
#Get Log
|
||||
Write-Output "Getting Data Sync Log from $StartTime to $EndTime"
|
||||
|
||||
if ($DS_ResourceGroupName -eq "")
|
||||
{
|
||||
$ResourceGroupName = Get-AzureRmResourceGroup | select -ExpandProperty ResourceGroupName
|
||||
}
|
||||
else
|
||||
{
|
||||
$ResourceGroupName = $DS_ResourceGroupName
|
||||
}
|
||||
|
||||
foreach ($ResourceGroup in $ResourceGroupName)
|
||||
{
|
||||
if ($DS_ServerName -eq "")
|
||||
{
|
||||
$ServerName = Get-AzureRmSqlServer -ResourceGroupName $ResourceGroup | select -ExpandProperty ServerName
|
||||
}
|
||||
else
|
||||
{
|
||||
$ServerName = $DS_ServerName
|
||||
}
|
||||
|
||||
foreach ($Server in $ServerName)
|
||||
{
|
||||
if ($DS_DatabaseName -eq "")
|
||||
{
|
||||
$DatabaseName = Get-AzureRmSqlDatabase -ResourceGroupName $ResourceGroup -ServerName $Server | select -ExpandProperty DatabaseName
|
||||
}
|
||||
else
|
||||
{
|
||||
$DatabaseName = $DS_DatabaseName
|
||||
}
|
||||
|
||||
foreach ($Database in $DatabaseName)
|
||||
{
|
||||
if ($Database -eq "master")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($DS_SyncGroupName -eq "")
|
||||
{
|
||||
$SyncGroupName = Get-AzureRmSqlSyncGroup -ResourceGroupName $ResourceGroup -ServerName $Server -DatabaseName $Database | select -ExpandProperty SyncGroupName
|
||||
}
|
||||
else
|
||||
{
|
||||
$SyncGroupName = $DS_SyncGroupName
|
||||
}
|
||||
|
||||
foreach ($SyncGroup in $SyncGroupName)
|
||||
{
|
||||
$Logs = Get-AzureRmSqlSyncGroupLog -ResourceGroupName $ResourceGroup `
|
||||
-ServerName $Server `
|
||||
-DatabaseName $Database `
|
||||
-SyncGroupName $SyncGroup `
|
||||
-starttime $StartTime `
|
||||
-endtime $EndTime;
|
||||
|
||||
if ($Logs.Length -gt 0)
|
||||
{
|
||||
foreach ($Log in $Logs)
|
||||
{
|
||||
$Log | Add-Member -Name "SubscriptionId" -Value $SubscriptionId -MemberType NoteProperty
|
||||
$Log | Add-Member -Name "ResourceGroupName" -Value $ResourceGroup -MemberType NoteProperty
|
||||
$Log | Add-Member -Name "ServerName" -Value $Server -MemberType NoteProperty
|
||||
$Log | Add-Member -Name "HubDatabaseName" -Value $Database -MemberType NoteProperty
|
||||
$Log | Add-Member -Name "SyncGroupName" -Value $SyncGroup -MemberType NoteProperty
|
||||
|
||||
#Filter out Successes to Reduce Data Volume to OMS
|
||||
#Include the 5 commented out line below to enable the filter
|
||||
#For($i=0; $i -lt $Log.Length; $i++ ) {
|
||||
# if($Log[$i].LogLevel -eq "Success") {
|
||||
# $Log[$i] =""
|
||||
# }
|
||||
# }
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
$json = ConvertTo-JSON $logs
|
||||
|
||||
|
||||
|
||||
Post-OMSData -customerId $customerId -sharedKey $sharedKey -body ([System.Text.Encoding]::UTF8.GetBytes($json)) -logType $logType
|
||||
if ($result -eq 200)
|
||||
{
|
||||
Write-Host "Success"
|
||||
}
|
||||
if ($result -ne 200)
|
||||
{
|
||||
throw
|
||||
@"
|
||||
Posting to OMS Failed
|
||||
Runbook Name: DataSyncOMSIntegration
|
||||
"@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
Set-AzureRmAutomationVariable -ResourceGroupName $AC_ResourceGroupName `
|
||||
–AutomationAccountName $AC_AccountName `
|
||||
-Name $AC_LastUpdatedTimeVariableName `
|
||||
-Value $EndTime `
|
||||
-Encrypted $False
|
||||
Reference in New Issue
Block a user