From 8a5085d46f9e5e889d78e55e55b2d06348a2085f Mon Sep 17 00:00:00 2001 From: MikeRayMSFT Date: Mon, 13 Mar 2017 18:18:45 -0700 Subject: [PATCH] Stage initial linux vdi sample --- samples/features/sqlvdi-linux/Makefile | 14 + samples/features/sqlvdi-linux/README.md | 50 +++ samples/features/sqlvdi-linux/vdi.h | 222 +++++++++++ samples/features/sqlvdi-linux/vdierror.h | 98 +++++ .../features/sqlvdi-linux/vdipipesample.cpp | 346 ++++++++++++++++++ 5 files changed, 730 insertions(+) create mode 100644 samples/features/sqlvdi-linux/Makefile create mode 100644 samples/features/sqlvdi-linux/README.md create mode 100644 samples/features/sqlvdi-linux/vdi.h create mode 100644 samples/features/sqlvdi-linux/vdierror.h create mode 100644 samples/features/sqlvdi-linux/vdipipesample.cpp diff --git a/samples/features/sqlvdi-linux/Makefile b/samples/features/sqlvdi-linux/Makefile new file mode 100644 index 00000000..0f4eb88c --- /dev/null +++ b/samples/features/sqlvdi-linux/Makefile @@ -0,0 +1,14 @@ +# A simple Makefile to build the VDI sample code +# + +EXECUTABLE=vdipipesample +LD_FLAGS=-luuid -lrt -lpthread -lsqlvdi +LD_LIBRARY_PATH=/opt/mssql/lib + +$(EXECUTABLE): vdipipesample.cpp vdi.h vdierror.h + clang++ -o $(EXECUTABLE) -g -std=c++11 vdipipesample.cpp $(LD_FLAGS) -L $(LD_LIBRARY_PATH) + +clean: + rm $(EXECUTABLE) + + diff --git a/samples/features/sqlvdi-linux/README.md b/samples/features/sqlvdi-linux/README.md new file mode 100644 index 00000000..5833f368 --- /dev/null +++ b/samples/features/sqlvdi-linux/README.md @@ -0,0 +1,50 @@ +# SQLVDI for Linux +This folder contains the latest files and samples required to build a SQL Server VDI based backup/restore application for Linux. + +## Files available +1. vdi.h +2. vdierror.h +3. vdipipesample.cpp +4. MAKEFILE + +## Known Bugs + + There is a problem with VDF_LikeDisk and VDF_RandomAccess where the backup/restore is aborted unexpectedly. + +## Steps +1. Install the mssql-server and mssql-tools packages  + + [Install SQL Server on Linux](http://docs.microsoft.com/sql/linux/sql-server-linux-setup)  + [Install SQL Server tools on Linux](http://docs.microsoft.com/sql/linux/sql-server-linux-setup-tools)  +  +1. Install the clang and uuid-dev packages in order to build the sample.  + +  Example (for Ubuntu):  + + ```bash + sudo apt-get install clang  + sudo apt-get install uuid-dev  + ``` + +1. Create a symbolic link to sqlcmd in /usr/bin + + ``` + sudo ln -s /opt/mssql-tools/bin/sqlcmd /usr/bin/sqlcmd + ``` + +1. Copy the vdi sample files to a directory on your Linux machine. + +1. Run make to build the sample code + + +1. Run the vdi client as the mssql user or follow these instructions: + + - Add the user running the vdi client to mssql group `sudo usermod -a -G mssql vdiuser`. + - Add the mssql user to the vdi client user's group `sudo usermod -a -G vdiuser mssql`. + - Reboot + +1. Run the following command to issue a database backup of pubs: + + ```bash + LD_LIBRARY_PATH="/opt/mssql/lib" ./vdipipesample B D pubs sa /tmp/pubs.bak + ``` diff --git a/samples/features/sqlvdi-linux/vdi.h b/samples/features/sqlvdi-linux/vdi.h new file mode 100644 index 00000000..8c593547 --- /dev/null +++ b/samples/features/sqlvdi-linux/vdi.h @@ -0,0 +1,222 @@ +//********************************************************************* +// Copyright (C) Microsoft Corporation. +// +// @File: vdi.h +// @Owner: +// +// Purpose: +// +// +// Notes: +// +// +//********************************************************************* +#ifndef __vdi_h__ +#define __vdi_h__ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#pragma pack(push, _vdi_h_) + +#include +#include + +// Errors that need to be passed back to SQL Server, originally defined +// in Windows.h or similar +// +#define ERROR_SUCCESS 0L +#define ERROR_ARENA_TRASHED 7L +#define ERROR_HANDLE_EOF 38L +#define ERROR_HANDLE_DISK_FULL 39L +#define ERROR_NOT_SUPPORTED 50L +#define ERROR_DISK_FULL 112L +#define ERROR_OPERATION_ABORTED 995L + +#pragma pack(8) +struct VDConfig +{ + uint32_t deviceCount; + uint32_t features; + uint32_t prefixZoneSize; + uint32_t alignment; + uint32_t softFileMarkBlockSize; + uint32_t EOMWarningSize; + uint32_t serverTimeOut; + uint32_t blockSize; + uint32_t maxIODepth; + uint32_t maxTransferSize; + uint32_t bufferAreaSize; +}; + +enum VDFeatures +{ VDF_Removable = 0x1, + VDF_Rewind = 0x2, + VDF_Position = 0x10, + VDF_SkipBlocks = 0x20, + VDF_ReversePosition = 0x40, + VDF_Discard = 0x80, + VDF_FileMarks = 0x100, + VDF_RandomAccess = 0x200, + VDF_SnapshotPrepare = 0x400, + VDF_EnumFrozenFiles = 0x800, + VDF_VSSWriter = 0x1000, + VDF_RequestComplete = 0x2000, + VDF_WriteMedia = 0x10000, + VDF_ReadMedia = 0x20000, + VDF_CompleteEnabled = 0x40000, + VDF_LatchStats = 0x80000000, + VDF_LikePipe = 0, + VDF_LikeTape = + ( ( ( ( ( VDF_FileMarks | VDF_Removable ) | VDF_Rewind ) | VDF_Position ) | + VDF_SkipBlocks ) | VDF_ReversePosition ), + VDF_LikeDisk = VDF_RandomAccess}; + +enum VDCommands +{ VDC_Read = 1, + VDC_Write = ( VDC_Read + 1 ), + VDC_ClearError = ( VDC_Write + 1 ), + VDC_Rewind = ( VDC_ClearError + 1 ), + VDC_WriteMark = ( VDC_Rewind + 1 ), + VDC_SkipMarks = ( VDC_WriteMark + 1 ), + VDC_SkipBlocks = ( VDC_SkipMarks + 1 ), + VDC_Load = ( VDC_SkipBlocks + 1 ), + VDC_GetPosition = ( VDC_Load + 1 ), + VDC_SetPosition = ( VDC_GetPosition + 1 ), + VDC_Discard = ( VDC_SetPosition + 1 ), + VDC_Flush = ( VDC_Discard + 1 ), + VDC_Snapshot = ( VDC_Flush + 1 ), + VDC_MountSnapshot = ( VDC_Snapshot + 1 ), + VDC_PrepareToFreeze = ( VDC_MountSnapshot + 1 ), + VDC_FileInfoBegin = ( VDC_PrepareToFreeze + 1 ), + VDC_FileInfoEnd = ( VDC_FileInfoBegin + 1 ), + VDC_GetError = (VDC_FileInfoEnd + 1), + VDC_Complete = (VDC_GetError + 1)}; + +enum VDWhence +{ VDC_Beginning = 0, + VDC_Current = ( VDC_Beginning + 1 ), + VDC_End = ( VDC_Current + 1 )}; + +struct VDC_Command +{ + int32_t commandCode; + int32_t size; + int64_t position; + uint8_t* buffer; +}; + +struct VDS_Command +{ + int32_t commandCode; + int32_t size; + int64_t inPosition; + int64_t outPosition; + uint8_t* buffer; + uint8_t* completionRoutine; + uint8_t* completionContext; + int32_t completionCode; + int32_t bytesTransferred; +}; + +//---------------------------------------------------------------------------- +// NAME: ClientVirtualDevice +// +// PURPOSE: +// +// Implement the ClientVirtualDevice component. +// +class CVDS; +class CVD; +class ClientVirtualDeviceSet; + +class ClientVirtualDevice +{ +public: + ClientVirtualDevice(); + + ~ClientVirtualDevice(); + + int + GetCommand( + time_t timeOut, + VDC_Command** ppCmd); + + int + CompleteCommand( + VDC_Command* pCmd, + int completionCode, + unsigned long bytesTransferred, + int64_t position); + +private: + CVD* cvd; + friend class ClientVirtualDeviceSet; +}; + +//---------------------------------------------------------------------------- +// NAME: ClientVirtualDeviceSet +// +// PURPOSE: +// +// Implement the ClientVirtualDeviceSet component. +// + +class ClientVirtualDeviceSet +{ +public: + int + Create( + char* name, + VDConfig* cfg); + + int + GetConfiguration( + time_t timeout, + VDConfig* cfg); + + int + OpenDevice( + char* name, + ClientVirtualDevice** ppVirtualDevice); + + int + Close(); + + int + SignalAbort(); + + int + OpenInSecondary( + char* setName); + + int + GetBufferHandle( + uint8_t* pBuffer, + unsigned int* pBufferHandle); + + int + MapBufferHandle( + int dwBuffer, + uint8_t** ppBuffer); + + void + RegisterDeviceClosed(); + + ClientVirtualDeviceSet (); + ~ClientVirtualDeviceSet (); + +private: + CVDS* cvds; +}; + +#pragma pack(pop, _vdi_h_) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/samples/features/sqlvdi-linux/vdierror.h b/samples/features/sqlvdi-linux/vdierror.h new file mode 100644 index 00000000..064c7f89 --- /dev/null +++ b/samples/features/sqlvdi-linux/vdierror.h @@ -0,0 +1,98 @@ +//********************************************************************* +// Copyright (C) Microsoft Corporation. +// +// @File: vdierror.h +// @Owner: +// +// Purpose: +// +// +// Notes: +// +// +//********************************************************************* +#ifndef VDIERROR_H_ +#define VDIERROR_H_ +//**************************************************************************** +// Copyright (c) Microsoft Corporation. +// +// vdierror.h +// +// Purpose: +// Declare the error codes emitted by the virtual device interface. +// +//**************************************************************************** + +// +// Create an int value from component pieces +// + +#define MAKE_HRESULT(sev, fac, code) \ + ((int) (((unsigned long)(sev) << 31) | ((unsigned long)(fac) << 16) | ((unsigned long)(code))) ) + +//--------------------------------------------------------------------------------------- +// Error code handling will be done in standard COM fashion: +// +// an int is returned and the caller can use +// SUCCEEDED(code) or FAILED(code) to determine +// if the function failed or not. +// + +// Form an error code mask. +// All VDI errors have 0x8077 as prefix. +// +#define VD_ERROR(code) MAKE_HRESULT(1, 0x77, code) + +// The object was not open +// +#define VD_E_NOTOPEN VD_ERROR(2) /* 0x80770002 */ + +// The api was waiting and the timeout interval had elapsed. +// +#define VD_E_TIMEOUT VD_ERROR(3) /* 0x80770003 */ + +// An abort request is preventing anything except termination actions. +// +#define VD_E_ABORT VD_ERROR(4) /* 0x80770004 */ + +// Failed to create security environment. +// +#define VD_E_SECURITY VD_ERROR(5) /* 0x80770005 */ + +// An invalid parameter was supplied +// +#define VD_E_INVALID VD_ERROR(6) /* 0x80770006 */ + +// Failed to recognize the SQL Server instance name +// +#define VD_E_INSTANCE_NAME VD_ERROR(7) /* 0x80770007 */ + +// The requested configuration is invalid +// +#define VD_E_NOTSUPPORTED VD_ERROR(9) /* 0x80770009 */ + +// Out of memory +// +#define VD_E_MEMORY VD_ERROR(10) /* 0x8077000a */ + +// Unexpected internal error +// +#define VD_E_UNEXPECTED VD_ERROR(11) /* 0x8077000b */ + +// Protocol error +// +#define VD_E_PROTOCOL VD_ERROR(12) /* 0x8077000c */ + +// All devices are open +// +#define VD_E_OPEN VD_ERROR(13) /* 0x8077000d */ + +// the object is now closed +// +#define VD_E_CLOSE VD_ERROR(14) /* 0x8077000e */ + +// the resource is busy +// +#define VD_E_BUSY VD_ERROR(15) /* 0x8077000f */ + +#endif diff --git a/samples/features/sqlvdi-linux/vdipipesample.cpp b/samples/features/sqlvdi-linux/vdipipesample.cpp new file mode 100644 index 00000000..c1e85fe3 --- /dev/null +++ b/samples/features/sqlvdi-linux/vdipipesample.cpp @@ -0,0 +1,346 @@ +/*********************************************************************** + Copyright (c) Microsoft Corporation + All Rights Reserved. +***********************************************************************/ +// This source code is an intended supplement to the Microsoft SQL +// Server online references and related electronic documentation. +// +// This sample is for instructional purposes only. +// Code contained herein is not intended to be used "as is" in real applications. +// +// vdipipesample.cpp +// +// This is a sample program used to demonstrate the Virtual Device Interface +// feature of Microsoft SQL Server. This code implements the client side of +// the Virtual Device Interface and uses sqlcmd to issue a T-SQL statement +// to start the server side of the backup or restore. +// +// The program will backup or restore a database. +// +// The program accepts 6 command line parameters. +// +// One of: +// b perform a backup +// r perform a restore +// One of: +// d perform a backup/restore on a database +// l perform a backup/restore on a log +// The DB Name +// databaseName +// The SQL username +// sa +// The SQL Password +// MyPassword1! +// And the filename: +// filename.bak +// + +#include // for file operations +#include // for toupper () +#include +#include +#include +#include // for memset +#include +#include +#include +#include + +#include "vdi.h" // interface declaration +#include "vdierror.h" // error constants + +using namespace std; + +void performTransfer( + ClientVirtualDevice* vd, + int backup, + char* fname); + +shared_ptr sendSQL(bool doBackup, + bool dataBackup, + char* databaseName, + char* userName, + char* password); + +// Using a GUID for the VDS Name is a good way to assure uniqueness. +// +static char wVdsName [50]; + +// +// main function +// +int main(int argc, char* argv[]) +{ + ClientVirtualDeviceSet* vds = NULL; + ClientVirtualDevice* vd = NULL; + int status; + + VDConfig config; + bool badParm = false; + bool doBackup = true; + bool dataBackup = true; + char* databaseName = nullptr; + char* userName = nullptr; + char* password = nullptr; + char* backupFile = nullptr; + shared_ptr processPipe; + + // Check the input parm + // + if (argc == 7) + { + if (toupper(argv[1][0]) == 'B') + { + doBackup = true; + } + else if (toupper(argv[1][0]) == 'R') + { + doBackup = false; + } + else + { + badParm = true; + } + + if (toupper(argv[2][0]) == 'D') + { + dataBackup = true; + } + else if (toupper(argv[2][0]) == 'L') + { + dataBackup = false; + } + else + { + badParm = true; + } + + databaseName = &argv[3][0]; + userName = &argv[4][0]; + password = &argv[5][0]; + backupFile = &argv[6][0]; + } + else + { + badParm = true; + } + + if (badParm) + { + printf("usage: vdipipesample {B|R} {D|L} \n" + "Demonstrate a Backup or Restore using the Virtual Device Interface\n"); + return 1; + } + + vds = new ClientVirtualDeviceSet(); + + // Setup the VDI configuration we want to use. + // This program doesn't use any fancy features, so the + // only field to setup is the deviceCount. + // + // The server will treat the virtual device just like a pipe: + // I/O will be strictly sequential with only the basic commands. + // + memset(&config, 0, sizeof(config)); + config.deviceCount = 1; + + // Create a GUID to use for a unique virtual device name + // + uuid_t vdsId; + uuid_generate(vdsId); + uuid_unparse(vdsId, wVdsName); + + // Create the virtual device set + // + status = vds->Create(wVdsName, &config); + if (status != 0) + { + printf("VDS::Create fails: x%X", status); + goto exit; + } + + // Send the SQL command, by starting 'sqlcmd' in a new process. + // + printf("\nSending the SQL...\n"); + + processPipe = sendSQL(doBackup, dataBackup, databaseName, userName, password); + if (!processPipe) + { + printf("sendSQL failed.\n"); + goto shutdown; + } + + printf("\nGetting configuration.\n"); + // Wait for the server to connect, completing the configuration. + // + status = vds->GetConfiguration(10000, &config); + if (status != 0) + { + printf("VDS::Getconfig fails: x%X\n", status); + if (status == VD_E_TIMEOUT) + { + printf("Timed out. Was Microsoft SQLServer running?\n"); + } + goto shutdown; + } + + printf("Features returned by SQL Server: 0x%x\n", config.features); + + printf("\nOpening the device.\n"); + // Open the single device in the set. + // + status = vds->OpenDevice(wVdsName, &vd); + if (status != 0) + { + printf("VDS::OpenDevice fails: x%X\n", status); + goto shutdown; + } + + printf("\nPerforming data transfer...\n"); + + performTransfer(vd, doBackup, backupFile); + +shutdown: + + // Close the set + // + vds->Close(); + +exit: + + // Print out any messages from SQLCMD + // + if (processPipe) + { + char line[1024]; + while (fgets(line, 1024, processPipe.get())) + { + printf("%s", line); + } + } + + return 0; +} + +// Execute a basic backup/restore, by spawning a process to execute 'sqlcmd'. +// +shared_ptr sendSQL(bool doBackup, + bool dataBackup, + char* databaseName, + char* userName, + char* password) +{ + printf("Connecting to SQL Server."); + char sqlCommand [1024]; // plenty of space for our purpose + + sprintf(sqlCommand, + "sqlcmd -U %s -P %s -S . -Q \"%s %s %s %s VIRTUAL_DEVICE='%s' WITH %s, MAXTRANSFERSIZE=1048576 \"", + userName, + password, + (doBackup) ? "BACKUP" : "RESTORE", + (dataBackup) ? "DATABASE" : "LOG", + databaseName, + (doBackup) ? "TO" : "FROM", + wVdsName, + (doBackup) ? "FORMAT" : "REPLACE"); + + shared_ptr pipe(popen(sqlCommand, "r"), pclose); + + return pipe; +} + +// This routine reads commands from the server until a 'Close' status is received. +// +void performTransfer( + ClientVirtualDevice* vd, + int backup, + char* fname) +{ + FILE* fh; + VDC_Command* cmd; + int completionCode; + size_t bytesTransferred; + int status; + + fh = fopen(fname, (backup) ? "wb" : "rb"); + if (fh == NULL) + { + printf("Failed to open: %s\n", fname); + return; + } + + // Timeout in seconds + // + int timeout = 90; + while ((status = vd->GetCommand(timeout, &cmd)) == 0) + { + bytesTransferred = 0; + switch (cmd->commandCode) + { + case VDC_Read: + bytesTransferred = fread(cmd->buffer, 1, cmd->size, fh); + if (bytesTransferred == (size_t)cmd->size) + { + completionCode = ERROR_SUCCESS; + } + else + { + // assume failure is eof + completionCode = ERROR_HANDLE_EOF; + } + break; + + case VDC_Write: + bytesTransferred = fwrite(cmd->buffer, 1, cmd->size, fh); + if (bytesTransferred == (size_t)cmd->size) + { + completionCode = ERROR_SUCCESS; + } + else + { + // assume failure is disk full + completionCode = ERROR_DISK_FULL; + } + break; + + case VDC_Flush: + fflush(fh); + completionCode = ERROR_SUCCESS; + break; + + case VDC_ClearError: + completionCode = ERROR_SUCCESS; + break; + + default: + // If command is unknown... + completionCode = ERROR_NOT_SUPPORTED; + } + + status = vd->CompleteCommand(cmd, completionCode, bytesTransferred, 0); + printf("Completed command code: %i, completionCode: %i, bytes; %li \n", + cmd->commandCode, completionCode, bytesTransferred); + if (status != 0) + { + printf("Completion Failed: x%X\n", status); + break; + } + } + + if (status != VD_E_CLOSE) + { + printf("Unexpected termination: x%X\n", status); + } + else + { + // As far as the data transfer is concerned, no + // errors occurred. The code which issues the SQL + // must determine if the backup/restore was + // really successful. + // + printf("Successfully completed data transfer.\n"); + } + + fclose(fh); +} +