diff --git a/samples/features/sql-clr/TransitiveClosure/.gitignore b/samples/features/sql-clr/TransitiveClosure/.gitignore new file mode 100644 index 00000000..407d0cc4 --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/.gitignore @@ -0,0 +1,7 @@ +*.cproj.user +.vs/* +.vscode/* +bin/* +obj/* +Properties/PublishProfiles/* +TransitiveClosure.sql \ No newline at end of file diff --git a/samples/features/sql-clr/TransitiveClosure/Properties/AssemblyInfo.cs b/samples/features/sql-clr/TransitiveClosure/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..09186f99 --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("TransitiveClosureAggregatorLibrary")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("TransitiveClosureAggregatorLibrary")] +[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("097ef341-926d-4dd2-a434-08e9980d6089")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/samples/features/sql-clr/TransitiveClosure/README.md b/samples/features/sql-clr/TransitiveClosure/README.md new file mode 100644 index 00000000..945ff5d9 --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/README.md @@ -0,0 +1,104 @@ +# Implementing Transitive Closure Clustering in SQL Server using CLR UDF +SQL Database don't have built-in support for transitive closure clustering, so the only workaround is to implement this algorithm in .Net framework and expose it as T-SQL function. +A discussion on the problem, the algorithm and a pure T-SQL based solution can be found here: +- [Transitive Closure Clustering with SQL Server, UDA and JSON](https://medium.com/@mauridb/transitive-closure-clustering-with-sql-server-uda-and-json-dade18953fd2) +- [T-SQL Puzzle Challenge Grouping Connected Items](http://www.itprotoday.com/microsoft-sql-server/t-sql-puzzle-challenge-grouping-connected-items) + +This code sample demonstrates how to create CLR User-Defined aggregate that implements clustering. + +### Contents + +[About this sample](#about-this-sample)
+[Build the CLR/TransitiveClosure aggregate](#build-functions)
+[Add RegEx functions to your SQL database](#add-functions)
+[Test the functions](#test)
+[Disclaimers](#disclaimers)
+ + + +## About this sample +1. **Applies to:** SQL Server 2016+ Enterprise / Developer / Evaluation Edition +2. **Key features:** + - CLR, JSON +3. **Programming Language:** .NET C# +4. **Author:** [Davide Mauri](https://github.com/yorek), Jovan Popovic [jovanpop-msft] + + + +## Build the CLR/TransitiveClosure aggregate + +1. Download the source code and open the solution using Visual Studio. +2. Change the password in .pfk file and rebuild the solution in **Release** mode. +3. Open and save TransitiveClosure.tt to generate output T-SQL file that will contain script that inserts .dll file with the Transitive closure clustering aggregate. + + +## Add Clustering aggregate to your SQL database + +File TransitiveClosure.sql contains the code that will import aggregate into SQL Database. + +If you have not added CLR assemblies in your database, you should use the following script to enable CLR: +``` +sp_configure @configname=clr_enabled, @configvalue=1 +GO +RECONFIGURE +GO +``` + +Once you enable CLR, you can use the T-SQL script to add the clustering aggregate. The script depends on the location where you have built the project, and might look like: +``` +CREATE ASSEMBLY TransitiveClosure FROM 'D:\GitHub\sql-server-samples\samples\features\sql-clr\TransitiveClosure\bin\Release\TransitiveClosureAggregatorLibrary.dll' WITH PERMISSION_SET = SAFE; +GO + +CREATE SCHEMA TC; +GO + +CREATE AGGREGATE TC.CLUSTERING(@id1 INT, @id2 INT) +RETURNS NVARCHAR(MAX) +EXTERNAL NAME TransitiveClosure.[TransitiveClosure.Aggregate]; +``` + +This code will import assembly in SQL Database and add an aggregate that provides clustering functionalities. + + + +## Test the function + +Once you create the assembly and expose the aggregate, you can use it to cluster some relational data in T-SQL code: + +``` +declare @edges table(n1 int, n2 int); + +insert into @edges +values + (1,2),(2,3),(3,4),(4,5),(2,21),(2,22), + (7,8),(8,9),(9,10); + +select TC.CLUSTERING(n1,n2) +from @edges; +``` +The result will be JSON document that groups the numbers that belong to the same cluster. +```javascript +{ + "0":[1,2,3,4,5,21,22], + "1":[7,8,9,10] +} +``` +You can transform this JSON document into relational formatusing **OPENJSON** function: +``` +select cluster = [key], elements = value +from openjson( + (select TC.CLUSTERING(n1,n2) from @edges) +); +``` +The result of this query is: + +|cluster|elements| +|---|---| +|0|[1,2,3,4,5,21,22]| +|1|[7,8,9,10]| + + + +## Disclaimers +The code included in this sample is not intended to be a set of best practices on how to build scalable enterprise grade applications. This is beyond the scope of this sample. + diff --git a/samples/features/sql-clr/TransitiveClosure/TransitiveClosure.tt b/samples/features/sql-clr/TransitiveClosure/TransitiveClosure.tt new file mode 100644 index 00000000..2216864e --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/TransitiveClosure.tt @@ -0,0 +1,23 @@ +<#@output extension=".sql"#> +<#@ template language="C#" hostspecific="True" #> + +DROP AGGREGATE IF EXISTS TC.CLUSTERING; +GO + +DROP SCHEMA IF EXISTS TC; +GO + +--Drop the assembly if it already exists +DROP ASSEMBLY IF EXISTS TransitiveClosure; +GO + +--Create the assembly +CREATE ASSEMBLY TransitiveClosure FROM '<#= this.Host.ResolvePath("bin\\Release\\TransitiveClosureAggregatorLibrary.dll") #>' WITH PERMISSION_SET = SAFE; +GO + +CREATE SCHEMA TC; +GO + +CREATE AGGREGATE TC.CLUSTERING(@id1 INT, @id2 INT) +RETURNS NVARCHAR(MAX) +EXTERNAL NAME TransitiveClosure.[TransitiveClosure.Aggregate]; \ No newline at end of file diff --git a/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregate.cs b/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregate.cs new file mode 100644 index 00000000..2969b22a --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregate.cs @@ -0,0 +1,316 @@ +using System; +using System.IO; +using System.Data.SqlTypes; +using System.Text; +using Microsoft.SqlServer.Server; +using System.Collections.Generic; +using System.Collections; + +namespace TransitiveClosure +{ + /// + /// Class that represents a group of numbers in the same cluster. + /// + public class Group: IEnumerable + { + private int? _groupRoot = null; + + private Dictionary _group = new Dictionary(); + public Dictionary.KeyCollection Elements => _group.Keys; + public int Count => _group.Keys.Count; + + public bool ContainsElement(int element) + { + return _group.ContainsKey(element); + } + + /// + /// Adds a pair of numbers to a group. + /// + /// + /// + public void AddUnique(int from, int to) + { + if (_groupRoot == null) _groupRoot = from; + this.AddIfNotExists(from); + this.AddIfNotExists(to); + } + /// + /// Adds the element into the current group. + /// + /// The number that should be added. + public void Add(int element) + { + if (_groupRoot == null) _groupRoot = element; + _group.Add(element, true); + } + + /// + /// Adds the element to a group if it is not already there. + /// + /// + public void AddIfNotExists(int element) + { + if (!_group.ContainsKey(element)) + { + _group.Add(element, true); + } + } + + public void MergeWith(Group source) + { + foreach (var e in source.Elements) + { + this.AddIfNotExists(e); + } + } + + public IEnumerator GetEnumerator() + { + foreach(var e in _group.Keys) + { + yield return e; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return (IEnumerator)GetEnumerator(); + } + + public override string ToString() + { + return string.Format($"[{_groupRoot.Value}]"); + } + } + + public class GroupSet: IEnumerable + { + private List _groupSet = new List(); + // This keeps about the distinct numbers and in which group they are + private Dictionary _numbers = new Dictionary(); + private int _merges = 0; + + public int Groups => _groupSet.Count; + public int Numbers => _numbers.Count; + public int Merges => _merges; + + public void Add(Group group) + { + _groupSet.Add(group); + + foreach(int e in group) + { + if (!_numbers.ContainsKey(e)) + _numbers.Add(e, group); + else + throw new ApplicationException("Element is already assigned to a group"); + } + } + + public List FindInGroups(int from, int to) + { + var result = new List(); + + if (_numbers.ContainsKey(from)) result.Add(_numbers[from]); + if (_numbers.ContainsKey(to)) result.Add(_numbers[to]); + + return result; + } + + public void AddPair(int from, int to) + { + //Find if the inputValue is already in a group + var foundInGroups = FindInGroups(from, to); + + // no item matches: create a new group and add both the values to it + if (foundInGroups.Count == 0) + { + var ng = new Group(); + ng.AddIfNotExists(from); + ng.AddIfNotExists(to); + + _groupSet.Add(ng); + + if (!_numbers.ContainsKey(from)) _numbers.Add(from, ng); + if (!_numbers.ContainsKey(to)) _numbers.Add(to, ng); + } + + // one item match, add the related item to the same group + if (foundInGroups.Count == 1) + { + var g = foundInGroups[0]; + g.AddUnique(from, to); + + if (!_numbers.ContainsKey(from)) _numbers.Add(from, g); else _numbers[from] = g; + if (!_numbers.ContainsKey(to)) _numbers.Add(to, g); else _numbers[to] = g; + } + + // if there is a match for both items but in two different groups + // merge them into just one group and delete the other + if (foundInGroups.Count == 2) + { + var g1 = foundInGroups[0]; + var g2 = foundInGroups[1]; + + if (g1 == g2) return; + + // Always move the smaller group + if (g2.Count > g1.Count) + { + var t = g1; + g1 = g2; + g2 = t; + } + + if (!_numbers.ContainsKey(from)) _numbers.Add(from, g1); else _numbers[from] = g1; + if (!_numbers.ContainsKey(to)) _numbers.Add(to, g1); else _numbers[to] = g1; + + g1.MergeWith(g2); + + foreach(var e in g2) + { + if (!_numbers.ContainsKey(e)) _numbers.Add(e, g1); else _numbers[e] = g1; + } + + _merges += 1; + + _groupSet.Remove(g2); + } + } + + public IEnumerator GetEnumerator() + { + foreach(var g in _groupSet) + { + yield return g; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return (IEnumerator)GetEnumerator(); + } + } + + /// + /// Aggregate that takes a pair of numbers that represents an edge in some graph/relation. + /// As an output, returns groups of reachable edges, for example: + /// { + /// "0":[1,2,3,4], + /// "1":[5,6,7], + /// "2":[8,9] + /// } + /// + [Serializable] + [SqlUserDefinedAggregateAttribute(Format.UserDefined, MaxByteSize = -1)] + public class Aggregate : IBinarySerialize + { + private GroupSet _groupSet; + + public int Groups => _groupSet.Groups; + public int Numbers => _groupSet.Numbers; + public int Merges => _groupSet.Merges; + + public void Init() + { + _groupSet = new GroupSet(); + } + + public void Accumulate(int inputValue1, int inputValue2) + { + _groupSet.AddPair(inputValue1, inputValue2); + } + + public void Merge(Aggregate value) + { + foreach (var g in value._groupSet) + { + int? pe = null; + foreach (var ce in g) + { + if (pe.HasValue) + { + this.Accumulate(pe.Value, ce); + } + pe = ce; + } + } + } + + public SqlString Terminate() + { + return this.ToString(); + } + + public override string ToString() + { + int c = 0; + StringBuilder sb = new StringBuilder(); + sb.Append("{"); + foreach (var g in this._groupSet) + { + sb.Append("\"" + c + "\":["); + + var ea = new int[g.Elements.Count]; + g.Elements.CopyTo(ea, 0); + + sb.Append(string.Join(",", ea)); + + sb.Append("],"); + + c += 1; + } + if (sb.Length > 1) sb.Remove(sb.Length - 1, 1); + sb.Append("}"); + return sb.ToString(); + } + + public void Read(BinaryReader r) + { + if (r == null) throw new ArgumentNullException("r"); + _groupSet = new GroupSet(); + + // Group Count + int g = r.ReadInt32(); + + // For Each Group + for (int j = 0; j < g; j++) + { + var l = new Group(); + + // List Size (or Values Count) + int s = r.ReadInt32(); + + // Read values and put them in the list + for (int i = 0; i < s; i++) + { + l.Add(r.ReadInt32()); + } + + // Add list to dictionary + _groupSet.Add(l); + } + } + + public void Write(BinaryWriter w) + { + if (w == null) throw new ArgumentNullException("w IS NULL"); + + // Group count + w.Write(_groupSet.Groups); + + foreach (var g in _groupSet) + { + // Values Count + w.Write(g.Count); + + // Values + foreach (var e in g) + { + w.Write(e); + } + } + } + } +} \ No newline at end of file diff --git a/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregatorLibrary.csproj b/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregatorLibrary.csproj new file mode 100644 index 00000000..68a5f113 --- /dev/null +++ b/samples/features/sql-clr/TransitiveClosure/TransitiveClosureAggregatorLibrary.csproj @@ -0,0 +1,73 @@ + + + + + Debug + AnyCPU + {097EF341-926D-4DD2-A434-08E9980D6089} + Library + Properties + TransitiveClosureAggregatorLibrary + TransitiveClosureAggregatorLibrary + v4.5.2 + 512 + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + false + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + false + + + true + + + tcc.pfx + + + + + + + + + + + + + + + + + + True + True + TransitiveClosure.tt + + + TextTemplatingFileGenerator + TransitiveClosure.sql + + + + + + + + + + \ No newline at end of file diff --git a/samples/features/sql-clr/TransitiveClosure/tcc.pfx b/samples/features/sql-clr/TransitiveClosure/tcc.pfx new file mode 100644 index 00000000..12006e67 Binary files /dev/null and b/samples/features/sql-clr/TransitiveClosure/tcc.pfx differ