diff --git a/samples/features/unicode/DataType_WesternMyth.sql b/samples/features/unicode/DataType_WesternMyth.sql index 2ef4b9d9..cfd1a3cb 100644 --- a/samples/features/unicode/DataType_WesternMyth.sql +++ b/samples/features/unicode/DataType_WesternMyth.sql @@ -7,7 +7,13 @@ -- Test Latin character strings with Latin collation -- Set size limit of data types to be the same under Basic Multilingual Plane (BMP) -- Characters between 1-byte (ASCII) and 3-bytes (East Asian) +USE master; +DROP DATABASE IF EXISTS UnicodeDatabase; +CREATE DATABASE UnicodeDatabase COLLATE LATIN1_GENERAL_100_CI_AS_SC_UTF8 +GO +USE UnicodeDatabase +GO DROP TABLE IF EXISTS t1; CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, c2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); @@ -46,7 +52,8 @@ GO -- uh-oh data loss on the varchar example. Why? --- varchar is bound to code page enconding, and these code points cannot be found in the Latin code page. +-- varchar is bound to code page enconding, +-- and these code points cannot be found in the Latin code page. SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63) SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63) @@ -152,10 +159,10 @@ GO -- But the majority of my data is set to Latin (ASCII) DROP TABLE IF EXISTS t4; -CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC); +CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') -SELECT LEN(c1) AS [varchar UTF16 LEN], - DATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1 +SELECT LEN(c1) AS [varchar UTF8 LEN], + DATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1 FROM t4; GO @@ -164,8 +171,8 @@ GO -- Where are the savings? SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], DATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], - DATALENGTH(N'👶👦') AS [SC_UTF16_4bytes] + DATALENGTH(N'👶') AS [SC_UTF16_4bytes] SELECT DATALENGTH('MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Latin_UTF8_1byte], DATALENGTH('敏捷的棕色狐狸跳' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Chinese_UTF8_3bytes], - DATALENGTH('👶👦' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes] + DATALENGTH('👶' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes] GO \ No newline at end of file diff --git a/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb b/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb index 92a4160f..c89e0f88 100644 --- a/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb +++ b/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb @@ -19,7 +19,7 @@ "# Data type sizes - a western myth" ], "metadata": { - "azdata_cell_guid": "9e602698-2ef8-4fed-872f-5bad8c440600" + "azdata_cell_guid": "da283803-97b1-44ca-af3a-037151ed5cef" } }, { @@ -30,15 +30,17 @@ "**Note:** My server default is SQL_Latin1_General_CP1_CI_AS\r\n", "\r\n", "Set size limit of data types to be the same under Basic Multilingual Plane (BMP)\r\n", - "Characters between 1-byte (ASCII) and 3-bytes (East Asian)" + "Characters: ranging from 1-byte (ASCII) to 3-bytes (East Asian) per character. So a max of 24-bytes for an East Asian 8 character string." ], "metadata": { - "azdata_cell_guid": "78fc4a30-52e8-407d-8807-04097ad12348" + "azdata_cell_guid": "d96941dc-8fac-4dc9-9f1d-32465d2a9569" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t1;\r\n", "CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n", "\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n", @@ -52,7 +54,7 @@ "GO" ], "metadata": { - "azdata_cell_guid": "4a7b199e-3117-4fcb-b149-776f4da6cf71" + "azdata_cell_guid": "00084952-2310-40b3-8007-54ae10924a85" }, "outputs": [ { @@ -79,14 +81,13 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.068" + "text/html": "Total execution time: 00:00:00.1996267" }, "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, - "execution_count": 15, + "execution_count": 1, "data": { "application/vnd.dataresource+json": { "schema": { @@ -111,12 +112,12 @@ ] }, "text/html": "
varchar LENvarchar DATALENGTHc1
88MyString
" - } + }, + "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, - "execution_count": 15, + "execution_count": 1, "data": { "application/vnd.dataresource+json": { "schema": { @@ -141,7 +142,8 @@ ] }, "text/html": "
nvarchar LENnvarchar DATALENGTHc2
816MyString
" - } + }, + "metadata": {} } ], "execution_count": 15 @@ -149,10 +151,12 @@ { "cell_type": "markdown", "source": [ - "That's as expected. So what was I talking about?" + "That's as expected on bothe cases. So what was I talking about?\r\n", + "\r\n", + "Run next example with Chinese characters." ], "metadata": { - "azdata_cell_guid": "8615ddba-64e7-4abc-9df6-ec5648f39788" + "azdata_cell_guid": "5ecddf1f-4b0f-460f-aa48-a97bcf2e8e62" } }, { @@ -161,12 +165,14 @@ "# Test Chinese character strings with Latin collation" ], "metadata": { - "azdata_cell_guid": "4a85f77d-fd22-46a6-b7aa-5d6fca65dd99" + "azdata_cell_guid": "772bd4b9-0ae0-4f52-ab34-f82226be4bca" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t1;\r\n", "CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n", "\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n", @@ -180,7 +186,7 @@ "GO" ], "metadata": { - "azdata_cell_guid": "f5d3d422-588c-4cc9-9bbd-2664b392b0fb" + "azdata_cell_guid": "cf4eb5a4-e2ae-40a8-a1ed-d476cf67fb2d" }, "outputs": [ { @@ -213,7 +219,6 @@ }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 16, "data": { "application/vnd.dataresource+json": { @@ -239,11 +244,11 @@ ] }, "text/html": "
varchar LENvarchar DATALENGTHc1
88????????
" - } + }, + "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 16, "data": { "application/vnd.dataresource+json": { @@ -269,7 +274,8 @@ ] }, "text/html": "
nvarchar LENnvarchar DATALENGTHc2
816敏捷的棕色狐狸跳
" - } + }, + "metadata": {} } ], "execution_count": 16 @@ -279,22 +285,37 @@ "source": [ "uh-oh data loss on the varchar example. Why?\r\n", "\r\n", - "varchar is bound to code page enconding, and these code points cannot be found in the Latin code page." + "varchar is bound to code page enconding by default, and these code points cannot be found in the Latin code page.\r\n", + "\r\n", + "But why didn't it happen in the nvarchar example? \r\n", + "\r\n", + "These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP), and nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page.\r\n", + "\r\n", + "Run the next example:" ], "metadata": { - "azdata_cell_guid": "d77295e2-75eb-487e-861e-ca1634b90740" + "azdata_cell_guid": "934db15a-7c57-4eb3-ac0a-48cce569ee33" } }, { "cell_type": "code", "source": [ - "SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63)\r\n", - "SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63)" + "USE UnicodeDatabase\r\n", + "GO\r\n", + "SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63);\r\n", + "SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63);" ], "metadata": { - "azdata_cell_guid": "9dd9bce0-c073-42ec-9faa-8ebc142fd025" + "azdata_cell_guid": "fa541bfb-134a-4a14-bb5d-47bd2cc55c24" }, "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, { "output_type": "display_data", "data": { @@ -312,14 +333,14 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.046" + "text/html": "Total execution time: 00:00:00.017" }, "metadata": {} }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 3, + "execution_count": 4, "data": { "application/vnd.dataresource+json": { "schema": { @@ -339,13 +360,18 @@ } ] }, - "text/html": "
(No column name)(No column name)
63?
" + "text/html": [ + "", + "", + "", + "
(No column name)(No column name)
63?
" + ] } }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 3, + "execution_count": 4, "data": { "application/vnd.dataresource+json": { "schema": { @@ -365,35 +391,47 @@ } ] }, - "text/html": "
(No column name)(No column name)
63?
" + "text/html": [ + "", + "", + "", + "
(No column name)(No column name)
63?
" + ] } } ], - "execution_count": 3 + "execution_count": 4 }, { "cell_type": "markdown", "source": [ - "But why didn't it happen in the nvarchar example?\r\n", + "The ASCII function returns the ASCII code value of the leftmost character of a character expression. We know the Latin code page that's chosen can't represent a double-byte character, so it can only read the first byte, which is incorrectly translated to code point 63. Using the CHAR function, we see that the 63 code point is a question mark character. \r\n", "\r\n", - "These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP)\r\n", - "\r\n", - "nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page" + "Run the next example:" ], "metadata": { - "azdata_cell_guid": "75416bff-a350-4f27-9de5-7b407b59ac81" + "azdata_cell_guid": "ae7aeb4c-d9e7-474a-b35d-fe62fe514ce4" } }, { "cell_type": "code", "source": [ - "SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935)\r\n", - "SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463)" + "USE UnicodeDatabase\r\n", + "GO\r\n", + "SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935);\r\n", + "SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463);" ], "metadata": { - "azdata_cell_guid": "ffed888f-da51-4cad-a88e-b0ccf2a38f10" + "azdata_cell_guid": "c7bc9d86-a4bc-47d6-aaed-967bd11ba378" }, "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, { "output_type": "display_data", "data": { @@ -411,7 +449,7 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.021" + "text/html": "Total execution time: 00:00:00.024" }, "metadata": {} }, @@ -438,7 +476,12 @@ } ] }, - "text/html": "
(No column name)(No column name)
25935
" + "text/html": [ + "", + "", + "", + "
(No column name)(No column name)
25935
" + ] } }, { @@ -464,7 +507,12 @@ } ] }, - "text/html": "
(No column name)(No column name)
25463
" + "text/html": [ + "", + "", + "", + "
(No column name)(No column name)
25463
" + ] } } ], @@ -473,110 +521,26 @@ { "cell_type": "markdown", "source": [ - "Irrespective of collation now. With a Unicode capable data type, collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase)" + "Works irrespective of collation now. By adding the N prefix we force the use of a [Unicode constant](https://docs.microsoft.com/sql/t-sql/data-types/constants-transact-sql#unicode-strings), and collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase), not the encoding. Using the UNICODE function, we can correctly identify the code points for the string characters, which the NCHAR function can represent accurately." ], "metadata": { - "azdata_cell_guid": "d6db32d2-033f-4800-a492-3494b65dbc36" + "azdata_cell_guid": "34ae3b63-602a-41c7-9b12-d8649e7e6e10" } }, - { - "cell_type": "code", - "source": [ - "SELECT UNICODE(N'敏' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25935)\r\n", - "SELECT UNICODE(N'捷' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25463)" - ], - "metadata": { - "azdata_cell_guid": "ba6107ed-50d8-4e3f-8e59-3eb1cb56e983" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/html": "(1 row affected)" - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/html": "(1 row affected)" - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/html": "Total execution time: 00:00:00.039" - }, - "metadata": {} - }, - { - "output_type": "execute_result", - "metadata": {}, - "execution_count": 9, - "data": { - "application/vnd.dataresource+json": { - "schema": { - "fields": [ - { - "name": "(No column name)" - }, - { - "name": "(No column name)" - } - ] - }, - "data": [ - { - "0": "25935", - "1": "敏" - } - ] - }, - "text/html": "
(No column name)(No column name)
25935
" - } - }, - { - "output_type": "execute_result", - "metadata": {}, - "execution_count": 9, - "data": { - "application/vnd.dataresource+json": { - "schema": { - "fields": [ - { - "name": "(No column name)" - }, - { - "name": "(No column name)" - } - ] - }, - "data": [ - { - "0": "25463", - "1": "捷" - } - ] - }, - "text/html": "
(No column name)(No column name)
25463
" - } - } - ], - "execution_count": 9 - }, { "cell_type": "markdown", "source": [ "## Now test Chinese character strings with Chinese collation" ], "metadata": { - "azdata_cell_guid": "041d9858-808d-4a6b-9bc7-302feddf4036" + "azdata_cell_guid": "bebfcf18-b6d8-4131-a783-51ebb3dfe4dd" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t2;\r\n", "CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI, \r\n", "\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI); \r\n", @@ -590,7 +554,7 @@ "GO" ], "metadata": { - "azdata_cell_guid": "d8fecb5c-9120-46b5-b34c-d75fc8664a80" + "azdata_cell_guid": "811086b3-779f-4d52-af63-c299830cd53d" }, "outputs": [ { @@ -623,7 +587,6 @@ }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 17, "data": { "application/vnd.dataresource+json": { @@ -649,11 +612,11 @@ ] }, "text/html": "
varchar LENvarchar DATALENGTHc1
816敏捷的棕色狐狸跳
" - } + }, + "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 17, "data": { "application/vnd.dataresource+json": { @@ -679,7 +642,8 @@ ] }, "text/html": "
nvarchar LENnvarchar DATALENGTHc2
816敏捷的棕色狐狸跳
" - } + }, + "metadata": {} } ], "execution_count": 17 @@ -687,15 +651,15 @@ { "cell_type": "markdown", "source": [ - "Now the varchar example is correct. But there's 2 bytes per character?...\r\n", + "Now the varchar example is correct because the code page can recognize Chinese characters. But there's 2 bytes per character, not 3?...\r\n", "\r\n", - "**Myth buster:** code page defines string length for varchar. It's not always 1 byte per character. \r\n", + "**Myth buster:** code page defines string length for varchar. Varchar is **not** always 1 byte per character. \r\n", "\r\n", - "Wasn't East-Asian 3 bytes? Yes, on UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n", + "Ok, but wasn't East-Asian 3 bytes? Yes, with UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n", "" ], "metadata": { - "azdata_cell_guid": "0bb36f33-1fb1-499e-97a9-a071f5332eba" + "azdata_cell_guid": "2ea7aa07-674a-407f-9bc3-66d0632c16e5" } }, { @@ -704,12 +668,14 @@ "## Test with Supplementary Characters (4 bytes)" ], "metadata": { - "azdata_cell_guid": "cdc78270-6de2-4bb9-9c49-8ac7aae92b8b" + "azdata_cell_guid": "c54c5ec7-a037-4ace-be26-6f773b600790" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t2;\r\n", "CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n", "\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n", @@ -723,13 +689,13 @@ "GO" ], "metadata": { - "azdata_cell_guid": "c8fa69ca-590b-414c-88eb-d4892816d324" + "azdata_cell_guid": "32789381-bee3-4c05-9247-fea7dbbb6f6b" }, "outputs": [ { "output_type": "error", - "evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.", "ename": "", + "evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.", "traceback": [] }, { @@ -762,7 +728,6 @@ }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 20, "data": { "application/vnd.dataresource+json": { @@ -782,11 +747,11 @@ "data": [] }, "text/html": "
varchar LENvarchar DATALENGTHc1
" - } + }, + "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 20, "data": { "application/vnd.dataresource+json": { @@ -806,7 +771,8 @@ "data": [] }, "text/html": "
nvarchar LENnvarchar DATALENGTHc2
" - } + }, + "metadata": {} } ], "execution_count": 20 @@ -814,15 +780,17 @@ { "cell_type": "markdown", "source": [ - "uh-oh, let's set the proper data type length" + "uh-oh, let's set the proper data type length from 8 to 16 byte-pairs (so a 32-byte encoding limit)" ], "metadata": { - "azdata_cell_guid": "0c643441-2b73-4880-82d7-0b019c14416c" + "azdata_cell_guid": "449fa293-8edd-41ea-b9b7-62d9af611a2a" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t2;\r\n", "CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n", "\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n", @@ -836,7 +804,7 @@ "GO" ], "metadata": { - "azdata_cell_guid": "ba341c74-d2ca-4477-82f2-0c39bcc38af7" + "azdata_cell_guid": "22e7a937-f154-4b21-988b-0e62d349b98a" }, "outputs": [ { @@ -869,7 +837,6 @@ }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 21, "data": { "application/vnd.dataresource+json": { @@ -895,11 +862,11 @@ ] }, "text/html": "
varchar LENvarchar DATALENGTHc1
1616????????????????
" - } + }, + "metadata": {} }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 21, "data": { "application/vnd.dataresource+json": { @@ -925,7 +892,8 @@ ] }, "text/html": "
nvarchar LENnvarchar DATALENGTHc2
832👶👦👧👨👩👴👵👨
" - } + }, + "metadata": {} } ], "execution_count": 21 @@ -933,16 +901,19 @@ { "cell_type": "markdown", "source": [ + "Nvarchar looks good. But varchar still doesn't encode? \r\n", "\r\n", - "Varchar still doesn't encode? " + "Set a larger data type length. For example double from 24 to 48 bytes. Now try again:" ], "metadata": { - "azdata_cell_guid": "d062a6e7-2a91-48d4-b81b-0bfb530df450" + "azdata_cell_guid": "3fa9b70f-57f1-4333-ad8c-23cd5b08de15" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t2;\r\n", "CREATE TABLE t2 (c1 varchar(48) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC_UTF8, \r\n", "\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n", @@ -956,9 +927,16 @@ "GO" ], "metadata": { - "azdata_cell_guid": "d8082437-485c-4c9d-84ca-16acd6e63849" + "azdata_cell_guid": "682a2d5f-43b4-4721-b5b8-6ddfea6f3b91" }, "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, { "output_type": "display_data", "data": { @@ -983,14 +961,14 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.071" + "text/html": "Total execution time: 00:00:00.033" }, "metadata": {} }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 22, + "execution_count": 3, "data": { "application/vnd.dataresource+json": { "schema": { @@ -1014,13 +992,18 @@ } ] }, - "text/html": "
varchar LENvarchar DATALENGTHc1
832👶👦👧👨👩👴👵👨
" + "text/html": [ + "", + "", + "", + "
varchar LENvarchar DATALENGTHc1
832👶👦👧👨👩👴👵👨
" + ] } }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 22, + "execution_count": 3, "data": { "application/vnd.dataresource+json": { "schema": { @@ -1044,26 +1027,33 @@ } ] }, - "text/html": "
nvarchar LENnvarchar DATALENGTHc2
832👶👦👧👨👩👴👵👨
" + "text/html": [ + "", + "", + "", + "
nvarchar LENnvarchar DATALENGTHc2
832👶👦👧👨👩👴👵👨
" + ] } } ], - "execution_count": 22 + "execution_count": 3 }, { "cell_type": "markdown", "source": [ "Finally!\r\n", "\r\n", - "What if I needed all these in one database? Easy, I could just use nvarchar." + "What if I needed all these characters in one database? Easy, I could just use nvarchar which encodes in UTF-16." ], "metadata": { - "azdata_cell_guid": "544ee63a-67fa-4b24-b974-4fa4e809949c" + "azdata_cell_guid": "2787fdcd-117c-47c5-962d-1fcf6f51df92" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t3;\r\n", "CREATE TABLE t3 (c1 nvarchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n", "INSERT INTO t3 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n", @@ -1073,7 +1063,7 @@ "GO" ], "metadata": { - "azdata_cell_guid": "87e6c383-d9ab-48da-a4d7-f1cc0cf3aee3" + "azdata_cell_guid": "da196747-8ef7-4844-895e-7fbf3496e7f0" }, "outputs": [ { @@ -1099,7 +1089,6 @@ }, { "output_type": "execute_result", - "metadata": {}, "execution_count": 23, "data": { "application/vnd.dataresource+json": { @@ -1125,7 +1114,8 @@ ] }, "text/html": "
nvarchar UTF16 LENnvarchar UTF16 DATALENGTHc1
65134MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦
" - } + }, + "metadata": {} } ], "execution_count": 23 @@ -1133,27 +1123,36 @@ { "cell_type": "markdown", "source": [ - "But the majority of my data is set to Latin (ASCII)" + "But wait. The majority of my data is set to Latin (ASCII), can we do better?" ], "metadata": { - "azdata_cell_guid": "808dc693-ca7b-4ee5-b71f-9cf9cbcef31f" + "azdata_cell_guid": "7c0e540f-265c-4e58-844d-14b51d731a6b" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "DROP TABLE IF EXISTS t4;\r\n", - "CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n", + "CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); \r\n", "INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n", - "SELECT LEN(c1) AS [varchar UTF16 LEN], \r\n", - "\tDATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1\r\n", + "SELECT LEN(c1) AS [varchar UTF8 LEN], \r\n", + "\tDATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1\r\n", "FROM t4; \r\n", "GO" ], "metadata": { - "azdata_cell_guid": "8597dfb2-ec28-4c82-865a-ec7a8d01c663" + "azdata_cell_guid": "89f77817-2cf3-493b-9d80-64d229103c70" }, "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, { "output_type": "display_data", "data": { @@ -1171,23 +1170,23 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.013" + "text/html": "Total execution time: 00:00:00.253" }, "metadata": {} }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 24, + "execution_count": 1, "data": { "application/vnd.dataresource+json": { "schema": { "fields": [ { - "name": "varchar UTF16 LEN" + "name": "varchar UTF8 LEN" }, { - "name": "varchar UTF16 DATALENGTH" + "name": "varchar UTF8 DATALENGTH" }, { "name": "c1" @@ -1196,30 +1195,37 @@ }, "data": [ { - "0": "67", - "1": "67", - "2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii????????????" + "0": "65", + "1": "87", + "2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦" } ] }, - "text/html": "
varchar UTF16 LENvarchar UTF16 DATALENGTHc1
6767MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii????????????
" + "text/html": [ + "", + "", + "", + "
varchar UTF8 LENvarchar UTF8 DATALENGTHc1
6587MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦
" + ] } } ], - "execution_count": 24 + "execution_count": 1 }, { "cell_type": "markdown", "source": [ - "Where are the savings?" + "With this data pattern the savings are obvious. Where are the savings? Let's compare breaking down to individual Latin, Chinese, and Emoji strings." ], "metadata": { - "azdata_cell_guid": "e8d09b41-c69d-43de-8b67-43ad7e25bb03" + "azdata_cell_guid": "db799749-7950-4b3b-a1e2-074e80b40aba" } }, { "cell_type": "code", "source": [ + "USE UnicodeDatabase\r\n", + "GO\r\n", "SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], \r\n", "\tDATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], \r\n", "\tDATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]\r\n", @@ -1229,9 +1235,16 @@ "GO" ], "metadata": { - "azdata_cell_guid": "79351524-1cba-412a-8111-2cc93871612d" + "azdata_cell_guid": "32f0463d-2224-4781-b2d1-76a6e0fd0b37" }, "outputs": [ + { + "output_type": "display_data", + "data": { + "text/html": "Commands completed successfully." + }, + "metadata": {} + }, { "output_type": "display_data", "data": { @@ -1249,14 +1262,14 @@ { "output_type": "display_data", "data": { - "text/html": "Total execution time: 00:00:00.043" + "text/html": "Total execution time: 00:00:00.024" }, "metadata": {} }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 25, + "execution_count": 2, "data": { "application/vnd.dataresource+json": { "schema": { @@ -1280,13 +1293,18 @@ } ] }, - "text/html": "
Latin_UTF16_2bytesChinese_UTF16_2bytesSC_UTF16_4bytes
110168
" + "text/html": [ + "", + "", + "", + "
Latin_UTF16_2bytesChinese_UTF16_2bytesSC_UTF16_4bytes
110168
" + ] } }, { "output_type": "execute_result", "metadata": {}, - "execution_count": 25, + "execution_count": 2, "data": { "application/vnd.dataresource+json": { "schema": { @@ -1305,16 +1323,21 @@ "data": [ { "0": "55", - "1": "8", - "2": "4" + "1": "24", + "2": "8" } ] }, - "text/html": "
Latin_UTF8_1byteChinese_UTF8_3bytesSC_UTF8_4bytes
5584
" + "text/html": [ + "", + "", + "", + "
Latin_UTF8_1byteChinese_UTF8_3bytesSC_UTF8_4bytes
55248
" + ] } } ], - "execution_count": 25 + "execution_count": 2 } ] } \ No newline at end of file