diff --git a/samples/features/unicode/DataType_WesternMyth.sql b/samples/features/unicode/DataType_WesternMyth.sql
index 2ef4b9d9..cfd1a3cb 100644
--- a/samples/features/unicode/DataType_WesternMyth.sql
+++ b/samples/features/unicode/DataType_WesternMyth.sql
@@ -7,7 +7,13 @@
-- Test Latin character strings with Latin collation
-- Set size limit of data types to be the same under Basic Multilingual Plane (BMP)
-- Characters between 1-byte (ASCII) and 3-bytes (East Asian)
+USE master;
+DROP DATABASE IF EXISTS UnicodeDatabase;
+CREATE DATABASE UnicodeDatabase COLLATE LATIN1_GENERAL_100_CI_AS_SC_UTF8
+GO
+USE UnicodeDatabase
+GO
DROP TABLE IF EXISTS t1;
CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI,
c2 nvarchar(8) COLLATE Latin1_General_100_CI_AI);
@@ -46,7 +52,8 @@ GO
-- uh-oh data loss on the varchar example. Why?
--- varchar is bound to code page enconding, and these code points cannot be found in the Latin code page.
+-- varchar is bound to code page enconding,
+-- and these code points cannot be found in the Latin code page.
SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63)
SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63)
@@ -152,10 +159,10 @@ GO
-- But the majority of my data is set to Latin (ASCII)
DROP TABLE IF EXISTS t4;
-CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC);
+CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8);
INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦')
-SELECT LEN(c1) AS [varchar UTF16 LEN],
- DATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1
+SELECT LEN(c1) AS [varchar UTF8 LEN],
+ DATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1
FROM t4;
GO
@@ -164,8 +171,8 @@ GO
-- Where are the savings?
SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes],
DATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes],
- DATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]
+ DATALENGTH(N'👶') AS [SC_UTF16_4bytes]
SELECT DATALENGTH('MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Latin_UTF8_1byte],
DATALENGTH('敏捷的棕色狐狸跳' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Chinese_UTF8_3bytes],
- DATALENGTH('👶👦' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]
+ DATALENGTH('👶' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]
GO
\ No newline at end of file
diff --git a/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb b/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb
index 92a4160f..c89e0f88 100644
--- a/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb
+++ b/samples/features/unicode/notebooks/DataType_WesternMyth.ipynb
@@ -19,7 +19,7 @@
"# Data type sizes - a western myth"
],
"metadata": {
- "azdata_cell_guid": "9e602698-2ef8-4fed-872f-5bad8c440600"
+ "azdata_cell_guid": "da283803-97b1-44ca-af3a-037151ed5cef"
}
},
{
@@ -30,15 +30,17 @@
"**Note:** My server default is SQL_Latin1_General_CP1_CI_AS\r\n",
"\r\n",
"Set size limit of data types to be the same under Basic Multilingual Plane (BMP)\r\n",
- "Characters between 1-byte (ASCII) and 3-bytes (East Asian)"
+ "Characters: ranging from 1-byte (ASCII) to 3-bytes (East Asian) per character. So a max of 24-bytes for an East Asian 8 character string."
],
"metadata": {
- "azdata_cell_guid": "78fc4a30-52e8-407d-8807-04097ad12348"
+ "azdata_cell_guid": "d96941dc-8fac-4dc9-9f1d-32465d2a9569"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t1;\r\n",
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
@@ -52,7 +54,7 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "4a7b199e-3117-4fcb-b149-776f4da6cf71"
+ "azdata_cell_guid": "00084952-2310-40b3-8007-54ae10924a85"
},
"outputs": [
{
@@ -79,14 +81,13 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.068"
+ "text/html": "Total execution time: 00:00:00.1996267"
},
"metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
- "execution_count": 15,
+ "execution_count": 1,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -111,12 +112,12 @@
]
},
"text/html": "
| varchar LEN | varchar DATALENGTH | c1 |
|---|
| 8 | 8 | MyString |
"
- }
+ },
+ "metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
- "execution_count": 15,
+ "execution_count": 1,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -141,7 +142,8 @@
]
},
"text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
| 8 | 16 | MyString |
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 15
@@ -149,10 +151,12 @@
{
"cell_type": "markdown",
"source": [
- "That's as expected. So what was I talking about?"
+ "That's as expected on bothe cases. So what was I talking about?\r\n",
+ "\r\n",
+ "Run next example with Chinese characters."
],
"metadata": {
- "azdata_cell_guid": "8615ddba-64e7-4abc-9df6-ec5648f39788"
+ "azdata_cell_guid": "5ecddf1f-4b0f-460f-aa48-a97bcf2e8e62"
}
},
{
@@ -161,12 +165,14 @@
"# Test Chinese character strings with Latin collation"
],
"metadata": {
- "azdata_cell_guid": "4a85f77d-fd22-46a6-b7aa-5d6fca65dd99"
+ "azdata_cell_guid": "772bd4b9-0ae0-4f52-ab34-f82226be4bca"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t1;\r\n",
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
@@ -180,7 +186,7 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "f5d3d422-588c-4cc9-9bbd-2664b392b0fb"
+ "azdata_cell_guid": "cf4eb5a4-e2ae-40a8-a1ed-d476cf67fb2d"
},
"outputs": [
{
@@ -213,7 +219,6 @@
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 16,
"data": {
"application/vnd.dataresource+json": {
@@ -239,11 +244,11 @@
]
},
"text/html": "| varchar LEN | varchar DATALENGTH | c1 |
|---|
| 8 | 8 | ???????? |
"
- }
+ },
+ "metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 16,
"data": {
"application/vnd.dataresource+json": {
@@ -269,7 +274,8 @@
]
},
"text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
| 8 | 16 | 敏捷的棕色狐狸跳 |
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 16
@@ -279,22 +285,37 @@
"source": [
"uh-oh data loss on the varchar example. Why?\r\n",
"\r\n",
- "varchar is bound to code page enconding, and these code points cannot be found in the Latin code page."
+ "varchar is bound to code page enconding by default, and these code points cannot be found in the Latin code page.\r\n",
+ "\r\n",
+ "But why didn't it happen in the nvarchar example? \r\n",
+ "\r\n",
+ "These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP), and nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page.\r\n",
+ "\r\n",
+ "Run the next example:"
],
"metadata": {
- "azdata_cell_guid": "d77295e2-75eb-487e-861e-ca1634b90740"
+ "azdata_cell_guid": "934db15a-7c57-4eb3-ac0a-48cce569ee33"
}
},
{
"cell_type": "code",
"source": [
- "SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63)\r\n",
- "SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63)"
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
+ "SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63);\r\n",
+ "SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63);"
],
"metadata": {
- "azdata_cell_guid": "9dd9bce0-c073-42ec-9faa-8ebc142fd025"
+ "azdata_cell_guid": "fa541bfb-134a-4a14-bb5d-47bd2cc55c24"
},
"outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": "Commands completed successfully."
+ },
+ "metadata": {}
+ },
{
"output_type": "display_data",
"data": {
@@ -312,14 +333,14 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.046"
+ "text/html": "Total execution time: 00:00:00.017"
},
"metadata": {}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 3,
+ "execution_count": 4,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -339,13 +360,18 @@
}
]
},
- "text/html": "| (No column name) | (No column name) |
|---|
| 63 | ? |
"
+ "text/html": [
+ "",
+ "| (No column name) | (No column name) |
",
+ "| 63 | ? |
",
+ "
"
+ ]
}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 3,
+ "execution_count": 4,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -365,35 +391,47 @@
}
]
},
- "text/html": "| (No column name) | (No column name) |
|---|
| 63 | ? |
"
+ "text/html": [
+ "",
+ "| (No column name) | (No column name) |
",
+ "| 63 | ? |
",
+ "
"
+ ]
}
}
],
- "execution_count": 3
+ "execution_count": 4
},
{
"cell_type": "markdown",
"source": [
- "But why didn't it happen in the nvarchar example?\r\n",
+ "The ASCII function returns the ASCII code value of the leftmost character of a character expression. We know the Latin code page that's chosen can't represent a double-byte character, so it can only read the first byte, which is incorrectly translated to code point 63. Using the CHAR function, we see that the 63 code point is a question mark character. \r\n",
"\r\n",
- "These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP)\r\n",
- "\r\n",
- "nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page"
+ "Run the next example:"
],
"metadata": {
- "azdata_cell_guid": "75416bff-a350-4f27-9de5-7b407b59ac81"
+ "azdata_cell_guid": "ae7aeb4c-d9e7-474a-b35d-fe62fe514ce4"
}
},
{
"cell_type": "code",
"source": [
- "SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935)\r\n",
- "SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463)"
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
+ "SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935);\r\n",
+ "SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463);"
],
"metadata": {
- "azdata_cell_guid": "ffed888f-da51-4cad-a88e-b0ccf2a38f10"
+ "azdata_cell_guid": "c7bc9d86-a4bc-47d6-aaed-967bd11ba378"
},
"outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": "Commands completed successfully."
+ },
+ "metadata": {}
+ },
{
"output_type": "display_data",
"data": {
@@ -411,7 +449,7 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.021"
+ "text/html": "Total execution time: 00:00:00.024"
},
"metadata": {}
},
@@ -438,7 +476,12 @@
}
]
},
- "text/html": "| (No column name) | (No column name) |
|---|
| 25935 | 敏 |
"
+ "text/html": [
+ "",
+ "| (No column name) | (No column name) |
",
+ "| 25935 | 敏 |
",
+ "
"
+ ]
}
},
{
@@ -464,7 +507,12 @@
}
]
},
- "text/html": "| (No column name) | (No column name) |
|---|
| 25463 | 捷 |
"
+ "text/html": [
+ "",
+ "| (No column name) | (No column name) |
",
+ "| 25463 | 捷 |
",
+ "
"
+ ]
}
}
],
@@ -473,110 +521,26 @@
{
"cell_type": "markdown",
"source": [
- "Irrespective of collation now. With a Unicode capable data type, collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase)"
+ "Works irrespective of collation now. By adding the N prefix we force the use of a [Unicode constant](https://docs.microsoft.com/sql/t-sql/data-types/constants-transact-sql#unicode-strings), and collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase), not the encoding. Using the UNICODE function, we can correctly identify the code points for the string characters, which the NCHAR function can represent accurately."
],
"metadata": {
- "azdata_cell_guid": "d6db32d2-033f-4800-a492-3494b65dbc36"
+ "azdata_cell_guid": "34ae3b63-602a-41c7-9b12-d8649e7e6e10"
}
},
- {
- "cell_type": "code",
- "source": [
- "SELECT UNICODE(N'敏' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25935)\r\n",
- "SELECT UNICODE(N'捷' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25463)"
- ],
- "metadata": {
- "azdata_cell_guid": "ba6107ed-50d8-4e3f-8e59-3eb1cb56e983"
- },
- "outputs": [
- {
- "output_type": "display_data",
- "data": {
- "text/html": "(1 row affected)"
- },
- "metadata": {}
- },
- {
- "output_type": "display_data",
- "data": {
- "text/html": "(1 row affected)"
- },
- "metadata": {}
- },
- {
- "output_type": "display_data",
- "data": {
- "text/html": "Total execution time: 00:00:00.039"
- },
- "metadata": {}
- },
- {
- "output_type": "execute_result",
- "metadata": {},
- "execution_count": 9,
- "data": {
- "application/vnd.dataresource+json": {
- "schema": {
- "fields": [
- {
- "name": "(No column name)"
- },
- {
- "name": "(No column name)"
- }
- ]
- },
- "data": [
- {
- "0": "25935",
- "1": "敏"
- }
- ]
- },
- "text/html": "| (No column name) | (No column name) |
|---|
| 25935 | 敏 |
"
- }
- },
- {
- "output_type": "execute_result",
- "metadata": {},
- "execution_count": 9,
- "data": {
- "application/vnd.dataresource+json": {
- "schema": {
- "fields": [
- {
- "name": "(No column name)"
- },
- {
- "name": "(No column name)"
- }
- ]
- },
- "data": [
- {
- "0": "25463",
- "1": "捷"
- }
- ]
- },
- "text/html": "| (No column name) | (No column name) |
|---|
| 25463 | 捷 |
"
- }
- }
- ],
- "execution_count": 9
- },
{
"cell_type": "markdown",
"source": [
"## Now test Chinese character strings with Chinese collation"
],
"metadata": {
- "azdata_cell_guid": "041d9858-808d-4a6b-9bc7-302feddf4036"
+ "azdata_cell_guid": "bebfcf18-b6d8-4131-a783-51ebb3dfe4dd"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t2;\r\n",
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI, \r\n",
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI); \r\n",
@@ -590,7 +554,7 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "d8fecb5c-9120-46b5-b34c-d75fc8664a80"
+ "azdata_cell_guid": "811086b3-779f-4d52-af63-c299830cd53d"
},
"outputs": [
{
@@ -623,7 +587,6 @@
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 17,
"data": {
"application/vnd.dataresource+json": {
@@ -649,11 +612,11 @@
]
},
"text/html": "| varchar LEN | varchar DATALENGTH | c1 |
|---|
| 8 | 16 | 敏捷的棕色狐狸跳 |
"
- }
+ },
+ "metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 17,
"data": {
"application/vnd.dataresource+json": {
@@ -679,7 +642,8 @@
]
},
"text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
| 8 | 16 | 敏捷的棕色狐狸跳 |
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 17
@@ -687,15 +651,15 @@
{
"cell_type": "markdown",
"source": [
- "Now the varchar example is correct. But there's 2 bytes per character?...\r\n",
+ "Now the varchar example is correct because the code page can recognize Chinese characters. But there's 2 bytes per character, not 3?...\r\n",
"\r\n",
- "**Myth buster:** code page defines string length for varchar. It's not always 1 byte per character. \r\n",
+ "**Myth buster:** code page defines string length for varchar. Varchar is **not** always 1 byte per character. \r\n",
"\r\n",
- "Wasn't East-Asian 3 bytes? Yes, on UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
+ "Ok, but wasn't East-Asian 3 bytes? Yes, with UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
""
],
"metadata": {
- "azdata_cell_guid": "0bb36f33-1fb1-499e-97a9-a071f5332eba"
+ "azdata_cell_guid": "2ea7aa07-674a-407f-9bc3-66d0632c16e5"
}
},
{
@@ -704,12 +668,14 @@
"## Test with Supplementary Characters (4 bytes)"
],
"metadata": {
- "azdata_cell_guid": "cdc78270-6de2-4bb9-9c49-8ac7aae92b8b"
+ "azdata_cell_guid": "c54c5ec7-a037-4ace-be26-6f773b600790"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t2;\r\n",
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
@@ -723,13 +689,13 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "c8fa69ca-590b-414c-88eb-d4892816d324"
+ "azdata_cell_guid": "32789381-bee3-4c05-9247-fea7dbbb6f6b"
},
"outputs": [
{
"output_type": "error",
- "evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.",
"ename": "",
+ "evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.",
"traceback": []
},
{
@@ -762,7 +728,6 @@
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 20,
"data": {
"application/vnd.dataresource+json": {
@@ -782,11 +747,11 @@
"data": []
},
"text/html": "| varchar LEN | varchar DATALENGTH | c1 |
|---|
"
- }
+ },
+ "metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 20,
"data": {
"application/vnd.dataresource+json": {
@@ -806,7 +771,8 @@
"data": []
},
"text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 20
@@ -814,15 +780,17 @@
{
"cell_type": "markdown",
"source": [
- "uh-oh, let's set the proper data type length"
+ "uh-oh, let's set the proper data type length from 8 to 16 byte-pairs (so a 32-byte encoding limit)"
],
"metadata": {
- "azdata_cell_guid": "0c643441-2b73-4880-82d7-0b019c14416c"
+ "azdata_cell_guid": "449fa293-8edd-41ea-b9b7-62d9af611a2a"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t2;\r\n",
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
@@ -836,7 +804,7 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "ba341c74-d2ca-4477-82f2-0c39bcc38af7"
+ "azdata_cell_guid": "22e7a937-f154-4b21-988b-0e62d349b98a"
},
"outputs": [
{
@@ -869,7 +837,6 @@
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 21,
"data": {
"application/vnd.dataresource+json": {
@@ -895,11 +862,11 @@
]
},
"text/html": "| varchar LEN | varchar DATALENGTH | c1 |
|---|
| 16 | 16 | ???????????????? |
"
- }
+ },
+ "metadata": {}
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 21,
"data": {
"application/vnd.dataresource+json": {
@@ -925,7 +892,8 @@
]
},
"text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
| 8 | 32 | 👶👦👧👨👩👴👵👨 |
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 21
@@ -933,16 +901,19 @@
{
"cell_type": "markdown",
"source": [
+ "Nvarchar looks good. But varchar still doesn't encode? \r\n",
"\r\n",
- "Varchar still doesn't encode? "
+ "Set a larger data type length. For example double from 24 to 48 bytes. Now try again:"
],
"metadata": {
- "azdata_cell_guid": "d062a6e7-2a91-48d4-b81b-0bfb530df450"
+ "azdata_cell_guid": "3fa9b70f-57f1-4333-ad8c-23cd5b08de15"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t2;\r\n",
"CREATE TABLE t2 (c1 varchar(48) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC_UTF8, \r\n",
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
@@ -956,9 +927,16 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "d8082437-485c-4c9d-84ca-16acd6e63849"
+ "azdata_cell_guid": "682a2d5f-43b4-4721-b5b8-6ddfea6f3b91"
},
"outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": "Commands completed successfully."
+ },
+ "metadata": {}
+ },
{
"output_type": "display_data",
"data": {
@@ -983,14 +961,14 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.071"
+ "text/html": "Total execution time: 00:00:00.033"
},
"metadata": {}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 22,
+ "execution_count": 3,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -1014,13 +992,18 @@
}
]
},
- "text/html": "| varchar LEN | varchar DATALENGTH | c1 |
|---|
| 8 | 32 | 👶👦👧👨👩👴👵👨 |
"
+ "text/html": [
+ "",
+ "| varchar LEN | varchar DATALENGTH | c1 |
",
+ "| 8 | 32 | 👶👦👧👨👩👴👵👨 |
",
+ "
"
+ ]
}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 22,
+ "execution_count": 3,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -1044,26 +1027,33 @@
}
]
},
- "text/html": "| nvarchar LEN | nvarchar DATALENGTH | c2 |
|---|
| 8 | 32 | 👶👦👧👨👩👴👵👨 |
"
+ "text/html": [
+ "",
+ "| nvarchar LEN | nvarchar DATALENGTH | c2 |
",
+ "| 8 | 32 | 👶👦👧👨👩👴👵👨 |
",
+ "
"
+ ]
}
}
],
- "execution_count": 22
+ "execution_count": 3
},
{
"cell_type": "markdown",
"source": [
"Finally!\r\n",
"\r\n",
- "What if I needed all these in one database? Easy, I could just use nvarchar."
+ "What if I needed all these characters in one database? Easy, I could just use nvarchar which encodes in UTF-16."
],
"metadata": {
- "azdata_cell_guid": "544ee63a-67fa-4b24-b974-4fa4e809949c"
+ "azdata_cell_guid": "2787fdcd-117c-47c5-962d-1fcf6f51df92"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t3;\r\n",
"CREATE TABLE t3 (c1 nvarchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n",
"INSERT INTO t3 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
@@ -1073,7 +1063,7 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "87e6c383-d9ab-48da-a4d7-f1cc0cf3aee3"
+ "azdata_cell_guid": "da196747-8ef7-4844-895e-7fbf3496e7f0"
},
"outputs": [
{
@@ -1099,7 +1089,6 @@
},
{
"output_type": "execute_result",
- "metadata": {},
"execution_count": 23,
"data": {
"application/vnd.dataresource+json": {
@@ -1125,7 +1114,8 @@
]
},
"text/html": "| nvarchar UTF16 LEN | nvarchar UTF16 DATALENGTH | c1 |
|---|
| 65 | 134 | MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦 |
"
- }
+ },
+ "metadata": {}
}
],
"execution_count": 23
@@ -1133,27 +1123,36 @@
{
"cell_type": "markdown",
"source": [
- "But the majority of my data is set to Latin (ASCII)"
+ "But wait. The majority of my data is set to Latin (ASCII), can we do better?"
],
"metadata": {
- "azdata_cell_guid": "808dc693-ca7b-4ee5-b71f-9cf9cbcef31f"
+ "azdata_cell_guid": "7c0e540f-265c-4e58-844d-14b51d731a6b"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"DROP TABLE IF EXISTS t4;\r\n",
- "CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n",
+ "CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); \r\n",
"INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
- "SELECT LEN(c1) AS [varchar UTF16 LEN], \r\n",
- "\tDATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1\r\n",
+ "SELECT LEN(c1) AS [varchar UTF8 LEN], \r\n",
+ "\tDATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1\r\n",
"FROM t4; \r\n",
"GO"
],
"metadata": {
- "azdata_cell_guid": "8597dfb2-ec28-4c82-865a-ec7a8d01c663"
+ "azdata_cell_guid": "89f77817-2cf3-493b-9d80-64d229103c70"
},
"outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": "Commands completed successfully."
+ },
+ "metadata": {}
+ },
{
"output_type": "display_data",
"data": {
@@ -1171,23 +1170,23 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.013"
+ "text/html": "Total execution time: 00:00:00.253"
},
"metadata": {}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 24,
+ "execution_count": 1,
"data": {
"application/vnd.dataresource+json": {
"schema": {
"fields": [
{
- "name": "varchar UTF16 LEN"
+ "name": "varchar UTF8 LEN"
},
{
- "name": "varchar UTF16 DATALENGTH"
+ "name": "varchar UTF8 DATALENGTH"
},
{
"name": "c1"
@@ -1196,30 +1195,37 @@
},
"data": [
{
- "0": "67",
- "1": "67",
- "2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii????????????"
+ "0": "65",
+ "1": "87",
+ "2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦"
}
]
},
- "text/html": "| varchar UTF16 LEN | varchar UTF16 DATALENGTH | c1 |
|---|
| 67 | 67 | MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii???????????? |
"
+ "text/html": [
+ "",
+ "| varchar UTF8 LEN | varchar UTF8 DATALENGTH | c1 |
",
+ "| 65 | 87 | MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦 |
",
+ "
"
+ ]
}
}
],
- "execution_count": 24
+ "execution_count": 1
},
{
"cell_type": "markdown",
"source": [
- "Where are the savings?"
+ "With this data pattern the savings are obvious. Where are the savings? Let's compare breaking down to individual Latin, Chinese, and Emoji strings."
],
"metadata": {
- "azdata_cell_guid": "e8d09b41-c69d-43de-8b67-43ad7e25bb03"
+ "azdata_cell_guid": "db799749-7950-4b3b-a1e2-074e80b40aba"
}
},
{
"cell_type": "code",
"source": [
+ "USE UnicodeDatabase\r\n",
+ "GO\r\n",
"SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], \r\n",
"\tDATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], \r\n",
"\tDATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]\r\n",
@@ -1229,9 +1235,16 @@
"GO"
],
"metadata": {
- "azdata_cell_guid": "79351524-1cba-412a-8111-2cc93871612d"
+ "azdata_cell_guid": "32f0463d-2224-4781-b2d1-76a6e0fd0b37"
},
"outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/html": "Commands completed successfully."
+ },
+ "metadata": {}
+ },
{
"output_type": "display_data",
"data": {
@@ -1249,14 +1262,14 @@
{
"output_type": "display_data",
"data": {
- "text/html": "Total execution time: 00:00:00.043"
+ "text/html": "Total execution time: 00:00:00.024"
},
"metadata": {}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 25,
+ "execution_count": 2,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -1280,13 +1293,18 @@
}
]
},
- "text/html": "| Latin_UTF16_2bytes | Chinese_UTF16_2bytes | SC_UTF16_4bytes |
|---|
| 110 | 16 | 8 |
"
+ "text/html": [
+ "",
+ "| Latin_UTF16_2bytes | Chinese_UTF16_2bytes | SC_UTF16_4bytes |
",
+ "| 110 | 16 | 8 |
",
+ "
"
+ ]
}
},
{
"output_type": "execute_result",
"metadata": {},
- "execution_count": 25,
+ "execution_count": 2,
"data": {
"application/vnd.dataresource+json": {
"schema": {
@@ -1305,16 +1323,21 @@
"data": [
{
"0": "55",
- "1": "8",
- "2": "4"
+ "1": "24",
+ "2": "8"
}
]
},
- "text/html": "| Latin_UTF8_1byte | Chinese_UTF8_3bytes | SC_UTF8_4bytes |
|---|
| 55 | 8 | 4 |
"
+ "text/html": [
+ "",
+ "| Latin_UTF8_1byte | Chinese_UTF8_3bytes | SC_UTF8_4bytes |
",
+ "| 55 | 24 | 8 |
",
+ "
"
+ ]
}
}
],
- "execution_count": 25
+ "execution_count": 2
}
]
}
\ No newline at end of file