mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Adding detail to UTF8 demo
This commit is contained in:
pmasl
@@ -7,7 +7,13 @@
|
||||
-- Test Latin character strings with Latin collation
|
||||
-- Set size limit of data types to be the same under Basic Multilingual Plane (BMP)
|
||||
-- Characters between 1-byte (ASCII) and 3-bytes (East Asian)
|
||||
USE master;
|
||||
DROP DATABASE IF EXISTS UnicodeDatabase;
|
||||
CREATE DATABASE UnicodeDatabase COLLATE LATIN1_GENERAL_100_CI_AS_SC_UTF8
|
||||
GO
|
||||
|
||||
USE UnicodeDatabase
|
||||
GO
|
||||
DROP TABLE IF EXISTS t1;
|
||||
CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI,
|
||||
c2 nvarchar(8) COLLATE Latin1_General_100_CI_AI);
|
||||
@@ -46,7 +52,8 @@ GO
|
||||
|
||||
|
||||
-- uh-oh data loss on the varchar example. Why?
|
||||
-- varchar is bound to code page enconding, and these code points cannot be found in the Latin code page.
|
||||
-- varchar is bound to code page enconding,
|
||||
-- and these code points cannot be found in the Latin code page.
|
||||
SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63)
|
||||
SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63)
|
||||
|
||||
@@ -152,10 +159,10 @@ GO
|
||||
|
||||
-- But the majority of my data is set to Latin (ASCII)
|
||||
DROP TABLE IF EXISTS t4;
|
||||
CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC);
|
||||
CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8);
|
||||
INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦')
|
||||
SELECT LEN(c1) AS [varchar UTF16 LEN],
|
||||
DATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1
|
||||
SELECT LEN(c1) AS [varchar UTF8 LEN],
|
||||
DATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1
|
||||
FROM t4;
|
||||
GO
|
||||
|
||||
@@ -164,8 +171,8 @@ GO
|
||||
-- Where are the savings?
|
||||
SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes],
|
||||
DATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes],
|
||||
DATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]
|
||||
DATALENGTH(N'👶') AS [SC_UTF16_4bytes]
|
||||
SELECT DATALENGTH('MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Latin_UTF8_1byte],
|
||||
DATALENGTH('敏捷的棕色狐狸跳' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Chinese_UTF8_3bytes],
|
||||
DATALENGTH('👶👦' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]
|
||||
DATALENGTH('👶' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]
|
||||
GO
|
||||
@@ -19,7 +19,7 @@
|
||||
"# Data type sizes - a western myth"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "9e602698-2ef8-4fed-872f-5bad8c440600"
|
||||
"azdata_cell_guid": "da283803-97b1-44ca-af3a-037151ed5cef"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -30,15 +30,17 @@
|
||||
"**Note:** My server default is SQL_Latin1_General_CP1_CI_AS\r\n",
|
||||
"\r\n",
|
||||
"Set size limit of data types to be the same under Basic Multilingual Plane (BMP)\r\n",
|
||||
"Characters between 1-byte (ASCII) and 3-bytes (East Asian)"
|
||||
"Characters: ranging from 1-byte (ASCII) to 3-bytes (East Asian) per character. So a max of 24-bytes for an East Asian 8 character string."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "78fc4a30-52e8-407d-8807-04097ad12348"
|
||||
"azdata_cell_guid": "d96941dc-8fac-4dc9-9f1d-32465d2a9569"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t1;\r\n",
|
||||
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
|
||||
@@ -52,7 +54,7 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "4a7b199e-3117-4fcb-b149-776f4da6cf71"
|
||||
"azdata_cell_guid": "00084952-2310-40b3-8007-54ae10924a85"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -79,14 +81,13 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.068"
|
||||
"text/html": "Total execution time: 00:00:00.1996267"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 15,
|
||||
"execution_count": 1,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -111,12 +112,12 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>8</td><td>8</td><td>MyString</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 15,
|
||||
"execution_count": 1,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -141,7 +142,8 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>16</td><td>MyString</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 15
|
||||
@@ -149,10 +151,12 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"That's as expected. So what was I talking about?"
|
||||
"That's as expected on bothe cases. So what was I talking about?\r\n",
|
||||
"\r\n",
|
||||
"Run next example with Chinese characters."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "8615ddba-64e7-4abc-9df6-ec5648f39788"
|
||||
"azdata_cell_guid": "5ecddf1f-4b0f-460f-aa48-a97bcf2e8e62"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -161,12 +165,14 @@
|
||||
"# Test Chinese character strings with Latin collation"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "4a85f77d-fd22-46a6-b7aa-5d6fca65dd99"
|
||||
"azdata_cell_guid": "772bd4b9-0ae0-4f52-ab34-f82226be4bca"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t1;\r\n",
|
||||
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
|
||||
@@ -180,7 +186,7 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "f5d3d422-588c-4cc9-9bbd-2664b392b0fb"
|
||||
"azdata_cell_guid": "cf4eb5a4-e2ae-40a8-a1ed-d476cf67fb2d"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -213,7 +219,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 16,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -239,11 +244,11 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>8</td><td>8</td><td>????????</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 16,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -269,7 +274,8 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>16</td><td>敏捷的棕色狐狸跳</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 16
|
||||
@@ -279,22 +285,37 @@
|
||||
"source": [
|
||||
"uh-oh data loss on the varchar example. Why?\r\n",
|
||||
"\r\n",
|
||||
"varchar is bound to code page enconding, and these code points cannot be found in the Latin code page."
|
||||
"varchar is bound to code page enconding by default, and these code points cannot be found in the Latin code page.\r\n",
|
||||
"\r\n",
|
||||
"But why didn't it happen in the nvarchar example? \r\n",
|
||||
"\r\n",
|
||||
"These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP), and nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page.\r\n",
|
||||
"\r\n",
|
||||
"Run the next example:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d77295e2-75eb-487e-861e-ca1634b90740"
|
||||
"azdata_cell_guid": "934db15a-7c57-4eb3-ac0a-48cce569ee33"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63)\r\n",
|
||||
"SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63)"
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63);\r\n",
|
||||
"SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63);"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "9dd9bce0-c073-42ec-9faa-8ebc142fd025"
|
||||
"azdata_cell_guid": "fa541bfb-134a-4a14-bb5d-47bd2cc55c24"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
@@ -312,14 +333,14 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.046"
|
||||
"text/html": "Total execution time: 00:00:00.017"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -339,13 +360,18 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>63</td><td>?</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>63</td><td>?</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -365,35 +391,47 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>63</td><td>?</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>63</td><td>?</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
"execution_count": 4
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"But why didn't it happen in the nvarchar example?\r\n",
|
||||
"The ASCII function returns the ASCII code value of the leftmost character of a character expression. We know the Latin code page that's chosen can't represent a double-byte character, so it can only read the first byte, which is incorrectly translated to code point 63. Using the CHAR function, we see that the 63 code point is a question mark character. \r\n",
|
||||
"\r\n",
|
||||
"These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP)\r\n",
|
||||
"\r\n",
|
||||
"nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page"
|
||||
"Run the next example:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "75416bff-a350-4f27-9de5-7b407b59ac81"
|
||||
"azdata_cell_guid": "ae7aeb4c-d9e7-474a-b35d-fe62fe514ce4"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935)\r\n",
|
||||
"SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463)"
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935);\r\n",
|
||||
"SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463);"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "ffed888f-da51-4cad-a88e-b0ccf2a38f10"
|
||||
"azdata_cell_guid": "c7bc9d86-a4bc-47d6-aaed-967bd11ba378"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
@@ -411,7 +449,7 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.021"
|
||||
"text/html": "Total execution time: 00:00:00.024"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
@@ -438,7 +476,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25935</td><td>敏</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>25935</td><td>敏</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -464,7 +507,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25463</td><td>捷</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>25463</td><td>捷</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -473,110 +521,26 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Irrespective of collation now. With a Unicode capable data type, collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase)"
|
||||
"Works irrespective of collation now. By adding the N prefix we force the use of a [Unicode constant](https://docs.microsoft.com/sql/t-sql/data-types/constants-transact-sql#unicode-strings), and collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase), not the encoding. Using the UNICODE function, we can correctly identify the code points for the string characters, which the NCHAR function can represent accurately."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d6db32d2-033f-4800-a492-3494b65dbc36"
|
||||
"azdata_cell_guid": "34ae3b63-602a-41c7-9b12-d8649e7e6e10"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"SELECT UNICODE(N'敏' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25935)\r\n",
|
||||
"SELECT UNICODE(N'捷' COLLATE Chinese_PRC_90_CI_AI), NCHAR(25463)"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "ba6107ed-50d8-4e3f-8e59-3eb1cb56e983"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "(1 row affected)"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "(1 row affected)"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.039"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 9,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "(No column name)"
|
||||
},
|
||||
{
|
||||
"name": "(No column name)"
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"0": "25935",
|
||||
"1": "敏"
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25935</td><td>敏</td></tr></table>"
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 9,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "(No column name)"
|
||||
},
|
||||
{
|
||||
"name": "(No column name)"
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"0": "25463",
|
||||
"1": "捷"
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25463</td><td>捷</td></tr></table>"
|
||||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 9
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Now test Chinese character strings with Chinese collation"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "041d9858-808d-4a6b-9bc7-302feddf4036"
|
||||
"azdata_cell_guid": "bebfcf18-b6d8-4131-a783-51ebb3dfe4dd"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI); \r\n",
|
||||
@@ -590,7 +554,7 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d8fecb5c-9120-46b5-b34c-d75fc8664a80"
|
||||
"azdata_cell_guid": "811086b3-779f-4d52-af63-c299830cd53d"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -623,7 +587,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 17,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -649,11 +612,11 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>8</td><td>16</td><td>敏捷的棕色狐狸跳</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 17,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -679,7 +642,8 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>16</td><td>敏捷的棕色狐狸跳</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 17
|
||||
@@ -687,15 +651,15 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Now the varchar example is correct. But there's 2 bytes per character?...\r\n",
|
||||
"Now the varchar example is correct because the code page can recognize Chinese characters. But there's 2 bytes per character, not 3?...\r\n",
|
||||
"\r\n",
|
||||
"**Myth buster:** code page defines string length for varchar. It's not always 1 byte per character. \r\n",
|
||||
"**Myth buster:** code page defines string length for varchar. Varchar is **not** always 1 byte per character. \r\n",
|
||||
"\r\n",
|
||||
"Wasn't East-Asian 3 bytes? Yes, on UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
|
||||
"Ok, but wasn't East-Asian 3 bytes? Yes, with UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
|
||||
""
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "0bb36f33-1fb1-499e-97a9-a071f5332eba"
|
||||
"azdata_cell_guid": "2ea7aa07-674a-407f-9bc3-66d0632c16e5"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -704,12 +668,14 @@
|
||||
"## Test with Supplementary Characters (4 bytes)"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "cdc78270-6de2-4bb9-9c49-8ac7aae92b8b"
|
||||
"azdata_cell_guid": "c54c5ec7-a037-4ace-be26-6f773b600790"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
@@ -723,13 +689,13 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "c8fa69ca-590b-414c-88eb-d4892816d324"
|
||||
"azdata_cell_guid": "32789381-bee3-4c05-9247-fea7dbbb6f6b"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "error",
|
||||
"evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.",
|
||||
"ename": "",
|
||||
"evalue": "Msg 2628, Level 16, State 1, Line 4\r\nString or binary data would be truncated in table 'master.dbo.t2', column 'c2'. Truncated value: '👶👦👧👨'.",
|
||||
"traceback": []
|
||||
},
|
||||
{
|
||||
@@ -762,7 +728,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 20,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -782,11 +747,11 @@
|
||||
"data": []
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 20,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -806,7 +771,8 @@
|
||||
"data": []
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 20
|
||||
@@ -814,15 +780,17 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"uh-oh, let's set the proper data type length"
|
||||
"uh-oh, let's set the proper data type length from 8 to 16 byte-pairs (so a 32-byte encoding limit)"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "0c643441-2b73-4880-82d7-0b019c14416c"
|
||||
"azdata_cell_guid": "449fa293-8edd-41ea-b9b7-62d9af611a2a"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
|
||||
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
@@ -836,7 +804,7 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "ba341c74-d2ca-4477-82f2-0c39bcc38af7"
|
||||
"azdata_cell_guid": "22e7a937-f154-4b21-988b-0e62d349b98a"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -869,7 +837,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 21,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -895,11 +862,11 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>16</td><td>16</td><td>????????????????</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 21,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -925,7 +892,8 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 21
|
||||
@@ -933,16 +901,19 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Nvarchar looks good. But varchar still doesn't encode? \r\n",
|
||||
"\r\n",
|
||||
"Varchar still doesn't encode? "
|
||||
"Set a larger data type length. For example double from 24 to 48 bytes. Now try again:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d062a6e7-2a91-48d4-b81b-0bfb530df450"
|
||||
"azdata_cell_guid": "3fa9b70f-57f1-4333-ad8c-23cd5b08de15"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(48) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC_UTF8, \r\n",
|
||||
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
@@ -956,9 +927,16 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d8082437-485c-4c9d-84ca-16acd6e63849"
|
||||
"azdata_cell_guid": "682a2d5f-43b4-4721-b5b8-6ddfea6f3b91"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
@@ -983,14 +961,14 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.071"
|
||||
"text/html": "Total execution time: 00:00:00.033"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 22,
|
||||
"execution_count": 3,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -1014,13 +992,18 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr>",
|
||||
"<tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 22,
|
||||
"execution_count": 3,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -1044,26 +1027,33 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr>",
|
||||
"<tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 22
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Finally!\r\n",
|
||||
"\r\n",
|
||||
"What if I needed all these in one database? Easy, I could just use nvarchar."
|
||||
"What if I needed all these characters in one database? Easy, I could just use nvarchar which encodes in UTF-16."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "544ee63a-67fa-4b24-b974-4fa4e809949c"
|
||||
"azdata_cell_guid": "2787fdcd-117c-47c5-962d-1fcf6f51df92"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t3;\r\n",
|
||||
"CREATE TABLE t3 (c1 nvarchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n",
|
||||
"INSERT INTO t3 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
|
||||
@@ -1073,7 +1063,7 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "87e6c383-d9ab-48da-a4d7-f1cc0cf3aee3"
|
||||
"azdata_cell_guid": "da196747-8ef7-4844-895e-7fbf3496e7f0"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -1099,7 +1089,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 23,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -1125,7 +1114,8 @@
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>nvarchar UTF16 LEN</th><th>nvarchar UTF16 DATALENGTH</th><th>c1</th></tr><tr><td>65</td><td>134</td><td>MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦</td></tr></table>"
|
||||
}
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 23
|
||||
@@ -1133,27 +1123,36 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"But the majority of my data is set to Latin (ASCII)"
|
||||
"But wait. The majority of my data is set to Latin (ASCII), can we do better?"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "808dc693-ca7b-4ee5-b71f-9cf9cbcef31f"
|
||||
"azdata_cell_guid": "7c0e540f-265c-4e58-844d-14b51d731a6b"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t4;\r\n",
|
||||
"CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n",
|
||||
"CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); \r\n",
|
||||
"INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar UTF16 LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar UTF16 DATALENGTH], c1\r\n",
|
||||
"SELECT LEN(c1) AS [varchar UTF8 LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1\r\n",
|
||||
"FROM t4; \r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "8597dfb2-ec28-4c82-865a-ec7a8d01c663"
|
||||
"azdata_cell_guid": "89f77817-2cf3-493b-9d80-64d229103c70"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
@@ -1171,23 +1170,23 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.013"
|
||||
"text/html": "Total execution time: 00:00:00.253"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 24,
|
||||
"execution_count": 1,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "varchar UTF16 LEN"
|
||||
"name": "varchar UTF8 LEN"
|
||||
},
|
||||
{
|
||||
"name": "varchar UTF16 DATALENGTH"
|
||||
"name": "varchar UTF8 DATALENGTH"
|
||||
},
|
||||
{
|
||||
"name": "c1"
|
||||
@@ -1196,30 +1195,37 @@
|
||||
},
|
||||
"data": [
|
||||
{
|
||||
"0": "67",
|
||||
"1": "67",
|
||||
"2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii????????????"
|
||||
"0": "65",
|
||||
"1": "87",
|
||||
"2": "MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦"
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>varchar UTF16 LEN</th><th>varchar UTF16 DATALENGTH</th><th>c1</th></tr><tr><td>67</td><td>67</td><td>MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii????????????</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>varchar UTF8 LEN</th><th>varchar UTF8 DATALENGTH</th><th>c1</th></tr>",
|
||||
"<tr><td>65</td><td>87</td><td>MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 24
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Where are the savings?"
|
||||
"With this data pattern the savings are obvious. Where are the savings? Let's compare breaking down to individual Latin, Chinese, and Emoji strings."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "e8d09b41-c69d-43de-8b67-43ad7e25bb03"
|
||||
"azdata_cell_guid": "db799749-7950-4b3b-a1e2-074e80b40aba"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], \r\n",
|
||||
"\tDATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], \r\n",
|
||||
"\tDATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]\r\n",
|
||||
@@ -1229,9 +1235,16 @@
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "79351524-1cba-412a-8111-2cc93871612d"
|
||||
"azdata_cell_guid": "32f0463d-2224-4781-b2d1-76a6e0fd0b37"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Commands completed successfully."
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
@@ -1249,14 +1262,14 @@
|
||||
{
|
||||
"output_type": "display_data",
|
||||
"data": {
|
||||
"text/html": "Total execution time: 00:00:00.043"
|
||||
"text/html": "Total execution time: 00:00:00.024"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 25,
|
||||
"execution_count": 2,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -1280,13 +1293,18 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>Latin_UTF16_2bytes</th><th>Chinese_UTF16_2bytes</th><th>SC_UTF16_4bytes</th></tr><tr><td>110</td><td>16</td><td>8</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>Latin_UTF16_2bytes</th><th>Chinese_UTF16_2bytes</th><th>SC_UTF16_4bytes</th></tr>",
|
||||
"<tr><td>110</td><td>16</td><td>8</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 25,
|
||||
"execution_count": 2,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
"schema": {
|
||||
@@ -1305,16 +1323,21 @@
|
||||
"data": [
|
||||
{
|
||||
"0": "55",
|
||||
"1": "8",
|
||||
"2": "4"
|
||||
"1": "24",
|
||||
"2": "8"
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": "<table><tr><th>Latin_UTF8_1byte</th><th>Chinese_UTF8_3bytes</th><th>SC_UTF8_4bytes</th></tr><tr><td>55</td><td>8</td><td>4</td></tr></table>"
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>Latin_UTF8_1byte</th><th>Chinese_UTF8_3bytes</th><th>SC_UTF8_4bytes</th></tr>",
|
||||
"<tr><td>55</td><td>24</td><td>8</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"execution_count": 25
|
||||
"execution_count": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user