mirror of
https://github.com/Microsoft/sql-server-samples.git
synced 2025-12-08 14:58:54 +00:00
Update DataType_WesternMyth.ipynb
This commit is contained in:
pmasl
@@ -15,47 +15,18 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Data type sizes - a western myth"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "da283803-97b1-44ca-af3a-037151ed5cef"
|
||||
}
|
||||
"source": "# Data type sizes - a western myth",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Test Latin character strings with Latin collation\r\n",
|
||||
"\r\n",
|
||||
"**Note:** My server default is SQL_Latin1_General_CP1_CI_AS\r\n",
|
||||
"\r\n",
|
||||
"Set size limit of data types to be the same under Basic Multilingual Plane (BMP)\r\n",
|
||||
"Characters: ranging from 1-byte (ASCII) to 3-bytes (East Asian) per character. So a max of 24-bytes for an East Asian 8 character string."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "d96941dc-8fac-4dc9-9f1d-32465d2a9569"
|
||||
}
|
||||
"source": "## Test Latin character strings with Latin collation\r\n\r\n**Note:** My server default is SQL_Latin1_General_CP1_CI_AS\r\n\r\nSet size limit of data types to be the same under Basic Multilingual Plane (BMP)\r\nCharacters: ranging from 1-byte (ASCII) to 3-bytes (East Asian) per character. So a max of 24-bytes for an East Asian 8 character string.",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t1;\r\n",
|
||||
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
|
||||
"INSERT INTO t1 VALUES (N'MyString', N'MyString') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t1; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t1;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "00084952-2310-40b3-8007-54ae10924a85"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t1;\r\nCREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\nINSERT INTO t1 VALUES (N'MyString', N'MyString') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t1; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t1;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -150,44 +121,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"That's as expected on bothe cases. So what was I talking about?\r\n",
|
||||
"\r\n",
|
||||
"Run next example with Chinese characters."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "5ecddf1f-4b0f-460f-aa48-a97bcf2e8e62"
|
||||
}
|
||||
"source": "That's as expected on bothe cases. So what was I talking about?\r\n\r\nRun next example with Chinese characters.",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Test Chinese character strings with Latin collation"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "772bd4b9-0ae0-4f52-ab34-f82226be4bca"
|
||||
}
|
||||
"source": "# Test Chinese character strings with Latin collation",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t1;\r\n",
|
||||
"CREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\n",
|
||||
"INSERT INTO t1 VALUES (N'敏捷的棕色狐狸跳', N'敏捷的棕色狐狸跳') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t1; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t1;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "cf4eb5a4-e2ae-40a8-a1ed-d476cf67fb2d"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t1;\r\nCREATE TABLE t1 (c1 varchar(24) COLLATE Latin1_General_100_CI_AI, \r\n\tc2 nvarchar(8) COLLATE Latin1_General_100_CI_AI); \r\nINSERT INTO t1 VALUES (N'敏捷的棕色狐狸跳', N'敏捷的棕色狐狸跳') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t1; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t1;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -282,32 +227,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"uh-oh data loss on the varchar example. Why?\r\n",
|
||||
"\r\n",
|
||||
"varchar is bound to code page enconding by default, and these code points cannot be found in the Latin code page.\r\n",
|
||||
"\r\n",
|
||||
"But why didn't it happen in the nvarchar example? \r\n",
|
||||
"\r\n",
|
||||
"These Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP), and nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page.\r\n",
|
||||
"\r\n",
|
||||
"Run the next example:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "934db15a-7c57-4eb3-ac0a-48cce569ee33"
|
||||
}
|
||||
"source": "uh-oh data loss on the varchar example. Why?\r\n\r\nVarchar is bound to code page encoding by default, and these code points cannot be found in the Latin code page.\r\n\r\nBut why didn't it happen in the nvarchar example? \r\n\r\nThese Chinese characters are double-byte and within the *Basic Multilingual Plane* (BMP), and nvarchar with this non-SC collation encodes in UCS-2 (BMP), not the code page.\r\n\r\nRun the next example:",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63);\r\n",
|
||||
"SELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63);"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "fa541bfb-134a-4a14-bb5d-47bd2cc55c24"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nSELECT ASCII('敏' COLLATE Latin1_General_100_CI_AI), CHAR(63);\r\nSELECT ASCII('捷' COLLATE Latin1_General_100_CI_AI), CHAR(63);",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -339,7 +265,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 4,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -360,17 +285,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>63</td><td>?</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>63</td><td>?</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 4,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -391,39 +311,22 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>63</td><td>?</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>63</td><td>?</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 4
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"The ASCII function returns the ASCII code value of the leftmost character of a character expression. We know the Latin code page that's chosen can't represent a double-byte character, so it can only read the first byte, which is incorrectly translated to code point 63. Using the CHAR function, we see that the 63 code point is a question mark character. \r\n",
|
||||
"\r\n",
|
||||
"Run the next example:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "ae7aeb4c-d9e7-474a-b35d-fe62fe514ce4"
|
||||
}
|
||||
"source": "The ASCII function returns the ASCII code value of the leftmost character of a character expression. We know the Latin code page that's chosen can't represent a double-byte character, so it can only read the first byte, which is incorrectly translated to code point 63. Using the CHAR function, we see that the 63 code point is a question mark character. \r\n\r\nRun the next example:",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935);\r\n",
|
||||
"SELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463);"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "c7bc9d86-a4bc-47d6-aaed-967bd11ba378"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nSELECT UNICODE(N'敏' COLLATE Latin1_General_100_CI_AI), NCHAR(25935);\r\nSELECT UNICODE(N'捷' COLLATE Latin1_General_100_CI_AI), NCHAR(25463);",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -455,7 +358,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 5,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -476,17 +378,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>25935</td><td>敏</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25935</td><td>敏</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 5,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -507,55 +404,27 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>(No column name)</th><th>(No column name)</th></tr>",
|
||||
"<tr><td>25463</td><td>捷</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>(No column name)</th><th>(No column name)</th></tr><tr><td>25463</td><td>捷</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 5
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Works irrespective of collation now. By adding the N prefix we force the use of a [Unicode constant](https://docs.microsoft.com/sql/t-sql/data-types/constants-transact-sql#unicode-strings), and collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase), not the encoding. Using the UNICODE function, we can correctly identify the code points for the string characters, which the NCHAR function can represent accurately."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "34ae3b63-602a-41c7-9b12-d8649e7e6e10"
|
||||
}
|
||||
"source": "Works irrespective of collation now. By adding the N prefix we force the use of a [Unicode constant](https://docs.microsoft.com/sql/t-sql/data-types/constants-transact-sql#unicode-strings), and collation only sets linguistic algorithms (Compare = sort; Case sensitivity = Upper/Lowercase), not the encoding. Using the UNICODE function, we can correctly identify the code points for the string characters, which the NCHAR function can represent accurately.",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Now test Chinese character strings with Chinese collation"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "bebfcf18-b6d8-4131-a783-51ebb3dfe4dd"
|
||||
}
|
||||
"source": "## Now test Chinese character strings with Chinese collation",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI); \r\n",
|
||||
"INSERT INTO t2 VALUES (N'敏捷的棕色狐狸跳', N'敏捷的棕色狐狸跳') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t2; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t2;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "811086b3-779f-4d52-af63-c299830cd53d"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t2;\r\nCREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI, \r\n\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI); \r\nINSERT INTO t2 VALUES (N'敏捷的棕色狐狸跳', N'敏捷的棕色狐狸跳') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t2; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t2;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -650,47 +519,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Now the varchar example is correct because the code page can recognize Chinese characters. But there's 2 bytes per character, not 3?...\r\n",
|
||||
"\r\n",
|
||||
"**Myth buster:** code page defines string length for varchar. Varchar is **not** always 1 byte per character. \r\n",
|
||||
"\r\n",
|
||||
"Ok, but wasn't East-Asian 3 bytes? Yes, with UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
|
||||
""
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "2ea7aa07-674a-407f-9bc3-66d0632c16e5"
|
||||
}
|
||||
"source": "Now the varchar example is correct because the code page can recognize Chinese characters. But there's 2 bytes per character, not 3?...\r\n\r\n**Myth buster:** code page defines string length for varchar. Varchar is **not** always 1 byte per character. \r\n\r\nOk, but wasn't East-Asian 3 bytes? Yes, with UTF-8, but under Chinese collation code page, they are encoded using 2 bytes just like UCS-2/UTF-16\r\n",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Test with Supplementary Characters (4 bytes)"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "c54c5ec7-a037-4ace-be26-6f773b600790"
|
||||
}
|
||||
"source": "## Test with Supplementary Characters (4 bytes)",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
|
||||
"\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
"INSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t2; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t2;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "32789381-bee3-4c05-9247-fea7dbbb6f6b"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t2;\r\nCREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n\tc2 nvarchar(8) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\nINSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t2; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t2;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "error",
|
||||
@@ -779,33 +619,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"uh-oh, let's set the proper data type length from 8 to 16 byte-pairs (so a 32-byte encoding limit)"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "449fa293-8edd-41ea-b9b7-62d9af611a2a"
|
||||
}
|
||||
"source": "uh-oh, let's set the proper data type length from 8 to 16 byte-pairs (so a 32-byte encoding limit)",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n",
|
||||
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
"INSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t2; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t2;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "22e7a937-f154-4b21-988b-0e62d349b98a"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t2;\r\nCREATE TABLE t2 (c1 varchar(24) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC, \r\n\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\nINSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t2; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t2;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -900,35 +720,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Nvarchar looks good. But varchar still doesn't encode? \r\n",
|
||||
"\r\n",
|
||||
"Set a larger data type length. For example double from 24 to 48 bytes. Now try again:"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "3fa9b70f-57f1-4333-ad8c-23cd5b08de15"
|
||||
}
|
||||
"source": "Nvarchar looks good. But varchar still doesn't encode? \r\n\r\nSet a larger data type length. For example double from 24 to 48 bytes. Now try again:",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t2;\r\n",
|
||||
"CREATE TABLE t2 (c1 varchar(48) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC_UTF8, \r\n",
|
||||
"\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\n",
|
||||
"INSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\n",
|
||||
"FROM t2; \r\n",
|
||||
"SELECT LEN(c2) AS [nvarchar LEN], \r\n",
|
||||
"\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\n",
|
||||
"FROM t2;\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "682a2d5f-43b4-4721-b5b8-6ddfea6f3b91"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t2;\r\nCREATE TABLE t2 (c1 varchar(48) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC_UTF8, \r\n\tc2 nvarchar(16) COLLATE Chinese_Traditional_Stroke_Order_100_CI_AI_SC); \r\nINSERT INTO t2 VALUES (N'👶👦👧👨👩👴👵👨', N'👶👦👧👨👩👴👵👨') \r\nSELECT LEN(c1) AS [varchar LEN], \r\n\tDATALENGTH(c1) AS [varchar DATALENGTH], c1\r\nFROM t2; \r\nSELECT LEN(c2) AS [nvarchar LEN], \r\n\tDATALENGTH(c2) AS [nvarchar DATALENGTH], c2 \r\nFROM t2;\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -967,7 +765,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 3,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -992,17 +789,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr>",
|
||||
"<tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>varchar LEN</th><th>varchar DATALENGTH</th><th>c1</th></tr><tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 3,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -1027,44 +819,22 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr>",
|
||||
"<tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>nvarchar LEN</th><th>nvarchar DATALENGTH</th><th>c2</th></tr><tr><td>8</td><td>32</td><td>👶👦👧👨👩👴👵👨</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 3
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"Finally!\r\n",
|
||||
"\r\n",
|
||||
"What if I needed all these characters in one database? Easy, I could just use nvarchar which encodes in UTF-16."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "2787fdcd-117c-47c5-962d-1fcf6f51df92"
|
||||
}
|
||||
"source": "Finally!\r\n\r\nWhat if I needed all these characters in one database? Easy, I could just use nvarchar which encodes in UTF-16.",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t3;\r\n",
|
||||
"CREATE TABLE t3 (c1 nvarchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\n",
|
||||
"INSERT INTO t3 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
|
||||
"SELECT LEN(c1) AS [nvarchar UTF16 LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [nvarchar UTF16 DATALENGTH], c1\r\n",
|
||||
"FROM t3; \r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "da196747-8ef7-4844-895e-7fbf3496e7f0"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t3;\r\nCREATE TABLE t3 (c1 nvarchar(110) COLLATE Latin1_General_100_CI_AI_SC); \r\nINSERT INTO t3 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\nSELECT LEN(c1) AS [nvarchar UTF16 LEN], \r\n\tDATALENGTH(c1) AS [nvarchar UTF16 DATALENGTH], c1\r\nFROM t3; \r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -1122,29 +892,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"But wait. The majority of my data is set to Latin (ASCII), can we do better?"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "7c0e540f-265c-4e58-844d-14b51d731a6b"
|
||||
}
|
||||
"source": "But wait. The majority of my data is set to Latin (ASCII), can we do better?",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"DROP TABLE IF EXISTS t4;\r\n",
|
||||
"CREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); \r\n",
|
||||
"INSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\n",
|
||||
"SELECT LEN(c1) AS [varchar UTF8 LEN], \r\n",
|
||||
"\tDATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1\r\n",
|
||||
"FROM t4; \r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "89f77817-2cf3-493b-9d80-64d229103c70"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nDROP TABLE IF EXISTS t4;\r\nCREATE TABLE t4 (c1 varchar(110) COLLATE Latin1_General_100_CI_AI_SC_UTF8); \r\nINSERT INTO t4 VALUES (N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦') \r\nSELECT LEN(c1) AS [varchar UTF8 LEN], \r\n\tDATALENGTH(c1) AS [varchar UTF8 DATALENGTH], c1\r\nFROM t4; \r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -1176,7 +930,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 1,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -1201,42 +954,22 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>varchar UTF8 LEN</th><th>varchar UTF8 DATALENGTH</th><th>c1</th></tr>",
|
||||
"<tr><td>65</td><td>87</td><td>MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>varchar UTF8 LEN</th><th>varchar UTF8 DATALENGTH</th><th>c1</th></tr><tr><td>65</td><td>87</td><td>MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii敏捷的棕色狐狸跳👶👦</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 1
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"With this data pattern the savings are obvious. Where are the savings? Let's compare breaking down to individual Latin, Chinese, and Emoji strings."
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "db799749-7950-4b3b-a1e2-074e80b40aba"
|
||||
}
|
||||
"source": "With this data pattern the savings are obvious. Where are the savings? Let's compare breaking down to individual Latin, Chinese, and Emoji strings.",
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"USE UnicodeDatabase\r\n",
|
||||
"GO\r\n",
|
||||
"SELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], \r\n",
|
||||
"\tDATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], \r\n",
|
||||
"\tDATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]\r\n",
|
||||
"SELECT DATALENGTH('MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Latin_UTF8_1byte], \r\n",
|
||||
"\tDATALENGTH('敏捷的棕色狐狸跳' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Chinese_UTF8_3bytes], \r\n",
|
||||
"\tDATALENGTH('👶👦' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]\r\n",
|
||||
"GO"
|
||||
],
|
||||
"metadata": {
|
||||
"azdata_cell_guid": "32f0463d-2224-4781-b2d1-76a6e0fd0b37"
|
||||
},
|
||||
"source": "USE UnicodeDatabase\r\nGO\r\nSELECT DATALENGTH(N'MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii') AS [Latin_UTF16_2bytes], \r\n\tDATALENGTH(N'敏捷的棕色狐狸跳') AS [Chinese_UTF16_2bytes], \r\n\tDATALENGTH(N'👶👦') AS [SC_UTF16_4bytes]\r\nSELECT DATALENGTH('MyStringThequickbrownfoxjumpsoverthelazydogIsLatinAscii' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Latin_UTF8_1byte], \r\n\tDATALENGTH('敏捷的棕色狐狸跳' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [Chinese_UTF8_3bytes], \r\n\tDATALENGTH('👶👦' COLLATE Latin1_General_100_CI_AI_SC_UTF8) AS [SC_UTF8_4bytes]\r\nGO",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "display_data",
|
||||
@@ -1268,7 +1001,6 @@
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 2,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -1293,17 +1025,12 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>Latin_UTF16_2bytes</th><th>Chinese_UTF16_2bytes</th><th>SC_UTF16_4bytes</th></tr>",
|
||||
"<tr><td>110</td><td>16</td><td>8</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>Latin_UTF16_2bytes</th><th>Chinese_UTF16_2bytes</th><th>SC_UTF16_4bytes</th></tr><tr><td>110</td><td>16</td><td>8</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
},
|
||||
{
|
||||
"output_type": "execute_result",
|
||||
"metadata": {},
|
||||
"execution_count": 2,
|
||||
"data": {
|
||||
"application/vnd.dataresource+json": {
|
||||
@@ -1328,13 +1055,9 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"text/html": [
|
||||
"<table>",
|
||||
"<tr><th>Latin_UTF8_1byte</th><th>Chinese_UTF8_3bytes</th><th>SC_UTF8_4bytes</th></tr>",
|
||||
"<tr><td>55</td><td>24</td><td>8</td></tr>",
|
||||
"</table>"
|
||||
]
|
||||
}
|
||||
"text/html": "<table><tr><th>Latin_UTF8_1byte</th><th>Chinese_UTF8_3bytes</th><th>SC_UTF8_4bytes</th></tr><tr><td>55</td><td>24</td><td>8</td></tr></table>"
|
||||
},
|
||||
"metadata": {}
|
||||
}
|
||||
],
|
||||
"execution_count": 2
|
||||
|
||||
Reference in New Issue
Block a user