diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md index d129e144..cecfe82c 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/README.md @@ -1,4 +1,4 @@ -# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS +# Create a Kubernetes cluster using Kubeadm on Ubuntu 20.04 LTS ## __[ubuntu](ubuntu/)__ diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh index 9d8d4154..4c2789fa 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm-ad/setup-bdc-ad.sh @@ -49,8 +49,8 @@ export DEBIAN_FRONTEND=noninteractive # Kube version. # -KUBE_DPKG_VERSION=1.16.3-00 -KUBE_VERSION=1.16.3 +KUBE_DPKG_VERSION=1.20.7-00 +KUBE_VERSION=1.20.7 # Wait for 5 minutes for the cluster to be ready. # @@ -61,7 +61,7 @@ RETRY_INTERVAL=5 # export DOCKER_REGISTRY="mcr.microsoft.com" export DOCKER_REPOSITORY="mssql/bdc" -export DOCKER_TAG="2019-CU8-ubuntu-16.04" +export DOCKER_TAG="2019-CU13-ubuntu-20.04" # Variables used for azdata cluster creation. # diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh index 3774e693..48b82e2e 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu-single-node-vm/setup-bdc.sh @@ -34,8 +34,8 @@ export DEBIAN_FRONTEND=noninteractive # Kube version. # -KUBE_DPKG_VERSION=1.18.3-00 -KUBE_VERSION=1.18.3 +KUBE_DPKG_VERSION=1.20.7-00 +KUBE_VERSION=1.20.7 # Wait for 5 minutes for the cluster to be ready. # @@ -46,7 +46,7 @@ RETRY_INTERVAL=5 # export DOCKER_REGISTRY="mcr.microsoft.com" export DOCKER_REPOSITORY="mssql/bdc" -export DOCKER_TAG="2019-CU8-ubuntu-16.04" +export DOCKER_TAG="2019-CU13-ubuntu-20.04" # Variables used for azdata cluster creation. # diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md index 3063bb97..1eef64d7 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/README.md @@ -1,6 +1,6 @@ -# Create a Kubernetes cluster using Kubeadm on Ubuntu 16.04 LTS or 18.04 LTS +# Create a Kubernetes cluster using Kubeadm on Ubuntu 20.04 LTS -In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu 16.04 LTS & 18.04 LTS versions. +In this example, we will deploy Kubernetes over multiple Linux machines (physical or virtualized) using kubeadm utility. These instructions have been tested primarily with Ubuntu20.04 LTS version. ## Pre-requisites diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh index 0bce8ab3..69cf78b4 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-master.sh @@ -2,7 +2,7 @@ # Initialize a kubernetes cluster on the current node. # -KUBE_VERSION=1.16.2 +KUBE_VERSION=1.20.7 sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --kubernetes-version=$KUBE_VERSION mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config diff --git a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh index 34bf5d20..fe58b0c6 100644 --- a/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh +++ b/samples/features/sql-big-data-cluster/deployment/kubeadm/ubuntu/setup-k8s-prereqs.sh @@ -10,7 +10,7 @@ cat </etc/apt/sources.list.d/kubernetes.list deb http://apt.kubernetes.io/ kubernetes-xenial main EOF -KUBE_DPKG_VERSION=1.16.2-00 +KUBE_DPKG_VERSION=1.20.7-00 apt-get update apt-get install -y ebtables ethtool apt-get install -y docker.io diff --git a/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh b/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh index cba195ca..6c3350ab 100644 --- a/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh +++ b/samples/features/sql-big-data-cluster/deployment/platform-ops/bdc-aks-ops/deploy-bdc-aks.sh @@ -19,7 +19,7 @@ done azdata bdc config init --source aks-dev-test --target bdc-aks --force #Configurations for BDC deployment -azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04" +azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default" diff --git a/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh b/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh index cba195ca..6c3350ab 100644 --- a/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh +++ b/samples/features/sql-big-data-cluster/deployment/platform-ops/deploy-bdc-aks.sh @@ -19,7 +19,7 @@ done azdata bdc config init --source aks-dev-test --target bdc-aks --force #Configurations for BDC deployment -azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04" +azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default" diff --git a/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh b/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh index a563299a..ce45532b 100644 --- a/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh +++ b/samples/features/sql-big-data-cluster/deployment/private-aks/scripts/deploy-bdc-private-aks.sh @@ -18,7 +18,7 @@ done azdata bdc config init --source aks-dev-test --target private-bdc-aks --force #Configurations for BDC deployment -azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU12-ubuntu-20.04" +azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.docker.imageTag=2019-CU13-ubuntu-20.04" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.data.className=default" azdata bdc config replace -p private-bdc-aks/control.json -j "$.spec.storage.logs.className=default" diff --git a/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb b/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb index afed0633..39964ad5 100644 --- a/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb +++ b/samples/features/sql-big-data-cluster/spark/config-install/installpackage_Spark.ipynb @@ -17,19 +17,21 @@ { "cell_type": "markdown", "source": [ - "

\n", - "\n", - "

\n", - "\n", - "# **Spark Package Management in SQL Server 2019 Big Data Clusters**\n", - "This guide covers installing packages and submitting jobs to a SQL Server 2019 Big Data Cluster using Spark.\n", - "* Built-In Tools\n", - "* Install Packages from a Maven Repository onto the Spark Cluster at Runtime\n", - "* Import .jar from HDFS for use at runtime\n", - "* Import .jar at runtime through Azure Data Studio notebook cell configuration\n", - "* Install Python Packages at Runtime for use with PySpark \n", - "* Submit local .jar or python file\n", - "" + "

\r\n", + "\r\n", + "

\r\n", + "\r\n", + "# **Spark Package Management in SQL Server 2019 Big Data Clusters**\r\n", + "This guide covers installing packages and submitting jobs to a SQL Server 2019 Big Data Cluster using Spark.\r\n", + "* Built-In Tools\r\n", + "* Install Packages from a Maven Repository onto the Spark Cluster at Runtime\r\n", + "* Import .jar from HDFS for use at runtime\r\n", + "* Import .jar at runtime through Azure Data Studio notebook cell configuration\r\n", + "* Install Python Packages at Runtime for use with PySpark \r\n", + "* Submit local .jar or python file\r\n", + "\r\n", + "\r\n", + "For more information on package managament, refer to [Spark library management](https://docs.microsoft.com/sql/big-data-cluster/spark-install-packages?view=sql-server-ver15)" ], "metadata": { "azdata_cell_guid": "cbc8ced8-8931-4302-b252-7e7e478a16d4" @@ -38,13 +40,10 @@ { "cell_type": "markdown", "source": [ - "# Built-in Tools\n", - "* Spark and Hadoop base packages\n", - "* Python 3.5 and Python 2.7\n", - "* Pandas, Sklearn, Numpy, and other data processing packages.\n", - "* R and MRO packages\n", - "* Sparklyr\n", - "" + "# Built-in Tools\r\n", + "* Spark and Hadoop base packages\r\n", + "* Python 3.8 with PySpark and Pandas, Sklearn, Numpy, and other data processing libraries.\r\n", + "* R 3.5 with Spark.R, sparklyr and MRO packages\r\n" ], "metadata": { "azdata_cell_guid": "2fc8a069-115e-4d9b-bedc-5c55f79466b1" @@ -59,8 +58,7 @@ "```\r\n", "%%configure -f \\\r\n", "{\"conf\": {\"spark.jars.packages\": \"com.microsoft.azure:azure-eventhubs-spark_2.11:2.3.1\"}}\r\n", - "```\r\n", - "" + "```\r\n" ], "metadata": { "azdata_cell_guid": "a0fecc05-f094-4dda-9afe-0de8ddad87eb" @@ -76,8 +74,7 @@ "```\n", "%%configure -f\n", "{\"conf\": {\"spark.jars\": \"/jar/mycodeJar.jar\"}}\n", - "```\n", - "" + "```\n" ], "metadata": { "azdata_cell_guid": "c5e65fa2-faf0-4e22-aac1-69d7ff8c9989" @@ -91,8 +88,7 @@ "```\n", "%%configure -f\n", "{\"conf\": {\"spark.jars\": \"/jar/mycodeJar.jar\"}}\n", - "```\n", - "" + "```\n" ], "metadata": { "azdata_cell_guid": "6fc4085f-e142-4355-b215-148dbf6c5b86" @@ -140,12 +136,11 @@ "\r\n", "* [Submit Spark jobs on SQL Server Big Data Clusters in Azure Data Studio](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-submit-job?view=sqlallproducts-allversions)\r\n", "* [Submit Spark jobs on SQL Server Big Data Clusters in IntelliJ](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-submit-job-intellij-tool-plugin?view=sqlallproducts-allversions)\r\n", - "* [Submit Spark jobs on SQL Server big data cluster in Visual Studio Code](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-hive-tools-vscode?view=sqlallproducts-allversions)\r\n", - "" + "* [Submit Spark jobs on SQL Server big data cluster in Visual Studio Code](https://docs.microsoft.com/en-us/sql/big-data-cluster/spark-hive-tools-vscode?view=sqlallproducts-allversions)\r\n" ], "metadata": { "azdata_cell_guid": "7d1b55c0-1961-45f7-8449-a24a913106e4" } } ] -} +} \ No newline at end of file