From 32235b0cb63030d9c2bbfaa601da86d4440a9666 Mon Sep 17 00:00:00 2001 From: Maddy <12754347+MaddyDev@users.noreply.github.com> Date: Tue, 25 Jun 2019 22:46:11 -0700 Subject: [PATCH] Cluster management/newdashboard task (#6060) * initial commit: added cluster status notebook and dashboard task * following the previous naming conventions * endpoint widget changes to accomodatw naming changes * management-proxy/mgmtproxy chnages * updates to address the comments and added the new copy image with hover text. * added user select for making the table selectable * localize changes * added the final documented notebook * reset execution_count to 0 for all cells * style changes * updated the url to point to private repo --- build/gulpfile.hygiene.js | 3 +- .../mssql/notebooks/TSG/cluster-status.ipynb | 159 ++++++++++++++++++ extensions/mssql/package.json | 15 +- extensions/mssql/package.nls.json | 1 + .../resources/dark/cluster_status_inverse.svg | 13 ++ .../mssql/resources/dark/copy_inverse.png | Bin 0 -> 288 bytes .../mssql/resources/light/cluster_status.svg | 9 + extensions/mssql/resources/light/copy.png | Bin 0 -> 277 bytes extensions/mssql/src/constants.ts | 1 + extensions/mssql/src/main.ts | 83 +++++++-- extensions/mssql/src/utils.ts | 29 ++++ .../workbench/api/node/extHostModelView.ts | 14 ++ .../modelComponents/componentWithIconBase.ts | 4 +- 13 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 extensions/mssql/notebooks/TSG/cluster-status.ipynb create mode 100644 extensions/mssql/resources/dark/cluster_status_inverse.svg create mode 100644 extensions/mssql/resources/dark/copy_inverse.png create mode 100644 extensions/mssql/resources/light/cluster_status.svg create mode 100644 extensions/mssql/resources/light/copy.png diff --git a/build/gulpfile.hygiene.js b/build/gulpfile.hygiene.js index 1b1b520d99..88330fa9d9 100644 --- a/build/gulpfile.hygiene.js +++ b/build/gulpfile.hygiene.js @@ -96,7 +96,8 @@ const indentationFilter = [ '!extensions/mssql/sqltoolsservice/**', '!extensions/import/flatfileimportservice/**', '!extensions/admin-tool-ext-win/ssmsmin/**', - '!extensions/resource-deployment/notebooks/**' + '!extensions/resource-deployment/notebooks/**', + '!extensions/mssql/notebooks/**' ]; const copyrightFilter = [ diff --git a/extensions/mssql/notebooks/TSG/cluster-status.ipynb b/extensions/mssql/notebooks/TSG/cluster-status.ipynb new file mode 100644 index 0000000000..ed2a6da2ac --- /dev/null +++ b/extensions/mssql/notebooks/TSG/cluster-status.ipynb @@ -0,0 +1,159 @@ +{ + "metadata": { + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python", + "version": "3.6.6", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + } + }, + "nbformat_minor": 2, + "nbformat": 4, + "cells": [ + { + "cell_type": "markdown", + "source": "\n\n# View the status of your big data cluster\nThis notebook allows you to see the status of the controller, master instance, and pools in your SQL Server big data cluster.\n\n## Important Instructions\n### **Before you begin, you will need:**\n* Big data cluster name\n* Controller username\n* Controller password\n* Controller endpoint \n\nYou can find the controller endpoint from the big data cluster dashboard in the Service Endpoints table. The endpoint is listed as **Cluster Management Service.**\n\nIf you do not know the credentials, ask the admin who deployed your cluster.\n\n### **Instructions**\n* For the best experience, click **Run Cells** on the toolbar above. This will automatically execute all code cells below and show the cluster status in each table.\n* When you click **Run Cells** for this Notebook, you will be prompted at the *Log in to your big data cluster* code cell to provide your login credentials. Follow the prompts and press enter to proceed.\n* **You won't need to modify any of the code cell contents** in this Notebook. If you accidentally made a change, you can reopen this Notebook from the bdc dashboard.\n\n\n", + "metadata": {} + }, + { + "cell_type": "markdown", + "source": "## **Dependencies**\r\n\r\n> This Notebook will try to install these dependencies for you.\r\n\r\n----------------------------------------------------------------\r\n
| Tool | \r\nRequired | \r\nDescription | \r\n
|---|---|---|
| mssqlctl | \r\nYes | \r\nCommand-line tool for installing and managing a big data cluster. | \r\n
| pandas | \r\nYes | \r\nPython Library for formatting data (More info). | \r\n
",
+ "metadata": {}
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Install latest version of mssqlctl**",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "import sys, platform\r\n\r\nif platform.system()==\"Windows\":\r\n user = ' --user'\r\nelse:\r\n user = ''\r\n\r\ndef executeCommand(cmd, successMsgs, printMsg):\r\n print(printMsg)\r\n cmdOutput = !{cmd}\r\n cmdOutput = ''.join(cmdOutput)\r\n if any(msg in cmdOutput for msg in successMsgs):\r\n print(f\"\\nSuccess >> \" + cmd)\r\n else:\r\n raise SystemExit(f'\\nFailed during:\\n\\n\\t{cmd}\\n\\nreturned: \\n' + ''.join(cmdOutput) + '.\\n')\r\n\r\ninstallPath = 'https://private-repo.microsoft.com/python/ctp3.1/mssqlctl/requirements.txt'\r\ncmd = f'{sys.executable} -m pip uninstall --yes mssqlctl-cli-storage'\r\ncmdOutput = !{cmd}\r\n\r\ncmd = f'{sys.executable} -m pip uninstall -r {installPath} --yes'\r\nexecuteCommand(cmd, ['is not installed', 'Successfully uninstalled mssqlctl'], 'Uninstalling mssqlctl:')\r\n\r\ncmd = f'{sys.executable} -m pip install -r {installPath}{user} --trusted-host helsinki'\r\ncmdOutput = !{cmd}\r\nexecuteCommand(cmd, ['Requirement already satisfied', 'Successfully installed mssqlctl'], 'Installing the latest version of mssqlctl:')",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Install latest version of pandas**",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "#install pandas\r\ncmd = f'{sys.executable} -m pip show pandas'\r\ncmdOutput = !{cmd}\r\nif len(cmdOutput) > 0 and '0.24' in cmdOutput[1]:\r\n print('Pandas required version is already installed!')\r\nelse:\r\n pandasVersion = 'pandas==0.24.2'\r\n cmd = f'{sys.executable} -m pip install {pandasVersion}'\r\n cmdOutput = !{cmd}\r\n print(f'\\nSuccess: Upgraded pandas.')",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "## **Log in to your big data cluster**\r\nTo view cluster status, you will need to connect to your big data cluster through mssqlctl. \r\n\r\nWhen you run this code cell, you will be prompted for:\r\n- Cluster name\r\n- Controller username\r\n- Controller password\r\n\r\nTo proceed:\r\n- **Click** on the input box\r\n- **Type** the login info\r\n- **Press** enter.\r\n\r\nIf your cluster is missing a configuration file, you will be asked to provide your controller endpoint. (Format: **https://00.00.00.000:00000**)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "import os, getpass, json\nimport pandas as pd\nimport numpy as np\nfrom IPython.display import *\n\ndef PromptForInfo(promptMsg, isPassword, errorMsg):\n if isPassword:\n promptResponse = getpass.getpass(prompt=promptMsg)\n else:\n promptResponse = input(promptMsg)\n if promptResponse == \"\":\n raise SystemExit(errorMsg + '\\n')\n return promptResponse\n\n# Prompt user inputs:\ncluster_name = PromptForInfo('Please provide your Cluster Name: ', False, 'Cluster Name is required!')\n\ncontroller_username = PromptForInfo('Please provide your Controller Username for login: ', False, 'Controller Username is required!')\n\ncontroller_password = PromptForInfo('Controller Password: ', True, 'Password is required!')\nprint('***********')\n\n!mssqlctl logout\n# Login in to your big data cluster \ncmd = f'mssqlctl login -n {cluster_name} -u {controller_username} -a yes'\nprint(\"Start \" + cmd)\nos.environ['CONTROLLER_USERNAME'] = controller_username\nos.environ['CONTROLLER_PASSWORD'] = controller_password\nos.environ['ACCEPT_EULA'] = 'yes'\n\nloginResult = !{cmd}\nif 'ERROR: Please check your kube config or specify the correct controller endpoint with: --controller-endpoint https:// ",
+ "metadata": {}
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Controller status**\nTo learn more about the controller, [read here.](https://docs.microsoft.com/sql/big-data-cluster/concept-controller?view=sql-server-ver15)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of controller\nresults = !mssqlctl bdc control status show\nshow_results(results)",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Master Instance status**\nTo learn more about the master instance, [read here.](https://docs.microsoft.com/sql/big-data-cluster/concept-master-instance?view=sqlallproducts-allversions)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of master instance\nresults = !mssqlctl bdc pool status show -k master -n default\nshow_results(results)",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Compute Pool status**\nTo learn more about compute pool, [read here.](https://docs.microsoft.com/sql/big-data-cluster/concept-compute-pool?view=sqlallproducts-allversions)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of compute pool\nresults = !mssqlctl bdc pool status show -k compute -n default\nshow_results(results)",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Storage Pool status**\nTo learn more about storage pool, [read here.](https://docs.microsoft.com/sql/big-data-cluster/concept-storage-pool?view=sqlallproducts-allversions)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of storage pools\nresults = !mssqlctl bdc pool status show -k storage -n default\nshow_results(results)",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Data Pool status**\nTo learn more about data pool, [read here.](https://docs.microsoft.com/sql/big-data-cluster/concept-data-pool?view=sqlallproducts-allversions)",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of data pools\nresults = !mssqlctl bdc pool status show -k data -n default\nshow_results(results)",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ },
+ {
+ "cell_type": "markdown",
+ "source": "### **Spark Pool status**\nDisplays status of spark pool if it exists. Otherwise, will show as \"No spark pool.\"",
+ "metadata": {}
+ },
+ {
+ "cell_type": "code",
+ "source": "# Display status of spark pool\nif spark_exists:\n results = !mssqlctl bdc pool status show -k spark -n default\n show_results(results)\nelse:\n print('No spark pool.')",
+ "metadata": {},
+ "outputs": [],
+ "execution_count": 0
+ }
+ ]
+}
\ No newline at end of file
diff --git a/extensions/mssql/package.json b/extensions/mssql/package.json
index fd97dd1bdd..dfc5612f03 100644
--- a/extensions/mssql/package.json
+++ b/extensions/mssql/package.json
@@ -89,11 +89,11 @@
}
},
{
- "command": "mssqlCluster.livy.task.openSparkHistory",
- "title": "%title.openSparkHistory%",
+ "command": "mssqlCluster.task.openClusterStatusNotebook",
+ "title": "%title.openClusterStatusNotebook%",
"icon": {
- "dark": "resources/dark/new_spark_job_inverse.svg",
- "light": "resources/light/new_spark_job.svg"
+ "dark": "resources/dark/cluster_status_inverse.svg",
+ "light": "resources/light/cluster_status.svg"
}
},
{
@@ -247,6 +247,10 @@
{
"command": "mssqlCluster.livy.task.submitSparkJob",
"when": "false"
+ },
+ {
+ "command": "mssqlCluster.task.openClusterStatusNotebook",
+ "when": "false"
}
],
"objectExplorer/item/context": [
@@ -405,7 +409,8 @@
"tasks-widget": [
"mssqlCluster.task.newNotebook",
"mssqlCluster.task.openNotebook",
- "mssqlCluster.livy.task.submitSparkJob"
+ "mssqlCluster.livy.task.submitSparkJob",
+ "mssqlCluster.task.openClusterStatusNotebook"
]
}
},
diff --git a/extensions/mssql/package.nls.json b/extensions/mssql/package.nls.json
index be3b10400b..4eeae5d413 100644
--- a/extensions/mssql/package.nls.json
+++ b/extensions/mssql/package.nls.json
@@ -25,6 +25,7 @@
"title.tasks": "Tasks",
"title.installPackages": "Install Packages",
"title.configurePython": "Configure Python for Notebooks",
+ "title.openClusterStatusNotebook": "Cluster Status",
"title.searchServers": "Search: Servers",
"title.clearSearchServerResult": "Search: Clear Search Server Results",
diff --git a/extensions/mssql/resources/dark/cluster_status_inverse.svg b/extensions/mssql/resources/dark/cluster_status_inverse.svg
new file mode 100644
index 0000000000..971d362195
--- /dev/null
+++ b/extensions/mssql/resources/dark/cluster_status_inverse.svg
@@ -0,0 +1,13 @@
+
+
+
diff --git a/extensions/mssql/resources/dark/copy_inverse.png b/extensions/mssql/resources/dark/copy_inverse.png
new file mode 100644
index 0000000000000000000000000000000000000000..1770281129be8f022ab97bb16f442d5b8c22f804
GIT binary patch
literal 288
zcmeAS@N?(olHy`uVBq!ia0vp^iXhCv1|-9u9Lfh$oCO|{#S9E$svykh8Km+7D0s=!
z#W5tJ_3iYFyoVKdT>7=Rm#p5RUGi$3L;6dObwWpjT9SC$tbRW5{HMTJvhn-^7NtWs
z?+7ubbZ{i({hSce(#qaY-YwwiqS0gB;J\r\n
\r\n\r\n \r\n\r\n\r\nColumn Name \r\nDescription \r\n\r\n \r\nKind \r\nIdentifies if component is a pod or a set. \r\n\r\n \r\nLogsURL \r\nLink to Kibana logs which is used for troubleshooting. \r\n\r\n \r\nName \r\nProvides the specific name of the pod or set. \r\n\r\n \r\nNodeMetricsURL \r\nLink to Grafana dashboard to view key metrics of the node. \r\n\r\n \r\nSQLMetricsURL \r\nLink to Grafana dashboard to view key metrics of the SQL instance \r\n\r\n \r\n\r\nState \r\nIndicates state of the pod or set.. \r\nOz^c`@WZsQu5;4s-<>dWF>q!v1`*10XRNDtnSQ3F%
+
+
diff --git a/extensions/mssql/resources/light/copy.png b/extensions/mssql/resources/light/copy.png
new file mode 100644
index 0000000000000000000000000000000000000000..776c79637812d7242b36cd455ad801233a5fb7d6
GIT binary patch
literal 277
zcmeAS@N?(olHy`uVBq!ia0vp^iXhCv1|-9u9Lfh$oCO|{#S9E$svykh8Km+7D0tk{
z#W5tJ_3hL^z7_)x7xvBKA9Sk!>}{%EmN&~|QKU$;#;=1aH8ae9c(Hpe6}huEliR^%
zsj`^TEMNAOQy;9DKX)U;jO{z4oLY9wTJW!e)u3^P!XAe+0&)s^0$~%>Is%vVHUv2C
zQjl8rLg<7_PoLl~rV8dC9nHEcgqAPkZM?8c{gQBf+SA(I;alwDUW-nA!kzm!