fixed perf tests and added more scenarios (#683)

* fixed perf tests and added more scenarios

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Program.cs

@@ -25,8 +25,7 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
             }
 
             Logger.Initialize("testdriver", LogLevel.Verbose);
-
+            return TestRunner.Instance.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
-            return TestRunner.RunTests(args, "Microsoft.SqlTools.ServiceLayer.PerfTests.").Result;
         }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ConnectionTests.cs

@@ -17,6 +17,8 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
         [Fact]
         [CreateTestDb(TestServerType.Azure)]
         public async Task ConnectAzureTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.Azure;
                 using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
@@ -43,14 +45,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     {
                         var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
                         return await testService.Connect(queryTempFile.FilePath, connectParams);
-                }, true);
+                    }, timer);
                     Assert.True(connected, "Connection was not successful");
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task ConnectOnPremTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -78,14 +83,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     {
                         var connectParams = testService.GetConnectionParameters(serverType, Common.PerfTestDatabaseName);
                         return await testService.Connect(queryTempFile.FilePath, connectParams);
-                }, true);
+                    }, timer);
                     Assert.True(connected, "Connection was not successful");
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task DisconnectTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -94,9 +102,10 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                 {
                     await testService.ConnectForQuery(serverType, Scripts.TestDbSimpleSelectQuery, queryTempFile.FilePath, Common.PerfTestDatabaseName);
                     Thread.Sleep(1000);
-                var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), true);
+                    var connected = await testService.CalculateRunTime(() => testService.Disconnect(queryTempFile.FilePath), timer);
                     Assert.True(connected);
                 }
+            });
         }
 
     }

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/IntellisenseTests.cs

@@ -20,6 +20,8 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task HoverTestOnPrem()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
@@ -27,15 +29,19 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                 {
                     const string query = Scripts.TestDbSimpleSelectQuery;
                     await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
-                Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), true);
+                    await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, waitForIntelliSense: true);
+                    Hover hover = await testService.CalculateRunTime(() => testService.RequestHover(queryTempFile.FilePath, query, 0, Scripts.TestDbComplexSelectQueries.Length + 1), timer);
                     Assert.NotNull(hover);
                     await testService.Disconnect(queryTempFile.FilePath);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task SuggestionsTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
@@ -43,15 +49,18 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                 {
                     const string query = Scripts.TestDbSimpleSelectQuery;
                     await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, Common.PerfTestDatabaseName);
-                await ValidateCompletionResponse(testService, queryTempFile.FilePath, false, Common.PerfTestDatabaseName, true);
+                    await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: null, waitForIntelliSense: true);
-                await ValidateCompletionResponse(testService, queryTempFile.FilePath, true, Common.PerfTestDatabaseName, false);
+                    await ValidateCompletionResponse(testService, queryTempFile.FilePath, Common.PerfTestDatabaseName, timer: timer, waitForIntelliSense: false);
                     await testService.Disconnect(queryTempFile.FilePath);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task DiagnosticsTests()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
                 SqlTestDb.CreateNew(serverType, doNotCleanupDb: true, databaseName: Common.PerfTestDatabaseName, query: Scripts.CreateDatabaseObjectsQuery);
@@ -91,7 +100,6 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                         }
                     };
 
-                TestTimer timer = new TestTimer() { PrintResult = true };
                     await testService.RequestChangeTextDocumentNotification(changeParams);
                     await testService.ExecuteWithTimeout(timer, 60000, async () =>
                     {
@@ -100,47 +108,59 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     });
                     await testService.Disconnect(queryTempFile.FilePath);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.Azure)]
         public async Task BindingCacheColdAzureSimpleQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.Azure;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
-                await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false);
+                    await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task BindingCacheColdOnPremSimpleQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
-                await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false);
+                    await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbSimpleSelectQuery, false, timer);
                 }
+            });
             
         }
 
         [Fact]
         [CreateTestDb(TestServerType.Azure)]
         public async Task BindingCacheWarmAzureSimpleQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.Azure;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
                 {
                     const string query = Scripts.TestDbSimpleSelectQuery;
-                await VerifyBindingLoadScenario(testService, serverType, query, true);
+                    await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task BindingCacheWarmOnPremSimpleQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -148,57 +168,70 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                 using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
                 {
                     const string query = Scripts.TestDbSimpleSelectQuery;
-                await VerifyBindingLoadScenario(testService, serverType, query, true);
+                    await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.Azure)]
         public async Task BindingCacheColdAzureComplexQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.Azure;
 
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
-                await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries,false);
+                    await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task BindingCacheColdOnPremComplexQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
-                await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false);
+                    await VerifyBindingLoadScenario(testService, serverType, Scripts.TestDbComplexSelectQueries, false, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.Azure)]
         public async Task BindingCacheWarmAzureComplexQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
                     string query = Scripts.TestDbComplexSelectQueries;
                     const TestServerType serverType = TestServerType.Azure;
-                await VerifyBindingLoadScenario(testService, serverType, query, true);
+                    await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
                 }
+            });
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task BindingCacheWarmOnPremComplexQuery()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
                 using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
                 {
                     string query = Scripts.TestDbComplexSelectQueries;
                     const TestServerType serverType = TestServerType.OnPrem;
-                await VerifyBindingLoadScenario(testService, serverType, query, true);
+                    await VerifyBindingLoadScenario(testService, serverType, query, true, timer);
                 }
+            });
         }
 
         #region Private Helper Methods
@@ -208,17 +241,18 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
             TestServerType serverType, 
             string query, 
             bool preLoad,
+            TestTimer timer,
             [CallerMemberName] string testName = "")
         {
             string databaseName = Common.PerfTestDatabaseName;
             if (preLoad)
             {
                 await VerifyCompletationLoaded(testService, serverType, Scripts.TestDbSimpleSelectQuery, 
-                    databaseName, printResult: false, testName: testName);
+                    databaseName, null, testName: testName);
                 Console.WriteLine("Intellisense cache loaded.");
             }
             await VerifyCompletationLoaded(testService, serverType, query, databaseName, 
-                printResult: true, testName: testName);
+                timer, testName: testName);
         }
 
         private async Task VerifyCompletationLoaded(
@@ -226,15 +260,16 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
             TestServerType serverType,
             string query,
             string databaseName,
-            bool printResult, 
+            TestTimer timer,
             string testName)
         {
+
             using (SelfCleaningTempFile testTempFile = new SelfCleaningTempFile())
             {
                 testService.WriteToFile(testTempFile.FilePath, query);
                 await testService.ConnectForQuery(serverType, query, testTempFile.FilePath, databaseName);
-                await ValidateCompletionResponse(testService, testTempFile.FilePath, printResult, databaseName, 
+                await ValidateCompletionResponse(testService, testTempFile.FilePath, databaseName,
-                    waitForIntelliSense: true, testName: testName);
+                    waitForIntelliSense: true, timer: timer, testName: testName);
                 await testService.Disconnect(testTempFile.FilePath);
             }
         }
@@ -242,14 +277,17 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
         private static async Task ValidateCompletionResponse(
             TestServiceDriverProvider testService, 
             string ownerUri, 
-            bool printResult, 
             string databaseName, 
             bool waitForIntelliSense,
+            TestTimer timer = null,
             [CallerMemberName] string testName = "")
         {
-            TestTimer timer = new TestTimer() { PrintResult = printResult };
+            if (timer == null)
+            {
+                timer = new TestTimer { PrintResult = false };
+            }
             bool isReady = !waitForIntelliSense;
-            await testService.ExecuteWithTimeout(timer, 150000, async () =>
+            await testService.ExecuteWithTimeout(timer, 550000, async () =>
             {
                 if (isReady)
                 {

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ObjectExplorerTests.cs

@@ -0,0 +1,146 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
+using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
+using Microsoft.SqlTools.ServiceLayer.Test.Common;
+using Xunit;
+
+namespace Microsoft.SqlTools.ServiceLayer.PerfTests
+{
+    public class ObjectExplorerTests
+    {
+        [Fact]
+        [CreateTestDb(TestServerType.Azure)]
+        public async Task CreateSessionAzure()
+        {
+            TestServerType serverType = TestServerType.Azure;
+            await VerifyCreateSession(serverType);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.OnPrem)]
+        public async Task CreateSessionOnPrem()
+        {
+            TestServerType serverType = TestServerType.OnPrem;
+            await VerifyCreateSession(serverType);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.OnPrem)]
+        public async Task ExpandDatabasesOnPrem()
+        {
+            TestServerType serverType = TestServerType.OnPrem;
+            await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.OnPrem)]
+        public async Task ExpandOneDatabaseOnPrem()
+        {
+            TestServerType serverType = TestServerType.OnPrem;
+            await VerifyExpand(serverType, Common.PerfTestDatabaseName);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.Azure)]
+        public async Task ExpandDatabasesAzure()
+        {
+            TestServerType serverType = TestServerType.Azure;
+            await VerifyExpand(serverType, SqlTestDb.MasterDatabaseName);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.Azure)]
+        public async Task ExpandOneDatabaseAzure()
+        {
+            TestServerType serverType = TestServerType.Azure;
+            await VerifyExpand(serverType, Common.PerfTestDatabaseName);
+        }
+
+        private async Task VerifyCreateSession(TestServerType serverType, [CallerMemberName] string testName = "")
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
+            {
+                ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, SqlTestDb.MasterDatabaseName);
+                using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
+                using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
+                {
+                    var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), timer);
+
+                    Assert.NotNull(result);
+                    Assert.True(result.Success);
+                    Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
+
+                    await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
+                    {
+                        SessionId = result.SessionId
+                    });
+                    await testService.Disconnect(queryTempFile.FilePath);
+                }
+            }, testName);
+        }
+
+
+        private async Task VerifyExpand(TestServerType serverType, string databaseName, [CallerMemberName] string testName = "")
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
+            {
+                ConnectParams connectParams = TestServiceProvider.Instance.ConnectionProfileService.GetConnectionParameters(serverType, databaseName);
+                using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
+                using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
+                {
+                    var result = await testService.CalculateRunTime(() => testService.RequestObjectExplorerCreateSession(connectParams.Connection), null);
+
+                    Assert.NotNull(result);
+                    Assert.True(result.Success);
+                    Assert.False(string.IsNullOrEmpty(result.SessionId), "Session id cannot be empty");
+
+                    await ExpandDatabase(testService, result.SessionId, result.RootNode, timer);
+
+                    await testService.RequestObjectExplorerCloseSession(new ObjectExplorer.Contracts.CloseSessionParams
+                    {
+                        SessionId = result.SessionId
+                    });
+                    await testService.Disconnect(queryTempFile.FilePath);
+                }
+            }, testName);
+        }
+
+        private async Task<bool> ExpandDatabase(TestServiceDriverProvider testService, string sessionId, NodeInfo nodeInfo, TestTimer timer)
+        {
+            if (nodeInfo == null) return false;
+            bool foundNode = nodeInfo.NodePath.Contains("Database") || nodeInfo.NodeType == "Database";
+            var expandResult = await testService.CalculateRunTime(() => testService.RequestObjectExplorerExpand(new ObjectExplorer.Contracts.ExpandParams
+            {
+                SessionId = sessionId,
+                NodePath = nodeInfo.NodePath
+            }, 50000), foundNode ? timer : null);
+
+            Assert.NotNull(expandResult);
+            Assert.NotNull(expandResult.Nodes);
+            Assert.False(expandResult.Nodes == null, "Nodes are not valid");
+            if (!foundNode)
+            {
+                foreach (var node in expandResult.Nodes)
+                {
+                    if (await ExpandDatabase(testService, sessionId, node, timer))
+                    {
+                        return true;
+                    }
+                }
+                return false;
+            }
+            else
+            {
+                Console.WriteLine("Node Expanded " + nodeInfo.NodePath);
+                return true;
+            }
+        }
+    }
+}

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/QueryExecutionTests.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Threading.Tasks;
 using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
 using Microsoft.SqlTools.ServiceLayer.Test.Common;
@@ -17,52 +18,56 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
         [Fact]
         public async Task QueryResultSummaryOnPremTest()
         {
-            TestServerType serverType = TestServerType.OnPrem;
+            await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
-
-            using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
-            using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
-            {
-                const string query = Scripts.MasterBasicQuery;
-
-                await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
-                var queryResult = await testService.CalculateRunTime(() => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query), true);
-
-                Assert.NotNull(queryResult);
-                Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
-
-                await testService.Disconnect(queryTempFile.FilePath);
-            }
         }
 
         [Fact]
         public async Task QueryResultFirstOnPremTest()
         {
-            TestServerType serverType = TestServerType.OnPrem;
+            await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterBasicQuery);
-
-            using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
-            using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
-            {
-                const string query = Scripts.MasterBasicQuery;
-
-                await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
-
-                var queryResult = await testService.CalculateRunTime(async () =>
-                {
-                    await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
-                    return await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100);
-                }, true);
-
-                Assert.NotNull(queryResult);
-                Assert.NotNull(queryResult.ResultSubset);
-                Assert.True(queryResult.ResultSubset.Rows.Any());
-
-                await testService.Disconnect(queryTempFile.FilePath);
         }
+
+        [Fact]
+        public async Task LongQueryResultSummaryOnPremTest()
+        {
+            await QueryResultSummaryOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
+        }
+
+        [Fact]
+        public async Task LongQueryResultFirstOnPremTest()
+        {
+            await QueryResultFirstOnPremTest(TestServerType.OnPrem, Scripts.MasterLongQuery);
+        }
+
+        [Fact]
+        public async Task QueryResultSummaryOnAzureTest()
+        {
+            await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
+        }
+
+        [Fact]
+        public async Task QueryResultFirstOnAzureTest()
+        {
+            await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterBasicQuery);
+        }
+
+        [Fact]
+        public async Task LongQueryResultSummaryOnAzureTest()
+        {
+            await QueryResultSummaryOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
+        }
+
+        [Fact]
+        public async Task LongQueryResultFirstOnAzureTest()
+        {
+            await QueryResultFirstOnPremTest(TestServerType.Azure, Scripts.MasterLongQuery);
         }
 
         [Fact]
         [CreateTestDb(TestServerType.OnPrem)]
         public async Task CancelQueryOnPremTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -76,10 +81,11 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                         QuerySelection = null
                     };
 
+                    testService.WriteToFile(queryTempFile.FilePath, Scripts.MasterLongQuery);
+
                     var result = await testService.Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
                     if (result != null)
                     {
-                    TestTimer timer = new TestTimer() { PrintResult = true };
                         await testService.ExecuteWithTimeout(timer, 100000, async () =>
                         {
                             var cancelQueryResult = await testService.CancelQuery(queryTempFile.FilePath);
@@ -89,10 +95,60 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     else
                     {
                         Assert.True(false, "Failed to run the query");
-                }
 
                         await testService.Disconnect(queryTempFile.FilePath);
                     }
                 }
+            });
+
+        }
+
+        private async Task QueryResultSummaryOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
+            {
+                using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
+                using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
+                {
+                    await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
+                    testService.WriteToFile(queryTempFile.FilePath, query);
+                    var queryResult = await testService.CalculateRunTime(
+                        () => testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, 50000),
+                        timer);
+
+                    Assert.NotNull(queryResult);
+                    Assert.True(queryResult.BatchSummaries.Any(x => x.ResultSetSummaries.Any(r => r.RowCount > 0)));
+
+                    await testService.Disconnect(queryTempFile.FilePath);
+                }
+            }, testName);
+        }
+
+        private async Task QueryResultFirstOnPremTest(TestServerType serverType, string query, [CallerMemberName] string testName = "")
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
+            {
+                using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
+                using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
+                {
+                    await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
+                    testService.WriteToFile(queryTempFile.FilePath, query);
+                    await testService.RunQueryAndWaitToStart(queryTempFile.FilePath, 50000);
+                    await testService.ExecuteWithTimeout(timer, 500000, async () =>
+                    {
+                        var queryResult = await testService.ExecuteSubset(queryTempFile.FilePath, 0, 0, 0, 100);
+                        if (queryResult != null)
+                        {
+                            Assert.NotNull(queryResult);
+                            Assert.NotNull(queryResult.ResultSubset);
+                            Assert.True(queryResult.ResultSubset.Rows.Any());
+                        }
+                        return queryResult != null;
+                    }, TimeSpan.FromMilliseconds(10));
+
+                    await testService.Disconnect(queryTempFile.FilePath);
+                }
+            }, testName);
+        }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/SaveResultsTests.cs

@@ -13,6 +13,8 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
     {
         [Fact]
         public async Task TestSaveResultsToCsvTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -25,13 +27,16 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     // Execute a query
                     await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
                     await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
-                await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true);
+                    await testService.CalculateRunTime(() => testService.SaveAsCsv(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
                     await testService.Disconnect(queryTempFile.FilePath);
                 }
+            });
         }
 
         [Fact]
         public async Task TestSaveResultsToJsonTest()
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
             {
                 TestServerType serverType = TestServerType.OnPrem;
 
@@ -44,9 +49,10 @@ namespace Microsoft.SqlTools.ServiceLayer.PerfTests
                     // Execute a query
                     await testService.ConnectForQuery(serverType, query, queryTempFile.FilePath, SqlTestDb.MasterDatabaseName);
                     await testService.RunQueryAndWaitToComplete(queryTempFile.FilePath, query);
-                await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), true);
+                    await testService.CalculateRunTime(() => testService.SaveAsJson(queryTempFile.FilePath, outputTempFile.FilePath, 0, 0), timer);
                     await testService.Disconnect(queryTempFile.FilePath);
                 }
+            });
         }
 
     }

test/Microsoft.SqlTools.ServiceLayer.PerfTests/Tests/ScriptinTests.cs

@@ -0,0 +1,79 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
+using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
+using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
+using Microsoft.SqlTools.ServiceLayer.Test.Common;
+using Xunit;
+
+namespace Microsoft.SqlTools.ServiceLayer.PerfTests
+{
+    public class ScriptingTests
+    {
+        [Fact]
+        [CreateTestDb(TestServerType.Azure)]
+        public async Task ScripTableAzure()
+        {
+            TestServerType serverType = TestServerType.Azure;
+            await VerifyScriptTable(serverType);
+        }
+
+        [Fact]
+        [CreateTestDb(TestServerType.OnPrem)]
+        public async Task ScripTableOnPrem()
+        {
+            TestServerType serverType = TestServerType.OnPrem;
+            await VerifyScriptTable(serverType);
+        }
+
+
+        private async Task VerifyScriptTable(TestServerType serverType, [CallerMemberName] string testName = "")
+        {
+            await TestServiceDriverProvider.RunTestIterations(async (timer) =>
+            {
+                using (TestServiceDriverProvider testService = new TestServiceDriverProvider())
+                using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
+                {
+                    await testService.ConnectForQuery(serverType, string.Empty, queryTempFile.FilePath, Common.PerfTestDatabaseName);
+                    List<ScriptingObject> scriptingObjects = new List<ScriptingObject>()
+                    {
+                        new ScriptingObject
+                        {
+                            Schema = "Person",
+                            Name = "Address",
+                            Type = "Table"
+                        }
+                    };
+                    ScriptingParams scriptingParams = new ScriptingParams
+                    {
+                        OwnerUri = queryTempFile.FilePath,
+                        Operation = ScriptingOperationType.Create,
+                        FilePath = queryTempFile.FilePath,
+                        ScriptOptions = new ScriptOptions
+                        {
+                            ScriptCreateDrop = "ScriptCreate",
+                        },
+                        ScriptDestination = "ToEditor",
+                        ScriptingObjects = scriptingObjects
+
+                    };
+                    var result = await testService.CalculateRunTime(() => testService.RequestScript(scriptingParams), timer);
+
+                    Assert.NotNull(result);
+                    Assert.NotNull(result.Script);
+                   
+                    Assert.False(string.IsNullOrEmpty(result.Script), "Script result is invalid");
+
+                    await testService.Disconnect(queryTempFile.FilePath);
+                }
+            }, testName);
+        }
+    }
+}

test/Microsoft.SqlTools.ServiceLayer.Test.Common/Scripts/Scripts.cs

@@ -14,6 +14,8 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
         private const string ResourceNameRefix = "Microsoft.SqlTools.ServiceLayer.Test.Common.Scripts.";
 
         public const string MasterBasicQuery = "SELECT * FROM sys.all_columns"; //basic queries should return at least 10000 rows
+        public const string MasterLongQuery = @"SELECT * FROM sys.all_columns a1 
+                                                JOIN sys.all_columns a2 on a1.object_id = a2.object_id";
 
         public const string DelayQuery = "WAITFOR DELAY '00:01:00'";
 

test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestResult.cs

@@ -3,10 +3,31 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
+using Newtonsoft.Json;
+
 namespace Microsoft.SqlTools.ServiceLayer.Test.Common
 {
     public class TestResult
     {
+        [JsonProperty("elapsedTime")]
         public double ElapsedTime { get; set; }
+
+        [JsonProperty("metricValue")]
+        public double MetricValue { get; set; }
+
+        [JsonProperty("iterations")]
+        public double[] Iterations { get; set; }
+
+        [JsonProperty("ninetiethPercentile")]
+        public double NinetiethPercentile { get; set; }
+
+        [JsonProperty("fiftiethPercentile")]
+        public double FiftiethPercentile { get; set; }
+
+        [JsonProperty("average")]
+        public double Average { get; set; }
+
+        [JsonProperty("primaryMetric")]
+        public string PrimaryMetric { get; set; }
     }
 }

test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestServiceDriverProvider.cs

@@ -9,11 +9,13 @@ using System.Threading.Tasks;
 using Microsoft.SqlTools.Hosting.Protocol.Contracts;
 using Microsoft.SqlTools.ServiceLayer.Connection.Contracts;
 using Microsoft.SqlTools.ServiceLayer.LanguageServices.Contracts;
+using Microsoft.SqlTools.ServiceLayer.ObjectExplorer.Contracts;
 using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts;
 using Microsoft.SqlTools.ServiceLayer.QueryExecution.Contracts.ExecuteRequests;
 using Microsoft.SqlTools.ServiceLayer.Scripting.Contracts;
 using Microsoft.SqlTools.ServiceLayer.SqlContext;
 using Microsoft.SqlTools.ServiceLayer.TestDriver.Driver;
+using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
 using Microsoft.SqlTools.ServiceLayer.Workspace.Contracts;
 using Xunit;
 
@@ -29,7 +31,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
 
         public TestServiceDriverProvider()
         {
-            Driver = new ServiceTestDriver();
+            Driver = new ServiceTestDriver(TestRunner.Instance.ExecutableFilePath);
             Driver.Start().Wait();
             this.isRunning = true;
         }
@@ -228,6 +230,18 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             return result;
         }
 
+        /// <summary>
+        /// Request a list of completion items for a position in a block of text
+        /// </summary>
+        public async Task RequestRebuildIntelliSense(string ownerUri)
+        {
+            var rebuildIntelliSenseParams = new RebuildIntelliSenseParams();
+            rebuildIntelliSenseParams.OwnerUri = ownerUri;
+
+            await Driver.SendEvent(RebuildIntelliSenseNotification.Type, rebuildIntelliSenseParams);
+        }
+
+
         /// <summary>
         /// Request a a hover tooltop
         /// </summary>
@@ -284,6 +298,15 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             // Write the query text to a backing file
             WriteToFile(ownerUri, query);
 
+            return await RunQueryAndWaitToComplete(ownerUri, timeoutMilliseconds);
+        }
+
+        /// <summary>
+        /// Run a query using a given connection bound to a URI
+        /// </summary>
+        public async Task<QueryCompleteParams> RunQueryAndWaitToComplete(string ownerUri, int timeoutMilliseconds = 5000)
+        {
+
             var queryParams = new ExecuteDocumentSelectionParams
             {
                 OwnerUri = ownerUri,
@@ -302,6 +325,79 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             }
         }
 
+        /// <summary>
+        /// Run a query using a given connection bound to a URI
+        /// </summary>
+        public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, string query, int timeoutMilliseconds = 5000)
+        {
+            // Write the query text to a backing file
+            WriteToFile(ownerUri, query);
+
+            return await RunQueryAndWaitToStart(ownerUri, timeoutMilliseconds);
+        }
+
+        /// <summary>
+        /// Run a query using a given connection bound to a URI
+        /// </summary>
+        public async Task<BatchEventParams> RunQueryAndWaitToStart(string ownerUri, int timeoutMilliseconds = 5000)
+        {
+            var queryParams = new ExecuteDocumentSelectionParams
+            {
+                OwnerUri = ownerUri,
+                QuerySelection = null
+            };
+
+            var result = await Driver.SendRequest(ExecuteDocumentSelectionRequest.Type, queryParams);
+            if (result != null)
+            {
+                var eventResult = await Driver.WaitForEvent(BatchStartEvent.Type, timeoutMilliseconds);
+                return eventResult;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        public async Task<SessionCreatedParameters> RequestObjectExplorerCreateSession(ConnectionDetails connectionDetails, int timeoutMilliseconds = 5000)
+        {
+            var result = await Driver.SendRequest(CreateSessionRequest.Type, connectionDetails);
+            if (result != null)
+            {
+                var eventResult = await Driver.WaitForEvent(CreateSessionCompleteNotification.Type, timeoutMilliseconds);
+                return eventResult;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        public async Task<ExpandResponse> RequestObjectExplorerExpand(ExpandParams expandParams, int timeoutMilliseconds = 5000)
+        {
+            var result = await Driver.SendRequest(ExpandRequest.Type, expandParams);
+            if (result)
+            {
+                var eventResult = await Driver.WaitForEvent(ExpandCompleteNotification.Type, timeoutMilliseconds);
+                return eventResult;
+            }
+            else
+            {
+                return null;
+            }
+        }
+
+        public async Task<ScriptingResult> RequestScript(ScriptingParams scriptingParams, int timeoutMilliseconds = 5000)
+        {
+            var result = await Driver.SendRequest(ScriptingRequest.Type, scriptingParams);
+            return result;
+        }
+
+        public async Task<CloseSessionResponse> RequestObjectExplorerCloseSession(CloseSessionParams closeSessionParams, int timeoutMilliseconds = 5000)
+        {
+            return await Driver.SendRequest(CloseSessionRequest.Type, closeSessionParams);
+        }
+
         /// <summary>
         /// Run a query using a given connection bound to a URI. This method only waits for the initial response from query
         /// execution (QueryExecuteResult). It is up to the caller to wait for the QueryCompleteEvent if they are interested.
@@ -324,7 +420,7 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             using (SelfCleaningTempFile queryTempFile = new SelfCleaningTempFile())
             {
                 await ConnectForQuery(serverType, query, queryTempFile.FilePath, databaseName);
-                var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000), false);
+                var queryResult = await CalculateRunTime(() => RunQueryAndWaitToComplete(queryTempFile.FilePath, query, 50000));
                 Assert.NotNull(queryResult);
                 Assert.NotNull(queryResult.BatchSummaries);
 
@@ -332,11 +428,29 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             }
         }
 
-        public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, bool printResult, [CallerMemberName] string testName = "")
+        public static async Task RunTestIterations(Func<TestTimer, Task> testToRun, [CallerMemberName] string testName = "")
         {
-            TestTimer timer = new TestTimer() { PrintResult = printResult };
+            TestTimer timer = new TestTimer() { PrintResult = true };
+            for (int i = 0; i < TestRunner.Instance.NumberOfRuns; i++)
+            {
+                Console.WriteLine("Iteration Number: " + i);
+               
+                await testToRun(timer);
+            }
+            timer.Print(testName);
+        }
+
+        public async Task<T> CalculateRunTime<T>(Func<Task<T>> testToRun, TestTimer timer = null)
+        {
+            if (timer != null)
+            {
+                timer.Start();
+            }
             T result = await testToRun();
-            timer.EndAndPrint(testName);
+            if (timer != null)
+            {
+                timer.End();
+            }
 
             return result;
         }
@@ -344,11 +458,12 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
         public async Task ExecuteWithTimeout(TestTimer timer, int timeout, Func<Task<bool>> repeatedCode,
             TimeSpan? delay = null, [CallerMemberName] string testName = "")
         {
+            timer.Start();
             while (true)
             {
                 if (await repeatedCode())
                 {
-                    timer.EndAndPrint(testName);
+                    timer.End();
                     break;
                 }
                 if (timer.TotalMilliSecondsUntilNow >= timeout)

test/Microsoft.SqlTools.ServiceLayer.Test.Common/TestTimer.cs

@@ -3,9 +3,12 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 //
 
+using Microsoft.SqlTools.ServiceLayer.TestDriver.Utility;
 using System;
+using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Runtime.CompilerServices;
 
 namespace Microsoft.SqlTools.ServiceLayer.Test.Common
@@ -17,10 +20,16 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
     {
         private static string resultFolder = InitResultFolder();
 
+        private List<double> iterations = new List<double>();
+
         private static string InitResultFolder()
         {
             string resultFodler = Environment.GetEnvironmentVariable("ResultFolder");
             if (string.IsNullOrEmpty(resultFodler))
+            {
+                resultFodler = TestRunner.Instance.ResultFolder;
+            }
+            if (string.IsNullOrEmpty(resultFodler))
             {
                 string assemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
                 resultFodler = Path.GetDirectoryName(assemblyLocation);
@@ -43,18 +52,39 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
         public void End()
         {
             EndDateTime = DateTime.UtcNow;
+            iterations.Add(TotalMilliSeconds);
+            if (PrintResult)
+            {
+                Console.WriteLine("Result: " + TotalMilliSeconds);
+            }
         }
 
         public void EndAndPrint([CallerMemberName] string testName = "")
         {
             End();
+            Print(testName);
+        }
+
+        public void Print([CallerMemberName] string testName = "")
+        {
             if (PrintResult)
             {
+                var iterationArray = iterations.ToArray();
+                double elapsed = Percentile(iterationArray, 0.5);
                 var currentColor = Console.ForegroundColor;
                 Console.ForegroundColor = ConsoleColor.Green;
                 Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "Test Name: {0} Run time in milliSeconds: {1}", testName, TotalMilliSeconds));
                 Console.ForegroundColor = currentColor;
-                string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult { ElapsedTime = TotalMilliSeconds });
+                string resultContent = Newtonsoft.Json.JsonConvert.SerializeObject(new TestResult
+                {
+                    ElapsedTime = TotalMilliSeconds,
+                    MetricValue = elapsed,
+                    PrimaryMetric = "ElapsedTimeMetric",
+                    Iterations = iterationArray,
+                    FiftiethPercentile = Percentile(iterationArray, 0.5),
+                    NinetiethPercentile = Percentile(iterationArray, 0.9),
+                    Average = iterations.Where(x => x > 0).Average()
+                });
                 string fileName = testName + ".json";
                 string resultFilePath = string.IsNullOrEmpty(resultFolder) ? fileName : Path.Combine(resultFolder, fileName);
                 File.WriteAllText(resultFilePath, resultContent);
@@ -62,6 +92,22 @@ namespace Microsoft.SqlTools.ServiceLayer.Test.Common
             }
         }
 
+        private static double Percentile(double[] sequence, double excelPercentile)
+        {
+            Array.Sort(sequence);
+            int N = sequence.Length;
+            double n = (N - 1) * excelPercentile + 1;
+            // Another method: double n = (N + 1) * excelPercentile;
+            if (n == 1d) return sequence[0];
+            else if (n == N) return sequence[N - 1];
+            else
+            {
+                int k = (int)n;
+                double d = n - k;
+                return sequence[k - 1] + d * (sequence[k] - sequence[k - 1]);
+            }
+        }
+
         public double TotalMilliSeconds
         {
             get

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Driver/ServiceTestDriver.cs

@@ -43,9 +43,9 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Driver
 
         private DateTime startTime;
 
-        public ServiceTestDriver()
+        public ServiceTestDriver(string executableFilePath = null)
         {
-            string serviceHostExecutable = Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
+            string serviceHostExecutable = executableFilePath != null ? executableFilePath : Environment.GetEnvironmentVariable(ServiceHostEnvironmentVariable);
             string serviceHostArguments = "--enable-logging";
             if (string.IsNullOrWhiteSpace(serviceHostExecutable))
             {

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/Constants.cs

@@ -0,0 +1,14 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
+{
+    public class Constants
+    {
+        public const string NumberOfRunsEnvironmentVariable = "NumberOfRuns";
+        public const string ExecutableFileEnvironmentVariable = "SQLTOOLSSERVICE_EXE";
+        public const string ResultFolderEnvironmentVariable = "ResultFolder";
+    }
+}

test/Microsoft.SqlTools.ServiceLayer.TestDriver/Utility/TestRunner.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Globalization;
+using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Threading.Tasks;
@@ -15,9 +16,86 @@ namespace Microsoft.SqlTools.ServiceLayer.TestDriver.Utility
 {
     public class TestRunner
     {
-        public static async Task<int> RunTests(string[] tests, string testNamespace)
+        private int numberOfRuns = 20;
+        protected int DefaultNumberOfRuns = 2;
+
+        public static TestRunner Instance { get; } = new TestRunner();
+        public string[] Tests { get; set; }
+        public int NumberOfRuns { get; set; }
+        public string ExecutableFilePath { get; set; }
+
+        private TestRunner()
         {
-            foreach (var test in tests)
+            InitParameters();
+        }
+
+        private void ParseArguments(string[] args)
+        {
+            int index = 0;
+            while (index < args.Length - 1)
+            {
+                string arg = args[index++];
+                string argValue = args[index++];
+                switch (arg)
+                {
+                    case "/t":
+                    case "/T":
+                    case "/tests":
+                        Tests = argValue.Split(" ");
+                        break;
+                    case "/n":
+                    case "/N":
+                    case "/numberOfRuns":
+                        int value;
+                        if (Int32.TryParse(argValue, out value))
+                        {
+                            NumberOfRuns = value;
+                        }
+                        break;
+                    case "/r":
+                    case "/R":
+                    case "/Result":
+                        ResultFolder = argValue;
+                        break;
+                    case "/s":
+                    case "/S":
+                    case "/Service":
+                        ExecutableFilePath = argValue;
+                        break;
+                }
+            }
+
+            if ((Tests == null || Tests.Length == 0) && args.Length >= 1)
+            {
+                Tests = args;
+            }
+        }
+
+        public string ResultFolder = InitResultFolder();
+
+        private static string InitResultFolder()
+        {
+            return Environment.GetEnvironmentVariable("ResultFolder");
+        }
+
+        private void InitParameters()
+        {
+            string numberOfRunsEnv = Environment.GetEnvironmentVariable(Constants.NumberOfRunsEnvironmentVariable);
+
+            if (!Int32.TryParse(numberOfRunsEnv, out numberOfRuns))
+            {
+                numberOfRuns = DefaultNumberOfRuns;
+            }
+
+            NumberOfRuns = numberOfRuns;
+
+            ExecutableFilePath = Environment.GetEnvironmentVariable(Constants.ExecutableFileEnvironmentVariable);
+        }
+
+        public async Task<int> RunTests(string[] args, string testNamespace)
+        {
+            ParseArguments(args);
+            foreach (var test in Tests)
             {
                 try
                 {