Skip to content

Commit f887c42

Browse files
Martin-MolineroMartin-Molinero
committed
Enable python and CSharp debugging
- Adding `DebuggerHelper` class, handles debugging initialization - Setting the "PYTHONPATH" will be handled by the `JobQueue`
1 parent 90ab859 commit f887c42

18 files changed

Lines changed: 321 additions & 39 deletions

‎AlgorithmFactory/DebuggerHelper.cs‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using System;
17+
using System.Diagnostics;
18+
using System.Threading;
19+
using Python.Runtime;
20+
using QuantConnect.Configuration;
21+
using QuantConnect.Logging;
22+
using QuantConnect.Python;
23+
24+
namespace QuantConnect.AlgorithmFactory
25+
{
26+
/// <summary>
27+
/// Helper class used to start a new debugging session
28+
/// </summary>
29+
public static class DebuggerHelper
30+
{
31+
/// <summary>
32+
/// The different implemented debugging methods
33+
/// </summary>
34+
public enum DebuggingMethod
35+
{
36+
/// <summary>
37+
/// Local debugging through cmdline.
38+
/// <see cref="Language.Python"/> will use built in 'pdb'
39+
/// </summary>
40+
LocalCmdline,
41+
42+
/// <summary>
43+
/// Visual studio local debugging.
44+
/// <see cref="Language.Python"/> will use 'Python Tools for Visual Studio',
45+
/// attach manually selecting `Python` code type.
46+
/// </summary>
47+
VisualStudio
48+
}
49+
50+
/// <summary>
51+
/// Will start a new debugging session
52+
/// </summary>
53+
public static void Initialize(Language language)
54+
{
55+
if (language == Language.Python)
56+
{
57+
DebuggingMethod debuggingType;
58+
Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType);
59+
60+
Log.Trace("DebuggerHelper.Initialize(): initializing python...");
61+
PythonInitializer.Initialize();
62+
Log.Trace("DebuggerHelper.Initialize(): python initialization done");
63+
64+
using (Py.GIL())
65+
{
66+
Log.Trace("DebuggerHelper.Initialize(): starting...");
67+
switch (debuggingType)
68+
{
69+
case DebuggingMethod.LocalCmdline:
70+
PythonEngine.RunSimpleString("import pdb; pdb.set_trace()");
71+
break;
72+
73+
case DebuggingMethod.VisualStudio:
74+
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
75+
PythonEngine.RunSimpleString(@"import sys; import time;
76+
while not sys.gettrace():
77+
time.sleep(0.25)");
78+
break;
79+
}
80+
Log.Trace("DebuggerHelper.Initialize(): started");
81+
}
82+
}
83+
else if(language == Language.CSharp)
84+
{
85+
if (Debugger.IsAttached)
86+
{
87+
Log.Trace("DebuggerHelper.Initialize(): debugger is already attached, triggering initial break.");
88+
Debugger.Break();
89+
}
90+
else
91+
{
92+
Log.Trace("DebuggerHelper.Initialize(): waiting for debugger to attach...");
93+
while (!Debugger.IsAttached)
94+
{
95+
Thread.Sleep(250);
96+
}
97+
Log.Trace("DebuggerHelper.Initialize(): debugger attached");
98+
}
99+
}
100+
else
101+
{
102+
throw new NotImplementedException($"DebuggerHelper.Initialize(): not implemented for {language}");
103+
}
104+
}
105+
}
106+
}

‎AlgorithmFactory/Loader.cs‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
using System.Reflection;
2121
using System.Runtime.InteropServices;
2222
using System.Security.Policy;
23+
using Python.Runtime;
2324
using QuantConnect.Interfaces;
2425
using QuantConnect.Logging;
2526
using QuantConnect.AlgorithmFactory.Python.Wrappers;
27+
using QuantConnect.Configuration;
2628
using QuantConnect.Python;
2729
using QuantConnect.Util;
2830

@@ -34,6 +36,9 @@ namespace QuantConnect.AlgorithmFactory
3436
[ClassInterface(ClassInterfaceType.AutoDual)]
3537
public class Loader : MarshalByRefObject
3638
{
39+
// True if we are in a debugging session
40+
private readonly bool _debugging;
41+
3742
// Defines the maximum amount of time we will allow for instantiating an instance of IAlgorithm
3843
private readonly TimeSpan _loaderTimeLimit;
3944

@@ -70,13 +75,14 @@ public class Loader : MarshalByRefObject
7075
/// Creates a new loader with a 10 second maximum load time that forces exactly one derived type to be found
7176
/// </summary>
7277
public Loader()
73-
: this(Language.CSharp, TimeSpan.FromSeconds(10), names => names.SingleOrDefault())
78+
: this(false, Language.CSharp, TimeSpan.FromSeconds(10), names => names.SingleOrDefault())
7479
{
7580
}
7681

7782
/// <summary>
7883
/// Creates a new loader with the specified configuration
7984
/// </summary>
85+
/// <param name="debugging">True if we are debugging</param>
8086
/// <param name="language">Which language are we trying to load</param>
8187
/// <param name="loaderTimeLimit">
8288
/// Used to limit how long it takes to create a new instance
@@ -89,8 +95,9 @@ public Loader()
8995
/// that's what this function does, it picks the correct type from the list of types found within the assembly.
9096
/// </param>
9197
/// <param name="workerThread">The worker thread instance the loader should use</param>
92-
public Loader(Language language, TimeSpan loaderTimeLimit, Func<List<string>, string> multipleTypeNameResolverFunction, WorkerThread workerThread = null)
98+
public Loader(bool debugging, Language language, TimeSpan loaderTimeLimit, Func<List<string>, string> multipleTypeNameResolverFunction, WorkerThread workerThread = null)
9399
{
100+
_debugging = debugging;
94101
_language = language;
95102
_workerThread = workerThread;
96103
if (multipleTypeNameResolverFunction == null)
@@ -159,28 +166,20 @@ private bool TryCreatePythonAlgorithm(string assemblyPath, out IAlgorithm algori
159166
var pythonFile = new FileInfo(assemblyPath);
160167
var moduleName = pythonFile.Name.Replace(".pyc", "").Replace(".py", "");
161168

162-
// Set the python path for loading python algorithms.
163-
var pythonPath = new List<string>
164-
{
165-
pythonFile.Directory.FullName,
166-
new DirectoryInfo(Environment.CurrentDirectory).FullName,
167-
};
168-
169-
// Don't include an empty environment variable in pythonPath, otherwise the PYTHONPATH
170-
// environment variable won't be used in the module import process
171-
var pythonPathEnvironmentVariable = Environment.GetEnvironmentVariable("PYTHONPATH");
172-
if (!string.IsNullOrEmpty(pythonPathEnvironmentVariable))
173-
{
174-
pythonPath.Add(pythonPathEnvironmentVariable);
175-
}
176-
177-
Environment.SetEnvironmentVariable("PYTHONPATH", string.Join(OS.IsLinux ? ":" : ";", pythonPath));
178-
179169
try
180170
{
181171
PythonInitializer.Initialize();
182172

183173
algorithmInstance = new AlgorithmPythonWrapper(moduleName);
174+
175+
// we need stdout for debugging
176+
if (!_debugging && Config.GetBool("mute-python-library-logging", true))
177+
{
178+
using (Py.GIL())
179+
{
180+
PythonEngine.Exec("import os, sys; sys.stdout = open(os.devnull, 'w')");
181+
}
182+
}
184183
}
185184
catch (Exception e)
186185
{

‎AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
using System;
3434
using System.Collections.Concurrent;
3535
using System.Collections.Generic;
36-
using QuantConnect.Configuration;
3736

3837
namespace QuantConnect.AlgorithmFactory.Python.Wrappers
3938
{
@@ -100,11 +99,6 @@ public AlgorithmPythonWrapper(string moduleName)
10099
{
101100
throw new Exception("Please ensure that one class inherits from QCAlgorithm.");
102101
}
103-
104-
if (Config.GetBool("mute-python-library-logging", true))
105-
{
106-
PythonEngine.Exec("import os, sys; sys.stdout = open(os.devnull, 'w')");
107-
}
108102
}
109103
}
110104
catch (Exception e)

‎AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
<Compile Include="..\Common\Properties\SharedAssemblyInfo.cs">
9090
<Link>Properties\SharedAssemblyInfo.cs</Link>
9191
</Compile>
92+
<Compile Include="DebuggerHelper.cs" />
9293
<Compile Include="Loader.cs" />
9394
<Compile Include="Properties\AssemblyInfo.cs" />
9495
<Compile Include="Python\Wrappers\AlgorithmPythonWrapper.cs" />

‎Common/Packets/BacktestNodePacket.cs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
*/
1616

1717
using System;
18+
using System.Collections.Generic;
19+
using System.Linq;
1820
using Newtonsoft.Json;
1921

2022
namespace QuantConnect.Packets
@@ -62,10 +64,27 @@ public class BacktestNodePacket : AlgorithmNodePacket
6264
[JsonProperty(PropertyName = "eRunMode")]
6365
public RunMode RunMode = RunMode.Series;
6466

67+
/// <summary>
68+
/// The initial breakpoints for debugging, if any
69+
/// </summary>
70+
[JsonProperty(PropertyName = "aBreakpoints")]
71+
public List<Breakpoint> Breakpoints = new List<Breakpoint>();
72+
73+
/// <summary>
74+
/// The initial Watchlist for debugging, if any
75+
/// </summary>
76+
[JsonProperty(PropertyName = "aWatchlist")]
77+
public List<string> Watchlist = new List<string>();
78+
79+
/// <summary>
80+
/// True, if this is a debugging backtest
81+
/// </summary>
82+
public bool IsDebugging => Breakpoints.Any();
83+
6584
/// <summary>
6685
/// Default constructor for JSON
6786
/// </summary>
68-
public BacktestNodePacket()
87+
public BacktestNodePacket()
6988
: base(PacketType.BacktestNode)
7089
{
7190
Controls = new Controls

‎Common/Packets/Breakpoint.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
*/
16+
17+
using Newtonsoft.Json;
18+
19+
namespace QuantConnect.Packets
20+
{
21+
/// <summary>
22+
/// A debugging breakpoint
23+
/// </summary>
24+
public class Breakpoint
25+
{
26+
/// <summary>
27+
/// The file name
28+
/// </summary>
29+
[JsonProperty(PropertyName = "fileName")]
30+
public string FileName { get; set; }
31+
32+
/// <summary>
33+
/// The line number
34+
/// </summary>
35+
[JsonProperty(PropertyName = "lineNumber")]
36+
public int LineNumber { get; set; }
37+
}
38+
}

‎Common/Packets/Packet.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ public enum PacketType
150150
RegressionAlgorithm,
151151

152152
/// Packet containing a heartbeat
153-
AlphaHeartbeat
153+
AlphaHeartbeat,
154+
155+
/// Used when debugging to send status updates
156+
DebuggingStatus
154157
}
155158
}

‎Common/QuantConnect.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@
275275
<Compile Include="Orders\Fees\FeeModel.cs" />
276276
<Compile Include="Orders\Fees\OrderFee.cs" />
277277
<Compile Include="Orders\Fees\OrderFeeParameters.cs" />
278+
<Compile Include="Packets\Breakpoint.cs" />
278279
<Compile Include="Python\BrokerageMessageHandlerPythonWrapper.cs" />
279280
<Compile Include="Python\MarginCallModelPythonWrapper.cs" />
280281
<Compile Include="Python\PythonInitializer.cs" />

‎Engine/Setup/BacktestingSetupHandler.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,16 @@ public virtual IAlgorithm CreateAlgorithmInstance(AlgorithmNodePacket algorithmN
116116
string error;
117117
IAlgorithm algorithm;
118118

119+
var debugNode = algorithmNodePacket as BacktestNodePacket;
120+
var debugging = debugNode != null && debugNode.IsDebugging || Config.GetBool("debugging", false);
121+
122+
if (debugging && !BaseSetupHandler.InitializeDebugging(algorithmNodePacket, WorkerThread))
123+
{
124+
throw new AlgorithmSetupException("Failed to initialize debugging");
125+
}
126+
119127
// limit load times to 60 seconds and force the assembly to have exactly one derived type
120-
var loader = new Loader(algorithmNodePacket.Language, TimeSpan.FromSeconds(60), names => names.SingleOrAlgorithmTypeName(Config.Get("algorithm-type-name")), WorkerThread);
128+
var loader = new Loader(debugging, algorithmNodePacket.Language, TimeSpan.FromSeconds(60), names => names.SingleOrAlgorithmTypeName(Config.Get("algorithm-type-name")), WorkerThread);
121129
var complete = loader.TryCreateAlgorithmInstanceWithIsolator(assemblyPath, algorithmNodePacket.RamAllocation, out algorithm, out error);
122130
if (!complete) throw new AlgorithmSetupException($"During the algorithm initialization, the following exception has occurred: {error}");
123131

‎Engine/Setup/BaseSetupHandler.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,16 @@
1717
using System;
1818
using System.Collections.Generic;
1919
using System.Linq;
20+
using QuantConnect.AlgorithmFactory;
21+
using QuantConnect.Configuration;
2022
using QuantConnect.Data;
2123
using QuantConnect.Data.UniverseSelection;
2224
using QuantConnect.Interfaces;
2325
using QuantConnect.Lean.Engine.DataFeeds;
2426
using QuantConnect.Logging;
27+
using QuantConnect.Packets;
28+
using QuantConnect.Util;
29+
using HistoryRequest = QuantConnect.Data.HistoryRequest;
2530

2631
namespace QuantConnect.Lean.Engine.Setup
2732
{
@@ -99,5 +104,20 @@ public static void SetupCurrencyConversions(
99104
Log.Trace("BaseSetupHandler.SetupCurrencyConversions():" +
100105
$"{Environment.NewLine}{algorithm.Portfolio.CashBook}");
101106
}
107+
108+
/// <summary>
109+
/// Initialize the debugger
110+
/// </summary>
111+
/// <param name="algorithmNodePacket">The algorithm node packet</param>
112+
/// <param name="workerThread">The worker thread instance to use</param>
113+
public static bool InitializeDebugging(AlgorithmNodePacket algorithmNodePacket, WorkerThread workerThread)
114+
{
115+
var isolator = new Isolator();
116+
return isolator.ExecuteWithTimeLimit(TimeSpan.FromMinutes(5),
117+
() => DebuggerHelper.Initialize(algorithmNodePacket.Language),
118+
algorithmNodePacket.RamAllocation,
119+
sleepIntervalMillis: 100,
120+
workerThread: workerThread);
121+
}
102122
}
103123
}

0 commit comments

Comments
 (0)