Skip to content

Commit 169c6ca

Browse files
RomazesRomazes
authored
Initialization via Job Packet and Rename DataQueueHandler to DataProvider (#2)
* rename: namespaces * rename: DataQueueHandler -> DataProvider * feat: Config -> Globals in ValidateSubscription * feat: SetJob fix: IsConnected Property feat: handle of null warnings * feat: reference on Lean\Tests + init Globals * refactor: use CanSubscribe for Symbol in GetHistory() * refactor: missed Lean in namespaces * rename: QueueHandler -> DataProvider Tests revert: commented code * fix: workflow dotnet test path * fix: testcase param forex instead of equity * refactor: return null in wrong request of GetHistory() * feat: add stop spamming flags feat: null sign refactor: remove extra else block * remove: duplicate validation from downloader * revert: IsConnected handle connection
1 parent 89db062 commit 169c6ca

26 files changed

+285
-148
lines changed

‎.github/workflows/build.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,14 @@ jobs:
4646
image: quantconnect/lean:foundation
4747
options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.CoinAPI/Lean.DataSource.CoinAPI -e QC_JOB_USER_ID=${{ secrets.QC_JOB_USER_ID }} -e QC_API_ACCESS_TOKEN=${{ secrets.QC_API_ACCESS_TOKEN }} -e QC_JOB_ORGANIZATION_ID=${{ secrets.QC_JOB_ORGANIZATION_ID }} -e QC_COINAPI_API_KEY=${{ secrets.QC_COINAPI_API_KEY }}
4848

49-
- name: Build QuantConnect.CoinAPI
50-
run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
49+
- name: Build QuantConnect.DataSource.CoinAPI
50+
run: dotnet build ./QuantConnect.CoinAPI/QuantConnect.DataSource.CoinAPI.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
5151

5252
- name: Build DataProcessing
5353
run: dotnet build ./DataProcessing/DataProcessing.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
5454

55-
- name: Build QuantConnect.CoinAPI.Tests
56-
run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
55+
- name: Build QuantConnect.DataSource.CoinAPI.Tests
56+
run: dotnet build ./QuantConnect.CoinAPI.Tests/QuantConnect.DataSource.CoinAPI.Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
5757

58-
- name: Run QuantConnect.CoinAPI.Tests
59-
run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.CoinAPI.Tests.dll
58+
- name: Run QuantConnect.DataSource.CoinAPI.Tests
59+
run: dotnet test ./QuantConnect.CoinAPI.Tests/bin/Release/QuantConnect.Lean.DataSource.CoinAPI.Tests.dll

‎DataProcessing/CoinApiDataConverter.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
using System.Diagnostics;
2020
using QuantConnect.Logging;
2121
using QuantConnect.ToolBox;
22-
using QuantConnect.CoinAPI;
22+
using QuantConnect.Lean.DataSource.CoinAPI;
2323

2424
namespace QuantConnect.DataProcessing
2525
{

‎DataProcessing/DataProcessing.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
</ItemGroup>
3030

3131
<ItemGroup>
32-
<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj" />
32+
<ProjectReference Include="..\QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj" />
3333
</ItemGroup>
3434

3535
<ItemGroup>

‎Lean.DataSource.CoinAPI.sln‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
33
# Visual Studio Version 17
44
VisualStudioVersion = 17.5.002.0
55
MinimumVisualStudioVersion = 10.0.40219.1
6-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}"
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI", "QuantConnect.CoinAPI\QuantConnect.DataSource.CoinAPI.csproj", "{2BEB31AD-5B1E-4D9B-A206-D67F3CA33A4C}"
77
EndProject
8-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}"
8+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.CoinAPI.Tests", "QuantConnect.CoinAPI.Tests\QuantConnect.DataSource.CoinAPI.Tests.csproj", "{337CEE6E-639A-448D-95ED-2C1628E26AF2}"
99
EndProject
1010
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{881514B4-641E-4EDC-8020-6BEA0CC8F48C}"
1111
EndProject

‎QuantConnect.CoinAPI.Tests/CoinAPIDataDownloaderTests.cs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@
1515

1616
using NUnit.Framework;
1717
using QuantConnect.Util;
18+
using QuantConnect.Tests;
1819
using QuantConnect.Logging;
1920

20-
namespace QuantConnect.CoinAPI.Tests
21+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2122
{
2223
[TestFixture]
2324
public class CoinAPIDataDownloaderTests
@@ -52,8 +53,9 @@ public void DownloadsHistoricalDataWithValidDataTestParameters(Symbol symbol, Re
5253
{
5354
var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, TickType.Trade);
5455

55-
var downloadResponse = _downloader.Get(parameters).ToList();
56+
var downloadResponse = _downloader.Get(parameters)?.ToList();
5657

58+
Assert.IsNotNull(downloadResponse);
5759
Assert.IsNotEmpty(downloadResponse);
5860

5961
Log.Trace($"{symbol}.{resolution}.[{startDateTimeUtc} - {endDateTimeUtc}]: Amount = {downloadResponse.Count}");
@@ -83,19 +85,20 @@ public void DownloadsHistoricalDataWithInvalidDataTestParameters(Symbol symbol,
8385
{
8486
var parameters = new DataDownloaderGetParameters(symbol, resolution, startDateTimeUtc, endDateTimeUtc, tickType);
8587

86-
var downloadResponse = _downloader.Get(parameters).ToList();
88+
var downloadResponse = _downloader.Get(parameters)?.ToList();
8789

88-
Assert.IsEmpty(downloadResponse);
90+
Assert.IsNull(downloadResponse);
8991
}
9092

9193
private static IEnumerable<TestCaseData> HistoricalInvalidDataThrowExceptionTestCases
9294
{
9395
get
9496
{
97+
TestGlobals.Initialize();
9598
yield return new TestCaseData(Symbol.Create("BTCBTC", SecurityType.Crypto, Market.Binance))
9699
.SetDescription($"Wrong Symbol - 'BTCBTC'");
97-
yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Equity, Market.Binance))
98-
.SetDescription($"Wrong SecurityType - {SecurityType.Equity}");
100+
yield return new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Forex, Market.Binance))
101+
.SetDescription($"Wrong SecurityType - {SecurityType.Forex}");
99102
}
100103
}
101104

@@ -104,7 +107,7 @@ public void DownloadsHistoricalDataWithInvalidDataTestParametersThrowException(S
104107
{
105108
var parameters = new DataDownloaderGetParameters(symbol, Resolution.Minute, new DateTime(2024, 1, 1), new DateTime(2024, 2, 1), TickType.Trade);
106109

107-
Assert.That(() => _downloader.Get(parameters).ToList(), Throws.Exception);
110+
Assert.That(() => _downloader.Get(parameters)?.ToList(), Throws.Exception);
108111
}
109112
}
110113
}

‎QuantConnect.CoinAPI.Tests/CoinAPIHistoryProviderTests.cs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
using QuantConnect.Data.Market;
2121
using QuantConnect.Securities;
2222

23-
namespace QuantConnect.CoinAPI.Tests
23+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2424
{
2525
[TestFixture]
2626
public class CoinAPIHistoryProviderTests
@@ -52,7 +52,7 @@ public void OneTimeSetUp()
5252

5353
[Test]
5454
[TestCaseSource(nameof(TestData))]
55-
public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNonEmptyResult)
55+
public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, int period, bool isNotNullResult)
5656
{
5757
_coinApiDataQueueHandler.SetUpHistDataLimit(100);
5858

@@ -67,14 +67,15 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
6767
resolution, true, false, DataNormalizationMode.Raw, TickType.Trade)
6868
};
6969

70-
var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc).ToArray();
70+
var slices = _coinApiDataQueueHandler.GetHistory(historyRequests, TimeZones.Utc)?.ToArray();
7171

72-
if (isNonEmptyResult)
72+
if (isNotNullResult)
7373
{
74+
Assert.IsNotNull(slices);
7475
// For resolution larger than second do more tests
7576
if (resolution > Resolution.Second)
7677
{
77-
Assert.AreEqual(period, slices.Length);
78+
Assert.That(slices.Length, Is.EqualTo(period));
7879

7980
var firstSliceTradeBars = slices.First().Bars.Values;
8081

@@ -83,38 +84,37 @@ public void CanGetHistory(Symbol symbol, Resolution resolution, Type dataType, i
8384
firstSliceTradeBars.DoForEach(tb =>
8485
{
8586
var resTimeSpan = resolution.ToTimeSpan();
86-
Assert.AreEqual(resTimeSpan, tb.Period);
87-
Assert.AreEqual(startTimeUtc.RoundUp(resTimeSpan), tb.Time);
87+
Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
88+
Assert.That(tb.Time, Is.EqualTo(startTimeUtc.RoundUp(resTimeSpan)));
8889
});
8990

9091
var lastSliceTradeBars = slices.Last().Bars.Values;
9192

9293
lastSliceTradeBars.DoForEach(tb =>
9394
{
9495
var resTimeSpan = resolution.ToTimeSpan();
95-
Assert.AreEqual(resTimeSpan, tb.Period);
96-
Assert.AreEqual(nowUtc.RoundDown(resTimeSpan), tb.Time);
96+
Assert.That(tb.Period, Is.EqualTo(resTimeSpan));
97+
Assert.That(tb.Time, Is.EqualTo(nowUtc.RoundDown(resTimeSpan)));
9798
});
9899
}
99100
// For res. second data counts, start/end dates may slightly vary from historical request's
100101
// Make sure just that resolution is correct and amount is positive numb.
101102
else
102103
{
103104
Assert.IsTrue(slices.Length > 0);
104-
Assert.AreEqual(resolution.ToTimeSpan(), slices.First().Bars.Values.FirstOrDefault()?.Period);
105+
Assert.That(slices.First().Bars.Values.FirstOrDefault()?.Period, Is.EqualTo(resolution.ToTimeSpan()));
105106
}
106107

107108
// Slices are ordered by time
108109
Assert.That(slices, Is.Ordered.By("Time"));
109110
}
110111
else
111112
{
112-
// Empty
113-
Assert.IsEmpty(slices);
113+
Assert.IsNull(slices);
114114
}
115115
}
116116

117-
public class CoinApiDataQueueHandlerMock : CoinApiDataQueueHandler
117+
public class CoinApiDataQueueHandlerMock : CoinApiDataProvider
118118
{
119119
public new void SetUpHistDataLimit(int limit)
120120
{

‎QuantConnect.CoinAPI.Tests/CoinAPISymbolMapperTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
using NUnit.Framework;
1818

19-
namespace QuantConnect.CoinAPI.Tests
19+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2020
{
2121
[TestFixture]
2222
public class CoinAPISymbolMapperTests

‎QuantConnect.CoinAPI.Tests/CoinApiAdditionalTests.cs‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@
1414
*/
1515

1616
using NUnit.Framework;
17+
using QuantConnect.Packets;
1718
using QuantConnect.Configuration;
1819

19-
namespace QuantConnect.CoinAPI.Tests
20+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2021
{
2122
[TestFixture]
2223
public class CoinApiAdditionalTests
@@ -28,11 +29,37 @@ public void ThrowsOnFailedAuthentication()
2829

2930
Assert.Throws<Exception>(() =>
3031
{
31-
using var _coinApiDataQueueHandler = new CoinApiDataQueueHandler();
32+
using var _coinApiDataQueueHandler = new CoinApiDataProvider();
3233
});
3334

3435
// reset api key
3536
TestSetup.GlobalSetup();
3637
}
38+
39+
[Test]
40+
public void CanInitializeUsingJobPacket()
41+
{
42+
var apiKey = Config.Get("coinapi-api-key");
43+
Config.Set("coinapi-api-key", "");
44+
45+
var job = new LiveNodePacket
46+
{
47+
BrokerageData = new Dictionary<string, string>() {
48+
{ "coinapi-api-key", "InvalidApiKeyThatWontBeUsed" },
49+
{ "coinapi-product", "Startup" }
50+
}
51+
};
52+
53+
using var iexDataProvider = new CoinApiDataProvider();
54+
55+
// Throw because CoinApiSymbolMapper makes request to API (we have invalid api key in LiveNodePacket)
56+
Assert.Throws<Exception>(() =>
57+
{
58+
iexDataProvider.SetJob(job);
59+
});
60+
61+
// revert Config of ApiKey for another tests
62+
Config.Set("coinapi-api-key", apiKey);
63+
}
3764
}
3865
}

‎QuantConnect.CoinAPI.Tests/CoinApiDataQueueHandlerTest.cs‎ renamed to ‎QuantConnect.CoinAPI.Tests/CoinApiDataProviderTests.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@
2121
using QuantConnect.Data.Market;
2222
using System.Collections.Concurrent;
2323

24-
namespace QuantConnect.CoinAPI.Tests
24+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2525
{
2626
[TestFixture]
27-
public class CoinApiDataQueueHandlerTest
27+
public class CoinApiDataProviderTests
2828
{
29-
private CoinApiDataQueueHandler _coinApiDataQueueHandler;
29+
private CoinApiDataProvider _coinApiDataQueueHandler;
3030
private CancellationTokenSource _cancellationTokenSource;
3131

3232
[SetUp]
@@ -61,12 +61,12 @@ public void SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest()
6161
_cancellationTokenSource.Token,
6262
tick =>
6363
{
64-
Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}");
64+
Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnCoinbaseDataStreamTest)}: {tick}");
6565
tradeBars.Add(tick);
6666

6767
if (tradeBars.Count > 5)
6868
{
69-
resetEvent.Set();
69+
resetEvent.Set();
7070
}
7171
},
7272
() => _cancellationTokenSource.Cancel());
@@ -111,7 +111,7 @@ public void SubscribeToBTCUSDSecondOnDifferentMarkets()
111111
_cancellationTokenSource.Token,
112112
tick =>
113113
{
114-
Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}");
114+
Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDSecondOnDifferentMarkets)}: {tick}");
115115
symbolBaseData[tick.Symbol].Add(tick);
116116
},
117117
() =>
@@ -165,7 +165,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance()
165165
_cancellationTokenSource.Token,
166166
tick =>
167167
{
168-
Log.Debug($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}");
168+
Log.Debug($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)}: {tick}");
169169
tickData.Add(tick);
170170

171171
if (tickData.Count > 5)
@@ -190,7 +190,7 @@ public void SubscribeToBTCUSDTFutureSecondBinance()
190190

191191
if (tickData.Count == 0)
192192
{
193-
Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}");
193+
Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(SubscribeToBTCUSDTFutureSecondBinance)} is nothing returned. {symbol}|{resolution}|tickData = {tickData.Count}");
194194
}
195195

196196
CoinApiTestHelper.AssertSymbol(tickData.First().Symbol, symbol);

‎QuantConnect.CoinAPI.Tests/CoinApiTestHelper.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
using QuantConnect.Logging;
2020
using QuantConnect.Data.Market;
2121

22-
namespace QuantConnect.CoinAPI.Tests
22+
namespace QuantConnect.Lean.DataSource.CoinAPI.Tests
2323
{
2424
public static class CoinApiTestHelper
2525
{
@@ -68,7 +68,7 @@ public static void AssertBaseData(List<BaseData> tradeBars, Resolution expectedR
6868
Assert.IsTrue(trade.Period.ToHigherResolutionEquivalent(true) == expectedResolution);
6969
break;
7070
default:
71-
Assert.Fail($"{nameof(CoinApiDataQueueHandlerTest)}.{nameof(AssertBaseData)}: The tick type doesn't support");
71+
Assert.Fail($"{nameof(CoinApiDataProviderTests)}.{nameof(AssertBaseData)}: The tick type doesn't support");
7272
break;
7373
}
7474
}

0 commit comments

Comments
 (0)