Skip to content

Commit 591c756

Browse files
tkstanczaktkstanczak
authored
Config 876 (#877)
* changed the config format, moved some config value sbetween config modules, simplified ExternalIP #876 * all config files in the new format #876 * proper fix for discovery (binding to local, advertising external) #876 * fixed tests and some diag fo #876 * fixed some config values after changes #876 * removed unnecessary prefix from metrics #876
1 parent 712d3ed commit 591c756

File tree

67 files changed

+1460
-2249
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1460
-2249
lines changed

‎src/Nethermind/Nethermind.Blockchain/BlockchainProcessor.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ public Block Process(Block suggestedBlock, ProcessingOptions options, IBlockTrac
346346
{
347347
if (block.Hash != null && _blockTree.WasProcessed(block.Number, block.Hash))
348348
{
349-
if (_logger.IsInfo) _logger.Info($"Rerunning block after reorg: {block.Hash}");
349+
if (_logger.IsInfo) _logger.Info($"Rerunning block after reorg: {block.ToString(Block.Format.FullHashAndNumber)}");
350350
}
351351

352352
blocksToProcess.Add(block);

‎src/Nethermind/Nethermind.Config.Test/ConfigProviderTests.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void Can_read_overwrites()
6666
Dictionary<string, string> args = new Dictionary<string, string>();
6767
if (bitArray.Get(4))
6868
{
69-
args.Add("StatsConfig.CaptureNodeStatsEventHistory", bitArray.Get(5).ToString());
69+
args.Add("Stats.CaptureNodeStatsEventHistory", bitArray.Get(5).ToString());
7070
}
7171

7272
Environment.SetEnvironmentVariable("NETHERMIND_STATSCONFIG_CAPTURENODESTATSEVENTHISTORY", null, EnvironmentVariableTarget.Process);
@@ -78,7 +78,7 @@ public void Can_read_overwrites()
7878
Dictionary<string, string> fakeJson = new Dictionary<string, string>();
7979
if (bitArray.Get(0))
8080
{
81-
fakeJson.Add("StatsConfig.CaptureNodeStatsEventHistory", bitArray.Get(1).ToString());
81+
fakeJson.Add("Stats.CaptureNodeStatsEventHistory", bitArray.Get(1).ToString());
8282
}
8383

8484
configProvider.AddSource(new ArgsConfigSource(args));

‎src/Nethermind/Nethermind.Config.Test/JsonConfigProviderTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void Can_load_config_from_file()
6666
Assert.AreEqual(2, jsonRpcConfig.EnabledModules.Count());
6767
new[] { ModuleType.Eth, ModuleType.Debug }.ToList().ForEach(x =>
6868
{
69-
Assert.IsTrue(jsonRpcConfig.EnabledModules.Contains(x));
69+
Assert.IsTrue(jsonRpcConfig.EnabledModules.Contains(x.ToString()));
7070
});
7171

7272
Assert.AreEqual(4, networkConfig.Concurrency);

‎src/Nethermind/Nethermind.Config.Test/SampleJsonConfig.cfg

+14-26
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,17 @@
1-
[
2-
{
3-
"ConfigModule": "KeYsToReCoNfIg",
4-
"ConfigItems": {
5-
"KdFpArAmSDklen": "100",
6-
"Cipher": "test"
7-
}
1+
{
2+
"KeYsToRe": {
3+
"KdFpArAmSDklen": "100",
4+
"Cipher": "test"
85
},
9-
{
10-
"ConfigModule": "JsonRpcConfig",
11-
"ConfigItems": {
12-
"EnabledModules": "Eth,Debug"
13-
}
6+
"JsonRpc": {
7+
"EnabledModules": "Eth,Debug"
148
},
15-
{
16-
"ConfigModule": "DiscoveryConfig",
17-
"ConfigItems": {
18-
"Concurrency": "4",
19-
"Bootnodes": "enode://04fb7acb86f47b64298374b5ccb3c2959f1e5e9362158e50e0793c261518ffe83759d8295ca4a88091d4726d5f85e6276d53ae9ef4f35b8c4c0cc6b99c8c0537@40.70.214.166:40303, enode://17de5580bbc1620081a21f82954731c7854305463630a0d677ed991487609829a6bf1ffcb8fb8ef269eff4829690625db176b498c629b9b13cb39b73b6e7b08b@213.186.16.82:1345"
20-
}
9+
"Discovery": {
10+
"Concurrency": "4",
11+
"Bootnodes": "enode://04fb7acb86f47b64298374b5ccb3c2959f1e5e9362158e50e0793c261518ffe83759d8295ca4a88091d4726d5f85e6276d53ae9ef4f35b8c4c0cc6b99c8c0537@40.70.214.166:40303, enode://17de5580bbc1620081a21f82954731c7854305463630a0d677ed991487609829a6bf1ffcb8fb8ef269eff4829690625db176b498c629b9b13cb39b73b6e7b08b@213.186.16.82:1345"
2112
},
22-
{
23-
"ConfigModule": "StatsConfig",
24-
"ConfigItems": {
25-
"PenalizedReputationLocalDisconnectReasons": "UnexpectedIdentity,IncompatibleP2PVersion,BreachOfProtocol",
26-
"CaptureNodeStatsEventHistory" : true
27-
}
28-
}
29-
]
13+
"Stats": {
14+
"PenalizedReputationLocalDisconnectReasons": "UnexpectedIdentity,IncompatibleP2PVersion,BreachOfProtocol",
15+
"CaptureNodeStatsEventHistory" : true
16+
}
17+
}

‎src/Nethermind/Nethermind.Config/ArgsConfigSource.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public ArgsConfigSource(Dictionary<string, string> args)
3232

3333
public (bool IsSet, object Value) GetValue(Type type, string category, string name)
3434
{
35-
(bool isSet, string value) = GetRawValue(category, name);
35+
(bool isSet, string value) = GetRawValue(category?.Replace("Config", string.Empty), name);
3636
return (isSet, isSet ? ConfigSourceHelper.ParseValue(type, value) : ConfigSourceHelper.GetDefault(type));
3737
}
3838

‎src/Nethermind/Nethermind.Config/JsonConfigSource.cs

+9-6
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ public JsonConfigSource(string configFilePath)
3333

3434
private void ApplyJsonConfig(string jsonContent)
3535
{
36-
var json = (JArray) JToken.Parse(jsonContent);
36+
var json = (JObject) JToken.Parse(jsonContent);
3737
foreach (var moduleEntry in json)
3838
{
39-
LoadModule(moduleEntry);
39+
LoadModule(moduleEntry.Key, (JObject)moduleEntry.Value);
4040
}
4141
}
4242

@@ -77,11 +77,9 @@ private void LoadJsonConfig(string configFilePath)
7777
ApplyJsonConfig(File.ReadAllText(configFilePath));
7878
}
7979

80-
private void LoadModule(JToken moduleEntry)
80+
private void LoadModule(string moduleName, JObject value)
8181
{
82-
var moduleName = (string) moduleEntry["ConfigModule"];
83-
84-
var configItems = (JObject) moduleEntry["ConfigItems"];
82+
var configItems = value;
8583
var itemsDict = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
8684

8785
foreach (var configItem in configItems)
@@ -105,6 +103,11 @@ private void LoadModule(JToken moduleEntry)
105103

106104
private void ApplyConfigValues(string configModule, Dictionary<string, string> items)
107105
{
106+
if (!configModule.EndsWith("Config"))
107+
{
108+
configModule = configModule + "Config";
109+
}
110+
108111
_values[configModule] = items;
109112
_parsedValues[configModule] = new Dictionary<string, object>(StringComparer.InvariantCultureIgnoreCase);
110113
}

‎src/Nethermind/Nethermind.DataMarketplace.Integration.Test/configs/ndm_consumer.cfg

+51-60
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,55 @@
1-
[
2-
{
3-
"ConfigModule": "InitConfig",
4-
"ConfigItems": {
5-
"PubSubEnabled": false,
6-
"JsonRpcEnabled": true,
7-
"NetworkEnabled": true,
8-
"DiscoveryEnabled": true,
9-
"SynchronizationEnabled": true,
10-
"PeerManagerEnabled": true,
11-
"ProcessingEnabled": true,
12-
"IsMining": true,
13-
"DiscoveryPort": 30377,
14-
"P2PPort": 30377,
15-
"HttpHost": "127.0.0.1",
16-
"HttpPort": 8577,
17-
"ChainSpecPath": "chainspec/nethdev.json",
18-
"GenesisHash": "",
19-
"BaseDbPath": "ndm/consumer/mainnet",
20-
"LogFileName": "ndm_consumer.logs.txt",
21-
"ObsoletePendingTransactionInterval": 15,
22-
"RemovePendingTransactionInterval": 600,
23-
"PeerNotificationThreshold": 20,
24-
"StoreReceipts": true
25-
}
1+
{
2+
"Init" : {
3+
"NetworkEnabled": true,
4+
"DiscoveryEnabled": true,
5+
"SynchronizationEnabled": true,
6+
"PeerManagerEnabled": true,
7+
"ProcessingEnabled": true,
8+
"IsMining": true,
9+
"ChainSpecPath": "chainspec/nethdev.json",
10+
"GenesisHash": "",
11+
"BaseDbPath": "ndm/consumer/mainnet",
12+
"LogFileName": "ndm_consumer.logs.txt",
13+
"ObsoletePendingTransactionInterval": 15,
14+
"RemovePendingTransactionInterval": 600,
15+
"PeerNotificationThreshold": 20,
16+
"StoreReceipts": true
2617
},
27-
{
28-
"ConfigModule": "DbConfig",
29-
"ConfigItems": {
30-
"WriteBufferSize": 67108864,
31-
"WriteBufferNumber": 6,
32-
"BlockCacheSize": 67108864,
33-
"CacheIndexAndFilterBlocks": true
34-
}
18+
"JsonRpc": {
19+
"Enabled": true,
20+
"Host": "127.0.0.1",
21+
"Port": 8577
3522
},
36-
{
37-
"ConfigModule": "KafkaConfig",
38-
"ConfigItems": {
39-
"Servers": "localhost:19092,localhost:29092,localhost:39092",
40-
"SecurityProtocol": "SASL_Plaintext",
41-
"SaslEnabled": true,
42-
"SaslUsername": "nethermind",
43-
"SaslPassword": "secret",
44-
"SslKeyLocation": "certs/nethermind.pem",
45-
"SchemaRegistryUrl": "http://localhost:8081",
46-
"TopicBlocks": "Nethermind.Blocks",
47-
"TopicReceipts": "Nethermind.Receipts",
48-
"TopicTransactions": "Nethermind.Transactions",
49-
"ProduceAvro": true,
50-
"ProduceJson": true,
51-
"ProduceUtf8Json": true
52-
}
23+
"Network": {
24+
"DiscoveryPort": 30377,
25+
"P2PPort": 30377,
5326
},
54-
{
55-
"ConfigModule": "NdmConfig",
56-
"ConfigItems": {
57-
"ProviderName": "Nethermind",
58-
"FilesPath": "ndm/files",
59-
"FileMaxSize": 67108864,
60-
"AccountAddress:": "0x2b5ad5c4795c026514f8317c7a215e218dccd6cf",
61-
"ContractAddress": "0x82c839fa4a41e158f613ec8a1a84be3c816d370f"
62-
}
27+
"Db": {
28+
"WriteBufferSize": 67108864,
29+
"WriteBufferNumber": 6,
30+
"BlockCacheSize": 67108864,
31+
"CacheIndexAndFilterBlocks": true
32+
},
33+
"Kafka": {
34+
"Servers": "localhost:19092,localhost:29092,localhost:39092",
35+
"SecurityProtocol": "SASL_Plaintext",
36+
"SaslEnabled": true,
37+
"SaslUsername": "nethermind",
38+
"SaslPassword": "secret",
39+
"SslKeyLocation": "certs/nethermind.pem",
40+
"SchemaRegistryUrl": "http://localhost:8081",
41+
"TopicBlocks": "Nethermind.Blocks",
42+
"TopicReceipts": "Nethermind.Receipts",
43+
"TopicTransactions": "Nethermind.Transactions",
44+
"ProduceAvro": true,
45+
"ProduceJson": true,
46+
"ProduceUtf8Json": true
47+
},
48+
"Ndm": {
49+
"ProviderName": "Nethermind",
50+
"FilesPath": "ndm/files",
51+
"FileMaxSize": 67108864,
52+
"AccountAddress:": "0x2b5ad5c4795c026514f8317c7a215e218dccd6cf",
53+
"ContractAddress": "0x82c839fa4a41e158f613ec8a1a84be3c816d370f"
6354
}
64-
]
55+
}

‎src/Nethermind/Nethermind.DataMarketplace.Integration.Test/configs/ndm_provider.cfg

+51-60
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,55 @@
1-
[
2-
{
3-
"ConfigModule": "InitConfig",
4-
"ConfigItems": {
5-
"PubSubEnabled": false,
6-
"JsonRpcEnabled": true,
7-
"NetworkEnabled": true,
8-
"DiscoveryEnabled": true,
9-
"SynchronizationEnabled": true,
10-
"PeerManagerEnabled": true,
11-
"ProcessingEnabled": true,
12-
"IsMining": true,
13-
"DiscoveryPort": 30388,
14-
"P2PPort": 30388,
15-
"HttpHost": "127.0.0.1",
16-
"HttpPort": 8588,
17-
"ChainSpecPath": "chainspec/nethdev.json",
18-
"GenesisHash": "",
19-
"BaseDbPath": "ndm/provider/mainnet",
20-
"LogFileName": "ndm_provider.logs.txt",
21-
"ObsoletePendingTransactionInterval": 15,
22-
"RemovePendingTransactionInterval": 600,
23-
"PeerNotificationThreshold": 20,
24-
"StoreReceipts": true
25-
}
1+
{
2+
"InitConfig": {
3+
"NetworkEnabled": true,
4+
"DiscoveryEnabled": true,
5+
"SynchronizationEnabled": true,
6+
"PeerManagerEnabled": true,
7+
"ProcessingEnabled": true,
8+
"IsMining": true,
9+
"ChainSpecPath": "chainspec/nethdev.json",
10+
"GenesisHash": "",
11+
"BaseDbPath": "ndm/provider/mainnet",
12+
"LogFileName": "ndm_provider.logs.txt",
13+
"ObsoletePendingTransactionInterval": 15,
14+
"RemovePendingTransactionInterval": 600,
15+
"PeerNotificationThreshold": 20,
16+
"StoreReceipts": true
2617
},
27-
{
28-
"ConfigModule": "DbConfig",
29-
"ConfigItems": {
30-
"WriteBufferSize": 67108864,
31-
"WriteBufferNumber": 6,
32-
"BlockCacheSize": 67108864,
33-
"CacheIndexAndFilterBlocks": true
34-
}
18+
"JsonRpc": {
19+
"Enabled": true,
20+
"Host": "127.0.0.1",
21+
"Port": 8588
3522
},
36-
{
37-
"ConfigModule": "KafkaConfig",
38-
"ConfigItems": {
39-
"Servers": "localhost:19092,localhost:29092,localhost:39092",
40-
"SecurityProtocol": "SASL_Plaintext",
41-
"SaslEnabled": true,
42-
"SaslUsername": "nethermind",
43-
"SaslPassword": "secret",
44-
"SslKeyLocation": "certs/nethermind.pem",
45-
"SchemaRegistryUrl": "http://localhost:8081",
46-
"TopicBlocks": "Nethermind.Blocks",
47-
"TopicReceipts": "Nethermind.Receipts",
48-
"TopicTransactions": "Nethermind.Transactions",
49-
"ProduceAvro": true,
50-
"ProduceJson": true,
51-
"ProduceUtf8Json": true
52-
}
23+
"Network": {
24+
"DiscoveryPort": 30388,
25+
"P2PPort": 30388
5326
},
54-
{
55-
"ConfigModule": "NdmConfig",
56-
"ConfigItems": {
57-
"ProviderName": "Nethermind",
58-
"FilesPath": "ndm/files",
59-
"FileMaxSize": 67108864,
60-
"AccountAddress:": "0x7e5f4552091a69125d5dfcb7b8c2659029395bdf",
61-
"ContractAddress": "0x82c839fa4a41e158f613ec8a1a84be3c816d370f"
62-
}
27+
"DbConfig": {
28+
"WriteBufferSize": 67108864,
29+
"WriteBufferNumber": 6,
30+
"BlockCacheSize": 67108864,
31+
"CacheIndexAndFilterBlocks": true
32+
},
33+
"KafkaConfig": {
34+
"Servers": "localhost:19092,localhost:29092,localhost:39092",
35+
"SecurityProtocol": "SASL_Plaintext",
36+
"SaslEnabled": true,
37+
"SaslUsername": "nethermind",
38+
"SaslPassword": "secret",
39+
"SslKeyLocation": "certs/nethermind.pem",
40+
"SchemaRegistryUrl": "http://localhost:8081",
41+
"TopicBlocks": "Nethermind.Blocks",
42+
"TopicReceipts": "Nethermind.Receipts",
43+
"TopicTransactions": "Nethermind.Transactions",
44+
"ProduceAvro": true,
45+
"ProduceJson": true,
46+
"ProduceUtf8Json": true
47+
},
48+
"NdmConfig": {
49+
"ProviderName": "Nethermind",
50+
"FilesPath": "ndm/files",
51+
"FileMaxSize": 67108864,
52+
"AccountAddress:": "0x7e5f4552091a69125d5dfcb7b8c2659029395bdf",
53+
"ContractAddress": "0x82c839fa4a41e158f613ec8a1a84be3c816d370f"
6354
}
64-
]
55+
}

‎src/Nethermind/Nethermind.Grpc/GrpcConfig.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ namespace Nethermind.Grpc
2020
{
2121
public class GrpcConfig : IGrpcConfig
2222
{
23-
public bool Enabled { get; set; } = true;
23+
public bool Enabled { get; set; }
2424
public string Host { get; set; } = "localhost";
2525
public int Port { get; set; } = 50000;
2626
public bool ProducerEnabled { get; set; } = false;

‎src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public void Initialize()
5555

5656
private JsonRpcResponse TestRequest<T>(T module, string method, params string[] parameters) where T : IModule
5757
{
58-
RpcModuleProvider moduleProvider = new RpcModuleProvider(_configurationProvider.GetConfig<IJsonRpcConfig>());
58+
RpcModuleProvider moduleProvider = new RpcModuleProvider(_configurationProvider.GetConfig<IJsonRpcConfig>(), LimboLogs.Instance);
5959
moduleProvider.Register(new SingletonModulePool<T>(new SingletonFactory<T>(module)));
6060
_jsonRpcService = new JsonRpcService(moduleProvider, _logManager);
6161
JsonRpcRequest request = RpcTest.GetJsonRequest(method, parameters);

0 commit comments

Comments
 (0)