diff --git a/ydb/core/config/init/init_impl.h b/ydb/core/config/init/init_impl.h index 146e6e730639..0b2536ffabae 100644 --- a/ydb/core/config/init/init_impl.h +++ b/ydb/core/config/init/init_impl.h @@ -469,9 +469,39 @@ struct TCommonAppOptions { ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::InterconnectConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); } - if (appConfig.HasGRpcConfig() && appConfig.GetGRpcConfig().HasCert()) { - appConfig.MutableGRpcConfig()->SetPathToCertificateFile(appConfig.GetGRpcConfig().GetCert()); - ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::GRpcConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + if (appConfig.HasGRpcConfig()) { + if (appConfig.GetGRpcConfig().HasCert()) { + appConfig.MutableGRpcConfig()->SetPathToCertificateFile(appConfig.GetGRpcConfig().GetCert()); + ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::GRpcConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + } + if (appConfig.GetGRpcConfig().HasXdsBootstrap()) { + auto* xdsBootstrapConfig = appConfig.MutableGRpcConfig()->MutableXdsBootstrap(); + if (xdsBootstrapConfig->GetNode().GetId().empty()) { + xdsBootstrapConfig->MutableNode()->SetId(env.FQDNHostName()); + ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::GRpcConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + } + if (xdsBootstrapConfig->GetNode().GetLocality().GetZone().empty()) { + TString dataCenter; + if (DataCenter) { + dataCenter = to_lower(DataCenter.GetRef()); + } else if (appConfig.HasNameserviceConfig()) { + for (const auto& node : appConfig.GetNameserviceConfig().GetNode()) { + if (node.GetNodeId() == NodeId) { + if (node.HasLocation()) { + dataCenter = to_lower(node.GetLocation().GetDataCenter()); + } else if (node.HasWalleLocation()) { + dataCenter = to_lower(node.GetWalleLocation().GetDataCenter()); + } + break; + } + } + } + if (!dataCenter.empty()) { + xdsBootstrapConfig->MutableNode()->MutableLocality()->SetZone(dataCenter); + ConfigUpdateTracer.AddUpdate(NKikimrConsole::TConfigItem::GRpcConfigItem, TConfigItemInfo::EUpdateKind::UpdateExplicitly); + } + } + } } if (!GrpcSslSettings.PathToGrpcCertFile.empty()) { diff --git a/ydb/core/config/init/init_ut.cpp b/ydb/core/config/init/init_ut.cpp index fc07db5ef2d0..86b971ce3126 100644 --- a/ydb/core/config/init/init_ut.cpp +++ b/ydb/core/config/init/init_ut.cpp @@ -56,11 +56,108 @@ Y_UNIT_TEST_SUITE(Init) { } } +namespace { + +TTempFileHandle CreateConfigFile(const TString& config) { + TTempFileHandle tempFile = TTempFileHandle::InCurrentDir("test_config", ".yaml"); + TUnbufferedFileOutput fileOutput(tempFile.Name()); + fileOutput.Write(config); + fileOutput.Finish(); + return tempFile; +} + +void PreFillArgs(std::vector& args, const TString& configPath) { + args.push_back("server"); + + args.push_back("--node"); + args.push_back("static"); + + args.push_back("--ic-port"); + args.push_back("9001"); + + args.push_back("--grpc-port"); + args.push_back("2135"); + + args.push_back("--mon-port"); + args.push_back("8765"); + + args.push_back("--yaml-config"); + args.push_back(configPath); +} + +NKikimrConfig::TAppConfig TransformConfig(const std::vector& args, std::unique_ptr envMock) { + auto errorCollector = NConfig::MakeDefaultErrorCollector(); + auto protoConfigFileProvider = NConfig::MakeDefaultProtoConfigFileProvider(); + auto configUpdateTracer = NConfig::MakeDefaultConfigUpdateTracer(); + auto memLogInit = NConfig::MakeNoopMemLogInitializer(); + auto nodeBrokerClient = NConfig::MakeNoopNodeBrokerClient(); + auto dynConfigClient = NConfig::MakeNoopDynConfigClient(); + auto configClient = NConfig::MakeNoopConfigClient(); + auto logger = NConfig::MakeNoopInitLogger(); + + NConfig::TInitialConfiguratorDependencies deps{ + *errorCollector, + *protoConfigFileProvider, + *configUpdateTracer, + *memLogInit, + *nodeBrokerClient, + *dynConfigClient, + *configClient, + *envMock, + *logger, + }; + auto initCfg = NConfig::MakeDefaultInitialConfigurator(deps); + + std::vector argv; + + for (const auto& arg : args) { + argv.push_back(arg.data()); + } + + NLastGetopt::TOpts opts; + initCfg->RegisterCliOptions(opts); + protoConfigFileProvider->RegisterCliOptions(opts); + + NLastGetopt::TOptsParseResult parseResult(&opts, argv.size(), argv.data()); + + initCfg->ValidateOptions(opts, parseResult); + initCfg->Parse(parseResult.GetFreeArgs(), nullptr); + + NKikimrConfig::TAppConfig appConfig; + ui32 nodeId; + TKikimrScopeId scopeId; + TString tenantName; + TBasicKikimrServicesMask servicesMask; + bool tinyMode; + TString clusterName; + NConfig::TConfigsDispatcherInitInfo configsDispatcherInitInfo; + + initCfg->Apply( + appConfig, + nodeId, + scopeId, + tenantName, + servicesMask, + tinyMode, + clusterName, + configsDispatcherInitInfo); + + return appConfig; +} + +std::unique_ptr GetEnvMock(const TString& hostName = "localhost", const TString& fqdnHostName = "localhost") { + auto envMock = std::make_unique(); + envMock->SavedHostName = hostName; + envMock->SavedFQDNHostName = fqdnHostName; + return envMock; +} + +} // namespace + Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { - TTempFileHandle CreateConfigFile() { - TString config = R"( + TString config = R"( metadata: kind: MainConfig version: 0 @@ -81,7 +178,7 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { - disk2 hosts: - host: localhost - port: 2135 + port: 9001 log_config: default_level: 5 @@ -134,103 +231,14 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { node_type: STORAGE cpu_count: 42 )"; - TTempFileHandle tempFile = TTempFileHandle::InCurrentDir("test_config", ".yaml"); - TUnbufferedFileOutput fileOutput(tempFile.Name()); - fileOutput.Write(config); - fileOutput.Finish(); - return tempFile; - } - - void PreFillArgs(std::vector& args, const TString& configPath) { - args.push_back("server"); - - args.push_back("--node"); - args.push_back("static"); - - args.push_back("--ic-port"); - args.push_back("2135"); - - args.push_back("--grpc-port"); - args.push_back("9001"); - - args.push_back("--mon-port"); - args.push_back("8765"); - - args.push_back("--yaml-config"); - args.push_back(configPath); - } - - NKikimrConfig::TAppConfig TransformConfig(const std::vector& args) { - auto errorCollector = NConfig::MakeDefaultErrorCollector(); - auto protoConfigFileProvider = NConfig::MakeDefaultProtoConfigFileProvider(); - auto configUpdateTracer = NConfig::MakeDefaultConfigUpdateTracer(); - auto memLogInit = NConfig::MakeNoopMemLogInitializer(); - auto nodeBrokerClient = NConfig::MakeNoopNodeBrokerClient(); - auto dynConfigClient = NConfig::MakeNoopDynConfigClient(); - auto configClient = NConfig::MakeNoopConfigClient(); - auto logger = NConfig::MakeNoopInitLogger(); - - auto envMock = std::make_unique(); - envMock->SavedHostName = "localhost"; - envMock->SavedFQDNHostName = "localhost"; - - NConfig::TInitialConfiguratorDependencies deps{ - *errorCollector, - *protoConfigFileProvider, - *configUpdateTracer, - *memLogInit, - *nodeBrokerClient, - *dynConfigClient, - *configClient, - *envMock, - *logger, - }; - auto initCfg = NConfig::MakeDefaultInitialConfigurator(deps); - - std::vector argv; - - for (const auto& arg : args) { - argv.push_back(arg.data()); - } - - NLastGetopt::TOpts opts; - initCfg->RegisterCliOptions(opts); - protoConfigFileProvider->RegisterCliOptions(opts); - - NLastGetopt::TOptsParseResult parseResult(&opts, argv.size(), argv.data()); - - initCfg->ValidateOptions(opts, parseResult); - initCfg->Parse(parseResult.GetFreeArgs(), nullptr); - - NKikimrConfig::TAppConfig appConfig; - ui32 nodeId; - TKikimrScopeId scopeId; - TString tenantName; - TBasicKikimrServicesMask servicesMask; - bool tinyMode; - TString clusterName; - NConfig::TConfigsDispatcherInitInfo configsDispatcherInitInfo; - - initCfg->Apply( - appConfig, - nodeId, - scopeId, - tenantName, - servicesMask, - tinyMode, - clusterName, - configsDispatcherInitInfo); - - return appConfig; - } Y_UNIT_TEST(TestStaticNodeSelectorForActorSystem) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=abc"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasActorSystemConfig()); const auto& actorConfig = appConfig.GetActorSystemConfig(); @@ -238,12 +246,12 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { } Y_UNIT_TEST(TestStaticNodeSelectorWithAnotherLabel) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=abd"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasActorSystemConfig()); const auto& actorConfig = appConfig.GetActorSystemConfig(); @@ -252,12 +260,12 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { } Y_UNIT_TEST(TestStaticNodeSelectorInheritance) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=abc"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasLogConfig()); const auto& logConfig = appConfig.GetLogConfig(); @@ -267,12 +275,12 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { } Y_UNIT_TEST(TestStaticNodeSelectorByNodeId) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=node_id"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasActorSystemConfig()); const auto& actorConfig = appConfig.GetActorSystemConfig(); @@ -280,13 +288,13 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { } Y_UNIT_TEST(TestStaticNodeSelectorByNodeHost) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=node_host"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasActorSystemConfig()); const auto& actorConfig = appConfig.GetActorSystemConfig(); @@ -294,15 +302,217 @@ Y_UNIT_TEST_SUITE(StaticNodeSelectorsInit) { } Y_UNIT_TEST(TestStaticNodeSelectorByNodeKind) { - TTempFileHandle configFile = CreateConfigFile(); + TTempFileHandle configFile = CreateConfigFile(config); TVector args; PreFillArgs(args, configFile.Name()); args.push_back("--label"); args.push_back("test=node_kind"); - NKikimrConfig::TAppConfig appConfig = TransformConfig(args); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock()); UNIT_ASSERT(appConfig.HasActorSystemConfig()); const auto& actorConfig = appConfig.GetActorSystemConfig(); UNIT_ASSERT_EQUAL(actorConfig.GetCpuCount(), 42); } } + +Y_UNIT_TEST_SUITE(XdsBootstrapConfig) { + TString config = R"( +metadata: + kind: MainConfig + version: 0 + cluster: test_cluster +config: + default_disk_type: NVME + self_management_config: + enabled: false + + channel_profile_config: + profile: + - channel: + - erasure_species: mirror-3-dc + pdisk_category: 1 + storage_pool_kind: ssdencrypted + vdisk_category: Default + - erasure_species: mirror-3-dc + pdisk_category: 1 + storage_pool_kind: ssdencrypted + vdisk_category: Default + - erasure_species: mirror-3-dc + pdisk_category: 1 + storage_pool_kind: ssdencrypted + vdisk_category: Default + profile_id: 0 + + actor_system_config: + use_auto_config: true + node_type: COMPUTE + cpu_count: 10 + + domains_config: + disable_builtin_security: true + domain: + - domain_id: 1 + name: testing + plan_resolution: '10' + scheme_root: '72057594046678944' + ssid: + - 1 + storage_pool_types: + - kind: ssd + pool_config: + box_id: '1' + erasure_species: mirror-3-dc + kind: ssd + pdisk_filter: + - property: + - type: SSD + vdisk_kind: Default + + host_configs: + - nvme: + - disk1 + - disk2 + hosts: + - host: sas-000-localhost + port: 9001 + node_id: 1 + walle_location: + body: 102526684 + data_center: SAS + rack: SAS-12#12.1.02 + - host: vla-000-localhost + port: 9001 + node_id: 2 + walle_location: + body: 102526684 + data_center: VLA + rack: VLA-12#12.1.02 + - host: klg-000-localhost + port: 9001 + node_id: 3 + walle_location: + body: 102526684 + data_center: KLG + rack: KLG-12#12.1.02 + + log_config: + default_level: 5 + entry: + - component: BS_CONTROLLER + level: 7 +)"; + + Y_UNIT_TEST(CanSetHostnameAndDataCenterFromYdbNode) { + TString tmpConfig(config); + tmpConfig.append(R"( + grpc_config: + xds_bootstrap: + xds_servers: + - server_uri: "xds-provider.bootstrap.cloud-testing.yandex.net:18000" + server_features: + - "xds_v3" + channel_creds: + - type: "insecure" + config: {"k1": "v1", "k2": "v2"} + node: + # id: Do not set id. Try set id from ydb node instance + cluster: "testing" + meta: {"service": "ydb"} + #locality: + #zone: Try set zone as data center of ydb node instance +)"); + TTempFileHandle configFile = CreateConfigFile(tmpConfig); + TVector args; + PreFillArgs(args, configFile.Name()); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock(/*hostname*/"vla-000-localhost", /*fqdnHostName*/"vla-000-localhost")); + + UNIT_ASSERT(appConfig.HasGRpcConfig()); + UNIT_ASSERT(appConfig.GetGRpcConfig().HasXdsBootstrap()); + const auto& xdsBootstrap = appConfig.GetGRpcConfig().GetXdsBootstrap(); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetId(), "vla-000-localhost"); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetLocality().GetZone(), "vla"); + } + + Y_UNIT_TEST(CanSetDataCenterFromYdbNodeArgument) { + TString tmpConfig(config); + tmpConfig.append(R"( + grpc_config: + xds_bootstrap: + xds_servers: + - server_uri: "xds-provider.bootstrap.cloud-testing.yandex.net:18000" + server_features: + - "xds_v3" + channel_creds: + - type: "insecure" + config: {"k1": "v1", "k2": "v2"} + node: + # id: Do not set id. Try set id from ydb node instance + cluster: "testing" + meta: {"service": "ydb"} + #locality: + #zone: Try set zone as data center of ydb node instance +)"); + TTempFileHandle configFile = CreateConfigFile(tmpConfig); + TVector args; + PreFillArgs(args, configFile.Name()); + args.push_back("--data-center"); + args.push_back("dc-klg"); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock(/*hostname*/"klg-000-localhost", /*fqdnHostName*/"klg-000-localhost")); + + UNIT_ASSERT(appConfig.HasGRpcConfig()); + UNIT_ASSERT(appConfig.GetGRpcConfig().HasXdsBootstrap()); + const auto& xdsBootstrap = appConfig.GetGRpcConfig().GetXdsBootstrap(); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetId(), "klg-000-localhost"); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetLocality().GetZone(), "dc-klg"); + } + + Y_UNIT_TEST(CanCheckThatXdsBootstrapIsAbsent) { + TString tmpConfig(config); + tmpConfig.append(R"( + grpc_config: + port: 2135 +)"); + TTempFileHandle configFile = CreateConfigFile(tmpConfig); + TVector args; + PreFillArgs(args, configFile.Name()); + args.push_back("--data-center"); + args.push_back("dc-klg"); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock(/*hostname*/"klg-000-localhost", /*fqdnHostName*/"klg-000-localhost")); + + UNIT_ASSERT(appConfig.HasGRpcConfig()); + UNIT_ASSERT(!appConfig.GetGRpcConfig().HasXdsBootstrap()); + } + + Y_UNIT_TEST(CanUseNodeIdFromYamlConfig) { + TString tmpConfig(config); + tmpConfig.append(R"( + grpc_config: + xds_bootstrap: + xds_servers: + - server_uri: "xds-provider.bootstrap.cloud-testing.yandex.net:18000" + server_features: + - "xds_v3" + channel_creds: + - type: "insecure" + config: {"k1": "v1", "k2": "v2"} + node: + id: "test-node-id" + cluster: "testing" + meta: {"service": "ydb"} + #locality: + #zone: Try set zone as data center of ydb node instance +)"); + TTempFileHandle configFile = CreateConfigFile(tmpConfig); + TVector args; + PreFillArgs(args, configFile.Name()); + args.push_back("--data-center"); + args.push_back("dc-klg"); + NKikimrConfig::TAppConfig appConfig = TransformConfig(args, GetEnvMock(/*hostname*/"klg-000-localhost", /*fqdnHostName*/"klg-000-localhost")); + + UNIT_ASSERT(appConfig.HasGRpcConfig()); + UNIT_ASSERT(appConfig.GetGRpcConfig().HasXdsBootstrap()); + const auto& xdsBootstrap = appConfig.GetGRpcConfig().GetXdsBootstrap(); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetId(), "test-node-id"); + UNIT_ASSERT_EQUAL(xdsBootstrap.GetNode().GetLocality().GetZone(), "dc-klg"); + } +} diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 7cdb8f8e14f6..9cf77390b530 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -148,6 +148,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1276,6 +1279,74 @@ TGRpcServers TKikimrRunner::CreateGRpcServers(const TKikimrRunConfig& runConfig) return grpcServers; } +void TKikimrRunner::InitializeXdsBootstrapConfig(const TKikimrRunConfig& runConfig) { + class TXdsBootstrapConfigBuilder { + private: + ::NKikimrConfig::TGRpcConfig::TXdsBootstrap ConfigYaml; + TString JsonConfig; + + public: + TXdsBootstrapConfigBuilder(const ::NKikimrConfig::TGRpcConfig::TXdsBootstrap& config) + : ConfigYaml(config) + { + NJson::TJsonValue xdsBootstrapConfigJson; + NProtobufJson::Proto2Json(ConfigYaml, xdsBootstrapConfigJson, {.FieldNameMode = NProtobufJson::TProto2JsonConfig::FldNameMode::FieldNameSnakeCaseDense}); + BuildFieldNode(&xdsBootstrapConfigJson); + BuildFieldXdsServers(&xdsBootstrapConfigJson); + JsonConfig = NJson::WriteJson(xdsBootstrapConfigJson, false); + } + + TString Build() const { + return JsonConfig; + } + + private: + void BuildFieldNode(NJson::TJsonValue* const json) const { + NJson::TJsonValue& nodeJson = (*json)["node"]; + if (ConfigYaml.GetNode().HasMeta()) { + // Message in protobuf can not contain field with name "metadata", so + // Create field "meta" with string in JSON format + // Convert string from field "meta" to JsonValue struct and write to field "metadata" + ConvertStringToJsonValue(nodeJson["meta"].GetString(), &nodeJson["metadata"]); + nodeJson.EraseValue("meta"); + } + } + + void BuildFieldXdsServers(NJson::TJsonValue* const json) const { + NJson::TJsonValue& xdsServersJson = *json; + NJson::TJsonValue::TArray xdsServers; + xdsServersJson["xds_servers"].GetArray(&xdsServers); + xdsServersJson.EraseValue("xds_servers"); + for (auto& xdsServerJson : xdsServers) { + NJson::TJsonValue::TArray channelCreds; + xdsServerJson["channel_creds"].GetArray(&channelCreds); + xdsServerJson.EraseValue("channel_creds"); + for (auto& channelCredJson : channelCreds) { + if (channelCredJson.Has("config")) { + ConvertStringToJsonValue(channelCredJson["config"].GetString(), &channelCredJson["config"]); + } + xdsServerJson["channel_creds"].AppendValue(channelCredJson); + } + xdsServersJson["xds_servers"].AppendValue(xdsServerJson); + } + } + + void ConvertStringToJsonValue(const TString& jsonString, NJson::TJsonValue* const out) const { + NJson::TJsonReaderConfig jsonConfig; + if (!NJson::ReadJsonTree(jsonString, &jsonConfig, out)) { + Cerr << "Warning: Failed to parse JSON string in ConvertStringToJsonValue: \"" << jsonString << "\"" << Endl; + *out = NJson::TJsonValue(); + } + } + }; + + static const TString XDS_BOOTSTRAP_ENV = "GRPC_XDS_BOOTSTRAP"; + static const TString XDS_BOOTSTRAP_CONFIG_ENV = "GRPC_XDS_BOOTSTRAP_CONFIG"; + if (GetEnv(XDS_BOOTSTRAP_ENV).empty() && GetEnv(XDS_BOOTSTRAP_CONFIG_ENV).empty() && runConfig.AppConfig.GetGRpcConfig().HasXdsBootstrap()) { + SetEnv(XDS_BOOTSTRAP_CONFIG_ENV, TXdsBootstrapConfigBuilder(runConfig.AppConfig.GetGRpcConfig().GetXdsBootstrap()).Build()); + } +} + void TKikimrRunner::InitializeAllocator(const TKikimrRunConfig& runConfig) { const auto& cfg = runConfig.AppConfig; const auto& allocConfig = cfg.GetAllocatorConfig(); @@ -2280,6 +2351,7 @@ TIntrusivePtr TKikimrRunner::CreateKikimrRunner( const TKikimrRunConfig& runConfig, std::shared_ptr factories) { TIntrusivePtr runner(new TKikimrRunner(factories)); + runner->InitializeXdsBootstrapConfig(runConfig); runner->InitializeAllocator(runConfig); runner->InitializeRegistries(runConfig); runner->InitializeMonitoring(runConfig); diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index e59a263addc2..56470826ba96 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -88,6 +88,16 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage { virtual void InitializeRegistries(const TKikimrRunConfig& runConfig); + /** + * Initializes the XDS bootstrap configuration for the runner. + * This method should be called during the startup sequence, before any components + * that depend on XDS configuration are initialized. It reads the relevant settings + * from the provided runConfig and sets up the XDS bootstrap environment accordingly. + * This is necessary for enabling gRPC XDS features such as dynamic service discovery + * and load balancing. + */ + void InitializeXdsBootstrapConfig(const TKikimrRunConfig& runConfig); + void InitializeAllocator(const TKikimrRunConfig& runConfig); void InitializeLogSettings(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run_ut.cpp b/ydb/core/driver_lib/run/run_ut.cpp new file mode 100644 index 000000000000..47fab16af9df --- /dev/null +++ b/ydb/core/driver_lib/run/run_ut.cpp @@ -0,0 +1,106 @@ +#include "run.h" +#include + +Y_UNIT_TEST_SUITE(XdsBootstrapConfigInitializer) { + +using namespace NKikimr; + +class TTestKikimrRunner : public TKikimrRunner { + TTestKikimrRunner() = default; + + void InitializeXdsBootstrapConfig(NKikimrConfig::TAppConfig& appConfig) { + TKikimrRunner::InitializeXdsBootstrapConfig(TKikimrRunConfig(appConfig)); + } + +public: + static void InitXdsBootstrapConfig(NKikimrConfig::TAppConfig& appConfig) { + TTestKikimrRunner runner; + runner.InitializeXdsBootstrapConfig(appConfig); + } +}; + +const TString XDS_BOOTSTRAP_ENV = "GRPC_XDS_BOOTSTRAP"; +const TString XDS_BOOTSTRAP_CONFIG_ENV = "GRPC_XDS_BOOTSTRAP_CONFIG"; + +Y_UNIT_TEST(CanNotSetEnvIfXdsBootstrapConfigIsAbsent) { + NKikimrConfig::TAppConfig appConfig; + TTestKikimrRunner::InitXdsBootstrapConfig(appConfig); + TString jsonXdsBootstrapConfig = GetEnv(XDS_BOOTSTRAP_CONFIG_ENV); + UNIT_ASSERT_STRINGS_EQUAL_C(jsonXdsBootstrapConfig, "", "The checked value: " + jsonXdsBootstrapConfig); +} + +Y_UNIT_TEST(CanSetGrpcXdsBootstrapConfigEnv) { + NKikimrConfig::TAppConfig appConfig; + auto* xdsBootstrapConfig = appConfig.MutableGRpcConfig()->MutableXdsBootstrap(); + auto* xdsServers = xdsBootstrapConfig->AddXdsServers(); + xdsServers->SetServerUri("xds-provider.bootstrap.my-company.net:18000"); + *xdsServers->AddServerFeatures() = "xds_v3"; + auto* channelCreds = xdsServers->AddChannelCreds(); + channelCreds->SetType("insecure"); + channelCreds->SetConfig("{\"k1\": \"v1\", \"k2\": \"v2\"}"); + auto* node = xdsBootstrapConfig->MutableNode(); + node->SetId("dc-000-host"); + node->SetCluster("testing"); + node->SetMeta("{\"service\": \"ydb\"}"); + node->MutableLocality()->SetZone("test-zone"); + + TTestKikimrRunner::InitXdsBootstrapConfig(appConfig); + const TString expectedJson = R"({"node":{"cluster":"testing","locality":{"zone":"test-zone"},"metadata":{"service":"ydb"},"id":"dc-000-host"},"xds_servers":[{"channel_creds":[{"config":{"k2":"v2","k1":"v1"},"type":"insecure"}],"server_uri":"xds-provider.bootstrap.my-company.net:18000","server_features":["xds_v3"]}]})"; + TString jsonXdsBootstrapConfig = GetEnv(XDS_BOOTSTRAP_CONFIG_ENV); + UNIT_ASSERT_STRINGS_EQUAL_C(jsonXdsBootstrapConfig, expectedJson, "The checked value: " + jsonXdsBootstrapConfig); +} + +Y_UNIT_TEST(CanSetGrpcXdsBootstrapConfigEnvWithSomeNumberOfXdsServers) { + NKikimrConfig::TAppConfig appConfig; + auto* xdsBootstrapConfig = appConfig.MutableGRpcConfig()->MutableXdsBootstrap(); + { + auto* xdsServers = xdsBootstrapConfig->AddXdsServers(); + xdsServers->SetServerUri("xds-provider-000.bootstrap.my-company.net:18000"); + *xdsServers->AddServerFeatures() = "xds_v3"; + auto* channelCreds = xdsServers->AddChannelCreds(); + channelCreds->SetType("insecure"); + channelCreds->SetConfig("{\"k1\": \"v1\", \"k2\": \"v2\"}"); + } + { + auto* xdsServers = xdsBootstrapConfig->AddXdsServers(); + xdsServers->SetServerUri("xds-provider-001.bootstrap.my-company.net:18000"); + *xdsServers->AddServerFeatures() = "xds_v3"; + auto* channelCreds = xdsServers->AddChannelCreds(); + channelCreds->SetType("secure"); + channelCreds->SetConfig("{\"k1\": \"v11\", \"k2\": \"v21\"}"); + } + auto* node = xdsBootstrapConfig->MutableNode(); + node->SetId("dc-000-host"); + node->SetCluster("testing"); + node->SetMeta("{\"service\": \"ydb\"}"); + node->MutableLocality()->SetZone("test-zone"); + + TTestKikimrRunner::InitXdsBootstrapConfig(appConfig); + const TString expectedJson = R"({"node":{"cluster":"testing","locality":{"zone":"test-zone"},"metadata":{"service":"ydb"},"id":"dc-000-host"},"xds_servers":[{"channel_creds":[{"config":{"k2":"v2","k1":"v1"},"type":"insecure"}],"server_uri":"xds-provider-000.bootstrap.my-company.net:18000","server_features":["xds_v3"]},{"channel_creds":[{"config":{"k2":"v21","k1":"v11"},"type":"secure"}],"server_uri":"xds-provider-001.bootstrap.my-company.net:18000","server_features":["xds_v3"]}]})"; + TString jsonXdsBootstrapConfig = GetEnv(XDS_BOOTSTRAP_CONFIG_ENV); + UNIT_ASSERT_STRINGS_EQUAL_C(jsonXdsBootstrapConfig, expectedJson, "The checked value: " + jsonXdsBootstrapConfig); +} + +Y_UNIT_TEST(CanNotSetGrpcXdsBootstrapConfigEnvIfVariableAlreadySet) { + NKikimrConfig::TAppConfig appConfig; + auto* xdsBootstrapConfig = appConfig.MutableGRpcConfig()->MutableXdsBootstrap(); + auto* xdsServers = xdsBootstrapConfig->AddXdsServers(); + xdsServers->SetServerUri("xds-provider.bootstrap.my-company.net:18000"); + *xdsServers->AddServerFeatures() = "xds_v3"; + auto* channelCreds = xdsServers->AddChannelCreds(); + channelCreds->SetType("insecure"); + channelCreds->SetConfig("{\"k1\": \"v1\", \"k2\": \"v2\"}"); + auto* node = xdsBootstrapConfig->MutableNode(); + node->SetId("dc-000-host"); + node->SetCluster("testing"); + node->SetMeta("{\"service\": \"ydb\"}"); + node->MutableLocality()->SetZone("test-zone"); + + SetEnv(XDS_BOOTSTRAP_CONFIG_ENV, "{xds bootstrap config already set}"); + + TTestKikimrRunner::InitXdsBootstrapConfig(appConfig); + TString jsonXdsBootstrapConfig = GetEnv(XDS_BOOTSTRAP_CONFIG_ENV); + UNIT_ASSERT_STRINGS_EQUAL_C(jsonXdsBootstrapConfig, "{xds bootstrap config already set}", "The checked value: " + jsonXdsBootstrapConfig); +} + +} // XdsBootstrapConfigInitializer diff --git a/ydb/core/driver_lib/run/ut/ya.make b/ydb/core/driver_lib/run/ut/ya.make index 753ee00cdd9f..59990313c7df 100644 --- a/ydb/core/driver_lib/run/ut/ya.make +++ b/ydb/core/driver_lib/run/ut/ya.make @@ -12,6 +12,7 @@ YQL_LAST_ABI_VERSION() SRCS( auto_config_initializer_ut.cpp + run_ut.cpp ) END() diff --git a/ydb/core/protos/auth.proto b/ydb/core/protos/auth.proto index 03983b031d81..b5dc1b50d959 100644 --- a/ydb/core/protos/auth.proto +++ b/ydb/core/protos/auth.proto @@ -60,6 +60,9 @@ message TAuthConfig { optional TAccountLockout AccountLockout = 84; optional string AccessServiceTokenName = 85; optional TTokenManager TokenManager = 86; + optional string AccessServiceSslTargetNameOverride = 87; + optional string UserAccountServiceSslTargetNameOverride = 88; + optional string ServiceAccountServiceSslTargetNameOverride = 89; } message TUserRegistryConfig { diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 34db62249d88..5b120e6eab35 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -812,6 +812,7 @@ message TGRpcConfig { } optional YdbGrpcCompressionLevel DefaultCompressionLevel = 31 [default = YDB_GRPC_COMPRESS_LEVEL_NONE]; + optional TXdsBootstrap XdsBootstrap = 32; // server socket options optional bool KeepAliveEnable = 100 [default = true]; // SO_KEEPALIVE @@ -827,6 +828,44 @@ message TGRpcConfig { optional string EndpointId = 108; repeated TGRpcConfig ExtEndpoints = 200; // run specific services on separate endpoints + + // TXdsBootstrap is used to configure XDS (Envoy's discovery service) support for gRPC endpoints. + // This enables dynamic service discovery, load balancing, and configuration via an XDS server. + // Configure these fields when you want your gRPC endpoints to use XDS for advanced traffic management. + // For more details, see: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/bootstrap/v3/bootstrap.proto + // Format XDS bootstrap file: https://grpc.github.io/grpc/core/md_doc_grpc_xds_bootstrap_format.html + message TXdsBootstrap { + // TXdsServer describes an XDS server to connect to for configuration and service discovery. + message TXdsServer { + // TChannelCred specifies the credentials used to connect to the XDS server. + message TChannelCred { + optional string Type = 1; // Type of channel credentials (e.g., "insecure", "tls") + optional string Config = 2; // String in JSON format containing config for the type + } + + optional string ServerUri = 1; // URI of the XDS server (e.g., "xds.example.com:443") + repeated TChannelCred ChannelCreds = 2; // Credentials for connecting to the XDS server + repeated string ServerFeatures = 3; // List of supported XDS server features (e.g., "xds_v3") + } + + // TNode describes the node identity and locality information sent to the XDS server. + message TNode { + // TLocality specifies the region, zone, and subzone of the node. + message TLocality { + optional string Region = 1; + optional string Zone = 2; + optional string SubZone = 3; + } + + optional string Id = 1; // Unique node identifier (e.g., "node-123") + optional string Cluster = 2; // Cluster name (e.g., "my-grpc-cluster") + optional TLocality Locality = 3; // Locality information for the node + optional string Meta = 4; // String in JSON format containing opaque metadata extending the node identifier. This field will be converted to 'metadata' in XDS bootstrap config + } + + repeated TXdsServer XdsServers = 1; // List of XDS servers to connect to + optional TNode Node = 2; // Node identity and locality information + } } message TDynamicNodeConfig { diff --git a/ydb/core/security/ticket_parser_impl.h b/ydb/core/security/ticket_parser_impl.h index 57a717f30ba9..9804be7a43a2 100644 --- a/ydb/core/security/ticket_parser_impl.h +++ b/ydb/core/security/ticket_parser_impl.h @@ -2100,6 +2100,7 @@ class TTicketParserImpl : public TActorBootstrapped { settings.Endpoint = Config.GetAccessServiceEndpoint(); if (Config.GetUseAccessServiceTLS()) { settings.CertificateRootCA = TUnbufferedFileInput(Config.GetPathToRootCA()).ReadAll(); + settings.SslTargetNameOverride = Config.GetAccessServiceSslTargetNameOverride(); } settings.GrpcKeepAliveTimeMs = Config.GetAccessServiceGrpcKeepAliveTimeMs(); settings.GrpcKeepAliveTimeoutMs = Config.GetAccessServiceGrpcKeepAliveTimeoutMs(); @@ -2146,6 +2147,7 @@ class TTicketParserImpl : public TActorBootstrapped { settings.Endpoint = Config.GetUserAccountServiceEndpoint(); if (Config.GetUseUserAccountServiceTLS()) { settings.CertificateRootCA = TUnbufferedFileInput(Config.GetPathToRootCA()).ReadAll(); + settings.SslTargetNameOverride = Config.GetUserAccountServiceSslTargetNameOverride(); } UserAccountService = Register(CreateUserAccountService(settings), TMailboxType::HTSwap, AppData()->UserPoolId); if (Config.GetCacheUserAccountService()) { @@ -2162,6 +2164,7 @@ class TTicketParserImpl : public TActorBootstrapped { settings.Endpoint = Config.GetServiceAccountServiceEndpoint(); if (Config.GetUseServiceAccountServiceTLS()) { settings.CertificateRootCA = TUnbufferedFileInput(Config.GetPathToRootCA()).ReadAll(); + settings.SslTargetNameOverride = Config.GetServiceAccountServiceSslTargetNameOverride(); } ServiceAccountService = Register(NCloud::CreateServiceAccountService(settings), TMailboxType::HTSwap, AppData()->UserPoolId); if (Config.GetCacheServiceAccountService()) { diff --git a/ydb/library/grpc/actor_client/grpc_service_client.h b/ydb/library/grpc/actor_client/grpc_service_client.h index 04d6b72454be..3414cc4a1c71 100644 --- a/ydb/library/grpc/actor_client/grpc_service_client.h +++ b/ydb/library/grpc/actor_client/grpc_service_client.h @@ -133,6 +133,9 @@ class TGrpcServiceClient { config.IntChannelParams[GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA] = 0; config.IntChannelParams[GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS] = settings.GrpcKeepAlivePingInterval; config.IntChannelParams[GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS] = settings.GrpcKeepAlivePingInterval; + if (!settings.SslTargetNameOverride.empty()) { + config.SslTargetNameOverride = settings.SslTargetNameOverride; + } return config; } diff --git a/ydb/library/grpc/actor_client/grpc_service_settings.h b/ydb/library/grpc/actor_client/grpc_service_settings.h index f48e0ae61e68..790b060a87c9 100644 --- a/ydb/library/grpc/actor_client/grpc_service_settings.h +++ b/ydb/library/grpc/actor_client/grpc_service_settings.h @@ -14,6 +14,7 @@ struct TGrpcClientSettings { bool EnableSsl = false; ui64 RequestTimeoutMs = 10000; // 10 seconds std::unordered_map Headers; + TString SslTargetNameOverride; }; } // namespace NGrpcActorClient