diff options
author | andrew-rykov <arykov@ydb.tech> | 2022-11-17 16:43:12 +0300 |
---|---|---|
committer | andrew-rykov <arykov@ydb.tech> | 2022-11-17 16:43:12 +0300 |
commit | 4a42af3f3b446ff6e05ca4503c96fcd1b501ebef (patch) | |
tree | ca073671982fa4367fce0d646dcfd1c39a21d891 | |
parent | f125f2c0683d82b24b9bd49de73f379d609d906a (diff) | |
download | ydb-4a42af3f3b446ff6e05ca4503c96fcd1b501ebef.tar.gz |
PR from branch users/andrew-rykov//audit-log-system
added PEERDIR 2
added PEERDIR
added comments
added test filePath
updated ss audit log
audit log system
-rw-r--r-- | ydb/core/CMakeLists.txt | 1 | ||||
-rw-r--r-- | ydb/core/audit/CMakeLists.txt | 45 | ||||
-rw-r--r-- | ydb/core/audit/audit_log.cpp | 42 | ||||
-rw-r--r-- | ydb/core/audit/audit_log.h | 150 | ||||
-rw-r--r-- | ydb/core/audit/audit_log_impl.h | 15 | ||||
-rw-r--r-- | ydb/core/audit/audit_log_json_impl.cpp | 67 | ||||
-rw-r--r-- | ydb/core/audit/audit_log_txt_impl.cpp | 64 | ||||
-rw-r--r-- | ydb/core/base/appdata.h | 1 | ||||
-rw-r--r-- | ydb/core/base/events.h | 3 | ||||
-rw-r--r-- | ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp | 2 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/CMakeLists.txt | 1 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/config.h | 1 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/kikimr_services_initializers.cpp | 33 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/kikimr_services_initializers.h | 7 | ||||
-rw-r--r-- | ydb/core/driver_lib/run/run.cpp | 8 | ||||
-rw-r--r-- | ydb/core/kqp/ut/kqp_scheme_ut.cpp | 75 | ||||
-rw-r--r-- | ydb/core/protos/config.proto | 11 | ||||
-rw-r--r-- | ydb/core/protos/services.proto | 3 | ||||
-rw-r--r-- | ydb/tests/library/harness/kikimr_config.py | 26 |
19 files changed, 478 insertions, 77 deletions
diff --git a/ydb/core/CMakeLists.txt b/ydb/core/CMakeLists.txt index a22589679e4..6050c26ab21 100644 --- a/ydb/core/CMakeLists.txt +++ b/ydb/core/CMakeLists.txt @@ -7,6 +7,7 @@ add_subdirectory(actorlib_impl) +add_subdirectory(audit) add_subdirectory(base) add_subdirectory(blob_depot) add_subdirectory(blobstorage) diff --git a/ydb/core/audit/CMakeLists.txt b/ydb/core/audit/CMakeLists.txt new file mode 100644 index 00000000000..976aa09e163 --- /dev/null +++ b/ydb/core/audit/CMakeLists.txt @@ -0,0 +1,45 @@ + +# This file was gererated by the build system used internally in the Yandex monorepo. +# Only simple modifications are allowed (adding source-files to targets, adding simple properties +# like target_include_directories). These modifications will be ported to original +# ya.make files by maintainers. Any complex modifications which can't be ported back to the +# original buildsystem will not be accepted. + + + +add_library(ydb-core-audit) +target_link_libraries(ydb-core-audit PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + library-cpp-json + library-cpp-logger + ydb-core-base + library-cpp-resource +) +target_sources(ydb-core-audit PRIVATE + ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log_json_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log_txt_impl.cpp + ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log.cpp +) + +add_global_library_for(ydb-core-audit.global ydb-core-audit) +target_link_libraries(ydb-core-audit.global PUBLIC + contrib-libs-cxxsupp + yutil + cpp-actors-core + library-cpp-json + library-cpp-logger + ydb-core-base + library-cpp-resource +) +target_sources(ydb-core-audit.global PRIVATE + ${CMAKE_BINARY_DIR}/ydb/core/audit/da10233abc65c3f178c7da0ce0d7a4ea.cpp +) +resources(ydb-core-audit.global + ${CMAKE_BINARY_DIR}/ydb/core/audit/da10233abc65c3f178c7da0ce0d7a4ea.cpp + INPUTS + ${CMAKE_SOURCE_DIR}/ydb/core/kqp/kqp_default_settings.txt + KEYS + kqp_default_settings.txt +) diff --git a/ydb/core/audit/audit_log.cpp b/ydb/core/audit/audit_log.cpp new file mode 100644 index 00000000000..d6a0a23445b --- /dev/null +++ b/ydb/core/audit/audit_log.cpp @@ -0,0 +1,42 @@ +#include "audit_log.h" +#include "audit_log_impl.h" + +#include <library/cpp/logger/record.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/string/builder.h> + +namespace NKikimr::NAudit { + +std::atomic<bool> AUDIT_LOG_ENABLED = false; + +THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format) +{ + AUDIT_LOG_ENABLED.store(true); + switch (format) { + case NKikimrConfig::TAuditConfig::JSON: + return MakeHolder<TAuditJsonLogActor>(std::move(auditFile)); + case NKikimrConfig::TAuditConfig::TXT: + return MakeHolder<TAuditTxtLogActor>(std::move(auditFile)); + default: + return MakeHolder<TAuditJsonLogActor>(std::move(auditFile)); + } +} + + +void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts) +{ + auto request = MakeHolder<TEvAuditLog::TEvWriteAuditLog>(Now(), parts); + sys->Send(MakeAuditServiceID(), request.Release()); +} + +const char* FormatLocalTimestamp(TInstant time, char* buf) { + struct tm localTime; + time.LocalTime(&localTime); + int r = strftime(buf, TimeBufSize, "%Y-%m-%d-%H-%M-%S", &localTime); + Y_VERIFY(r != 0); + return buf; +} + +} // namespace NKikimr::NAudit diff --git a/ydb/core/audit/audit_log.h b/ydb/core/audit/audit_log.h new file mode 100644 index 00000000000..d5bc7e6bed3 --- /dev/null +++ b/ydb/core/audit/audit_log.h @@ -0,0 +1,150 @@ +#pragma once + +#include <ydb/core/base/events.h> + +#include <library/cpp/actors/core/actor.h> +#include <library/cpp/actors/core/events.h> +#include <library/cpp/logger/backend.h> +#include <ydb/core/protos/config.pb.h> + +#include <library/cpp/logger/record.h> +#include <library/cpp/actors/core/hfunc.h> +#include <library/cpp/actors/core/log.h> + +#include <util/generic/strbuf.h> +#include <util/datetime/base.h> + +#define AUDIT_LOG_S(sys, expr) \ + do { \ + if (::NKikimr::NAudit::AUDIT_LOG_ENABLED.load()) { \ + TVector<std::pair<TStringBuf, TString>> auditParts; \ + expr \ + ::NKikimr::NAudit::SendAuditLog(sys, auditParts); \ + } \ + } while (0) /**/ + +#define AUDIT_LOG(expr) AUDIT_LOG_S((TlsActivationContext->ExecutorThread.ActorSystem), expr) + +#define AUDIT_PART_NO_COND(key, value) auditParts.push_back({key, value}); +#define AUDIT_PART_COND(key, value, condition) \ + do { \ + if (condition && !value.Empty()) { \ + auditParts.push_back({key, value}); \ + } \ + } while (0); + +#define GET_AUDIT_PART_MACRO(_1, _2, _3, NAME,...) NAME +#define AUDIT_PART(...) GET_AUDIT_PART_MACRO(__VA_ARGS__, AUDIT_PART_COND, AUDIT_PART_NO_COND)(__VA_ARGS__) + +namespace NKikimr::NAudit { + +extern std::atomic<bool> AUDIT_LOG_ENABLED; + +struct TEvAuditLog +{ + // + // Events declaration + // + + enum EEvents + { + EvBegin = EventSpaceBegin(TKikimrEvents::ES_YDB_AUDIT_LOG), + + // Request actors + EvWriteAuditLog = EvBegin + 0, + + EvEnd + }; + + static_assert(EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG), + "expected EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG)"); + + struct TEvWriteAuditLog + : public NActors::TEventLocal<TEvWriteAuditLog, EvWriteAuditLog> + { + TInstant Time; + TVector<std::pair<TStringBuf, TString>> Parts; + + TEvWriteAuditLog(TInstant time, TVector<std::pair<TStringBuf, TString>> parts) + : Time(time) + , Parts(std::move(parts)) + {} + }; +}; + +class TAuditJsonLogActor final + : public TActor<TAuditJsonLogActor> +{ +private: + const THolder<TLogBackend> AuditFile; +public: + TAuditJsonLogActor(THolder<TLogBackend> auditFile) + : TActor(&TThis::StateWork) + , AuditFile(std::move(auditFile)) + { + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::AUDIT_WRITER_ACTOR; + } + +private: + STFUNC(StateWork); + + void HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx); + + void HandleWriteAuditLog( + const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, + const TActorContext& ctx); + + void HandleUnexpectedEvent(STFUNC_SIG); +}; + +class TAuditTxtLogActor final + : public TActor<TAuditTxtLogActor> +{ +private: + const THolder<TLogBackend> AuditFile; +public: + TAuditTxtLogActor(THolder<TLogBackend> auditFile) + : TActor(&TThis::StateWork) + , AuditFile(std::move(auditFile)) + { + } + + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::AUDIT_WRITER_ACTOR; + } + +private: + STFUNC(StateWork); + + void HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx); + + void HandleWriteAuditLog( + const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, + const TActorContext& ctx); + + void HandleUnexpectedEvent(STFUNC_SIG); +}; + +//////////////////////////////////////////////////////////////////////////////// + +void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts); + +inline NActors::TActorId MakeAuditServiceID() { + return NActors::TActorId(0, TStringBuf("YDB_AUDIT")); +} + +THolder<NActors::IActor> CreateAuditWriter( + THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format); + +constexpr size_t TimeBufSize = 512; + +const char* FormatLocalTimestamp(TInstant time, char* buf); + +} // namespace NKikimr::NAudit diff --git a/ydb/core/audit/audit_log_impl.h b/ydb/core/audit/audit_log_impl.h new file mode 100644 index 00000000000..91de44ddead --- /dev/null +++ b/ydb/core/audit/audit_log_impl.h @@ -0,0 +1,15 @@ +#if defined LOG_T || \ + defined LOG_D || \ + defined LOG_I || \ + defined LOG_N || \ + defined LOG_W || \ + defined LOG_E +# error log macro redefinition +#endif + +#define LOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) +#define LOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) +#define LOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) +#define LOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) +#define LOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) +#define LOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream) diff --git a/ydb/core/audit/audit_log_json_impl.cpp b/ydb/core/audit/audit_log_json_impl.cpp new file mode 100644 index 00000000000..689c5648d5a --- /dev/null +++ b/ydb/core/audit/audit_log_json_impl.cpp @@ -0,0 +1,67 @@ +#include "audit_log.h" +#include "audit_log_impl.h" + +#include <library/cpp/json/json_value.h> +#include <library/cpp/json/json_writer.h> + +namespace NKikimr::NAudit { + +using namespace NActors; + +void TAuditJsonLogActor::HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + AUDIT_LOG_ENABLED.store(false); + Die(ctx); +} + +STFUNC(TAuditJsonLogActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + HFunc(TEvAuditLog::TEvWriteAuditLog, HandleWriteAuditLog); + default: + HandleUnexpectedEvent(ev, ctx); + break; + } +} + +void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + const auto* msg = ev->Get(); + try { + TStringStream ss; + char buf[TimeBufSize]; + ss << FormatLocalTimestamp(msg->Time, buf) << ": "; + + NJson::TJsonMap m; + for (auto& [k, v] : msg->Parts) { + m[k] = v; + } + NJson::WriteJson(&ss, &m, false, false); + ss << Endl; + auto json = ss.Str(); + + AuditFile->WriteData( + TLogRecord( + ELogPriority::TLOG_INFO, + json.data(), + json.length())); + } catch (const TFileError& e) { + LOG_W("TAuditJsonLogActor:" + << " unable to write audit log (error: " << e.what() << ")"); + } +} + +void TAuditJsonLogActor::HandleUnexpectedEvent(STFUNC_SIG) +{ + Y_UNUSED(ctx); + + LOG_W("TAuditJsonLogActor:" + << " unhandled event type: " << ev->GetTypeRewrite() + << " event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?")); +} + +} // namespace NKikimr::NAudit diff --git a/ydb/core/audit/audit_log_txt_impl.cpp b/ydb/core/audit/audit_log_txt_impl.cpp new file mode 100644 index 00000000000..972dadfb03e --- /dev/null +++ b/ydb/core/audit/audit_log_txt_impl.cpp @@ -0,0 +1,64 @@ +#include "audit_log.h" +#include "audit_log_impl.h" + +namespace NKikimr::NAudit { + +using namespace NActors; + +void TAuditTxtLogActor::HandlePoisonPill( + const TEvents::TEvPoisonPill::TPtr& ev, + const TActorContext& ctx) +{ + Y_UNUSED(ev); + AUDIT_LOG_ENABLED.store(false); + Die(ctx); +} + +STFUNC(TAuditTxtLogActor::StateWork) +{ + switch (ev->GetTypeRewrite()) { + HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); + HFunc(TEvAuditLog::TEvWriteAuditLog, HandleWriteAuditLog); + default: + HandleUnexpectedEvent(ev, ctx); + break; + } +} + +void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) { + Y_UNUSED(ctx); + const auto* msg = ev->Get(); + try { + TStringStream ss; + char buf[TimeBufSize]; + ss << FormatLocalTimestamp(msg->Time, buf) << ": "; + + for (auto it = msg->Parts.begin(); it != msg->Parts.end(); it++) { + if (it != msg->Parts.begin()) + ss << ", "; + ss << it->first << "=" << it->second; + } + ss << Endl; + auto text = ss.Str(); + + AuditFile->WriteData( + TLogRecord( + ELogPriority::TLOG_INFO, + text.data(), + text.length())); + } catch (const TFileError& e) { + LOG_W("TAuditTxtLogActor:" + << " unable to write audit log (error: " << e.what() << ")"); + } +} + +void TAuditTxtLogActor::HandleUnexpectedEvent(STFUNC_SIG) +{ + Y_UNUSED(ctx); + + LOG_W("TAuditTxtLogActor:" + << " unhandled event type: " << ev->GetTypeRewrite() + << " event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?")); +} + +} // namespace NKikimr::NAudit diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h index d70f2955b1c..8be75810dc5 100644 --- a/ydb/core/base/appdata.h +++ b/ydb/core/base/appdata.h @@ -137,6 +137,7 @@ struct TAppData { NKikimrConfig::TDataShardConfig DataShardConfig; NKikimrConfig::TSchemeShardConfig SchemeShardConfig; NKikimrConfig::TMeteringConfig MeteringConfig; + NKikimrConfig::TAuditConfig AuditConfig; NKikimrConfig::TCompactionConfig CompactionConfig; NKikimrConfig::TDomainsConfig DomainsConfig; NKikimrConfig::TBootstrap BootstrapConfig; diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h index 4f4ec8e3f53..af59ea23c73 100644 --- a/ydb/core/base/events.h +++ b/ydb/core/base/events.h @@ -152,7 +152,8 @@ struct TKikimrEvents : TEvents { ES_INTERNAL_REQUEST, ES_BACKGROUND_TASKS, ES_TIERING, - ES_METADATA_INITIALIZER + ES_METADATA_INITIALIZER, + ES_YDB_AUDIT_LOG, }; }; diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp index 1d562e4b24b..5ce4d782706 100644 --- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp @@ -208,6 +208,7 @@ protected: config.Opts->AddLongOption("feature-flags-file", "File with feature flags to turn new features on/off").OptionalArgument("PATH"); config.Opts->AddLongOption("rb-file", "File with resource broker customizations").OptionalArgument("PATH"); config.Opts->AddLongOption("metering-file", "File with metering config").OptionalArgument("PATH"); + config.Opts->AddLongOption("audit-file", "File with audit config").OptionalArgument("PATH"); config.Opts->AddLongOption('r', "restarts-count-file", "State for restarts monitoring counter,\nuse empty string to disable\n") .OptionalArgument("PATH").DefaultValue(RestartsCountFile).StoreResult(&RestartsCountFile); config.Opts->AddLongOption("compile-inflight-limit", "Limit on parallel programs compilation").OptionalArgument("NUM").StoreResult(&CompileInflightLimit); @@ -437,6 +438,7 @@ protected: OPTION("feature-flags-file", FeatureFlags); OPTION("rb-file", ResourceBrokerConfig); OPTION("metering-file", MeteringConfig); + OPTION("audit-file", AuditConfig); OPTION("kqp-file", KQPConfig); OPTION("incrhuge-file", IncrHugeConfig); OPTION("alloc-file", AllocatorConfig); diff --git a/ydb/core/driver_lib/run/CMakeLists.txt b/ydb/core/driver_lib/run/CMakeLists.txt index 0b09685d4ab..ecc840965dd 100644 --- a/ydb/core/driver_lib/run/CMakeLists.txt +++ b/ydb/core/driver_lib/run/CMakeLists.txt @@ -36,6 +36,7 @@ target_link_libraries(run PUBLIC cpp-string_utils-parse_size library-cpp-svnversion ydb-core-actorlib_impl + ydb-core-audit ydb-core-base ydb-core-blob_depot ydb-core-blobstorage diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h index f90503bf411..9863b0aefbf 100644 --- a/ydb/core/driver_lib/run/config.h +++ b/ydb/core/driver_lib/run/config.h @@ -58,6 +58,7 @@ union TBasicKikimrServicesMask { bool EnablePersQueueClusterTracker:1; bool EnableSysViewService:1; bool EnableMeteringWriter:1; + bool EnableAuditWriter:1; bool EnableSchemeBoardMonitoring:1; bool EnableConfigsCache:1; bool EnableLongTxService:1; diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 4ea7f2a0d63..10be71c0407 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -8,6 +8,8 @@ #include <ydb/core/actorlib_impl/mad_squirrel.h> #include <ydb/core/actorlib_impl/node_identifier.h> +#include "ydb/core/audit/audit_log.h" + #include <ydb/core/base/appdata.h> #include <ydb/core/base/config_units.h> #include <ydb/core/base/counters.h> @@ -2274,7 +2276,7 @@ TMeteringWriterInitializer::TMeteringWriterInitializer(const TKikimrRunConfig &r { } -void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup *setup, const TAppData *appData) +void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) { if (!Config.HasMeteringConfig() || !Config.GetMeteringConfig().HasMeteringFilePath()) { return; @@ -2297,6 +2299,35 @@ void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup *setup, co TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->IOPoolId))); } +TAuditWriterInitializer::TAuditWriterInitializer(const TKikimrRunConfig &runConfig) + : IKikimrServicesInitializer(runConfig) +{ +} + +void TAuditWriterInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData) +{ + if (!Config.HasAuditConfig() || !Config.GetAuditConfig().HasAuditFilePath() || !Config.GetAuditConfig().HasFormat()) { + return; + } + + const auto& filePath = Config.GetAuditConfig().GetAuditFilePath(); + const auto format = Config.GetAuditConfig().GetFormat(); + + THolder<TFileLogBackend> fileBackend; + try { + fileBackend = MakeHolder<TFileLogBackend>(filePath); + } catch (const TFileError& ex) { + Cerr << "TAuditWriterInitializer: failed to open file '" << filePath << "': " << ex.what() << Endl; + exit(1); + } + + auto actor = NAudit::CreateAuditWriter(std::move(fileBackend), format); + + setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>( + NAudit::MakeAuditServiceID(), + TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->IOPoolId))); +} + TSchemeBoardMonitoringInitializer::TSchemeBoardMonitoringInitializer(const TKikimrRunConfig &runConfig) : IKikimrServicesInitializer(runConfig) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h index f4bf8d00bd4..4d08368384d 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.h +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h @@ -483,6 +483,13 @@ public: void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; }; +class TAuditWriterInitializer : public IKikimrServicesInitializer { +public: + TAuditWriterInitializer(const TKikimrRunConfig& runConfig); + + void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override; +}; + class TSchemeBoardMonitoringInitializer : public IKikimrServicesInitializer { public: TSchemeBoardMonitoringInitializer(const TKikimrRunConfig& runConfig); diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 21e0a25dcee..7c7ba7509e8 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1032,6 +1032,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig) AppData->MeteringConfig = runConfig.AppConfig.GetMeteringConfig(); } + if (runConfig.AppConfig.HasAuditConfig()) { + AppData->AuditConfig = runConfig.AppConfig.GetAuditConfig(); + } + AppData->TenantName = runConfig.TenantName; if (runConfig.AppConfig.HasBootstrapConfig()) { @@ -1464,6 +1468,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers sil->AddServiceInitializer(new TMeteringWriterInitializer(runConfig)); } + if (serviceMask.EnableAuditWriter) { + sil->AddServiceInitializer(new TAuditWriterInitializer(runConfig)); + } + if (serviceMask.EnableLongTxService) { sil->AddServiceInitializer(new TLongTxServiceInitializer(runConfig)); } diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp index 0775f128245..1062f6c1d49 100644 --- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp @@ -100,81 +100,6 @@ Y_UNIT_TEST_SUITE(KqpScheme) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - Y_UNIT_TEST(CreateAndDropTableCheckAuditLog) { - TStringStream logStream; - { - TKikimrRunner kikimr(TKikimrSettings().SetLogStream(&logStream)); - - auto driverConfig = TDriverConfig() - .SetEndpoint(kikimr.GetEndpoint()) - .SetAuthToken("user0@builtin"); - auto driver = TDriver(driverConfig); - auto db = NYdb::NTable::TTableClient(driver); - auto session = db.CreateSession().GetValueSync().GetSession(); - kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_INFO); - - { - auto schemeClient = kikimr.GetSchemeClient(); - - NYdb::NScheme::TPermissions permissions("user0@builtin", {"ydb.deprecated.create_table"}); - AssertSuccessResult(schemeClient.ModifyPermissions("/Root", - NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions) - ).ExtractValueSync() - ); - } - - { - const static TString createTableQuery = R"( - CREATE TABLE `/Root/Test1234/KeyValue` ( - Key Uint32, - Value String, - PRIMARY KEY(Key) - ); - )"; - auto result = session.ExecuteSchemeQuery(createTableQuery).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - { - const static TString dropTableQuery = R"( - DROP TABLE `/Root/Test1234/KeyValue`; - )"; - auto result = session.ExecuteSchemeQuery(dropTableQuery).ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); - } - - driver.Stop(true); - } - - TString line; - int mtc = 0; - int ctc = 0; - int dtc = 0; - while (logStream.ReadLine(line)) { - if (line.find("AUDIT:") == line.npos) - continue; - - const TString modifyAclTablePattern("operation: MODIFY ACL"); - if (line.find(modifyAclTablePattern) != line.npos) { - mtc += 1; - } - - const TString createTablePattern("operation: CREATE TABLE"); - if (line.find(createTablePattern) != line.npos) { - ctc += 1; - } - - const TString dropTablePattern("operation: DROP TABLE"); - if (line.find(dropTablePattern) != line.npos) { - dtc += 1; - } - } - - UNIT_ASSERT_VALUES_EQUAL_C(mtc, 1, mtc); - UNIT_ASSERT_VALUES_EQUAL_C(ctc, 1, ctc); - UNIT_ASSERT_VALUES_EQUAL_C(dtc, 1, dtc); - } - Y_UNIT_TEST(CreateDropTableMultipleTime) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index fcb5e64f79c..97b4da95c4e 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1374,6 +1374,16 @@ message TMeteringConfig { repeated string SystemBackupSIDs = 2; }; +message TAuditConfig { + enum EFormat { + JSON = 1; + TXT = 2; + } + + optional string AuditFilePath = 1; + optional EFormat Format = 2; +}; + message THiveTabletLimit { optional NKikimrTabletBase.TTabletTypes.EType Type = 1; optional uint64 MaxCount = 2; @@ -1660,6 +1670,7 @@ message TAppConfig { optional THttpProxyConfig PublicHttpConfig = 57; optional TMetadataProviderConfig MetadataProviderConfig = 59; optional TBackgroundTasksConfig BackgroundTasksConfig = 60; + optional TAuditConfig AuditConfig = 61; optional NYq.NConfig.TConfig YandexQueryConfig = 50; // TODO: remove after migration to FederatedQueryConfig diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto index d770c45ca18..2c515042c24 100644 --- a/ydb/core/protos/services.proto +++ b/ydb/core/protos/services.proto @@ -283,6 +283,8 @@ enum EServiceKikimr { BUILD_INDEX = 1000; + AUDIT_LOG_WRITER = 1005; + METERING_WRITER = 1010; // Streaming @@ -939,5 +941,6 @@ message TActivity { HTTP_MON_SERVICE_NODE_REQUEST = 593; HTTP_MON_SERVICE_MON_REQUEST = 594; HTTP_MON_SERVICE_NODE_PROXY = 595; + AUDIT_WRITER_ACTOR = 596; }; }; diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py index 4e7e93c5af2..f436bd14147 100644 --- a/ydb/tests/library/harness/kikimr_config.py +++ b/ydb/tests/library/harness/kikimr_config.py @@ -126,6 +126,7 @@ class KikimrConfigGenerator(object): use_in_memory_pdisks=False, enable_pqcd=True, enable_metering=False, + enable_audit_log=False, grpc_tls_data_path=None, fq_config_path=None, public_http_config_path=None, @@ -237,6 +238,9 @@ class KikimrConfigGenerator(object): if enable_metering: self.__set_enable_metering() + if enable_audit_log: + self.__set_enable_audit_log() + self.naming_config = config_pb2.TAppConfig() dc_it = itertools.cycle(self._dcs) rack_it = itertools.count(start=1) @@ -353,11 +357,33 @@ class KikimrConfigGenerator(object): metering_file.write('') self.yaml_config['metering_config'] = {'metering_file_path': metering_file_path} + def __set_enable_audit_log(self): + def ensure_path_exists(path): + if not os.path.isdir(path): + os.makedirs(path) + return path + + def get_cwd_for_test(output_path): + test_name = yatest_common.context.test_name or "" + test_name = test_name.replace(':', '_') + return os.path.join(output_path, test_name) + + cwd = get_cwd_for_test(self.__output_path) + ensure_path_exists(cwd) + audit_file_path = os.path.join(cwd, 'audit.txt') + with open(audit_file_path, "w") as audit_file: + audit_file.write('') + self.yaml_config['audit_config'] = {'audit_file_path': audit_file_path} + @property def metering_file_path(self): return self.yaml_config.get('metering_config', {}).get('metering_file_path') @property + def audit_file_path(self): + return self.yaml_config.get('audit_config', {}).get('audit_file_path') + + @property def nbs_enable(self): return self._enable_nbs |