aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorandrew-rykov <arykov@ydb.tech>2022-11-17 16:43:12 +0300
committerandrew-rykov <arykov@ydb.tech>2022-11-17 16:43:12 +0300
commit4a42af3f3b446ff6e05ca4503c96fcd1b501ebef (patch)
treeca073671982fa4367fce0d646dcfd1c39a21d891
parentf125f2c0683d82b24b9bd49de73f379d609d906a (diff)
downloadydb-4a42af3f3b446ff6e05ca4503c96fcd1b501ebef.tar.gz
PR from branch users/andrew-rykov//audit-log-system
added PEERDIR 2 added PEERDIR added comments added test filePath updated ss audit log audit log system
-rw-r--r--ydb/core/CMakeLists.txt1
-rw-r--r--ydb/core/audit/CMakeLists.txt45
-rw-r--r--ydb/core/audit/audit_log.cpp42
-rw-r--r--ydb/core/audit/audit_log.h150
-rw-r--r--ydb/core/audit/audit_log_impl.h15
-rw-r--r--ydb/core/audit/audit_log_json_impl.cpp67
-rw-r--r--ydb/core/audit/audit_log_txt_impl.cpp64
-rw-r--r--ydb/core/base/appdata.h1
-rw-r--r--ydb/core/base/events.h3
-rw-r--r--ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp2
-rw-r--r--ydb/core/driver_lib/run/CMakeLists.txt1
-rw-r--r--ydb/core/driver_lib/run/config.h1
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.cpp33
-rw-r--r--ydb/core/driver_lib/run/kikimr_services_initializers.h7
-rw-r--r--ydb/core/driver_lib/run/run.cpp8
-rw-r--r--ydb/core/kqp/ut/kqp_scheme_ut.cpp75
-rw-r--r--ydb/core/protos/config.proto11
-rw-r--r--ydb/core/protos/services.proto3
-rw-r--r--ydb/tests/library/harness/kikimr_config.py26
19 files changed, 478 insertions, 77 deletions
diff --git a/ydb/core/CMakeLists.txt b/ydb/core/CMakeLists.txt
index a22589679e4..6050c26ab21 100644
--- a/ydb/core/CMakeLists.txt
+++ b/ydb/core/CMakeLists.txt
@@ -7,6 +7,7 @@
add_subdirectory(actorlib_impl)
+add_subdirectory(audit)
add_subdirectory(base)
add_subdirectory(blob_depot)
add_subdirectory(blobstorage)
diff --git a/ydb/core/audit/CMakeLists.txt b/ydb/core/audit/CMakeLists.txt
new file mode 100644
index 00000000000..976aa09e163
--- /dev/null
+++ b/ydb/core/audit/CMakeLists.txt
@@ -0,0 +1,45 @@
+
+# This file was gererated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(ydb-core-audit)
+target_link_libraries(ydb-core-audit PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ library-cpp-json
+ library-cpp-logger
+ ydb-core-base
+ library-cpp-resource
+)
+target_sources(ydb-core-audit PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log_json_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log_txt_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/core/audit/audit_log.cpp
+)
+
+add_global_library_for(ydb-core-audit.global ydb-core-audit)
+target_link_libraries(ydb-core-audit.global PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ cpp-actors-core
+ library-cpp-json
+ library-cpp-logger
+ ydb-core-base
+ library-cpp-resource
+)
+target_sources(ydb-core-audit.global PRIVATE
+ ${CMAKE_BINARY_DIR}/ydb/core/audit/da10233abc65c3f178c7da0ce0d7a4ea.cpp
+)
+resources(ydb-core-audit.global
+ ${CMAKE_BINARY_DIR}/ydb/core/audit/da10233abc65c3f178c7da0ce0d7a4ea.cpp
+ INPUTS
+ ${CMAKE_SOURCE_DIR}/ydb/core/kqp/kqp_default_settings.txt
+ KEYS
+ kqp_default_settings.txt
+)
diff --git a/ydb/core/audit/audit_log.cpp b/ydb/core/audit/audit_log.cpp
new file mode 100644
index 00000000000..d6a0a23445b
--- /dev/null
+++ b/ydb/core/audit/audit_log.cpp
@@ -0,0 +1,42 @@
+#include "audit_log.h"
+#include "audit_log_impl.h"
+
+#include <library/cpp/logger/record.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/string/builder.h>
+
+namespace NKikimr::NAudit {
+
+std::atomic<bool> AUDIT_LOG_ENABLED = false;
+
+THolder<NActors::IActor> CreateAuditWriter(THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format)
+{
+ AUDIT_LOG_ENABLED.store(true);
+ switch (format) {
+ case NKikimrConfig::TAuditConfig::JSON:
+ return MakeHolder<TAuditJsonLogActor>(std::move(auditFile));
+ case NKikimrConfig::TAuditConfig::TXT:
+ return MakeHolder<TAuditTxtLogActor>(std::move(auditFile));
+ default:
+ return MakeHolder<TAuditJsonLogActor>(std::move(auditFile));
+ }
+}
+
+
+void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts)
+{
+ auto request = MakeHolder<TEvAuditLog::TEvWriteAuditLog>(Now(), parts);
+ sys->Send(MakeAuditServiceID(), request.Release());
+}
+
+const char* FormatLocalTimestamp(TInstant time, char* buf) {
+ struct tm localTime;
+ time.LocalTime(&localTime);
+ int r = strftime(buf, TimeBufSize, "%Y-%m-%d-%H-%M-%S", &localTime);
+ Y_VERIFY(r != 0);
+ return buf;
+}
+
+} // namespace NKikimr::NAudit
diff --git a/ydb/core/audit/audit_log.h b/ydb/core/audit/audit_log.h
new file mode 100644
index 00000000000..d5bc7e6bed3
--- /dev/null
+++ b/ydb/core/audit/audit_log.h
@@ -0,0 +1,150 @@
+#pragma once
+
+#include <ydb/core/base/events.h>
+
+#include <library/cpp/actors/core/actor.h>
+#include <library/cpp/actors/core/events.h>
+#include <library/cpp/logger/backend.h>
+#include <ydb/core/protos/config.pb.h>
+
+#include <library/cpp/logger/record.h>
+#include <library/cpp/actors/core/hfunc.h>
+#include <library/cpp/actors/core/log.h>
+
+#include <util/generic/strbuf.h>
+#include <util/datetime/base.h>
+
+#define AUDIT_LOG_S(sys, expr) \
+ do { \
+ if (::NKikimr::NAudit::AUDIT_LOG_ENABLED.load()) { \
+ TVector<std::pair<TStringBuf, TString>> auditParts; \
+ expr \
+ ::NKikimr::NAudit::SendAuditLog(sys, auditParts); \
+ } \
+ } while (0) /**/
+
+#define AUDIT_LOG(expr) AUDIT_LOG_S((TlsActivationContext->ExecutorThread.ActorSystem), expr)
+
+#define AUDIT_PART_NO_COND(key, value) auditParts.push_back({key, value});
+#define AUDIT_PART_COND(key, value, condition) \
+ do { \
+ if (condition && !value.Empty()) { \
+ auditParts.push_back({key, value}); \
+ } \
+ } while (0);
+
+#define GET_AUDIT_PART_MACRO(_1, _2, _3, NAME,...) NAME
+#define AUDIT_PART(...) GET_AUDIT_PART_MACRO(__VA_ARGS__, AUDIT_PART_COND, AUDIT_PART_NO_COND)(__VA_ARGS__)
+
+namespace NKikimr::NAudit {
+
+extern std::atomic<bool> AUDIT_LOG_ENABLED;
+
+struct TEvAuditLog
+{
+ //
+ // Events declaration
+ //
+
+ enum EEvents
+ {
+ EvBegin = EventSpaceBegin(TKikimrEvents::ES_YDB_AUDIT_LOG),
+
+ // Request actors
+ EvWriteAuditLog = EvBegin + 0,
+
+ EvEnd
+ };
+
+ static_assert(EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG),
+ "expected EvEnd <= EventSpaceEnd(TKikimrEvents::ES_YDB_AUDIT_LOG)");
+
+ struct TEvWriteAuditLog
+ : public NActors::TEventLocal<TEvWriteAuditLog, EvWriteAuditLog>
+ {
+ TInstant Time;
+ TVector<std::pair<TStringBuf, TString>> Parts;
+
+ TEvWriteAuditLog(TInstant time, TVector<std::pair<TStringBuf, TString>> parts)
+ : Time(time)
+ , Parts(std::move(parts))
+ {}
+ };
+};
+
+class TAuditJsonLogActor final
+ : public TActor<TAuditJsonLogActor>
+{
+private:
+ const THolder<TLogBackend> AuditFile;
+public:
+ TAuditJsonLogActor(THolder<TLogBackend> auditFile)
+ : TActor(&TThis::StateWork)
+ , AuditFile(std::move(auditFile))
+ {
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::AUDIT_WRITER_ACTOR;
+ }
+
+private:
+ STFUNC(StateWork);
+
+ void HandlePoisonPill(
+ const TEvents::TEvPoisonPill::TPtr& ev,
+ const TActorContext& ctx);
+
+ void HandleWriteAuditLog(
+ const TEvAuditLog::TEvWriteAuditLog::TPtr& ev,
+ const TActorContext& ctx);
+
+ void HandleUnexpectedEvent(STFUNC_SIG);
+};
+
+class TAuditTxtLogActor final
+ : public TActor<TAuditTxtLogActor>
+{
+private:
+ const THolder<TLogBackend> AuditFile;
+public:
+ TAuditTxtLogActor(THolder<TLogBackend> auditFile)
+ : TActor(&TThis::StateWork)
+ , AuditFile(std::move(auditFile))
+ {
+ }
+
+ static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
+ return NKikimrServices::TActivity::AUDIT_WRITER_ACTOR;
+ }
+
+private:
+ STFUNC(StateWork);
+
+ void HandlePoisonPill(
+ const TEvents::TEvPoisonPill::TPtr& ev,
+ const TActorContext& ctx);
+
+ void HandleWriteAuditLog(
+ const TEvAuditLog::TEvWriteAuditLog::TPtr& ev,
+ const TActorContext& ctx);
+
+ void HandleUnexpectedEvent(STFUNC_SIG);
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
+void SendAuditLog(const NActors::TActorSystem* sys, TVector<std::pair<TStringBuf, TString>>& parts);
+
+inline NActors::TActorId MakeAuditServiceID() {
+ return NActors::TActorId(0, TStringBuf("YDB_AUDIT"));
+}
+
+THolder<NActors::IActor> CreateAuditWriter(
+ THolder<TLogBackend> auditFile, NKikimrConfig::TAuditConfig_EFormat format);
+
+constexpr size_t TimeBufSize = 512;
+
+const char* FormatLocalTimestamp(TInstant time, char* buf);
+
+} // namespace NKikimr::NAudit
diff --git a/ydb/core/audit/audit_log_impl.h b/ydb/core/audit/audit_log_impl.h
new file mode 100644
index 00000000000..91de44ddead
--- /dev/null
+++ b/ydb/core/audit/audit_log_impl.h
@@ -0,0 +1,15 @@
+#if defined LOG_T || \
+ defined LOG_D || \
+ defined LOG_I || \
+ defined LOG_N || \
+ defined LOG_W || \
+ defined LOG_E
+# error log macro redefinition
+#endif
+
+#define LOG_T(stream) LOG_TRACE_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
+#define LOG_D(stream) LOG_DEBUG_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
+#define LOG_I(stream) LOG_INFO_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
+#define LOG_N(stream) LOG_NOTICE_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
+#define LOG_W(stream) LOG_WARN_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
+#define LOG_E(stream) LOG_ERROR_S((TlsActivationContext->AsActorContext()), NKikimrServices::AUDIT_LOG_WRITER, stream)
diff --git a/ydb/core/audit/audit_log_json_impl.cpp b/ydb/core/audit/audit_log_json_impl.cpp
new file mode 100644
index 00000000000..689c5648d5a
--- /dev/null
+++ b/ydb/core/audit/audit_log_json_impl.cpp
@@ -0,0 +1,67 @@
+#include "audit_log.h"
+#include "audit_log_impl.h"
+
+#include <library/cpp/json/json_value.h>
+#include <library/cpp/json/json_writer.h>
+
+namespace NKikimr::NAudit {
+
+using namespace NActors;
+
+void TAuditJsonLogActor::HandlePoisonPill(
+ const TEvents::TEvPoisonPill::TPtr& ev,
+ const TActorContext& ctx)
+{
+ Y_UNUSED(ev);
+ AUDIT_LOG_ENABLED.store(false);
+ Die(ctx);
+}
+
+STFUNC(TAuditJsonLogActor::StateWork)
+{
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvents::TEvPoisonPill, HandlePoisonPill);
+ HFunc(TEvAuditLog::TEvWriteAuditLog, HandleWriteAuditLog);
+ default:
+ HandleUnexpectedEvent(ev, ctx);
+ break;
+ }
+}
+
+void TAuditJsonLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ const auto* msg = ev->Get();
+ try {
+ TStringStream ss;
+ char buf[TimeBufSize];
+ ss << FormatLocalTimestamp(msg->Time, buf) << ": ";
+
+ NJson::TJsonMap m;
+ for (auto& [k, v] : msg->Parts) {
+ m[k] = v;
+ }
+ NJson::WriteJson(&ss, &m, false, false);
+ ss << Endl;
+ auto json = ss.Str();
+
+ AuditFile->WriteData(
+ TLogRecord(
+ ELogPriority::TLOG_INFO,
+ json.data(),
+ json.length()));
+ } catch (const TFileError& e) {
+ LOG_W("TAuditJsonLogActor:"
+ << " unable to write audit log (error: " << e.what() << ")");
+ }
+}
+
+void TAuditJsonLogActor::HandleUnexpectedEvent(STFUNC_SIG)
+{
+ Y_UNUSED(ctx);
+
+ LOG_W("TAuditJsonLogActor:"
+ << " unhandled event type: " << ev->GetTypeRewrite()
+ << " event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?"));
+}
+
+} // namespace NKikimr::NAudit
diff --git a/ydb/core/audit/audit_log_txt_impl.cpp b/ydb/core/audit/audit_log_txt_impl.cpp
new file mode 100644
index 00000000000..972dadfb03e
--- /dev/null
+++ b/ydb/core/audit/audit_log_txt_impl.cpp
@@ -0,0 +1,64 @@
+#include "audit_log.h"
+#include "audit_log_impl.h"
+
+namespace NKikimr::NAudit {
+
+using namespace NActors;
+
+void TAuditTxtLogActor::HandlePoisonPill(
+ const TEvents::TEvPoisonPill::TPtr& ev,
+ const TActorContext& ctx)
+{
+ Y_UNUSED(ev);
+ AUDIT_LOG_ENABLED.store(false);
+ Die(ctx);
+}
+
+STFUNC(TAuditTxtLogActor::StateWork)
+{
+ switch (ev->GetTypeRewrite()) {
+ HFunc(TEvents::TEvPoisonPill, HandlePoisonPill);
+ HFunc(TEvAuditLog::TEvWriteAuditLog, HandleWriteAuditLog);
+ default:
+ HandleUnexpectedEvent(ev, ctx);
+ break;
+ }
+}
+
+void TAuditTxtLogActor::HandleWriteAuditLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev, const TActorContext& ctx) {
+ Y_UNUSED(ctx);
+ const auto* msg = ev->Get();
+ try {
+ TStringStream ss;
+ char buf[TimeBufSize];
+ ss << FormatLocalTimestamp(msg->Time, buf) << ": ";
+
+ for (auto it = msg->Parts.begin(); it != msg->Parts.end(); it++) {
+ if (it != msg->Parts.begin())
+ ss << ", ";
+ ss << it->first << "=" << it->second;
+ }
+ ss << Endl;
+ auto text = ss.Str();
+
+ AuditFile->WriteData(
+ TLogRecord(
+ ELogPriority::TLOG_INFO,
+ text.data(),
+ text.length()));
+ } catch (const TFileError& e) {
+ LOG_W("TAuditTxtLogActor:"
+ << " unable to write audit log (error: " << e.what() << ")");
+ }
+}
+
+void TAuditTxtLogActor::HandleUnexpectedEvent(STFUNC_SIG)
+{
+ Y_UNUSED(ctx);
+
+ LOG_W("TAuditTxtLogActor:"
+ << " unhandled event type: " << ev->GetTypeRewrite()
+ << " event: " << (ev->HasEvent() ? ev->GetBase()->ToString().data() : "serialized?"));
+}
+
+} // namespace NKikimr::NAudit
diff --git a/ydb/core/base/appdata.h b/ydb/core/base/appdata.h
index d70f2955b1c..8be75810dc5 100644
--- a/ydb/core/base/appdata.h
+++ b/ydb/core/base/appdata.h
@@ -137,6 +137,7 @@ struct TAppData {
NKikimrConfig::TDataShardConfig DataShardConfig;
NKikimrConfig::TSchemeShardConfig SchemeShardConfig;
NKikimrConfig::TMeteringConfig MeteringConfig;
+ NKikimrConfig::TAuditConfig AuditConfig;
NKikimrConfig::TCompactionConfig CompactionConfig;
NKikimrConfig::TDomainsConfig DomainsConfig;
NKikimrConfig::TBootstrap BootstrapConfig;
diff --git a/ydb/core/base/events.h b/ydb/core/base/events.h
index 4f4ec8e3f53..af59ea23c73 100644
--- a/ydb/core/base/events.h
+++ b/ydb/core/base/events.h
@@ -152,7 +152,8 @@ struct TKikimrEvents : TEvents {
ES_INTERNAL_REQUEST,
ES_BACKGROUND_TASKS,
ES_TIERING,
- ES_METADATA_INITIALIZER
+ ES_METADATA_INITIALIZER,
+ ES_YDB_AUDIT_LOG,
};
};
diff --git a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
index 1d562e4b24b..5ce4d782706 100644
--- a/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
+++ b/ydb/core/driver_lib/cli_utils/cli_cmds_server.cpp
@@ -208,6 +208,7 @@ protected:
config.Opts->AddLongOption("feature-flags-file", "File with feature flags to turn new features on/off").OptionalArgument("PATH");
config.Opts->AddLongOption("rb-file", "File with resource broker customizations").OptionalArgument("PATH");
config.Opts->AddLongOption("metering-file", "File with metering config").OptionalArgument("PATH");
+ config.Opts->AddLongOption("audit-file", "File with audit config").OptionalArgument("PATH");
config.Opts->AddLongOption('r', "restarts-count-file", "State for restarts monitoring counter,\nuse empty string to disable\n")
.OptionalArgument("PATH").DefaultValue(RestartsCountFile).StoreResult(&RestartsCountFile);
config.Opts->AddLongOption("compile-inflight-limit", "Limit on parallel programs compilation").OptionalArgument("NUM").StoreResult(&CompileInflightLimit);
@@ -437,6 +438,7 @@ protected:
OPTION("feature-flags-file", FeatureFlags);
OPTION("rb-file", ResourceBrokerConfig);
OPTION("metering-file", MeteringConfig);
+ OPTION("audit-file", AuditConfig);
OPTION("kqp-file", KQPConfig);
OPTION("incrhuge-file", IncrHugeConfig);
OPTION("alloc-file", AllocatorConfig);
diff --git a/ydb/core/driver_lib/run/CMakeLists.txt b/ydb/core/driver_lib/run/CMakeLists.txt
index 0b09685d4ab..ecc840965dd 100644
--- a/ydb/core/driver_lib/run/CMakeLists.txt
+++ b/ydb/core/driver_lib/run/CMakeLists.txt
@@ -36,6 +36,7 @@ target_link_libraries(run PUBLIC
cpp-string_utils-parse_size
library-cpp-svnversion
ydb-core-actorlib_impl
+ ydb-core-audit
ydb-core-base
ydb-core-blob_depot
ydb-core-blobstorage
diff --git a/ydb/core/driver_lib/run/config.h b/ydb/core/driver_lib/run/config.h
index f90503bf411..9863b0aefbf 100644
--- a/ydb/core/driver_lib/run/config.h
+++ b/ydb/core/driver_lib/run/config.h
@@ -58,6 +58,7 @@ union TBasicKikimrServicesMask {
bool EnablePersQueueClusterTracker:1;
bool EnableSysViewService:1;
bool EnableMeteringWriter:1;
+ bool EnableAuditWriter:1;
bool EnableSchemeBoardMonitoring:1;
bool EnableConfigsCache:1;
bool EnableLongTxService:1;
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
index 4ea7f2a0d63..10be71c0407 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp
@@ -8,6 +8,8 @@
#include <ydb/core/actorlib_impl/mad_squirrel.h>
#include <ydb/core/actorlib_impl/node_identifier.h>
+#include "ydb/core/audit/audit_log.h"
+
#include <ydb/core/base/appdata.h>
#include <ydb/core/base/config_units.h>
#include <ydb/core/base/counters.h>
@@ -2274,7 +2276,7 @@ TMeteringWriterInitializer::TMeteringWriterInitializer(const TKikimrRunConfig &r
{
}
-void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup *setup, const TAppData *appData)
+void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData)
{
if (!Config.HasMeteringConfig() || !Config.GetMeteringConfig().HasMeteringFilePath()) {
return;
@@ -2297,6 +2299,35 @@ void TMeteringWriterInitializer::InitializeServices(TActorSystemSetup *setup, co
TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->IOPoolId)));
}
+TAuditWriterInitializer::TAuditWriterInitializer(const TKikimrRunConfig &runConfig)
+ : IKikimrServicesInitializer(runConfig)
+{
+}
+
+void TAuditWriterInitializer::InitializeServices(TActorSystemSetup* setup, const TAppData* appData)
+{
+ if (!Config.HasAuditConfig() || !Config.GetAuditConfig().HasAuditFilePath() || !Config.GetAuditConfig().HasFormat()) {
+ return;
+ }
+
+ const auto& filePath = Config.GetAuditConfig().GetAuditFilePath();
+ const auto format = Config.GetAuditConfig().GetFormat();
+
+ THolder<TFileLogBackend> fileBackend;
+ try {
+ fileBackend = MakeHolder<TFileLogBackend>(filePath);
+ } catch (const TFileError& ex) {
+ Cerr << "TAuditWriterInitializer: failed to open file '" << filePath << "': " << ex.what() << Endl;
+ exit(1);
+ }
+
+ auto actor = NAudit::CreateAuditWriter(std::move(fileBackend), format);
+
+ setup->LocalServices.push_back(std::pair<TActorId, TActorSetupCmd>(
+ NAudit::MakeAuditServiceID(),
+ TActorSetupCmd(actor.Release(), TMailboxType::HTSwap, appData->IOPoolId)));
+}
+
TSchemeBoardMonitoringInitializer::TSchemeBoardMonitoringInitializer(const TKikimrRunConfig &runConfig)
: IKikimrServicesInitializer(runConfig)
{
diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.h b/ydb/core/driver_lib/run/kikimr_services_initializers.h
index f4bf8d00bd4..4d08368384d 100644
--- a/ydb/core/driver_lib/run/kikimr_services_initializers.h
+++ b/ydb/core/driver_lib/run/kikimr_services_initializers.h
@@ -483,6 +483,13 @@ public:
void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
};
+class TAuditWriterInitializer : public IKikimrServicesInitializer {
+public:
+ TAuditWriterInitializer(const TKikimrRunConfig& runConfig);
+
+ void InitializeServices(NActors::TActorSystemSetup* setup, const NKikimr::TAppData* appData) override;
+};
+
class TSchemeBoardMonitoringInitializer : public IKikimrServicesInitializer {
public:
TSchemeBoardMonitoringInitializer(const TKikimrRunConfig& runConfig);
diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp
index 21e0a25dcee..7c7ba7509e8 100644
--- a/ydb/core/driver_lib/run/run.cpp
+++ b/ydb/core/driver_lib/run/run.cpp
@@ -1032,6 +1032,10 @@ void TKikimrRunner::InitializeAppData(const TKikimrRunConfig& runConfig)
AppData->MeteringConfig = runConfig.AppConfig.GetMeteringConfig();
}
+ if (runConfig.AppConfig.HasAuditConfig()) {
+ AppData->AuditConfig = runConfig.AppConfig.GetAuditConfig();
+ }
+
AppData->TenantName = runConfig.TenantName;
if (runConfig.AppConfig.HasBootstrapConfig()) {
@@ -1464,6 +1468,10 @@ TIntrusivePtr<TServiceInitializersList> TKikimrRunner::CreateServiceInitializers
sil->AddServiceInitializer(new TMeteringWriterInitializer(runConfig));
}
+ if (serviceMask.EnableAuditWriter) {
+ sil->AddServiceInitializer(new TAuditWriterInitializer(runConfig));
+ }
+
if (serviceMask.EnableLongTxService) {
sil->AddServiceInitializer(new TLongTxServiceInitializer(runConfig));
}
diff --git a/ydb/core/kqp/ut/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/kqp_scheme_ut.cpp
index 0775f128245..1062f6c1d49 100644
--- a/ydb/core/kqp/ut/kqp_scheme_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_scheme_ut.cpp
@@ -100,81 +100,6 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
}
- Y_UNIT_TEST(CreateAndDropTableCheckAuditLog) {
- TStringStream logStream;
- {
- TKikimrRunner kikimr(TKikimrSettings().SetLogStream(&logStream));
-
- auto driverConfig = TDriverConfig()
- .SetEndpoint(kikimr.GetEndpoint())
- .SetAuthToken("user0@builtin");
- auto driver = TDriver(driverConfig);
- auto db = NYdb::NTable::TTableClient(driver);
- auto session = db.CreateSession().GetValueSync().GetSession();
- kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::FLAT_TX_SCHEMESHARD, NActors::NLog::PRI_INFO);
-
- {
- auto schemeClient = kikimr.GetSchemeClient();
-
- NYdb::NScheme::TPermissions permissions("user0@builtin", {"ydb.deprecated.create_table"});
- AssertSuccessResult(schemeClient.ModifyPermissions("/Root",
- NYdb::NScheme::TModifyPermissionsSettings().AddGrantPermissions(permissions)
- ).ExtractValueSync()
- );
- }
-
- {
- const static TString createTableQuery = R"(
- CREATE TABLE `/Root/Test1234/KeyValue` (
- Key Uint32,
- Value String,
- PRIMARY KEY(Key)
- );
- )";
- auto result = session.ExecuteSchemeQuery(createTableQuery).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- {
- const static TString dropTableQuery = R"(
- DROP TABLE `/Root/Test1234/KeyValue`;
- )";
- auto result = session.ExecuteSchemeQuery(dropTableQuery).ExtractValueSync();
- UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
- }
-
- driver.Stop(true);
- }
-
- TString line;
- int mtc = 0;
- int ctc = 0;
- int dtc = 0;
- while (logStream.ReadLine(line)) {
- if (line.find("AUDIT:") == line.npos)
- continue;
-
- const TString modifyAclTablePattern("operation: MODIFY ACL");
- if (line.find(modifyAclTablePattern) != line.npos) {
- mtc += 1;
- }
-
- const TString createTablePattern("operation: CREATE TABLE");
- if (line.find(createTablePattern) != line.npos) {
- ctc += 1;
- }
-
- const TString dropTablePattern("operation: DROP TABLE");
- if (line.find(dropTablePattern) != line.npos) {
- dtc += 1;
- }
- }
-
- UNIT_ASSERT_VALUES_EQUAL_C(mtc, 1, mtc);
- UNIT_ASSERT_VALUES_EQUAL_C(ctc, 1, ctc);
- UNIT_ASSERT_VALUES_EQUAL_C(dtc, 1, dtc);
- }
-
Y_UNIT_TEST(CreateDropTableMultipleTime) {
TKikimrRunner kikimr;
auto db = kikimr.GetTableClient();
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
index fcb5e64f79c..97b4da95c4e 100644
--- a/ydb/core/protos/config.proto
+++ b/ydb/core/protos/config.proto
@@ -1374,6 +1374,16 @@ message TMeteringConfig {
repeated string SystemBackupSIDs = 2;
};
+message TAuditConfig {
+ enum EFormat {
+ JSON = 1;
+ TXT = 2;
+ }
+
+ optional string AuditFilePath = 1;
+ optional EFormat Format = 2;
+};
+
message THiveTabletLimit {
optional NKikimrTabletBase.TTabletTypes.EType Type = 1;
optional uint64 MaxCount = 2;
@@ -1660,6 +1670,7 @@ message TAppConfig {
optional THttpProxyConfig PublicHttpConfig = 57;
optional TMetadataProviderConfig MetadataProviderConfig = 59;
optional TBackgroundTasksConfig BackgroundTasksConfig = 60;
+ optional TAuditConfig AuditConfig = 61;
optional NYq.NConfig.TConfig YandexQueryConfig = 50; // TODO: remove after migration to FederatedQueryConfig
diff --git a/ydb/core/protos/services.proto b/ydb/core/protos/services.proto
index d770c45ca18..2c515042c24 100644
--- a/ydb/core/protos/services.proto
+++ b/ydb/core/protos/services.proto
@@ -283,6 +283,8 @@ enum EServiceKikimr {
BUILD_INDEX = 1000;
+ AUDIT_LOG_WRITER = 1005;
+
METERING_WRITER = 1010;
// Streaming
@@ -939,5 +941,6 @@ message TActivity {
HTTP_MON_SERVICE_NODE_REQUEST = 593;
HTTP_MON_SERVICE_MON_REQUEST = 594;
HTTP_MON_SERVICE_NODE_PROXY = 595;
+ AUDIT_WRITER_ACTOR = 596;
};
};
diff --git a/ydb/tests/library/harness/kikimr_config.py b/ydb/tests/library/harness/kikimr_config.py
index 4e7e93c5af2..f436bd14147 100644
--- a/ydb/tests/library/harness/kikimr_config.py
+++ b/ydb/tests/library/harness/kikimr_config.py
@@ -126,6 +126,7 @@ class KikimrConfigGenerator(object):
use_in_memory_pdisks=False,
enable_pqcd=True,
enable_metering=False,
+ enable_audit_log=False,
grpc_tls_data_path=None,
fq_config_path=None,
public_http_config_path=None,
@@ -237,6 +238,9 @@ class KikimrConfigGenerator(object):
if enable_metering:
self.__set_enable_metering()
+ if enable_audit_log:
+ self.__set_enable_audit_log()
+
self.naming_config = config_pb2.TAppConfig()
dc_it = itertools.cycle(self._dcs)
rack_it = itertools.count(start=1)
@@ -353,11 +357,33 @@ class KikimrConfigGenerator(object):
metering_file.write('')
self.yaml_config['metering_config'] = {'metering_file_path': metering_file_path}
+ def __set_enable_audit_log(self):
+ def ensure_path_exists(path):
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ return path
+
+ def get_cwd_for_test(output_path):
+ test_name = yatest_common.context.test_name or ""
+ test_name = test_name.replace(':', '_')
+ return os.path.join(output_path, test_name)
+
+ cwd = get_cwd_for_test(self.__output_path)
+ ensure_path_exists(cwd)
+ audit_file_path = os.path.join(cwd, 'audit.txt')
+ with open(audit_file_path, "w") as audit_file:
+ audit_file.write('')
+ self.yaml_config['audit_config'] = {'audit_file_path': audit_file_path}
+
@property
def metering_file_path(self):
return self.yaml_config.get('metering_config', {}).get('metering_file_path')
@property
+ def audit_file_path(self):
+ return self.yaml_config.get('audit_config', {}).get('audit_file_path')
+
+ @property
def nbs_enable(self):
return self._enable_nbs