Audit log json envelope (ydb-platform#11466)

(cherry picked from commit 0f7cf47)

ydb/core/audit/audit_log_impl.cpp

@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
                 log.length()
             ));
         } catch (const yexception& e) {
-            LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
+            LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
         }
     }
 }

ydb/core/log_backend/json_envelope.cpp

@@ -0,0 +1,116 @@
+#include "json_envelope.h"
+
+#include <util/charset/utf8.h>
+
+namespace NKikimr {
+
+const TStringBuf PLACEHOLDER = "%message%";
+
+void TJsonEnvelope::Parse() {
+    ReadJsonTree(TemplateString, &Value, true);
+    std::vector<TReplace::TPathComponent> path;
+    Parse(Value, path);
+}
+
+bool TJsonEnvelope::Parse(const NJson::TJsonValue& value, std::vector<TReplace::TPathComponent>& path) {
+    Y_ASSERT(!Replace);
+
+    switch (value.GetType()) {
+    case NJson::JSON_STRING: {
+        if (auto replacePair = Parse(value.GetStringSafe())) {
+            Replace.emplace(std::move(path), std::move(replacePair->first), std::move(replacePair->second));
+            return true;
+        }
+        return false;
+    }
+    case NJson::JSON_ARRAY: {
+        const auto& arr = value.GetArraySafe();
+        path.emplace_back(size_t(0));
+        for (size_t i = 0; i < arr.size(); ++i) {
+            path.back() = i;
+            if (Parse(arr[i], path)) {
+                return true;
+            }
+        }
+        path.pop_back();
+        return false;
+    }
+    case NJson::JSON_MAP: {
+        const auto& map = value.GetMapSafe();
+        path.emplace_back(TString());
+        for (const auto& [key, el] : map) {
+            path.back() = key;
+            if (Parse(el, path)) {
+                return true;
+            }
+        }
+        path.pop_back();
+        return false;
+    }
+    default:
+        return false;
+    }
+}
+
+std::optional<std::pair<TString, TString>> TJsonEnvelope::Parse(const TString& stringValue) {
+    std::optional<std::pair<TString, TString>> result;
+    size_t pos = stringValue.find(PLACEHOLDER);
+    if (pos == TString::npos) {
+        return result;
+    }
+
+    TString prefix, suffix;
+    if (pos != 0) {
+        prefix = stringValue.substr(0, pos);
+    }
+    if (pos + PLACEHOLDER.size() < stringValue.size()) {
+        suffix = stringValue.substr(pos + PLACEHOLDER.size());
+    }
+    result.emplace(std::move(prefix), std::move(suffix));
+    return result;
+}
+
+TString TJsonEnvelope::ApplyJsonEnvelope(const TStringBuf& message) const {
+    if (!IsUtf(message)) {
+        throw std::runtime_error("Attempt to write non utf-8 string");
+    }
+
+    const NJson::TJsonValue* value = &Value;
+    NJson::TJsonValue valueCopy;
+    if (Replace) {
+        valueCopy = Value;
+        value = &valueCopy;
+        Replace->Apply(&valueCopy, message);
+    }
+
+    TStringStream ss;
+    NJson::WriteJson(&ss, value, NJson::TJsonWriterConfig().SetValidateUtf8(true));
+    ss << Endl;
+    return ss.Str();
+}
+
+void TJsonEnvelope::TReplace::Apply(NJson::TJsonValue* value, const TStringBuf& message) const {
+    size_t currentEl = 0;
+    while (currentEl < Path.size()) {
+        std::visit(
+            [&](const auto& child) {
+                value = &(*value)[child];
+            },
+            Path[currentEl++]
+        );
+    }
+
+    Y_ASSERT(value && value->GetType() == NJson::JSON_STRING);
+    TString result;
+    result.reserve(Prefix.size() + Suffix.size() + message.size());
+    if (Prefix) {
+        result += Prefix;
+    }
+    result += message;
+    if (Suffix) {
+        result += Suffix;
+    }
+    *value = result;
+}
+
+} // namespace NKikimr

ydb/core/log_backend/json_envelope.h

@@ -0,0 +1,54 @@
+#pragma once
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_writer.h>
+
+#include <util/generic/string.h>
+
+#include <optional>
+#include <variant>
+#include <vector>
+
+namespace NKikimr {
+
+class TJsonEnvelope {
+    struct TReplace {
+        using TPathComponent = std::variant<size_t, TString>; // field or index
+        std::vector<TPathComponent> Path;
+        // replace
+        TString Prefix;
+        TString Suffix;
+
+        TReplace(std::vector<TPathComponent> path, TString prefix, TString suffix)
+            : Path(std::move(path))
+            , Prefix(std::move(prefix))
+            , Suffix(std::move(suffix))
+        {}
+
+        void Apply(NJson::TJsonValue* value, const TStringBuf& message) const;
+    };
+
+public:
+    explicit TJsonEnvelope(const TString& templateString)
+        : TemplateString(templateString)
+    {
+        Parse(); // can throw
+    }
+
+    TJsonEnvelope() = delete;
+    TJsonEnvelope(const TJsonEnvelope&) = delete;
+    TJsonEnvelope(TJsonEnvelope&&) = delete;
+
+    TString ApplyJsonEnvelope(const TStringBuf& message) const;
+
+private:
+    void Parse();
+    bool Parse(const NJson::TJsonValue& value, std::vector<TReplace::TPathComponent>& path);
+    std::optional<std::pair<TString, TString>> Parse(const TString& stringValue); // returns prefix/suffix pair for replace
+
+private:
+    TString TemplateString;
+    NJson::TJsonValue Value;
+    std::optional<TReplace> Replace;
+};
+
+} // namespace NKikimr

ydb/core/log_backend/json_envelope_ut.cpp

@@ -0,0 +1,105 @@
+#include "json_envelope.h"
+
+#include <library/cpp/json/json_reader.h>
+#include <library/cpp/json/json_writer.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+
+#define UNIT_ASSERT_JSONS_EQUAL(j1, j2) {                    \
+    const TString js1 = (j1), js2 = (j2);                    \
+    UNIT_ASSERT(!js1.empty());                               \
+    UNIT_ASSERT_C(js1.back() == '\n', js1);                  \
+    NJson::TJsonValue jv1, jv2;                              \
+    UNIT_ASSERT(ReadJsonTree(j1, &jv1));                     \
+    UNIT_ASSERT(ReadJsonTree(j2, &jv2));                     \
+    const TString jsn1 = NJson::WriteJson(&jv1, true, true); \
+    const TString jsn2 = NJson::WriteJson(&jv2, true, true); \
+    UNIT_ASSERT_VALUES_EQUAL(jsn1, jsn2);                    \
+}
+
+Y_UNIT_TEST_SUITE(JsonEnvelopeTest) {
+    Y_UNIT_TEST(Simple) {
+        TJsonEnvelope env1(R"json({
+            "a": "b",
+            "m": "abc%message%def"
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsgdef"})json");
+        UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyzdef"})json");
+
+
+        TJsonEnvelope env2(R"json({
+            "a": "b",
+            "m": "%message%def"
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"msgdef"})json");
+        UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"xyzdef"})json");
+
+
+        TJsonEnvelope env3(R"json({
+            "a": "b",
+            "m": "abc%message%"
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsg"})json");
+        UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyz"})json");
+    }
+
+    Y_UNIT_TEST(NoReplace) {
+        TJsonEnvelope env(R"json({
+            "a": "b",
+            "x": "%y%",
+            "subfield": {
+                "s": "% message %",
+                "t": "%Message%",
+                "x": 42,
+                "a": [
+                    42
+                ]
+            }
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
+    }
+
+    Y_UNIT_TEST(ArrayItem) {
+        TJsonEnvelope env(R"json({
+            "a": "b",
+            "subfield": {
+                "a": [
+                    42,
+                    "%message%",
+                    53
+                ]
+            }
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","subfield":{"a":[42,"msg",53]}})json");
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","subfield":{"a":[42,"xyz",53]}})json");
+    }
+
+    Y_UNIT_TEST(Escape) {
+        TJsonEnvelope env(R"json({
+            "a": "%message%"
+        })json");
+
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"msg"})json");
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("\"\n\""), R"json({"a":"\"\n\""})json");
+    }
+
+    Y_UNIT_TEST(BinaryData) {
+        TJsonEnvelope env(R"json({
+            "a": "%message%"
+        })json");
+
+        const ui64 binaryData = 0xABCDEFFF87654321;
+        const TStringBuf data(reinterpret_cast<const char*>(&binaryData), sizeof(binaryData));
+        UNIT_ASSERT_EXCEPTION(env.ApplyJsonEnvelope(data), std::exception);
+        UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("text"), R"json({"a":"text"})json");
+    }
+}
+
+} // namespace NKikimr