intermediate changes

ref:cde9a383711a11544ce7e107a78147fb96cc4029

5 files changed, 345 insertions, 0 deletions

diff --git a/library/cpp/unicode/punycode/punycode.cpp b/library/cpp/unicode/punycode/punycode.cpp
new file mode 100644
index 0000000000..800d1f19fe
--- /dev/null
+++ b/library/cpp/unicode/punycode/punycode.cpp
@@ -0,0 +1,143 @@
+#include "punycode.h"
+#include <contrib/libs/libidn/idna.h>
+#include <contrib/libs/libidn/punycode.h>
+#include <util/charset/wide.h>
+#include <util/generic/ptr.h>
+#include <util/generic/vector.h>
+
+#include <cstdlib>
+
+static inline void CheckPunycodeResult(int rc) {
+    if (rc != PUNYCODE_SUCCESS)
+        ythrow TPunycodeError() << punycode_strerror(static_cast<Punycode_status>(rc));
+}
+
+static inline void CheckIdnaResult(int rc) {
+    if (rc != IDNA_SUCCESS)
+        ythrow TPunycodeError() << idna_strerror(static_cast<Idna_rc>(rc));
+}
+
+// UTF-32 helpers
+
+static inline void AppendWideToUtf32(const TWtringBuf& in, TVector<ui32>& out) {
+    out.reserve(out.size() + in.size() + 1);
+
+    const wchar16* b = in.begin();
+    const wchar16* e = in.end();
+    while (b < e) {
+        out.push_back(ReadSymbolAndAdvance(b, e));
+    }
+}
+
+static inline void AppendUtf32ToWide(const ui32* in, size_t len, TUtf16String& out) {
+    out.reserve(out.size() + len);
+
+    const ui32* b = in;
+    const ui32* e = in + len;
+    for (; b != e; ++b) {
+        WriteSymbol(wchar32(*b), out);
+    }
+}
+
+TStringBuf WideToPunycode(const TWtringBuf& in16, TString& out) {
+    TVector<ui32> in32;
+    AppendWideToUtf32(in16, in32);
+    size_t outlen = in32.size();
+
+    int rc;
+    do {
+        outlen *= 2;
+        out.ReserveAndResize(outlen);
+        rc = punycode_encode(in32.size(), in32.data(), nullptr, &outlen, out.begin());
+    } while (rc == PUNYCODE_BIG_OUTPUT);
+
+    CheckPunycodeResult(rc);
+
+    out.resize(outlen);
+    return out;
+}
+
+TWtringBuf PunycodeToWide(const TStringBuf& in, TUtf16String& out16) {
+    size_t outlen = in.size();
+    TVector<ui32> out32(outlen);
+
+    int rc = punycode_decode(in.size(), in.data(), &outlen, out32.begin(), nullptr);
+    CheckPunycodeResult(rc);
+
+    AppendUtf32ToWide(out32.begin(), outlen, out16);
+    return out16;
+}
+
+namespace {
+    template <typename TChar>
+    struct TIdnaResult {
+        TChar* Data = nullptr;
+
+        ~TIdnaResult() {
+            free(Data);
+        }
+    };
+}
+
+TString HostNameToPunycode(const TWtringBuf& unicodeHost) {
+    TVector<ui32> in32;
+    AppendWideToUtf32(unicodeHost, in32);
+    in32.push_back(0);
+
+    TIdnaResult<char> out;
+    int rc = idna_to_ascii_4z(in32.begin(), &out.Data, 0);
+    CheckIdnaResult(rc);
+
+    return out.Data;
+}
+
+TUtf16String PunycodeToHostName(const TStringBuf& punycodeHost) {
+    if (!IsStringASCII(punycodeHost.begin(), punycodeHost.end()))
+        ythrow TPunycodeError() << "Non-ASCII punycode input";
+
+    size_t len = punycodeHost.size();
+    TVector<ui32> in32(len + 1, 0);
+    for (size_t i = 0; i < len; ++i)
+        in32[i] = static_cast<ui8>(punycodeHost[i]);
+    in32[len] = 0;
+
+    TIdnaResult<ui32> out;
+    int rc = idna_to_unicode_4z4z(in32.begin(), &out.Data, 0);
+    CheckIdnaResult(rc);
+
+    TUtf16String decoded;
+    AppendUtf32ToWide(out.Data, std::char_traits<ui32>::length(out.Data), decoded);
+    return decoded;
+}
+
+TString ForceHostNameToPunycode(const TWtringBuf& unicodeHost) {
+    try {
+        return HostNameToPunycode(unicodeHost);
+    } catch (const TPunycodeError&) {
+        return WideToUTF8(unicodeHost);
+    }
+}
+
+TUtf16String ForcePunycodeToHostName(const TStringBuf& punycodeHost) {
+    try {
+        return PunycodeToHostName(punycodeHost);
+    } catch (const TPunycodeError&) {
+        return UTF8ToWide(punycodeHost);
+    }
+}
+
+bool CanBePunycodeHostName(const TStringBuf& host) {
+    if (!IsStringASCII(host.begin(), host.end()))
+        return false;
+
+    static constexpr TStringBuf ACE = "xn--";
+
+    TStringBuf tail(host);
+    while (tail) {
+        const TStringBuf label = tail.NextTok('.');
+        if (label.StartsWith(ACE))
+            return true;
+    }
+
+    return false;
+}
diff --git a/library/cpp/unicode/punycode/punycode.h b/library/cpp/unicode/punycode/punycode.h
new file mode 100644
index 0000000000..af4acc25c1
--- /dev/null
+++ b/library/cpp/unicode/punycode/punycode.h
@@ -0,0 +1,46 @@
+#pragma once
+
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/yexception.h>
+
+// Simplified arcadia wrappers for contrib/libs/libidn/
+
+// Raw strings encoder/decoder: does not prepend with ACE prefix ("xn--"),
+// does not limit input length. Throws TPunycodeError on any internal error.
+// Returned strbuf points to @out data.
+TStringBuf WideToPunycode(const TWtringBuf& in, TString& out);
+TWtringBuf PunycodeToWide(const TStringBuf& in, TUtf16String& out);
+
+inline TString WideToPunycode(const TWtringBuf& in) {
+    TString out;
+    WideToPunycode(in, out);
+    return out;
+}
+
+inline TUtf16String PunycodeToWide(const TStringBuf& in) {
+    TUtf16String out;
+    PunycodeToWide(in, out);
+    return out;
+}
+
+// Encode a sequence of point-separated domain labels
+// into a sequence of corresponding punycode labels.
+// Labels containing non-ASCII characters are prefixed with ACE prefix ("xn--").
+// Limits maximal encoded domain label length to IDNA_LABEL_MAX_LENGTH (255 by default).
+// Throws TPunycodeError on failure.
+TString HostNameToPunycode(const TWtringBuf& unicodeHost);
+TUtf16String PunycodeToHostName(const TStringBuf& punycodeHost);
+
+// Robust versions: on failure return original input, converted to/from UTF8
+TString ForceHostNameToPunycode(const TWtringBuf& unicodeHost);
+TUtf16String ForcePunycodeToHostName(const TStringBuf& punycodeHost);
+
+// True if @host looks like punycode domain label sequence,
+// containing at least one ACE-prefixed label.
+// Note that this function does not check all requied IDNA constraints
+// (max label length, empty non-root domains, etc.)
+bool CanBePunycodeHostName(const TStringBuf& host);
+
+class TPunycodeError: public yexception {
+};
diff --git a/library/cpp/unicode/punycode/punycode_ut.cpp b/library/cpp/unicode/punycode/punycode_ut.cpp
new file mode 100644
index 0000000000..97271cf0d8
--- /dev/null
+++ b/library/cpp/unicode/punycode/punycode_ut.cpp
@@ -0,0 +1,126 @@
+#include "punycode.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <util/charset/wide.h>
+
+namespace {
+        template<typename T1, typename T2>
+        inline bool HasSameBuffer(const T1& s1, const T2& s2) {
+                return s1.begin() == s2.begin();
+        }
+}
+
+Y_UNIT_TEST_SUITE(TPunycodeTest) {
+    static bool TestRaw(const TString& utf8, const TString& punycode) {
+        TUtf16String unicode = UTF8ToWide(utf8);
+        TString buf1;
+        TUtf16String buf2;
+        return HasSameBuffer(WideToPunycode(unicode, buf1), buf1) && buf1 == punycode && HasSameBuffer(PunycodeToWide(punycode, buf2), buf2) && buf2 == unicode && WideToPunycode(unicode) == punycode && PunycodeToWide(punycode) == unicode;
+    }
+
+    Y_UNIT_TEST(RawEncodeDecode) {
+        UNIT_ASSERT(TestRaw("", ""));
+        UNIT_ASSERT(TestRaw(" ", " -"));
+        UNIT_ASSERT(TestRaw("-", "--"));
+        UNIT_ASSERT(TestRaw("!@#$%", "!@#$%-"));
+        UNIT_ASSERT(TestRaw("xn-", "xn--"));
+        UNIT_ASSERT(TestRaw("xn--", "xn---"));
+        UNIT_ASSERT(TestRaw("abc", "abc-"));
+        UNIT_ASSERT(TestRaw("Latin123", "Latin123-"));
+
+        UNIT_ASSERT(TestRaw("München", "Mnchen-3ya"));
+        UNIT_ASSERT(TestRaw("bücher", "bcher-kva"));
+        UNIT_ASSERT(TestRaw("BüüchEr", "BchEr-kvaa"));
+
+        UNIT_ASSERT(TestRaw("президент", "d1abbgf6aiiy"));
+        UNIT_ASSERT(TestRaw("Президент", "r0a6bcbig1bsy"));
+        UNIT_ASSERT(TestRaw("ПРЕЗИДЕНТ", "g0abbgf6aiiy"));
+        UNIT_ASSERT(TestRaw("рф", "p1ai"));
+        UNIT_ASSERT(TestRaw("пример", "e1afmkfd"));
+
+        {
+            const wchar16 tmp[] = {0x82, 0x81, 0x80, 0};
+            UNIT_ASSERT(PunycodeToWide("abc") == tmp); // "abc" is still valid punycode
+        }
+
+        UNIT_ASSERT_EXCEPTION(PunycodeToWide("     "), TPunycodeError);
+        UNIT_ASSERT_EXCEPTION(PunycodeToWide("абвгд"), TPunycodeError);
+        UNIT_ASSERT_EXCEPTION(PunycodeToWide("-"), TPunycodeError);
+
+        {
+            TString longIn;
+            for (size_t i = 0; i < 1024; ++i)
+                longIn += "Qй";
+
+            TString longOut = "QQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ-lo11fbabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+            UNIT_ASSERT(TestRaw(longIn, longOut));
+        }
+    }
+
+    static bool TestHostName(const TString& utf8, const TString& punycode, bool canBePunycode = false) {
+        TUtf16String unicode = UTF8ToWide(utf8);
+        TString buf1;
+        TUtf16String buf2;
+        //Cerr << "Testing " << utf8 << Endl;
+        return HostNameToPunycode(unicode) == punycode && HostNameToPunycode(UTF8ToWide(punycode)) == punycode // repeated encoding should give same result
+               && PunycodeToHostName(punycode) == unicode && CanBePunycodeHostName(punycode) == canBePunycode;
+    }
+
+    static bool TestForced(const TString& bad) {
+        return ForceHostNameToPunycode(UTF8ToWide(bad)) == bad && ForcePunycodeToHostName(bad) == UTF8ToWide(bad);
+    }
+
+    Y_UNIT_TEST(HostNameEncodeDecode) {
+        UNIT_ASSERT(TestHostName("президент.рф", "xn--d1abbgf6aiiy.xn--p1ai", true));
+        UNIT_ASSERT(TestHostName("яндекс.ru", "xn--d1acpjx3f.ru", true));
+        UNIT_ASSERT(TestHostName("пример", "xn--e1afmkfd", true));
+        UNIT_ASSERT(TestHostName("ascii.test", "ascii.test"));
+
+        UNIT_ASSERT(TestHostName("", ""));
+        UNIT_ASSERT(TestHostName(".", "."));
+        UNIT_ASSERT(TestHostName("a.", "a.")); // empty root domain is ok
+        UNIT_ASSERT(TestHostName("a.b.c.д.e.f", "a.b.c.xn--d1a.e.f", true));
+        UNIT_ASSERT(TestHostName("а.б.в.г.д", "xn--80a.xn--90a.xn--b1a.xn--c1a.xn--d1a", true));
+
+        UNIT_ASSERT(TestHostName("-", "-"));
+        UNIT_ASSERT(TestHostName("xn--", "xn--", true));
+        UNIT_ASSERT(TestHostName("xn--aaa.-", "xn--aaa.-", true));
+        UNIT_ASSERT(TestHostName("xn--xn--d1acpjx3f.xn--ru", "xn--xn--d1acpjx3f.xn--ru", true));
+
+        {
+            // non-ascii
+            TString bad = "президент.рф";
+            UNIT_ASSERT_EXCEPTION(PunycodeToHostName("президент.рф"), TPunycodeError);
+            UNIT_ASSERT(ForcePunycodeToHostName(bad) == UTF8ToWide(bad));
+        }
+        {
+            // too long domain label
+            TString bad(500, 'a');
+            UNIT_ASSERT_EXCEPTION(HostNameToPunycode(UTF8ToWide(bad)), TPunycodeError);
+            UNIT_ASSERT(TestForced(bad)); // but can decode it
+        }
+        {
+            // already has ACE prefix
+            TString bad("xn--яндекс.xn--рф");
+            UNIT_ASSERT_EXCEPTION(HostNameToPunycode(UTF8ToWide(bad)), TPunycodeError);
+            UNIT_ASSERT(TestForced(bad));
+        }
+        {
+            // empty non-root domain is not allowed (?)
+            TString bad(".яндекс.рф");
+            UNIT_ASSERT_EXCEPTION(HostNameToPunycode(UTF8ToWide(bad)), TPunycodeError);
+            UNIT_ASSERT(TestForced(bad));
+        }
+
+        UNIT_ASSERT(CanBePunycodeHostName("xn--"));
+        UNIT_ASSERT(CanBePunycodeHostName("yandex.xn--p1ai"));
+        UNIT_ASSERT(CanBePunycodeHostName("xn--d1acpjx3f.xn--p1ai"));
+        UNIT_ASSERT(CanBePunycodeHostName("a.b.c.d.xn--e"));
+        UNIT_ASSERT(CanBePunycodeHostName("xn--a.b.c.xn--d.e"));
+        UNIT_ASSERT(!CanBePunycodeHostName("yandex.ru"));       // no xn--
+        UNIT_ASSERT(!CanBePunycodeHostName("яндекс.рф"));       // non-ascii
+        UNIT_ASSERT(!CanBePunycodeHostName("яндекс.xn--p1ai")); // non-ascii
+        UNIT_ASSERT(!CanBePunycodeHostName(""));
+        UNIT_ASSERT(!CanBePunycodeHostName("http://xn--a.b")); // scheme prefix is not detected here
+    }
+}
diff --git a/library/cpp/unicode/punycode/ut/ya.make b/library/cpp/unicode/punycode/ut/ya.make
new file mode 100644
index 0000000000..74272102a8
--- /dev/null
+++ b/library/cpp/unicode/punycode/ut/ya.make
@@ -0,0 +1,13 @@
+UNITTEST_FOR(library/cpp/unicode/punycode)
+
+OWNER(
+    g:base
+    g:middle
+    g:upper
+)
+
+SRCS(
+    punycode_ut.cpp
+)
+
+END()
diff --git a/library/cpp/unicode/punycode/ya.make b/library/cpp/unicode/punycode/ya.make
new file mode 100644
index 0000000000..62b41b07b7
--- /dev/null
+++ b/library/cpp/unicode/punycode/ya.make
@@ -0,0 +1,17 @@
+LIBRARY()
+
+OWNER(
+    g:base
+    g:middle
+    g:upper
+)
+
+PEERDIR(
+    contrib/libs/libidn
+)
+
+SRCS(
+    punycode.cpp
+)
+
+END()