#include <library/cpp/charset/recyr.hh>

#include <util/system/sys_alloc.h>

TMallocPtr<char> TUri::IDNToAscii(const wchar32* idna) {

// XXX: don't use punycode_encode directly as it doesn't include

// proper stringprep and splitting on dot-equivalent characters

TMallocPtr<char> TUri::IDNToAscii(const TStringBuf& host, ECharset enc) {

TTempBuf buf(sizeof(wchar32) * (1 + host.length()));

wchar32* wbuf = reinterpret_cast<wchar32*>(buf.Data());

const size_t written = NDetail::NBaseOps::Recode(host, wbuf, enc).length();

wbuf[written] = 0;

return IDNToAscii(wbuf);

}

TStringBuf TUri::HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc) {

TStringBuf outhost; // store the result here before returning it, to get RVO

size_t buflen = 0;

if (hasExtended && !allowIDN)

return outhost; // definitely can't convert

// charset-recode: RFC 3986, 3.2.2, requires percent-encoded non-ASCII

// chars in reg-name to be UTF-8 so convert to UTF-8 prior to decoding

const bool recoding = CODES_UTF8 != enc && hasExtended;

return outhost;

host = TStringBuf(buf.Get(), nwr);

}

// percent-decode

if (0 == buflen) {

buflen = host.length();

// check again

if (hasExtended && !allowIDN)

host = out.Str();

// convert to punycode if needed

if (!hasExtended) {

outhost = host;

return outhost;

}

TMallocPtr<char> puny;

try {

puny = IDNToAscii(host);

} catch (const yexception& /* exc */) {

}

if (!puny) {

// XXX: try user charset unless UTF8 or converted to it

if (CODES_UTF8 == enc || recoding)

if (!puny)

return outhost;

}

buf = puny;

outhost = buf.Get();

return outhost;

TStringBuf TUri::HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc) {

// find what we have

long haveFlags = 0;

for (size_t i = 0; i != host.length(); ++i)

haveFlags |= TEncoder::GetFlags(host[i]).FeatFlags;

// interested in encoded characters or (if IDN is allowed) extended ascii

TStringBuf outhost;

const bool haveExtended = haveFlags & FeatureEncodeExtendedASCII;

if (!haveExtended || allowIDN) {

if (!haveExtended && 0 == (haveFlags & FeatureDecodeANY))

outhost = host;

else

outhost = HostToAscii(host, buf, haveExtended, allowIDN, enc);

}

return outhost;

}

static inline bool AppendField(TMemoryWriteBuffer& out, TField::EField fld, const TStringBuf& val, long flags) {

if (val.empty())

return false;

if (flags & TFeature::FeaturesAllEncoder)

TUri::ReEncodeField(out, val, fld, flags);

out << val;

return true;

TState::EParsed TUri::AssignImpl(const TParser& parser, TScheme::EKind defscheme) {

Clear();

TState::EParsed ret = parser.State;

if (ParsedBadFormat <= ret)

return ret;

const TSection& scheme = parser.Get(FieldScheme);

const TSchemeInfo& schemeInfo = SetSchemeImpl(parser.Scheme);

long flags = parser.Flags.Allow;

if (convertIDN)

flags |= FeatureAllowHostIDN | FeatureCheckHost;

// process non-ASCII host for punycode

TMallocPtr<char> hostptr;

TStringBuf hostascii; // empty: use host field; non-empty: ascii

bool hostConverted = false; // hostascii is empty or the original

if (host.IsSet() && !FldIsSet(FieldHost)) {

const bool allowIDN = (flags & FeatureAllowHostIDN);

const TStringBuf hostbuf = host.Get();

// if we know we have and allow extended-ASCII chars, no need to check further

if (allowIDN && (host.GetFlagsAllPlaintext() & FeatureEncodeExtendedASCII))

hostascii = HostToAscii(hostbuf, hostptr, true, true, parser.Enc);

else

hostascii = HostToAscii(hostbuf, hostptr, allowIDN, parser.Enc);

if (hostascii.empty())

ret = ParsedBadHost; // exists but cannot be converted

else if (hostbuf.data() != hostascii.data()) {

FldMarkSet(FieldHost); // so that we don't process host below

}

}

// add unprocessed fields

for (int idx = 0; idx < FieldUrlMAX; ++idx) {

const EField fld = EField(idx);

const TSection& section = parser.Get(fld);

if (section.IsSet() && !FldIsSet(fld))

buflen += 1 + section.EncodedLen(); // includes null

if (0 == buflen) // no more sections set?

return ret;

// process #! fragments

// https://developers.google.com/webmasters/ajax-crawling/docs/specification

static const TStringBuf escFragPrefix(TStringBuf("_escaped_fragment_="));

bool encHashBangFrag = false;

do {

if (FldIsSet(FieldFrag) || FldIsSet(FieldQuery))

break;

const TSection& frag = parser.Get(FieldFrag);

if (frag.IsSet()) {

if (0 == (parser.Flags & FeatureHashBangToEscapedFragment))

buflen -= escFragPrefix.length();

}

} while (false);

// now set all fields prior to validating

Alloc(buflen);

TMemoryWriteBuffer out(Buffer.data(), Buffer.size());

for (int idx = 0; idx < FieldUrlMAX; ++idx) {

const EField fld = EField(idx);

const TSection& section = parser.Get(fld);

if (!section.IsSet() || FldIsSet(fld))

continue;

if (FieldQuery == fld && encHashBangFrag)

continue;

if (FieldFrag == fld && qryEscapedFragment.IsInited())

continue;

char* beg = out.Buf();

TStringBuf val = section.Get();

long careFlags = section.GetFlagsEncode();

switch (fld) {

default:

break;

case FieldQuery:

if (qryEscapedFragment.IsInited()) {

const EField dstfld = FieldFrag; // that's where we will store

val = qryBeforeEscapedFragment;

}

break;

case FieldFrag:

if (encHashBangFrag) {

const EField dstfld = FieldQuery; // that's where we will store

}

break;

}

AppendField(out, fld, val, careFlags);

char* end = out.Buf();

Y_ASSERT(beg >= out.Beg());

out.SetPos(end);

}

FldSetNoDirty(fld, TStringBuf(beg, end));

// special character case

const long allowChars = parser.GetFieldFlags(fld) & checkChars;

if (checkChars != allowChars)

ret = ParsedBadFormat;

out << '\0';

}

}

Buffer.Resize(out.Len());

if (GetScheme() == SchemeEmpty && SchemeEmpty != defscheme) {

if (SchemeUnknown == defscheme)

ret = ParsedBadScheme;

else

SetSchemeImpl(defscheme);

}

if (0 == (parser.Flags & FeatureAllowEmptyPath))

CheckMissingFields();

const TStringBuf& port = GetField(FieldPort);

if (!port.empty()) {

if (!TryFromString<ui16>(port, Port))

ret = ParsedBadPort;

if (ParsedOK != ret)

return ret;

// run validity checks now that all fields are set

// check the host for DNS compliance

do {

if (0 == (flags & FeatureCheckHost))

} while (false);

return ret;

TState::EParsed TUri::ParseImpl(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) {

Clear();

if (url.empty())

return ParsedEmpty;

if (maxlen > 0 && url.length() > maxlen)

return ParsedTooLong;

const TParser parser(flags, url, enc);

return AssignImpl(parser, defscheme);

}

TState::EParsed TUri::Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& url_base, ui32 maxlen, ECharset enc) {

const TParseFlags flags1 = flags.Exclude(FeatureNoRelPath);

TState::EParsed ret = ParseImpl(url, url_base.empty() ? flags : flags1, maxlen, SchemeEmpty, enc);

if (!url_base.empty() && !IsValidAbs()) {

TUri base;

ret = base.ParseImpl(url_base, flags, maxlen, SchemeEmpty, enc);

}

Rewrite();

}

TState::EParsed TUri::Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags, ui32 maxlen, ECharset enc) {

const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc);

if (!IsValidAbs())

Merge(base, PathOperationFlag(flags));

Rewrite();

}

TState::EParsed TUri::ParseAbsUri(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) {

url, flags | FeatureNoRelPath, maxlen, defscheme, enc);

if (ParsedOK != ret)

return ret;

if (IsNull(FlagHost))

return ParsedBadHost;

Rewrite();

return ParsedOK;

}

#include <util/generic/singleton.h>

static_assert(TFeature::FeatureMAX <= sizeof(unsigned long) * 8, "expect TFeature::FeatureMAX <= sizeof(unsigned long) * 8");

const TSchemeInfo TSchemeInfo::Registry[] = {

TSchemeInfo(TScheme::SchemeEmpty, TStringBuf()), // scheme is empty and inited

TSchemeInfo(TScheme::SchemeHTTP, TStringBuf("http"), TField::FlagHost | TField::FlagPath, 80),

// add above

TSchemeInfo(TScheme::SchemeUnknown, TStringBuf()) // scheme is empty and uninited

};

namespace {

struct TLessNoCase {

bool operator()(const TStringBuf& lt, const TStringBuf& rt) const {

return 0 > CompareNoCase(lt, rt);

}

};

class TSchemeInfoMap {

typedef TMap<TStringBuf, TScheme::EKind, TLessNoCase> TdMap;

TdMap Map_;

public:

TSchemeInfoMap() {

for (int i = TScheme::SchemeEmpty; i < TScheme::SchemeUnknown; ++i) {

Map_.insert(std::make_pair(info.Str, info.Kind));

}

}

TScheme::EKind Get(const TStringBuf& scheme) const {

const TdMap::const_iterator it = Map_.find(scheme);

return Map_.end() == it ? TScheme::SchemeUnknown : it->second;

const TSchemeInfo& TSchemeInfo::Get(const TStringBuf& scheme) {

return Registry[TSchemeInfoMap::Instance().Get(scheme)];

const char* ParsedStateToString(const TState::EParsed& t) {

switch (t) {

return "Parsed[Unknown]";

}

}

const char* FieldToString(const TField::EField& t) {

switch (t) {

case TField::FieldScheme:

#pragma once

#include <util/generic/strbuf.h>

namespace NEncode {

class TEncoder;

class TEncodeMapperBase;

namespace NParse {

class TRange;

}

class TParser;

struct TField {

#define FIELD_NAME(f) Field##f

#define FIELD_FLAG(f) Flag##f = 1U << FIELD_NAME(f)

enum EField {

FIELD_NAME(Scheme),

FIELD_NAME(User),

FIELD_NAME(Path),

FIELD_NAME(Query),

FIELD_NAME(Frag),

// add fields above

FieldUrlMAX,

// reset count so actual field offsets are not interrupted

FieldUrlLast = FieldUrlMAX - 1,

// add extra fields below

FIELD_NAME(HostAscii),

// add extra fields above

FieldAllMAX,

// add aliases below

FieldUsername = FieldUser,

FieldPassword = FieldPass,

FieldFragment = FieldFrag,

};

enum EFlags {

FIELD_FLAG(Scheme),

FIELD_FLAG(User),

FlagAll = FlagUrlFields, // obsolete, for backwards compatibility

FlagAllFields = FlagAllMAX - 1

};

};

struct TState {

enum EParsed {

ParsedOK = 0,

ParsedBadAuth,

ParsedBadScheme,

ParsedBadHost,

// add before this line

ParsedMAX

};

#define FEATURE_NAME(f) _BitFeature##f

#define FEATURE_FLAG_NAME(f) Feature##f

protected:

enum EBit {

//==============================

// Cases interpreted as errors:

//==============================

// allows authorization user/password in URL

FEATURE_NAME(AuthSupported),

// allows all known schemes in URL

FEATURE_NAME(SchemeKnown),

// allow opaque (RFC 2396) or rootless (RFC 3986) urls

FEATURE_NAME(AllowRootless),

//==============================

// Cases interpreted for processing (if required):

// (effects on result of Parse method)

// path needs normalization

// (simplification of directory tree: /../, /./, etc.

FEATURE_NAME(PathOperation),

// don't force empty path to "/"

FEATURE_NAME(AllowEmptyPath),

// in scheme and host segments:

// change upper case letters onto lower case ones

FEATURE_NAME(ToLower),

// decode unreserved symbols

FEATURE_NAME(DecodeUnreserved),

// legacy: decode standard symbols which may be safe for some fields

FEATURE_NAME(DecodeStandardExtra),

// decode symbols allowed (not necessarily safe to decode) only for a given field

// (do not use directly, instead use FeatureDecodeSafe mask below)

FEATURE_NAME(DecodeFieldAllowed),

// handling of spaces

FEATURE_NAME(EncodeSpace),

// in query segment: change escaped space to '+'

FEATURE_NAME(EncodeSpaceAsPlus),

// escape all string 'markup' symbols

FEATURE_NAME(EncodeForSQL),

// encoding of extended ascii symbols (8-bit)

FEATURE_NAME(EncodeExtendedASCII),

// decoding of extended ascii symbols (8-bit)

FEATURE_NAME(DecodeExtendedASCII),

// encoding of extended delimiter set

FEATURE_NAME(EncodeExtendedDelim),

// decoding of extended delimiter set

FEATURE_NAME(DecodeExtendedDelim),

// control characters [0x00 .. 0x20)

FEATURE_NAME(EncodeCntrl),

// raw percent character

FEATURE_NAME(EncodePercent),

// hash fragments

// https://developers.google.com/webmasters/ajax-crawling/docs/specification

// move and encode #! fragments to the query

FEATURE_NAME(HashBangToEscapedFragment),

// move and decode _escaped_fragment_ to the fragment

FEATURE_NAME(EscapedToHashBangFragment),

// reject absolute paths started by "/../"

FEATURE_NAME(PathDenyRootParent),

// paths started by "/../" - ignore head

FEATURE_NAME(PathStripRootParent),

// tries to fix errors (in particular, in fragment)

FEATURE_NAME(TryToFix),

// check host for DNS compliance

FEATURE_NAME(CheckHost),

// allow IDN hosts

// host is converted to punycode and stored in FieldHostAscii

// @note host contains characters in the charset of the document

// percent-decoding cannot be converted from UTF-8 to UCS-4,

// try to recode from the document charset (if not UTF-8)

FEATURE_NAME(AllowHostIDN),

// forces AllowHostIDN, but host is replaced with punycode

// forces CheckHost since this replacement is irreversible

FEATURE_NAME(ConvertHostIDN),

// robot interpreted network paths as BadFormat urls

FEATURE_NAME(DenyNetworkPath),

// robot interprets URLs without a host as BadFormat

FEATURE_NAME(RemoteOnly),

/* non-RFC use case:

FEATURE_NAME(NoRelPath),

// standard prefers that all hex escapes were using uppercase A-F

FEATURE_NAME(UpperEncoded),

// internal usage: decode all encoded symbols

FEATURE_NAME(DecodeANY),

// add before this line

_FeatureMAX

};

protected:

enum EPrivate : ui32 {

FEATURE_FLAG(DecodeANY),

FEATURE_FLAG(DecodeFieldAllowed),

FEATURE_FLAG(DecodeStandardExtra),

};

public:

enum EPublic : ui32 {

FeatureMAX = _FeatureMAX,

FEATURE_FLAG_NAME(HierURI) = FEATURE_FLAG_NAME(NoRelPath),

FEATURE_FLAG(UpperEncoded),

};

public:

//==============================

enum ESets {

// these are guaranteed and will change buffer size

FeatureDecodeStandard = 0 | FeatureDecodeUnreserved | FeatureDecodeStandardExtra,

FeaturesDecodeExtended = 0 | FeatureDecodeExtendedASCII | FeatureDecodeExtendedDelim,

FeaturesDecode = 0 | FeatureDecodeUnreserved | FeatureDecodeStandard | FeaturesDecodeExtended,

FeaturesEncodeExtended = 0 | FeatureEncodeExtendedASCII | FeatureEncodeExtendedDelim,

FeaturesEncode = 0 | FeatureEncodeForSQL | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent | FeaturesEncodeExtended,

// these are not guaranteed to apply to a given field

FeatureDecodeAllowed = 0 | FeatureDecodeUnreserved | FeatureDecodeFieldAllowed,

FeaturesMaybeDecode = 0 | FeaturesDecode | FeatureDecodeAllowed,

FeaturesMaybeEncode = 0 | FeaturesEncode,

FeaturesEncodeDecode = 0 | FeaturesMaybeEncode | FeaturesMaybeDecode,

FeaturesAllEncoder = 0 | FeaturesEncodeDecode | FeatureDecodeANY | FeatureToLower | FeatureUpperEncoded | FeatureEncodeSpaceAsPlus,

//==============================

FeaturesNormalizeSet = 0 | FeaturePathOperation | FeatureToLower | FeatureDecodeAllowed | FeatureEncodeSpaceAsPlus | FeatureEncodeForSQL | FeaturePathStripRootParent | FeatureTryToFix | FeatureUpperEncoded,

FeaturesDefault = 0 // it reproduces old parsedURL

| FeaturePathOperation | FeaturePathDenyRootParent | FeatureCheckHost,

// essentially allows all valid RFC urls and keeps them as-is

FeaturesBare = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureAllowEmptyPath,

FeaturesAll = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureCheckHost | FeaturesNormalizeSet,

// Deprecated, use FeaturesRecommended

| FeatureDecodeUnreserved // 6.2.2.2

| FeaturePathOperation // 6.2.2.3

| FeaturePathDenyRootParent | FeatureSchemeKnown | FeatureConvertHostIDN | FeatureRemoteOnly | FeatureHashBangToEscapedFragment | FeatureCheckHost,

// these are mutually exclusive

FeaturesPath = 0 | FeaturePathDenyRootParent | FeaturePathStripRootParent,

FeaturesEscapedFragment = 0 | FeatureEscapedToHashBangFragment | FeatureHashBangToEscapedFragment,

FeaturesCheckSpecialChar = 0 | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent,

FeaturesEncodePChar = 0 | FeatureUpperEncoded | FeaturesEncodeDecode | FeaturesCheckSpecialChar,

// http://wiki.yandex-team.ru/robot/newDesign/dups/normolization

FeaturesRobot = FeaturesRecommended

};

static inline int strnicmp(const char* lt, const char* rt, size_t len) {

return lt == rt ? 0 : ::strnicmp(lt, rt, len);

}

static inline int CompareNoCasePrefix(const TStringBuf& lt, const TStringBuf& rt) {

return strnicmp(lt.data(), rt.data(), rt.length());

}

static inline bool EqualNoCase(const TStringBuf& lt, const TStringBuf& rt) {

return lt.length() == rt.length() && 0 == CompareNoCasePrefix(lt, rt);

static inline int CompareNoCase(const TStringBuf& lt, const TStringBuf& rt) {

if (lt.length() == rt.length())

return CompareNoCasePrefix(lt, rt);

return lt.length() < rt.length() ? -1 : 1;

class TSchemeInfo {

public:

bool Matches(const TStringBuf& scheme) const {

return EqualNoCase(scheme, Str);

}

public:

static const TSchemeInfo& Get(const TStringBuf& scheme);

static const TSchemeInfo& Get(TScheme::EKind scheme) {

const char* ParsedStateToString(const TState::EParsed& t);

const char* SchemeKindToString(const TScheme::EKind& t);

Y_DECLARE_OUT_SPEC(inline, NUri::TField::EField, out, t) {

}

Y_DECLARE_OUT_SPEC(inline, NUri::TScheme::EKind, out, t) {

}

Y_DECLARE_OUT_SPEC(inline, NUri::TState::EParsed, out, t) {

}

static inline ui16 DefaultPort(NUri::TScheme::EKind scheme) {

}

static inline NUri::TScheme::EKind SchemeKind(const TStringBuf& scheme) {

}

#include <util/generic/singleton.h>

namespace NEncode {

const TEncoder::TGrammar& TEncoder::Grammar() {

return *Singleton<TEncoder::TGrammar>();

}

// initialize the grammar map

TEncoder::TGrammar::TGrammar() {

// first set up unreserved characters safe in any field

AddRng('A', 'Z', ECFUpper, featUnres | TFeature::FeatureToLower);

AddRng('a', 'z', ECFLower, featUnres);

Add(UNRESERVED, ECFUnres, featUnres);

// XXX: standard "safe" set used previously "-_.!~*();/:@$,", with comment:

// alnum + reserved + mark + ( '[', ']') - ('=' '+' '&' '\'' '"' '\\' '?')

Add("!*();/:@$,", ECFStdrd, TFeature::FeatureDecodeStandardExtra);

// now field-specific subsets of reserved characters (gen-delims + sub-delims)

const ui64 featSafe = TFeature::FeatureDecodeFieldAllowed;

Add(GENDELIMS1, 0, featSafe, TField::FlagPath | TField::FlagQuery | TField::FlagFrag);

Add(GENDELIMS2, 0, featSafe, TField::FlagQuery | TField::FlagFrag);

Add(SUBDELIMS1, 0, featSafe, TField::FlagUser);

Add(SUBDELIMS2, 0, featSafe, TField::FlagUser | TField::FlagQuery | TField::FlagFrag);

// control chars

AddRng(0x00, 0x20, TFeature::FeatureEncodeCntrl);

Add(0x7f, TFeature::FeatureEncodeCntrl);

// '%' starts a percent-encoded sequence

Add('%', TFeature::FeatureDecodeANY | TFeature::FeatureEncodePercent);

// extended ASCII

AddRng(128, 255, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeExtendedASCII);

// extended delims

Add("\"<>[\\]^`{|}", TFeature::FeatureEncodeExtendedDelim | TFeature::FeatureDecodeExtendedDelim);

// add characters with other features

Add(' ', TFeature::FeatureEncodeSpace | TFeature::FeatureEncodeSpaceAsPlus);

Add("'\"\\", TFeature::FeatureEncodeForSQL);

GetMutable(':').EncodeFld |= TField::FlagUser;

GetMutable('?').EncodeFld |= TField::FlagPath;

GetMutable('#').EncodeFld |= TField::FlagPath | TField::FlagQuery;

return true;

return (fldmask & DecodeFld) && (flags & TFeature::FeatureDecodeFieldAllowed);

}

const int dD = 'a' - 'A';

int TEncodeMapper::EncodeSym(unsigned char& ch) const {

const TCharFlags& chflags = TEncoder::GetFlags(ch);

const ui64 flags = Flags & chflags.FeatFlags;

if (flags & TFeature::FeatureToLower)

ch += dD;

if (Q_DecodeAny)

return -1;

if (flags & TFeature::FeaturesEncode)

return 1;

if (' ' == ch) {

if (Q_EncodeSpcAsPlus)

ch = '+';

return 0;

}

return 0;

}

int TEncodeMapper::EncodeHex(unsigned char& ch) const {

const TCharFlags& chflags = TEncoder::GetFlags(ch);

const ui64 flags = Flags & chflags.FeatFlags;

if (flags & TFeature::FeatureToLower)

ch += dD;

if (Q_DecodeAny)

return -1;

if (chflags.IsDecode(FldMask, Flags))

return 0;

if (' ' == ch) {

if (!Q_EncodeSpcAsPlus)

return 1;

return 0;

}

}

bool TEncodeToMapper::Encode(unsigned char ch) const {

if (Q_DecodeAny)

return false;

const TCharFlags& chflags = TEncoder::GetFlags(ch);

if (FldMask & chflags.EncodeFld)

return true;

const ui64 flags = Flags & chflags.FeatFlags;

return (flags & TFeature::FeaturesEncode);

}

TEncoder::TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst)

: Out(out)

, FldSrc(fldsrc)

, HexValue(0)

{

}

IOutputStream& TEncoder::Hex(IOutputStream& out, unsigned char val) {

static const char sHexCodes[] = "0123456789ABCDEF";

return out << sHexCodes[(val >> 4) & 0xF] << sHexCodes[val & 0xF];

}

IOutputStream& TEncoder::EncodeAll(IOutputStream& out, const TStringBuf& val) {

for (size_t i = 0; i != val.length(); ++i)

Encode(out, val[i]);

}

return out;

}

IOutputStream& TEncoder::EncodeField(

IOutputStream& out, const TStringBuf& val, TField::EField fld) {

const ui32 fldmask = ui32(1) << fld;

Out << ch;

return;

}

Out << '%';

if (escapepct)

Out.Write("25", 2); // '%'

Hex(Out, ch);

}

}

namespace NEncode {

#define CHAR_TYPE_NAME(f) _ECT##f

#define CHAR_TYPE_FLAG(f) ECF##f = 1u << CHAR_TYPE_NAME(f)

CHAR_TYPE_NAME(Unres),

CHAR_TYPE_NAME(Stdrd),

};

enum ECharFlag {

CHAR_TYPE_FLAG(Digit),

CHAR_TYPE_FLAG(Lower),

ECGStdrd = ECGUnres | ECFStdrd,

};

struct TCharFlags {

ui32 TypeFlags;

// should we decode an encoded character

bool IsDecode(ui32 fldmask, ui64 flags) const;

};

class TEncodeMapperBase {

protected:

TEncodeMapperBase()

, Q_DecodeAny(flags & TFeature::FeatureDecodeANY)

{

}

protected:

const ui64 Flags;

const ui32 FldMask;

const bool Q_DecodeAny; // this is a special option for username/password

};

// maps a sym or hex character and indicates whether it has to be encoded

class TEncodeMapper

: public TEncodeMapperBase {

// negative=sym, positive=hex, zero=maybesym

int EncodeSym(unsigned char&) const;

int EncodeHex(unsigned char&) const;

protected:

const bool Q_EncodeSpcAsPlus;

};

// indicates whether a character has to be encoded when copying to a field

class TEncodeToMapper

: public TEncodeMapperBase {

}

bool Encode(unsigned char) const;

};

class TEncoder {

public:

TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst = TEncodeToMapper());

ui64 ReEncode(const TStringBuf& url);

ui64 ReEncode(const char* str, size_t len) {

return ReEncode(TStringBuf(str, len));

}

protected:

static bool IsType(unsigned char c, ui64 flags) {

return GetFlags(c).TypeFlags & flags;

static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld);

static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld, ui64 flags);

static IOutputStream& Encode(IOutputStream& out, const TStringBuf& val) {

return EncodeField(out, val, TField::FieldAllMAX);

}

public:

class TGrammar {

TCharFlags Map_[256];

public:

TGrammar();

const TCharFlags& Get(unsigned char ch) const {

return Map_[ch];

}

TCharFlags& GetMutable(unsigned char ch) {

return Map_[ch];

}

TCharFlags& Add(unsigned char ch, const TCharFlags& val) {

return GetMutable(ch).Add(val);

}

void AddRng(unsigned char lo, unsigned char hi, const TCharFlags& val) {

for (unsigned i = lo; i <= hi; ++i)

Add(i, val);

void AddRng(unsigned char lo, unsigned char hi, ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) {

AddRng(lo, hi, TCharFlags(type, feat, decmask, encmask));

}

void Add(const TStringBuf& set, const TCharFlags& val) {

for (size_t i = 0; i != set.length(); ++i)

Add(set[i], val);

Add(set, TCharFlags(type, feat, decmask, encmask));

}

};

static const TGrammar& Grammar();

protected:

IOutputStream& Out;

const TEncodeMapper FldSrc;

void Do(unsigned char, int);

};

}

using TEncoder = NEncode::TEncoder;

}

#pragma clang diagnostic ignored "-Wunused-variable"

#endif

%%{

bad_escaped = ( "%" hex )

% {

main := ( escaped | bad_escaped | sym )**;

}%%

{

int cs;

%% write init;

%% write exec;

}

#pragma once

class THttpURL

: public NUri::TUri {

FeatureEscSpace = TFeature::FeatureEncodeSpaceAsPlus,

FeatureEscapeUnescaped = TFeature::FeatureEncodeExtendedASCII,

FeatureNormalPath = TFeature::FeaturePathStripRootParent,

};

{

}

THttpURL(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0)

{

}

{

}

const char* Get(EField fld) const {

void Set(EField field, const TStringBuf& value) {

void Set(EField field, const char (&value)[size]) {

bool SetInMemory(EField field, const TStringBuf& value) {

void Reset(EField field) {

};

static inline const char* HttpURLParsedStateToString(const NUri::TState::EParsed& t) {

static inline const char* HttpUrlSchemeKindToString(const NUri::TScheme::EKind& t) {

#include <util/system/yassert.h>

/********************************************************/

/********************************************************/

static const Tr InvertTr(".:/?#", "\005\004\003\002\001");

static const Tr RevertTr("\005\004\003\002\001", ".:/?#");

#include "parse.h"

const TParseFlags TParser::FieldFlags[] =

{

TParseFlags(0 // FieldScheme

TParseFlags(0 // FieldHost

| TFeature::FeatureToLower | TFeature::FeatureUpperEncoded | (TFeature::FeaturesMaybeEncode & ~TFeature::FeatureEncodeExtendedDelim),

0 | TFeature::FeaturesMaybeDecode)

,

TParseFlags(0 // FieldPort

,

0)

,

TParseFlags(0 // FieldPath

| TFeature::FeaturesEncodePChar | TFeature::FeaturePathOperation,

0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus)

,

TParseFlags(0 // FieldQuery

| TFeature::FeaturesEncodePChar | TFeature::FeatureEncodeSpaceAsPlus,

0 | TFeature::FeatureToLower)

,

TParseFlags(0 // FieldFragment

| TFeature::FeaturesEncodePChar,

0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus)};

namespace NParse {

void TRange::AddRange(const TRange& range, ui64 mask) {

FlagsAllPlaintext |= range.FlagsAllPlaintext;

}

}

void TParser::copyRequirementsImpl(const char* ptr) {

Y_ASSERT(0 != CurRange.FlagsAllPlaintext);

Y_UNUSED(ptr);

PrintHead(ptr, __FUNCTION__)

<< " & " << IntToString<16>(Flags.Allow | Flags.Extra) << "]";

PrintTail(CurRange.Beg, ptr);

for (int i = 0; i < TField::FieldUrlMAX; ++i) {

const TField::EField fld = TField::EField(i);

TSection& section = Sections[fld];

}

void TParser::PctEndImpl(const char* ptr) {

PrintHead(PctBegin, __FUNCTION__);

PrintTail(PctBegin, ptr);

Y_UNUSED(ptr);

setRequirement(PctBegin, TEncoder::GetFlags('%').FeatFlags);

PctBegin = nullptr;

}

void TParser::HexSet(const char* ptr) {

Y_ASSERT(nullptr != PctBegin);

PrintHead(ptr, __FUNCTION__);

PrintTail(PctBegin, ptr + 1);

PctBegin = nullptr;

const unsigned char ch = HexValue;

ui64 flags = TEncoder::GetFlags('%').FeatFlags | TEncoder::GetFlags(ch).FeatFlags;

}

TState::EParsed TParser::ParseImpl() {

PrintHead(UriStr.data(), "[Parsing]") << "URL";

PrintTail(UriStr);

const bool ok = doParse(UriStr.data(), UriStr.length());

Cdbg << (ok ? "[Parsed]" : "[Failed]");

for (int idx = 0; idx < TField::FieldUrlMAX; ++idx) {

const TSection& section = Sections[idx];

Cdbg << ' ' << TField::EField(idx) << "=[" << section.Get() << ']';

}

Cdbg << Endl;

if (!ok) {

if (!(Flags & TFeature::FeatureTryToFix) || !Sections[TField::FieldFrag].Beg)

return TState::ParsedBadFormat;

}

if ((Flags & TFeature::FeatureDenyNetworkPath) && IsNetPath())

const TSection& scheme = Sections[TField::FieldScheme];

Scheme = scheme.IsSet() ? TSchemeInfo::GetKind(scheme.Get()) : TScheme::SchemeEmpty;

// opaque case happens

if (schemeInfo.FldReq & TField::FlagHost)

return TState::ParsedBadFormat;

if (TScheme::SchemeEmpty == Scheme)

return TState::ParsedBadScheme;

if (Flags & TFeature::FeatureAllowRootless)

return TState::ParsedOK;

if (!(Flags & TFeature::FeatureSchemeFlexible))

return TState::ParsedBadScheme;

return TState::ParsedRootless;

}

checkSectionCollision(TField::FieldUser, TField::FieldHost);

checkSectionCollision(TField::FieldPass, TField::FieldPort);

if ((schemeInfo.FldReq & TField::FlagHost) || (Flags & TFeature::FeatureRemoteOnly))

if (!host.IsSet() || 0 == host.Len())

return TState::ParsedBadFormat;

return TState::ParsedOK;

}

#include <library/cpp/charset/doccodes.h>

class TParser;

namespace NParse {

class TRange {

public:

ui64 FlagsAllPlaintext;

ui32 Encode;

ui32 Decode;

public:

TRange(const char* beg = nullptr)

: Beg(beg)

, Decode(0)

{

}

void Reset(const char* beg = nullptr) {

*this = TRange(beg);

}

void AddRange(const TRange& range, ui64 mask);

void AddFlag(const char* ptr, ui64 mask, ui64 flag) {

if (0 != flag)

AddFlagImpl(ptr, mask, flag, flag);

}

void AddFlagExcept(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag) {

if (0 != flag)

AddFlagImpl(ptr, mask, flag & ~exclflag, flag);

}

void AddFlagUnless(const char* ptr, ui64 mask, ui64 flag, ui64 exclmask) {

if (0 != flag)

AddFlagImpl(ptr, mask, flag, flag, exclmask);

}

void AddFlag(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag, ui64 exclmask) {

if (0 != flag)

AddFlagImpl(ptr, mask, flag & ~exclflag, flag, exclmask);

}

private:

void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag) {

AddFlagAllPlaintextImpl(ptr, plainflag);

AddFlagEncodeMaskedImpl(encflag & mask);

}

void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag, ui64 exclmask) {

AddFlagAllPlaintextImpl(ptr, plainflag);

if (0 == (mask & exclmask))

AddFlagEncodeMaskedImpl(encflag & mask);

}

void AddFlagAllPlaintextImpl(const char* ptr, ui64 flag) {

if (nullptr == Beg)

Beg = ptr;

FlagsAllPlaintext |= flag;

}

void AddFlagEncodeMaskedImpl(ui64 flag) {

if (0 == flag)

return;

++Decode;

}

};

class TSection

: protected NParse::TRange {

private:

friend class TParser;

private:

const char* End;

TSection(const char* beg = nullptr)

: NParse::TRange(beg)

, End(nullptr)

{

}

void Reset() {

Enter(nullptr);

}

void Reset(const char* pc) {

Y_ASSERT(!Beg || !pc || Beg < pc);

Reset();

}

void Enter(const char* pc) {

*this = TSection(pc);

}

bool Leave(const char* pc) {

Y_ASSERT(Beg);

End = pc;

bool IsSet() const {

return End;

}

TStringBuf Get() const {

return TStringBuf(Beg, End);

}

size_t Len() const {

return End - Beg;

}

size_t DecodedLen() const {

return Len() - 2 * Decode;

}

size_t EncodedLen() const {

return 2 * Encode + DecodedLen();

}

ui32 GetEncode() const {

return Encode;

}

ui32 GetDecode() const {

return Decode;

}

ui64 GetFlagsEncode() const {

return FlagsEncodeMasked;

}

ui64 GetFlagsAllPlaintext() const {

return FlagsAllPlaintext;

}

};

class TParser {

public:

TSection Sections[TField::FieldUrlMAX];

const TStringBuf UriStr;

TState::EParsed State;

ECharset Enc;

public:

TParser(const TParseFlags& flags, const TStringBuf& uri, ECharset enc = CODES_UTF8)

: Scheme(TScheme::SchemeEmpty)

ui64 GetFieldFlags(TField::EField fld) const {

return GetFieldFlags(fld, Flags);

}

protected:

static const TParseFlags FieldFlags[TField::FieldUrlMAX];

TSection::TRange CurRange;

unsigned HexValue;

const char* PctBegin;

IOutputStream& PrintAddr(const char* ptr) const {

return Cdbg << "[" << IntToString<16>(ui64(ptr)) << "] ";

}

IOutputStream& PrintHead(const char* ptr, const char* func) const {

return PrintAddr(ptr) << func << " ";

}

IOutputStream& PrintHead(const char* ptr, const char* func, const TField::EField& fld) const {

return PrintHead(ptr, func) << fld;

}

IOutputStream& PrintTail(const TStringBuf& val) const {

return Cdbg << " [" << val << "]" << Endl;

}

IOutputStream& PrintTail(const char* beg, const char* end) const {

return PrintTail(TStringBuf(beg, end));

}

void ResetSection(TField::EField fld, const char* pc = nullptr) {

PrintHead(pc, __FUNCTION__, fld);

PrintTail(pc);

Sections[fld].Reset(pc);

}

void storeSection(const TStringBuf& val, TField::EField fld) {

PrintHead(val.data(), __FUNCTION__, fld);

PrintTail(val);

Sections[fld].Set(val);

}

void startSection(const char* pc, TField::EField fld) {

PrintHead(pc, __FUNCTION__, fld);

PrintTail(pc);

copyRequirements(pc);

Sections[fld].Enter(pc);

}

void finishSection(const char* pc, TField::EField fld) {

PrintHead(pc, __FUNCTION__, fld);

PrintTail(pc);

if (Sections[fld].Leave(pc))

copyRequirements(pc);

}

void setRequirement(const char* ptr, ui64 flags) {

PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags)

<< " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra);

PrintTail(ptr);

CurRange.AddFlag(ptr, Flags.Allow | Flags.Extra, flags);

}

void setRequirementExcept(const char* ptr, ui64 flags, ui64 exclflag) {

PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags)

<< " & exclflag=" << IntToString<16>(exclflag)

<< " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra);

PrintTail(ptr);

CurRange.AddFlagExcept(ptr, Flags.Allow | Flags.Extra, flags, exclflag);

}

void setRequirementUnless(const char* ptr, ui64 flags, ui64 exclmask) {

PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags)

<< " & exclmask=" << IntToString<16>(exclmask)

<< " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra);

PrintTail(ptr);

CurRange.AddFlagUnless(ptr, Flags.Allow | Flags.Extra, flags, exclmask);

}

void copyRequirementsImpl(const char* ptr);

void copyRequirements(const char* ptr) {

PctEnd(ptr);

HexReset();

PctBegin = ptr;

}

void checkSectionCollision(TField::EField fld1, TField::EField fld2) {

if (Sections[fld1].IsSet() && Sections[fld2].IsSet() && Sections[fld1].Beg == Sections[fld2].Beg) {

Sections[fld1].Reset();

bool doParse(const char* str_beg, size_t length);

TState::EParsed ParseImpl();

};

}

#endif

%%{

# with some modifications

#================================================

# The RegEx

# $8 = #Related

# $9 = Related

#

#================================================

#================================================

) >{ REQ(fpc, FeatureEncodeExtendedDelim) }; # " fix hilite

pct_maybe_encoded = PCT (HEXDIG | HEXNUM)? ;

# MOD: non-empty; will use host?

### todo: allow ':' (need to fix grammar to disambiguate port)

achar = any - (0x00 .. 0x20) - '/' - '#' - '?' - ':' - '%';

upperhalf = any - (0x00 .. 0x7F);

hostname = (((achar | pct_encoded)+) & (any* (alnum | upperhalf) any*));

reg_name = hostname - IPv4address - IP_literal;

#================================================

# checkPath rules

#================================================

#================================================

write data;

}%%

{

const char* p = str_beg;

int cs;

if (0 == (Flags & TFeature::FeatureNoRelPath)) {

} else if (0 == (Flags & TFeature::FeatureAllowRootless)) {

} else {

}

%% write exec;

}

#include <library/cpp/charset/recyr.hh>

#include <library/cpp/html/entity/htmlentity.h>

#include <util/system/maxlen.h>

namespace {

TString AsWin1251(const TString& s) {

return Recode(CODES_UTF8, CODES_WIN, s);

UNIT_ASSERT(url.IsNull(TField::FlagHost));

UNIT_ASSERT(!url.IsNull(TField::FlagPath));

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagPath), "www.ya.ru/index.html");

UNIT_ASSERT_VALUES_EQUAL(url.Parse(AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10")), TState::ParsedOK);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10"));

UNIT_ASSERT_VALUES_EQUAL(url.Parse(AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10"),

TFeature::FeaturesDefault | TFeature::FeatureEncodeExtendedASCII),

TState::ParsedOK);

Y_UNIT_TEST(testRuIDNA) {

{

#define DEC "\xD7\xE5\xF0\xE5\xEf\xEE\xE2\xE5\xF6.\xF0\xF4" /* "Череповец.рф" in Windows-1251 */

TTest test = {

"http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC};

TUri url;

UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC);

}

}

// Should be properly null-terminated

UNIT_ASSERT_VALUES_EQUAL(host.size(), strlen(host.data()));

}

}

#include <util/system/yassert.h>

TState::EParsed TUri::CheckHost(const TStringBuf& host) {

if (host.empty())

return ParsedOK;

unsigned domainLevel = 0;

unsigned domainLevelOfUnderscore = 0;

bool isAlnum = false;

bool startLabel = true;

for (size_t i = 0; i != host.length(); ++i) {

const char ch = host[i];

if ('.' == ch) { // label separator

if (!isAlnum || startLabel) // previous label must end in alnum

return ParsedBadHost;

startLabel = true;

continue;

}

isAlnum = isalnum((const unsigned char)ch);

if (startLabel) { // label is starting

if (!isAlnum && '_' != ch) // new label must start with alnum or '_'

return ParsedBadHost;

domainLevelOfUnderscore = domainLevel;

continue;

}

if (isAlnum || '-' == ch)

continue;

}

return ParsedBadHost;

if (0 < domainLevelOfUnderscore && domainLevel < 2 + domainLevelOfUnderscore)

return ParsedBadHost;

return ParsedOK;

}

if (!scheme.empty()) {

if (SetSchemeImpl(TSchemeInfo::Get(scheme)).Str.empty())

FldSet(FieldScheme, scheme);

if (0 < defaultPort) // override the scheme's default port

DefaultPort = static_cast<ui16>(defaultPort);

char sport[6]; // enough for ui16

if (0 != port) {

const size_t len = ToString(port, sport, sizeof(sport));

FldSet(FieldPort, TStringBuf(sport, len));

}

FldTrySet(FieldHost, host);

FldTrySet(FieldPath, path);

FldTrySet(FieldQuery, query);

Rewrite();

/********************************************************/

bool TUri::FldSetImpl(

EField field, TStringBuf value, bool strconst, bool nocopy) {

if (!FldIsValid(field))

return false;

switch (field) {

case FieldScheme:

if (!SetScheme(TSchemeInfo::Get(value)).Str.empty())

return false;

break;

case FieldPort:

Port = value.empty() ? 0 : FromString<ui16>(value);

break;

default:

break;

}

FldClr(field);

return false;

}

if (strconst) { // string constants don't need to be saved in the buffer

FldMarkClean(field);

FldSetNoDirty(field, value);

}

if (nocopy) {

FldSet(field, value);

return true;

}

return FldTryCpy(field, value);

}

do {

if (!FldIsSet(field))

break;

TStringBuf& fld = Fields[field];

if (fld.length() < value.length())

break;

FldMarkDirty(field);

}

/********************************************************/

void TUri::RewriteImpl() {

size_t len = 0;

const EField fld = EField(i);

if (!FldIsSet(fld))

continue;

const char* beg = out.Buf();

const TStringBuf& val = Fields[fld];

out << val;

}

Buffer = std::move(newbuf);

}

CheckMissingFields();

FieldsDirty = 0;

void TUri::CheckMissingFields() {

// if host is set but path is not...

if (FldSetCmp(FlagPath | FlagHost, FlagHost))

void TUri::Merge(const TUri& base, int correctAbs) {

if (base.Scheme == SchemeUnknown)

return;

if (!base.IsValidGlobal())

return;

const bool noscheme = !selfscheme.IsInited();

if (!noscheme && !EqualNoCase(selfscheme, basescheme))

return;

const ui32 cleanFields = ~FieldsDirty;

do {

static constexpr TStringBuf rootPath = "/";

if (noscheme) {

if (!basescheme.empty()) {

FldSetNoDirty(FieldScheme, basescheme);

Scheme = base.Scheme;

DefaultPort = base.DefaultPort;

}

if (!IsNull(FlagHost))

break; // no merge

if (noscheme && IsNull(FlagQuery) && IsNull(FlagPath))

FldTrySet(FieldQuery, base);

if (IsNull(FlagAuth) && !base.IsNull(FlagAuth)) {

FldChkSet(FieldUser, base);

FldChkSet(FieldPass, base);

if (IsValidAbs())

break;

TStringBuf p0 = base.GetField(FieldPath);

if (!p0.IsInited())

p0 = rootPath;

else if (p1.empty() || '.' != p1[0])

pathop = false;

out << p1;

char* beg = out.Data();

char* end = beg + out.Filled();

if (pathop && !PathOperation(beg, end, correctAbs)) {

Clear();

break;

}

// Needs immediate forced rewrite because of TTempBuf

FldSetNoDirty(FieldPath, TStringBuf(beg, end));

RewriteImpl();

} while (false);

CheckMissingFields();

// rewrite only if borrowed fields from base

if (cleanFields & FieldsDirty)

RewriteImpl();

}

/********************************************************/

TUri::TLinkType TUri::Normalize(const TUri& base,

const TStringBuf& link, const TStringBuf& codebase, long careFlags, ECharset enc) {

// parse URL

if (ParsedOK != ParseImpl(link, careFlags, 0, SchemeEmpty, enc))

return LinkIsBad;

const TStringBuf& host = GetHost();

// merge with base URL

else

len += v.length() + 1;

}

}

return len;

if (!v.empty())

out << v << ':';

}

TStringBuf host;

if (flags & FlagHost) {

const EField fldhost =

TStringBuf port;

if ((flags & FlagPort) && 0 != Port && Port != DefaultPort)

port = Fields[FieldPort];

if (host) {

if (wantFlags & FlagScheme)

out << "//";

if (flags & FlagAuth) {

if (flags & FlagUser) {

v = Fields[FieldUser];

TEncoder::EncodeAll(out, v);

}

}

out << '@';

}

return false;

if (pathPtr == pathEnd)

return true;

if ((pathEnd - pathPtr) >= 2 && *(pathEnd - 2) == '/' && *(pathEnd - 1) == '.') {

--pathEnd;

}

char* p_wr = pathEnd;

int upCount = 0;

#pragma once

#include <library/cpp/charset/doccodes.h>

#include <util/generic/buffer.h>

#include <util/generic/string.h>

#include <util/stream/mem.h>

#include <util/stream/str.h>

#include <util/system/yassert.h>

#include <cstdlib>

/********************************************************/

class TUri

: public TFeature,

LinkIsLocal,

LinkIsGlobal

};

private:

TBuffer Buffer;

TStringBuf Fields[FieldAllMAX];

Scheme = SchemeEmpty;

FieldsDirty = 0;

}

void CopyData(const TUri& url) {

FieldsSet = url.FieldsSet;

Port = url.Port;

Scheme = url.Scheme;

FieldsDirty = url.FieldsDirty;

}

void CopyImpl(const TUri& url) {

for (int i = 0; i < FieldAllMAX; ++i)

Fields[i] = url.Fields[i];

RewriteImpl();

}

private:

static ui32 FldFlag(EField fld) {

return 1 << fld;

}

public:

static bool FldIsValid(EField fld) {

return 0 <= fld && FieldAllMAX > fld;

}

bool FldSetCmp(ui32 chk, ui32 exp) const {

return (FieldsSet & chk) == exp;

}

bool FldSetCmp(ui32 chk) const {

return FldSetCmp(chk, chk);

}

bool FldIsSet(EField fld) const {

return !FldSetCmp(FldFlag(fld), 0);

}

private:

void FldMarkSet(EField fld) {

FieldsSet |= FldFlag(fld);

}

void FldMarkUnset(EField fld) {

FieldsSet &= ~FldFlag(fld);

}

// use when we know the field is dirty or RewriteImpl will be called

void FldSetNoDirty(EField fld, const TStringBuf& value) {

Fields[fld] = value;

FldMarkSet(fld);

}

void FldSet(EField fld, const TStringBuf& value) {

FldSetNoDirty(fld, value);

FldMarkDirty(fld);

const TStringBuf& FldGet(EField fld) const {

return Fields[fld];

}

private:

/// depending on value, clears or sets it

void FldChkSet(EField fld, const TStringBuf& value) {

void FldChkSet(EField fld, const TUri& other) {

FldChkSet(fld, other.GetField(fld));

}

/// set only if initialized

bool FldTrySet(EField fld, const TStringBuf& value) {

const bool ok = value.IsInited();

private:

/// copies the value if it fits

bool FldTryCpy(EField fld, const TStringBuf& value);

// main method: sets the field value, possibly copies, etc.

bool FldSetImpl(EField fld, TStringBuf value, bool strconst = false, bool nocopy = false);

public: // clear a field

void FldClr(EField fld) {

Fields[fld].Clear();

FldMarkUnset(fld);

FldMarkClean(fld);

}

bool FldTryClr(EField field) {

const bool ok = FldIsSet(field);

if (ok)

FldClr(field);

return ok;

}

public: // set a field value: might leave state dirty and require a Rewrite()

// copies if fits and not dirty, sets and marks dirty otherwise

bool FldMemCpy(EField field, const TStringBuf& value) {

return FldSetImpl(field, value, false);

}

// uses directly, marks dirty

/// @note client MUST guarantee value will be alive until Rewrite is called

bool FldMemSet(EField field, const TStringBuf& value) {

return FldSetImpl(field, value, false, true);

}

// uses directly, doesn't mark dirty (value scope exceeds "this")

bool FldMemUse(EField field, const TStringBuf& value) {

return FldSetImpl(field, value, true);

}

// uses directly, doesn't mark dirty

template <size_t size>

bool FldMemSet(EField field, const char (&value)[size]) {

static_assert(size > 0);

return FldSetImpl(field, TStringBuf(value, size - 1), true);

}

// duplicate one field to another

bool FldDup(EField src, EField dst) {

if (!FldIsSet(src) || !FldIsValid(dst))

FldMarkClean(dst);

return true;

}

// move one field to another

bool FldMov(EField src, EField dst) {

if (!FldDup(src, dst))

FldClr(src);

return true;

}

private:

bool IsInBuffer(const char* buf) const {

return buf >= Buffer.data() && buf < Buffer.data() + Buffer.size();

}

public:

bool FldIsDirty() const {

return 0 != FieldsDirty;

}

bool FldIsDirty(EField fld) const {

return 0 != (FieldsDirty & FldFlag(fld));

}

private:

void FldMarkDirty(EField fld) {

FieldsDirty |= FldFlag(fld);

}

void FldMarkClean(EField fld) {

FieldsDirty &= ~FldFlag(fld);

}

void RewriteImpl();

public:

static TState::EParsed CheckHost(const TStringBuf& host);

// convert a [potential] IDN to ascii

static TMallocPtr<char> IDNToAscii(const wchar32* idna);

static TMallocPtr<char> IDNToAscii(const TStringBuf& host, ECharset enc = CODES_UTF8);

// convert hosts with percent-encoded or extended chars

// returns non-empty string if host can be converted to ASCII with given parameters

static TStringBuf HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc = CODES_UTF8);

// returns host if already ascii, or non-empty if it can be converted

static TStringBuf HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc = CODES_UTF8);

public:

explicit TUri(unsigned defaultPort = 0)

: FieldsSet(0)

, FieldsDirty(0)

{

}

TUri(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0);

TUri(const TUri& url)

: FieldsSet(url.FieldsSet)

, Port(url.Port)

void Clear() {

Dealloc();

ClearImpl();

ui32 GetFieldMask() const {

return FieldsSet;

}

ui32 GetUrlFieldMask() const {

return GetFieldMask() & FlagUrlFields;

}

ui32 GetDirtyMask() const {

return FieldsDirty;

}

void CheckMissingFields();

// Process methods

void Rewrite() {

if (FldIsDirty())

RewriteImpl();

TState::EParsed AssignImpl(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty);

TState::EParsed ParseImpl(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeEmpty, ECharset enc = CODES_UTF8);

public:

TState::EParsed Assign(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty) {

const TState::EParsed ret = AssignImpl(parser, defscheme);

Rewrite();

return ret;

}

TState::EParsed ParseUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) {

const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc);

if (ParsedOK == ret)

Rewrite();

return ret;

}

// parses absolute URIs

// prepends default scheme (unless unknown) if URI has none

TState::EParsed ParseAbsUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeUnknown, ECharset enc = CODES_UTF8);

TState::EParsed ParseAbsOrHttpUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) {

return ParseAbsUri(url, flags, maxlen, SchemeHTTP, enc);

}

TState::EParsed Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8);

TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault) {

return ParseUri(url, flags);

}

TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& base_url, ui32 maxlen = 0, ECharset enc = CODES_UTF8);

TState::EParsed ParseAbs(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, const TStringBuf& base_url = TStringBuf(), ui32 maxlen = 0, ECharset enc = CODES_UTF8) {

const TState::EParsed result = Parse(url, flags, base_url, maxlen, enc);

return ParsedOK != result || IsValidGlobal() ? result : ParsedBadFormat;

}

// correctAbs works with head "/.." portions:

// 1 - reject URL

// 0 - keep portions

// -1 - ignore portions

void Merge(const TUri& base, int correctAbs = -1);

TLinkType Normalize(const TUri& base, const TStringBuf& link, const TStringBuf& codebase = TStringBuf(), long careFlags = FeaturesDefault, ECharset enc = CODES_UTF8);

protected:

size_t PrintSize(ui32 flags) const;

// Output method, prints to stream

IOutputStream& PrintImpl(IOutputStream& out, int flags) const;

char* PrintImpl(char* str, size_t size, int flags) const {

TMemoryOutput out(str, size);

PrintImpl(out, flags) << '\0';

return str;

}

static bool IsAbsPath(const TStringBuf& path) {

return 1 <= path.length() && path[0] == '/';

}

bool IsAbsPathImpl() const {

return IsAbsPath(GetField(FieldPath));

}

public:

// Output method, prints to stream

IOutputStream& Print(IOutputStream& out, int flags = FlagUrlFields) const {

return PrintImpl(out, PrintFlags(flags));

}

// Output method, print to str, allocate memory if str is NULL

// Should be deprecated

char* Print(char* str, size_t size, int flags = FlagUrlFields) const {

return nullptr == str ? Serialize(flags) : Serialize(str, size, flags);

}

char* Serialize(char* str, size_t size, int flags = FlagUrlFields) const {

Y_ASSERT(str);

flags = PrintFlags(flags);

const size_t size = PrintSize(flags) + 1;

return PrintImpl(static_cast<char*>(malloc(size)), size, flags);

}

// Output method to str

void Print(TString& str, int flags = FlagUrlFields) const {

flags = PrintFlags(flags);

TStringOutput out(str);

PrintImpl(out, flags);

}

TString PrintS(int flags = FlagUrlFields) const {

TString str;

Print(str, flags);

TString PrintHostS() const {

return PrintS((Scheme != SchemeHTTP ? FlagScheme : 0) | FlagHostPort);

}

// Info methods

int Compare(const TUri& A, int flags = FlagUrlFields) const;

const TStringBuf& GetField(EField fld) const {

return FldIsValid(fld) && FldIsSet(fld) ? FldGet(fld) : Default<TStringBuf>();

}

ui16 GetPort() const {

return 0 == Port ? DefaultPort : Port;

}

bool UseHostAscii() {

return FldMov(FieldHostAscii, FieldHost);

}

TScheme::EKind GetScheme() const {

return Scheme;

}

const TSchemeInfo& GetSchemeInfo() const {

return TSchemeInfo::Get(Scheme);

}

bool IsNull(ui32 flags = FlagScheme | FlagHost | FlagPath) const {

return !FldSetCmp(flags);

}

return false;

return IsAbsPathImpl();

}

bool IsValidGlobal() const {

if (IsNull(FlagScheme | FlagHost))

return false;

bool IsOpaque() const {

return IsRootless();

}

// Inline helpers

TUri& operator=(const TUri& u) {

Copy(u);

return *this;

}

bool operator!() const {

return IsNull();

}

static IOutputStream& ReEncode(IOutputStream& out, const TStringBuf& val, long flags = FeaturesEncodeDecode) {

return ReEncodeField(out, val, FieldAllMAX, flags);

}

static int PathOperationFlag(const TParseFlags& flags) {

return flags & FeaturePathDenyRootParent ? 1

: flags & FeaturePathStripRootParent ? -1 : 0;

}

static bool PathOperation(char*& pathBeg, char*& pathEnd, int correctAbs);

private:

const TSchemeInfo& SetSchemeImpl(const TSchemeInfo& info) {

Scheme = info.Kind;

const TSchemeInfo& SetSchemeImpl(TScheme::EKind scheme) {

return SetSchemeImpl(TSchemeInfo::Get(scheme));

}

public:

const TSchemeInfo& SetScheme(const TSchemeInfo& info) {

SetSchemeImpl(info);

return SetScheme(TSchemeInfo::Get(scheme));

}

};

class TUriUpdate {

TUri& Uri_;

~TUriUpdate() {

Uri_.Rewrite();

}

public:

bool Set(TField::EField field, const TStringBuf& value) {

return Uri_.FldMemSet(field, value);

}

template <size_t size>

bool Set(TField::EField field, const char (&value)[size]) {

return Uri_.FldMemSet(field, value);

}

void Clr(TField::EField field) {

Uri_.FldClr(field);

}

};

const char* LinkTypeToString(const TUri::TLinkType& t);

Y_DECLARE_OUT_SPEC(inline, NUri::TUri, out, url) {

Y_DECLARE_OUT_SPEC(inline, NUri::TUri::TLinkType, out, t) {

}

#include "qargs.h"

#include <library/cpp/html/entity/htmlentity.h>

Y_UNIT_TEST_SUITE(URLTest) {

static const char* urls[] = {

"http://a/b/c/d;p?q#r",

"%20y", "http://a/b/c/%20y",

// "%2zy", "http://a/b/c/%2zy",

nullptr};

Y_UNIT_TEST(test_httpURL) {

TUri rel, base, abs;

TState::EParsed er = base.Parse(urls[0]);

UNIT_ASSERT_EQUAL_C(rel, abs, errbuf);

}

}

Y_UNIT_TEST(test_Schemes) {

TUri url;

UNIT_ASSERT_VALUES_EQUAL(url.Parse("www.ya.ru/index.html"), TState::ParsedOK);

UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpsssss://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK);

UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown);

}

struct Link4Norm {

const char* const base;

const char* const link;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "host:8080");

}

Y_UNIT_TEST(test06) {

TTest test = {

"http://user:pass@host?q", TFeature::FeaturesAll, TState::ParsedOK, "http", "user", "pass", "host", 80, "/", "q", ""};

UNIT_ASSERT(!url.FldIsDirty());

UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldScheme), "https");

UNIT_ASSERT_VALUES_EQUAL(url.GetPort(), 443);

// test copying

TUri url2(url);

// make sure strings are equal...

url2.GetField(TField::FieldUser));

// and urls compare the same

URL_EQ(url, url2);

// cause a dirty field

url.FldMemSet(TField::FieldUser, "use"); // it is now shorter

UNIT_ASSERT(!url.FldIsDirty());

url.FldMemSet(TField::FieldUser, TStringBuf("user"));

UNIT_ASSERT(url.FldIsDirty());

// copy again

url2 = url;

UNIT_ASSERT(url.FldIsDirty());

url.GetField(TField::FieldUser).data(),

url2.GetField(TField::FieldUser).data());

URL_EQ(url, url2);

// make query empty

url.FldMemSet(TField::FieldQuery, "");

url2 = url;

url2.FldMemSet(TField::FieldPort, "443");

URL_EQ(url, url2);

}

Y_UNIT_TEST(test07) {

{

TTest test = {

UNIT_ASSERT_EQUAL(TScheme::SchemeHTTP, url.GetScheme());

UNIT_ASSERT_EQUAL("http", url.GetField(TField::FieldScheme));

}

{

const TString scheme = "http";

const TString host = "host.com";

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), urlstr + "/");

}

}

Y_UNIT_TEST(test08) {

{

TTest test = {

URL_TEST(url, test);

}

}

Y_UNIT_TEST(test09) {

{

TTest test = {

URL_TEST(url, test);

}

}

Y_UNIT_TEST(test10) {

// test some escaping madness, note the ehost vs host

{

TUri url;

URL_TEST(url, test);

}

{

TString host = "%D0%BF%D1%80%D0%B5%D0%B7%D0%B8%D0%B4%D0%B5%D0%BD%D1%82.%D1%80%D1%84";

const TString urlstr = TString::Join("http://", host, "/");

TUri url;

URL_TEST(url, test);

}

{

TString host = "Фilip.ru";

TString ehost = "%D0%A4ilip.ru";

TUri url;

URL_TEST(url, test);

}

{

TString host = "%D0%A4ilip.ru";

const TString urlstr = TString::Join("http://", host);

TUri url;

URL_TEST(url, test);

}

{

TString host = "Filip%90.rЯ";

TString ehost = "Filip%90.r%D0%AF";

TUri url;

URL_TEST(url, test);

}

{

TString host = "Filip%90.r%D0%AF";

const TString urlstr = TString::Join(host, ":8080");

TUri url;

URL_TEST(url, test);

}

{

TTest test = {

"HtTp://HoSt/%50a%54h/?Query#Frag", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TFeature::FeatureToLower), TState::ParsedOK, "http", "", "", "host", 80, "/path/", "query", "frag"};

Y_UNIT_TEST(test12) {

// test characters which are not always safe

{

TTest test = {

"http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW);

}

{

TTest test = {

"http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW);

}

{

TTest test = {

"http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, ENC, ENC};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC);

}

{

TTest test = {

"http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, DEC, DEC};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" DEC "#" DEC);

}

{

TTest test = {

"http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC};

TUri url;

UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC);

}

{

TTest test = {

"http://" DEC "/" DEC "?" DEC "#" DEC, TParseFlags(TFeature::FeaturesRobot | TFeature::FeatureEncodeExtendedASCII), TState::ParsedOK, "http", "", "", PNC, 80, "/" ENC, ENC, ENC};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" PNC "/" ENC "?" ENC "#" ENC);

}

{

#define DEC "независимая-экспертиза-оценка-ущерба-авто-дтп.рф"

#undef DEC

}

}

Y_UNIT_TEST(testFlexibleAuthority) {

TUri uri;

UNIT_ASSERT_EQUAL(uri.Parse("http://hello_world", TFeature::FeatureCheckHost), TState::ParsedBadHost);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%thA");

}

}

Y_UNIT_TEST(testIPv6) {

{

TTest test = {

"http://" RAW "/" RAW "?" RAW "#" RAW, TParseFlags(TFeature::FeaturesAll), TState::ParsedOK, "http", "", "", DEC, 80, "/" RAW, RAW, RAW};

TUri url;

URL_TEST(url, test);

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" RAW "?" RAW "#" RAW);

}

}

Y_UNIT_TEST(testEscapedFragment) {

{

TTest test = {

UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host.com/#!a=b&c=d#e+g%25");

}

}

Y_UNIT_TEST(testReEncode) {

{

TStringStream out;

UNIT_ASSERT_VALUES_EQUAL(out.Str(), "foo%20bar");

}

}

static const TStringBuf NonRfcUrls[] = {

"http://deshevle.ru/price/price=&SrchTp=1&clID=24&BL=SrchTp=0|clID=24&frmID=75&SortBy=P&PreSort=&NmDir=0&VndDir=0&PrDir=0&SPP=44",

"http://secure.rollerwarehouse.com/skates/aggressive/skates/c/11[03]/tx/$$$+11[03][a-z]",

"http://www.trinity.by/?section_id=46,47,48&cat=1&filters[]=2^_^Sony",

"http://translate.yandex.net/api/v1/tr.json/translate?lang=en-ru&text=>",

nullptr};

Y_UNIT_TEST(test_NonRfcUrls) {

TUri url;

const long flags = TFeature::FeaturesRobot;

UNIT_ASSERT_VALUES_EQUAL(TState::ParsedOK, url.Parse(buf, flags));

}

}

static const TStringBuf CheckParseException[] = {

"http://www.'>'.com/?.net/",

nullptr};

Y_UNIT_TEST(test_CheckParseException) {

TUri url;

const long flags = TFeature::FeaturesRobot | TFeature::FeaturesEncode;

}

ythrow yexception() << "failed to parse URL [" << buf << "]: " << what;

}

Y_UNIT_TEST(test_PrintPort) {

TUri uri;

{

UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldPath), "/path");

}

}

Y_UNIT_TEST_SUITE(TInvertDomainTest) {

Y_UNIT_TEST(TestInvert) {

TString a;

TString h("www.yandex.ru:8080/redir.pl?url=https://google.com/");

UNIT_ASSERT_EQUAL(InvertDomain(h), "ru.yandex.www:8080/redir.pl?url=https://google.com/");

}

TQueryArg::EProcessed ProcessQargs(TString url, TString& processed, TQueryArgFilter filter = 0, void* filterData = 0) {

TUri uri;

uri.Parse(url, NUri::TFeature::FeaturesRecommended);

TQueryArgProcessing processing(TQueryArg::FeatureSortByName | (filter ? TQueryArg::FeatureFilter : 0) | TQueryArg::FeatureRewriteDirty, filter, filterData);

auto result = processing.Process(uri);

processed = uri.PrintS();

return result;

}

TString SortQargs(TString url) {

TString r;

ProcessQargs(url, r);

return r;

}

bool QueryArgsFilter(const TQueryArg& arg, void* filterData) {

const char* skipName = static_cast<const char*>(filterData);

return arg.Name != skipName;

#include <library/cpp/testing/unittest/registar.h>

struct TTest {

TStringBuf Val;

TParseFlags Flags;

TStringBuf Query;

TStringBuf Frag;

};

(TString("[") + url1.PrintS() + ("] " cmp " [") + url2.PrintS() + "]")

UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::Field##fld), test.fld)

#define CMP_URL(url, test) \

do { \

CMP_FLD(url, test, Scheme); \

CMP_FLD(url, test, Query); \

CMP_FLD(url, test, Frag); \

} while (false)

#define URL_TEST_ENC(url, test, enc) \

do { \

TState::EParsed st = url.ParseUri(test.Val, test.Flags, 0, enc); \

CMP_URL(_url, test2); \

UNIT_ASSERT_VALUES_EQUAL(url.GetUrlFieldMask(), _url.GetUrlFieldMask()); \

} while (false)

LIBRARY()

OWNER(

mvel

g:base

)

http_url.h

location.cpp

qargs.cpp

encodefsm.rl6

parsefsm.rl6

library/cpp/charset

RECURSE(

benchmark