Update contrib/libs/flatbuffers to 23.5.9

1 files changed, 908 insertions, 463 deletions

diff --git a/contrib/libs/flatbuffers/src/idl_parser.cpp b/contrib/libs/flatbuffers/src/idl_parser.cpp
index ad642d79a9..bddc736a5f 100644
--- a/contrib/libs/flatbuffers/src/idl_parser.cpp
+++ b/contrib/libs/flatbuffers/src/idl_parser.cpp
@@ -16,11 +16,15 @@
 
 #include <algorithm>
 #include <cmath>
+#include <iostream>
 #include <list>
 #include <string>
 #include <utility>
 
+#include "flatbuffers/base.h"
+#include "flatbuffers/buffer.h"
 #include "flatbuffers/idl.h"
+#include "flatbuffers/reflection_generated.h"
 #include "flatbuffers/util.h"
 
 namespace flatbuffers {
@@ -35,28 +39,14 @@ const char *FLATBUFFERS_VERSION() {
   // clang-format on
 }
 
-const double kPi = 3.14159265358979323846;
+namespace {
 
-// clang-format off
-const char *const kTypeNames[] = {
-  #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \
-    IDLTYPE,
-    FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
-  #undef FLATBUFFERS_TD
-  nullptr
-};
-
-const char kTypeSizes[] = {
-  #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \
-    sizeof(CTYPE),
-    FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
-  #undef FLATBUFFERS_TD
-};
-// clang-format on
+static const double kPi = 3.14159265358979323846;
 
 // The enums in the reflection schema should match the ones we use internally.
 // Compare the last element to check if these go out of sync.
-static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union),
+static_assert(BASE_TYPE_VECTOR64 ==
+                  static_cast<BaseType>(reflection::MaxBaseType - 1),
               "enums don't match");
 
 // Any parsing calls have to be wrapped in this macro, which automates
@@ -92,117 +82,36 @@ static bool IsLowerSnakeCase(const std::string &str) {
   return true;
 }
 
-// Convert an underscore_based_identifier in to camelCase.
-// Also uppercases the first character if first is true.
-std::string MakeCamel(const std::string &in, bool first) {
-  std::string s;
-  for (size_t i = 0; i < in.length(); i++) {
-    if (!i && first)
-      s += CharToUpper(in[0]);
-    else if (in[i] == '_' && i + 1 < in.length())
-      s += CharToUpper(in[++i]);
-    else
-      s += in[i];
-  }
-  return s;
-}
-
-// Convert an underscore_based_identifier in to screaming snake case.
-std::string MakeScreamingCamel(const std::string &in) {
-  std::string s;
-  for (size_t i = 0; i < in.length(); i++) {
-    if (in[i] != '_')
-      s += CharToUpper(in[i]);
-    else
-      s += in[i];
-  }
-  return s;
-}
-
-void DeserializeDoc(std::vector<std::string> &doc,
-                    const Vector<Offset<String>> *documentation) {
+static void DeserializeDoc(std::vector<std::string> &doc,
+                           const Vector<Offset<String>> *documentation) {
   if (documentation == nullptr) return;
   for (uoffset_t index = 0; index < documentation->size(); index++)
     doc.push_back(documentation->Get(index)->str());
 }
 
-void Parser::Message(const std::string &msg) {
-  if (!error_.empty()) error_ += "\n";  // log all warnings and errors
-  error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
-  // clang-format off
-
-  #ifdef _WIN32  // MSVC alike
-    error_ +=
-        "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")";
-  #else  // gcc alike
-    if (file_being_parsed_.length()) error_ += ":";
-    error_ += NumToString(line_) + ": " + NumToString(CursorPosition());
-  #endif
-  // clang-format on
-  error_ += ": " + msg;
-}
-
-void Parser::Warning(const std::string &msg) {
-  if (!opts.no_warnings) Message("warning: " + msg);
-}
-
-CheckedError Parser::Error(const std::string &msg) {
-  Message("error: " + msg);
-  return CheckedError(true);
-}
-
-inline CheckedError NoError() { return CheckedError(false); }
+static CheckedError NoError() { return CheckedError(false); }
 
-CheckedError Parser::RecurseError() {
-  return Error("maximum parsing depth " + NumToString(parse_depth_counter_) +
-               " reached");
-}
-
-class Parser::ParseDepthGuard {
- public:
-  explicit ParseDepthGuard(Parser *parser_not_null)
-      : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) {
-    FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) &&
-                       "Check() must be called to prevent stack overflow");
-    parser_.parse_depth_counter_ += 1;
-  }
-
-  ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; }
-
-  CheckedError Check() {
-    return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH)
-               ? parser_.RecurseError()
-               : CheckedError(false);
-  }
-
-  FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &));
-  FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &));
-
- private:
-  Parser &parser_;
-  const int caller_depth_;
-};
-
-template<typename T> std::string TypeToIntervalString() {
+template<typename T> static std::string TypeToIntervalString() {
   return "[" + NumToString((flatbuffers::numeric_limits<T>::lowest)()) + "; " +
          NumToString((flatbuffers::numeric_limits<T>::max)()) + "]";
 }
 
 // atot: template version of atoi/atof: convert a string to an instance of T.
 template<typename T>
-bool atot_scalar(const char *s, T *val, bool_constant<false>) {
+static bool atot_scalar(const char *s, T *val, bool_constant<false>) {
   return StringToNumber(s, val);
 }
 
 template<typename T>
-bool atot_scalar(const char *s, T *val, bool_constant<true>) {
+static bool atot_scalar(const char *s, T *val, bool_constant<true>) {
   // Normalize NaN parsed from fbs or json to unsigned NaN.
   if (false == StringToNumber(s, val)) return false;
   *val = (*val != *val) ? std::fabs(*val) : *val;
   return true;
 }
 
-template<typename T> CheckedError atot(const char *s, Parser &parser, T *val) {
+template<typename T>
+static CheckedError atot(const char *s, Parser &parser, T *val) {
   auto done = atot_scalar(s, val, bool_constant<is_floating_point<T>::value>());
   if (done) return NoError();
   if (0 == *val)
@@ -212,33 +121,26 @@ template<typename T> CheckedError atot(const char *s, Parser &parser, T *val) {
                         ", constant does not fit " + TypeToIntervalString<T>());
 }
 template<>
-inline CheckedError atot<Offset<void>>(const char *s, Parser &parser,
-                                       Offset<void> *val) {
+CheckedError atot<Offset<void>>(const char *s, Parser &parser,
+                                Offset<void> *val) {
   (void)parser;
   *val = Offset<void>(atoi(s));
   return NoError();
 }
 
-std::string Namespace::GetFullyQualifiedName(const std::string &name,
-                                             size_t max_components) const {
-  // Early exit if we don't have a defined namespace.
-  if (components.empty() || !max_components) { return name; }
-  std::string stream_str;
-  for (size_t i = 0; i < std::min(components.size(), max_components); i++) {
-    stream_str += components[i];
-    stream_str += '.';
-  }
-  if (!stream_str.empty()) stream_str.pop_back();
-  if (name.length()) {
-    stream_str += '.';
-    stream_str += name;
-  }
-  return stream_str;
+template<>
+CheckedError atot<Offset64<void>>(const char *s, Parser &parser,
+                                  Offset64<void> *val) {
+  (void)parser;
+  *val = Offset64<void>(atoi(s));
+  return NoError();
 }
 
 template<typename T>
-T *LookupTableByName(const SymbolTable<T> &table, const std::string &name,
-                     const Namespace &current_namespace, size_t skip_top) {
+static T *LookupTableByName(const SymbolTable<T> &table,
+                            const std::string &name,
+                            const Namespace &current_namespace,
+                            size_t skip_top) {
   const auto &components = current_namespace.components;
   if (table.dict.empty()) return nullptr;
   if (components.size() < skip_top) return nullptr;
@@ -297,6 +199,254 @@ static std::string TokenToString(int t) {
 }
 // clang-format on
 
+static bool IsIdentifierStart(char c) { return is_alpha(c) || (c == '_'); }
+
+static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b,
+                                     const FieldDef &key) {
+  switch (key.value.type.base_type) {
+#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...)                       \
+  case BASE_TYPE_##ENUM: {                                              \
+    CTYPE def = static_cast<CTYPE>(0);                                  \
+    if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \
+    const auto av = a ? ReadScalar<CTYPE>(a) : def;                     \
+    const auto bv = b ? ReadScalar<CTYPE>(b) : def;                     \
+    return av < bv;                                                     \
+  }
+    FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
+#undef FLATBUFFERS_TD
+    default: {
+      FLATBUFFERS_ASSERT(false && "scalar type expected");
+      return false;
+    }
+  }
+}
+
+static bool CompareTablesByScalarKey(const Offset<Table> *_a,
+                                     const Offset<Table> *_b,
+                                     const FieldDef &key) {
+  const voffset_t offset = key.value.offset;
+  // Indirect offset pointer to table pointer.
+  auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
+  auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
+  // Fetch field address from table.
+  a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
+  b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
+  return CompareSerializedScalars(a, b, key);
+}
+
+static bool CompareTablesByStringKey(const Offset<Table> *_a,
+                                     const Offset<Table> *_b,
+                                     const FieldDef &key) {
+  const voffset_t offset = key.value.offset;
+  // Indirect offset pointer to table pointer.
+  auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
+  auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
+  // Fetch field address from table.
+  a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
+  b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
+  if (a && b) {
+    // Indirect offset pointer to string pointer.
+    a += ReadScalar<uoffset_t>(a);
+    b += ReadScalar<uoffset_t>(b);
+    return *reinterpret_cast<const String *>(a) <
+           *reinterpret_cast<const String *>(b);
+  } else {
+    return a ? true : false;
+  }
+}
+
+static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) {
+  // These are serialized offsets, so are relative where they are
+  // stored in memory, so compute the distance between these pointers:
+  ptrdiff_t diff = (b - a) * sizeof(Offset<Table>);
+  FLATBUFFERS_ASSERT(diff >= 0);  // Guaranteed by SimpleQsort.
+  auto udiff = static_cast<uoffset_t>(diff);
+  a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff);
+  b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff);
+  std::swap(*a, *b);
+}
+
+// See below for why we need our own sort :(
+template<typename T, typename F, typename S>
+static void SimpleQsort(T *begin, T *end, size_t width, F comparator,
+                        S swapper) {
+  if (end - begin <= static_cast<ptrdiff_t>(width)) return;
+  auto l = begin + width;
+  auto r = end;
+  while (l < r) {
+    if (comparator(begin, l)) {
+      r -= width;
+      swapper(l, r);
+    } else {
+      l += width;
+    }
+  }
+  l -= width;
+  swapper(begin, l);
+  SimpleQsort(begin, l, width, comparator, swapper);
+  SimpleQsort(r, end, width, comparator, swapper);
+}
+
+template<typename T> static inline void SingleValueRepack(Value &e, T val) {
+  // Remove leading zeros.
+  if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); }
+}
+
+#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
+// Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from
+// hex-float literal.
+static void SingleValueRepack(Value &e, float val) {
+  if (val != val) e.constant = "nan";
+}
+static void SingleValueRepack(Value &e, double val) {
+  if (val != val) e.constant = "nan";
+}
+#endif
+
+template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) {
+  if (e1 < e2) { std::swap(e1, e2); }  // use std for scalars
+  // Signed overflow may occur, use unsigned calculation.
+  // The unsigned overflow is well-defined by C++ standard (modulo 2^n).
+  return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2);
+}
+
+static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
+  auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
+  auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
+  return a_id < b_id;
+}
+
+static Namespace *GetNamespace(
+    const std::string &qualified_name, std::vector<Namespace *> &namespaces,
+    std::map<std::string, Namespace *> &namespaces_index) {
+  size_t dot = qualified_name.find_last_of('.');
+  std::string namespace_name = (dot != std::string::npos)
+                                   ? std::string(qualified_name.c_str(), dot)
+                                   : "";
+  Namespace *&ns = namespaces_index[namespace_name];
+
+  if (!ns) {
+    ns = new Namespace();
+    namespaces.push_back(ns);
+
+    size_t pos = 0;
+
+    for (;;) {
+      dot = qualified_name.find('.', pos);
+      if (dot == std::string::npos) { break; }
+      ns->components.push_back(qualified_name.substr(pos, dot - pos));
+      pos = dot + 1;
+    }
+  }
+
+  return ns;
+}
+
+// Generate a unique hash for a file based on its name and contents (if any).
+static uint64_t HashFile(const char *source_filename, const char *source) {
+  uint64_t hash = 0;
+
+  if (source_filename)
+    hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str());
+
+  if (source && *source) hash ^= HashFnv1a<uint64_t>(source);
+
+  return hash;
+}
+
+template<typename T> static bool compareName(const T *a, const T *b) {
+  return a->defined_namespace->GetFullyQualifiedName(a->name) <
+         b->defined_namespace->GetFullyQualifiedName(b->name);
+}
+
+template<typename T> static void AssignIndices(const std::vector<T *> &defvec) {
+  // Pre-sort these vectors, such that we can set the correct indices for them.
+  auto vec = defvec;
+  std::sort(vec.begin(), vec.end(), compareName<T>);
+  for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
+}
+
+}  // namespace
+
+void Parser::Message(const std::string &msg) {
+  if (!error_.empty()) error_ += "\n";  // log all warnings and errors
+  error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
+  // clang-format off
+
+  #ifdef _WIN32  // MSVC alike
+    error_ +=
+        "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")";
+  #else  // gcc alike
+    if (file_being_parsed_.length()) error_ += ":";
+    error_ += NumToString(line_) + ": " + NumToString(CursorPosition());
+  #endif
+  // clang-format on
+  error_ += ": " + msg;
+}
+
+void Parser::Warning(const std::string &msg) {
+  if (!opts.no_warnings) {
+    Message("warning: " + msg);
+    has_warning_ = true;  // for opts.warnings_as_errors
+  }
+}
+
+CheckedError Parser::Error(const std::string &msg) {
+  Message("error: " + msg);
+  return CheckedError(true);
+}
+
+CheckedError Parser::RecurseError() {
+  return Error("maximum parsing depth " + NumToString(parse_depth_counter_) +
+               " reached");
+}
+
+const std::string &Parser::GetPooledString(const std::string &s) const {
+  return *(string_cache_.insert(s).first);
+}
+
+class Parser::ParseDepthGuard {
+ public:
+  explicit ParseDepthGuard(Parser *parser_not_null)
+      : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) {
+    FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) &&
+                       "Check() must be called to prevent stack overflow");
+    parser_.parse_depth_counter_ += 1;
+  }
+
+  ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; }
+
+  CheckedError Check() {
+    return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH)
+               ? parser_.RecurseError()
+               : CheckedError(false);
+  }
+
+  FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &));
+  FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &));
+
+ private:
+  Parser &parser_;
+  const int caller_depth_;
+};
+
+std::string Namespace::GetFullyQualifiedName(const std::string &name,
+                                             size_t max_components) const {
+  // Early exit if we don't have a defined namespace.
+  if (components.empty() || !max_components) { return name; }
+  std::string stream_str;
+  for (size_t i = 0; i < std::min(components.size(), max_components); i++) {
+    stream_str += components[i];
+    stream_str += '.';
+  }
+  if (!stream_str.empty()) stream_str.pop_back();
+  if (name.length()) {
+    stream_str += '.';
+    stream_str += name;
+  }
+  return stream_str;
+}
+
 std::string Parser::TokenToStringId(int t) const {
   return t == kTokenIdentifier ? attribute_ : TokenToString(t);
 }
@@ -326,12 +476,9 @@ CheckedError Parser::SkipByteOrderMark() {
   return NoError();
 }
 
-static inline bool IsIdentifierStart(char c) {
-  return is_alpha(c) || (c == '_');
-}
-
 CheckedError Parser::Next() {
   doc_comment_.clear();
+  prev_cursor_ = cursor_;
   bool seen_newline = cursor_ == source_;
   attribute_.clear();
   attr_is_trivial_ascii_string_ = true;
@@ -356,6 +503,8 @@ CheckedError Parser::Next() {
       case ')':
       case '[':
       case ']':
+      case '<':
+      case '>':
       case ',':
       case ':':
       case ';':
@@ -508,10 +657,21 @@ CheckedError Parser::Next() {
         }
 
         const auto has_sign = (c == '+') || (c == '-');
-        if (has_sign && IsIdentifierStart(*cursor_)) {
-          // '-'/'+' and following identifier - it could be a predefined
-          // constant. Return the sign in token_, see ParseSingleValue.
-          return NoError();
+        if (has_sign) {
+          // Check for +/-inf which is considered a float constant.
+          if (strncmp(cursor_, "inf", 3) == 0 &&
+              !(IsIdentifierStart(cursor_[3]) || is_digit(cursor_[3]))) {
+            attribute_.assign(cursor_ - 1, cursor_ + 3);
+            token_ = kTokenFloatConstant;
+            cursor_ += 3;
+            return NoError();
+          }
+
+          if (IsIdentifierStart(*cursor_)) {
+            // '-'/'+' and following identifier - it could be a predefined
+            // constant. Return the sign in token_, see ParseSingleValue.
+            return NoError();
+          }
         }
 
         auto dot_lvl =
@@ -753,6 +913,15 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
   ECHECK(ParseType(type));
 
   if (struct_def.fixed) {
+    if (IsIncompleteStruct(type) ||
+        (IsArray(type) && IsIncompleteStruct(type.VectorType()))) {
+      std::string type_name = IsArray(type) ? type.VectorType().struct_def->name
+                                            : type.struct_def->name;
+      return Error(
+          std::string("Incomplete type in struct is not allowed, type name: ") +
+          type_name);
+    }
+
     auto valid = IsScalar(type.base_type) || IsStruct(type);
     if (!valid && IsArray(type)) {
       const auto &elem_type = type.VectorType();
@@ -799,6 +968,14 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
   FieldDef *field;
   ECHECK(AddField(struct_def, name, type, &field));
 
+  if (typefield) {
+    // We preserve the relation between the typefield
+    // and field, so we can easily map it in the code
+    // generators.
+    typefield->sibling_union_field = field;
+    field->sibling_union_field = typefield;
+  }
+
   if (token_ == '=') {
     NEXT();
     ECHECK(ParseSingleValue(&field->name, field->value, true));
@@ -808,8 +985,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
           "or in structs.");
     if (IsString(type) || IsVector(type)) {
       advanced_features_ |= reflection::DefaultVectorsAndStrings;
-      if (field->value.constant != "0" && field->value.constant != "null" &&
-          !SupportsDefaultVectorsAndStrings()) {
+      if (field->value.constant != "0" && !SupportsDefaultVectorsAndStrings()) {
         return Error(
             "Default values for strings and vectors are not supported in one "
             "of the specified programming languages");
@@ -872,6 +1048,65 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
     }
   }
 
+  if (field->attributes.Lookup("vector64") != nullptr) {
+    if (!IsVector(type)) {
+      return Error("`vector64` attribute can only be applied on vectors.");
+    }
+
+    // Upgrade the type to be a BASE_TYPE_VECTOR64, since the attributes are
+    // parsed after the type.
+    const BaseType element_base_type = type.element;
+    type = Type(BASE_TYPE_VECTOR64, type.struct_def, type.enum_def);
+    type.element = element_base_type;
+
+    // Since the field was already added to the parent object, update the type
+    // in place.
+    field->value.type = type;
+
+    // 64-bit vectors imply the offset64 attribute.
+    field->offset64 = true;
+  }
+
+  // Record that this field uses 64-bit offsets.
+  if (field->attributes.Lookup("offset64") != nullptr) {
+    // TODO(derekbailey): would be nice to have this be a recommendation or hint
+    // instead of a warning.
+    if (type.base_type == BASE_TYPE_VECTOR64) {
+      Warning("attribute `vector64` implies `offset64` and isn't required.");
+    }
+
+    field->offset64 = true;
+  }
+
+  // Check for common conditions with Offset64 fields.
+  if (field->offset64) {
+    // TODO(derekbailey): this is where we can disable string support for
+    // offset64, as that is not a hard requirement to have.
+    if (!IsString(type) && !IsVector(type)) {
+      return Error(
+          "only string and vectors can have `offset64` attribute applied");
+    }
+
+    // If this is a Vector, only scalar and scalar-like (structs) items are
+    // allowed.
+    // TODO(derekbailey): allow vector of strings, just require that the strings
+    // are Offset64<string>.
+    if (IsVector(type) &&
+        !((IsScalar(type.element) && !IsEnum(type.VectorType())) ||
+          IsStruct(type.VectorType()))) {
+      return Error("only vectors of scalars are allowed to be 64-bit.");
+    }
+
+    // Lastly, check if it is supported by the specified generated languages. Do
+    // this last so the above checks can inform the user of schema errors to fix
+    // first.
+    if (!Supports64BitOffsets()) {
+      return Error(
+          "fields using 64-bit offsets are not yet supported in at least one "
+          "of the specified programming languages.");
+    }
+  }
+
   // For historical convenience reasons, string keys are assumed required.
   // Scalars are kDefault unless otherwise specified.
   // Nonscalars are kOptional unless required;
@@ -894,8 +1129,16 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
   if (field->key) {
     if (struct_def.has_key) return Error("only one field may be set as 'key'");
     struct_def.has_key = true;
-    if (!IsScalar(type.base_type) && !IsString(type)) {
-      return Error("'key' field must be string or scalar type");
+    auto is_valid =
+        IsScalar(type.base_type) || IsString(type) || IsStruct(type);
+    if (IsArray(type)) {
+      is_valid |=
+          IsScalar(type.VectorType().base_type) || IsStruct(type.VectorType());
+    }
+    if (!is_valid) {
+      return Error(
+          "'key' field must be string, scalar type or fixed size array of "
+          "scalars");
     }
   }
 
@@ -914,7 +1157,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
     }
     if (!SupportsOptionalScalars()) {
       return Error(
-          "Optional scalars are not yet supported in at least one the of "
+          "Optional scalars are not yet supported in at least one of "
           "the specified programming languages.");
     }
   }
@@ -978,15 +1221,19 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
         "definition");
 
   field->native_inline = field->attributes.Lookup("native_inline") != nullptr;
-  if (field->native_inline && !IsStruct(field->value.type))
-    return Error("native_inline can only be defined on structs");
+  if (field->native_inline && !IsStruct(field->value.type) &&
+      !IsVectorOfStruct(field->value.type) &&
+      !IsVectorOfTable(field->value.type))
+    return Error(
+        "'native_inline' can only be defined on structs, vector of structs or "
+        "vector of tables");
 
   auto nested = field->attributes.Lookup("nested_flatbuffer");
   if (nested) {
     if (nested->type.base_type != BASE_TYPE_STRING)
       return Error(
           "nested_flatbuffer attribute must be a string (the root type)");
-    if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR)
+    if (!IsVector(type.base_type) || type.element != BASE_TYPE_UCHAR)
       return Error(
           "nested_flatbuffer attribute may only apply to a vector of ubyte");
     // This will cause an error if the root type of the nested flatbuffer
@@ -1055,7 +1302,7 @@ CheckedError Parser::ParseComma() {
 CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
                                    size_t parent_fieldn,
                                    const StructDef *parent_struct_def,
-                                   uoffset_t count, bool inside_vector) {
+                                   size_t count, bool inside_vector) {
   switch (val.type.base_type) {
     case BASE_TYPE_UNION: {
       FLATBUFFERS_ASSERT(field);
@@ -1121,7 +1368,11 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
       }
       uint8_t enum_idx;
       if (vector_of_union_types) {
-        enum_idx = vector_of_union_types->Get(count);
+        if (vector_of_union_types->size() <= count)
+          return Error(
+              "union types vector smaller than union values vector for: " +
+              field->name);
+        enum_idx = vector_of_union_types->Get(static_cast<uoffset_t>(count));
       } else {
         ECHECK(atot(constant.c_str(), *this, &enum_idx));
       }
@@ -1150,9 +1401,10 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field,
       ECHECK(ParseString(val, field->shared));
       break;
     }
+    case BASE_TYPE_VECTOR64:
     case BASE_TYPE_VECTOR: {
       uoffset_t off;
-      ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn));
+      ECHECK(ParseVector(val.type, &off, field, parent_fieldn));
       val.constant = NumToString(off);
       break;
     }
@@ -1324,6 +1576,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
   for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size;
        size /= 2) {
     // Go through elements in reverse, since we're building the data backwards.
+    // TODO(derekbailey): this doesn't work when there are Offset64 fields, as
+    // those have to be built first. So this needs to be changed to iterate over
+    // Offset64 then Offset32 fields.
     for (auto it = field_stack_.rbegin();
          it != field_stack_.rbegin() + fieldn_outer; ++it) {
       auto &field_value = it->first;
@@ -1340,10 +1595,18 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
                 ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
                 builder_.PushElement(val); \
               } else { \
-                CTYPE val, valdef; \
-                ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
-                ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
-                builder_.AddElement(field_value.offset, val, valdef); \
+                if (field->IsScalarOptional()) { \
+                  if (field_value.constant != "null") { \
+                    CTYPE val; \
+                    ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                    builder_.AddElement(field_value.offset, val); \
+                  } \
+                } else { \
+                  CTYPE val, valdef; \
+                  ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                  ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \
+                  builder_.AddElement(field_value.offset, val, valdef); \
+                } \
               } \
               break;
             FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
@@ -1354,9 +1617,16 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
               if (IsStruct(field->value.type)) { \
                 SerializeStruct(*field->value.type.struct_def, field_value); \
               } else { \
-                CTYPE val; \
-                ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
-                builder_.AddOffset(field_value.offset, val); \
+                /* Special case for fields that use 64-bit addressing */ \
+                if(field->offset64) { \
+                  Offset64<void> offset; \
+                  ECHECK(atot(field_value.constant.c_str(), *this, &offset)); \
+                  builder_.AddOffset(field_value.offset, offset); \
+                } else { \
+                  CTYPE val; \
+                  ECHECK(atot(field_value.constant.c_str(), *this, &val)); \
+                  builder_.AddOffset(field_value.offset, val); \
+                } \
               } \
               break;
             FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD)
@@ -1394,7 +1664,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value,
 }
 
 template<typename F>
-CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
+CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) {
   EXPECT('[');
   for (;;) {
     if ((!opts.strict_json || !count) && Is(']')) break;
@@ -1407,91 +1677,6 @@ CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) {
   return NoError();
 }
 
-static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b,
-                                     const FieldDef &key) {
-  switch (key.value.type.base_type) {
-#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...)                       \
-  case BASE_TYPE_##ENUM: {                                              \
-    CTYPE def = static_cast<CTYPE>(0);                                  \
-    if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \
-    const auto av = a ? ReadScalar<CTYPE>(a) : def;                     \
-    const auto bv = b ? ReadScalar<CTYPE>(b) : def;                     \
-    return av < bv;                                                     \
-  }
-    FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD)
-#undef FLATBUFFERS_TD
-    default: {
-      FLATBUFFERS_ASSERT(false && "scalar type expected");
-      return false;
-    }
-  }
-}
-
-static bool CompareTablesByScalarKey(const Offset<Table> *_a,
-                                     const Offset<Table> *_b,
-                                     const FieldDef &key) {
-  const voffset_t offset = key.value.offset;
-  // Indirect offset pointer to table pointer.
-  auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
-  auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
-  // Fetch field address from table.
-  a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
-  b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
-  return CompareSerializedScalars(a, b, key);
-}
-
-static bool CompareTablesByStringKey(const Offset<Table> *_a,
-                                     const Offset<Table> *_b,
-                                     const FieldDef &key) {
-  const voffset_t offset = key.value.offset;
-  // Indirect offset pointer to table pointer.
-  auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a);
-  auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b);
-  // Fetch field address from table.
-  a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset);
-  b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset);
-  if (a && b) {
-    // Indirect offset pointer to string pointer.
-    a += ReadScalar<uoffset_t>(a);
-    b += ReadScalar<uoffset_t>(b);
-    return *reinterpret_cast<const String *>(a) <
-           *reinterpret_cast<const String *>(b);
-  } else {
-    return a ? true : false;
-  }
-}
-
-static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) {
-  // These are serialized offsets, so are relative where they are
-  // stored in memory, so compute the distance between these pointers:
-  ptrdiff_t diff = (b - a) * sizeof(Offset<Table>);
-  FLATBUFFERS_ASSERT(diff >= 0);  // Guaranteed by SimpleQsort.
-  auto udiff = static_cast<uoffset_t>(diff);
-  a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff);
-  b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff);
-  std::swap(*a, *b);
-}
-
-// See below for why we need our own sort :(
-template<typename T, typename F, typename S>
-void SimpleQsort(T *begin, T *end, size_t width, F comparator, S swapper) {
-  if (end - begin <= static_cast<ptrdiff_t>(width)) return;
-  auto l = begin + width;
-  auto r = end;
-  while (l < r) {
-    if (comparator(begin, l)) {
-      r -= width;
-      swapper(l, r);
-    } else {
-      l += width;
-    }
-  }
-  l -= width;
-  swapper(begin, l);
-  SimpleQsort(begin, l, width, comparator, swapper);
-  SimpleQsort(r, end, width, comparator, swapper);
-}
-
 CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
                                          size_t min_align, size_t *align) {
   // Use uint8_t to avoid problems with size_t==`unsigned long` on LP64.
@@ -1509,10 +1694,11 @@ CheckedError Parser::ParseAlignAttribute(const std::string &align_constant,
                NumToString(FLATBUFFERS_MAX_ALIGNMENT));
 }
 
-CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
+CheckedError Parser::ParseVector(const Type &vector_type, uoffset_t *ovalue,
                                  FieldDef *field, size_t fieldn) {
-  uoffset_t count = 0;
-  auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
+  Type type = vector_type.VectorType();
+  size_t count = 0;
+  auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
     Value val;
     val.type = type;
     ECHECK(ParseAnyValue(val, field, fieldn, nullptr, count, true));
@@ -1521,6 +1707,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
   });
   ECHECK(err);
 
+  const size_t alignment = InlineAlignment(type);
   const size_t len = count * InlineSize(type) / InlineAlignment(type);
   const size_t elemsize = InlineAlignment(type);
   const auto force_align = field->attributes.Lookup("force_align");
@@ -1530,8 +1717,15 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
     if (align > 1) { builder_.ForceVectorAlignment(len, elemsize, align); }
   }
 
-  builder_.StartVector(len, elemsize);
-  for (uoffset_t i = 0; i < count; i++) {
+  // TODO Fix using element alignment as size (`elemsize`)!
+  if (vector_type.base_type == BASE_TYPE_VECTOR64) {
+    // TODO(derekbailey): this requires a 64-bit builder.
+    // builder_.StartVector<Offset64, uoffset64_t>(len, elemsize, alignment);
+    builder_.StartVector(len, elemsize, alignment);
+  } else {
+    builder_.StartVector(len, elemsize, alignment);
+  }
+  for (size_t i = 0; i < count; i++) {
     // start at the back, since we're building the data backwards.
     auto &val = field_stack_.back().first;
     switch (val.type.base_type) {
@@ -1553,7 +1747,11 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue,
   }
 
   builder_.ClearOffsets();
-  *ovalue = builder_.EndVector(count);
+  if (vector_type.base_type == BASE_TYPE_VECTOR64) {
+    *ovalue = builder_.EndVector<uoffset64_t>(count);
+  } else {
+    *ovalue = builder_.EndVector(count);
+  }
 
   if (type.base_type == BASE_TYPE_STRUCT && type.struct_def->has_key) {
     // We should sort this vector. Find the key first.
@@ -1621,9 +1819,9 @@ CheckedError Parser::ParseArray(Value &array) {
   FlatBufferBuilder builder;
   const auto &type = array.type.VectorType();
   auto length = array.type.fixed_length;
-  uoffset_t count = 0;
-  auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
-    vector_emplace_back(&stack, Value());
+  size_t count = 0;
+  auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
+    stack.emplace_back(Value());
     auto &val = stack.back();
     val.type = type;
     if (IsStruct(type)) {
@@ -1667,7 +1865,13 @@ CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field,
                                            size_t fieldn,
                                            const StructDef *parent_struct_def) {
   if (token_ == '[') {  // backwards compat for 'legacy' ubyte buffers
-    ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0));
+    if (opts.json_nested_legacy_flatbuffers) {
+      ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0));
+    } else {
+      return Error(
+          "cannot parse nested_flatbuffer as bytes unless"
+          " --json-nested-bytes is set");
+    }
   } else {
     auto cursor_at_value_begin = cursor_;
     ECHECK(SkipAnyJsonValue());
@@ -1812,22 +2016,6 @@ CheckedError Parser::TokenError() {
   return Error("cannot parse value starting with: " + TokenToStringId(token_));
 }
 
-// Re-pack helper (ParseSingleValue) to normalize defaults of scalars.
-template<typename T> inline void SingleValueRepack(Value &e, T val) {
-  // Remove leading zeros.
-  if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); }
-}
-#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0)
-// Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from
-// hex-float literal.
-static inline void SingleValueRepack(Value &e, float val) {
-  if (val != val) e.constant = "nan";
-}
-static inline void SingleValueRepack(Value &e, double val) {
-  if (val != val) e.constant = "nan";
-}
-#endif
-
 CheckedError Parser::ParseFunction(const std::string *name, Value &e) {
   ParseDepthGuard depth_guard(this);
   ECHECK(depth_guard.Check());
@@ -1836,8 +2024,8 @@ CheckedError Parser::ParseFunction(const std::string *name, Value &e) {
   const auto functionname = attribute_;
   if (!IsFloat(e.type.base_type)) {
     return Error(functionname + ": type of argument mismatch, expecting: " +
-                 kTypeNames[BASE_TYPE_DOUBLE] +
-                 ", found: " + kTypeNames[e.type.base_type] +
+                 TypeName(BASE_TYPE_DOUBLE) +
+                 ", found: " + TypeName(e.type.base_type) +
                  ", name: " + (name ? *name : "") + ", value: " + e.constant);
   }
   NEXT();
@@ -1883,8 +2071,7 @@ CheckedError Parser::TryTypedValue(const std::string *name, int dtoken,
       e.type.base_type = req;
     } else {
       return Error(std::string("type mismatch: expecting: ") +
-                   kTypeNames[e.type.base_type] +
-                   ", found: " + kTypeNames[req] +
+                   TypeName(e.type.base_type) + ", found: " + TypeName(req) +
                    ", name: " + (name ? *name : "") + ", value: " + e.constant);
     }
   }
@@ -1945,7 +2132,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e,
       return Error(
           std::string("type mismatch or invalid value, an initializer of "
                       "non-string field must be trivial ASCII string: type: ") +
-          kTypeNames[in_type] + ", name: " + (name ? *name : "") +
+          TypeName(in_type) + ", name: " + (name ? *name : "") +
           ", value: " + attribute_);
     }
 
@@ -2017,7 +2204,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e,
   if (!match) {
     std::string msg;
     msg += "Cannot assign token starting with '" + TokenToStringId(token_) +
-           "' to value of <" + std::string(kTypeNames[in_type]) + "> type.";
+           "' to value of <" + std::string(TypeName(in_type)) + "> type.";
     return Error(msg);
   }
   const auto match_type = e.type.base_type;  // may differ from in_type
@@ -2099,13 +2286,6 @@ const EnumVal *EnumDef::MaxValue() const {
   return vals.vec.empty() ? nullptr : vals.vec.back();
 }
 
-template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) {
-  if (e1 < e2) { std::swap(e1, e2); }  // use std for scalars
-  // Signed overflow may occur, use unsigned calculation.
-  // The unsigned overflow is well-defined by C++ standard (modulo 2^n).
-  return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2);
-}
-
 uint64_t EnumDef::Distance(const EnumVal *v1, const EnumVal *v2) const {
   return IsUInt64() ? EnumDistanceImpl(v1->GetAsUInt64(), v2->GetAsUInt64())
                     : EnumDistanceImpl(v1->GetAsInt64(), v2->GetAsInt64());
@@ -2148,10 +2328,14 @@ void EnumDef::SortByValue() {
   auto &v = vals.vec;
   if (IsUInt64())
     std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
+      if (e1->GetAsUInt64() == e2->GetAsUInt64()) {
+        return e1->name < e2->name;
+      }
       return e1->GetAsUInt64() < e2->GetAsUInt64();
     });
   else
     std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) {
+      if (e1->GetAsInt64() == e2->GetAsInt64()) { return e1->name < e2->name; }
       return e1->GetAsInt64() < e2->GetAsInt64();
     });
 }
@@ -2281,13 +2465,18 @@ struct EnumValBuilder {
   bool user_value;
 };
 
-CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) {
+CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest,
+                               const char *filename) {
   std::vector<std::string> enum_comment = doc_comment_;
   NEXT();
   std::string enum_name = attribute_;
   EXPECT(kTokenIdentifier);
   EnumDef *enum_def;
   ECHECK(StartEnum(enum_name, is_union, &enum_def));
+  if (filename != nullptr && !opts.project_root.empty()) {
+    enum_def->declaration_file =
+        &GetPooledString(RelativeToRootPath(opts.project_root, filename));
+  }
   enum_def->doc_comment = enum_comment;
   if (!is_union && !opts.proto_mode) {
     // Give specialized error message, since this type spec used to
@@ -2314,6 +2503,9 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) {
     // todo: Convert to the Error in the future?
     Warning("underlying type of bit_flags enum must be unsigned");
   }
+  if (enum_def->attributes.Lookup("force_align")) {
+    return Error("`force_align` is not a valid attribute for Enums. ");
+  }
   EnumValBuilder evb(*this, *enum_def);
   EXPECT('{');
   // A lot of code generatos expect that an enum is not-empty.
@@ -2360,14 +2552,17 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) {
         EXPECT(kTokenIntegerConstant);
       }
 
-      ECHECK(evb.AcceptEnumerator());
-
       if (opts.proto_mode && Is('[')) {
         NEXT();
         // ignore attributes on enums.
         while (token_ != ']') NEXT();
         NEXT();
+      } else {
+        // parse attributes in fbs schema
+        ECHECK(ParseMetaData(&ev.attributes));
       }
+
+      ECHECK(evb.AcceptEnumerator());
     }
     if (!Is(opts.proto_mode ? ';' : ',')) break;
     NEXT();
@@ -2407,14 +2602,18 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) {
   }
 
   if (dest) *dest = enum_def;
-  types_.Add(current_namespace_->GetFullyQualifiedName(enum_def->name),
-             new Type(BASE_TYPE_UNION, nullptr, enum_def));
+  const auto qualified_name =
+      current_namespace_->GetFullyQualifiedName(enum_def->name);
+  if (types_.Add(qualified_name, new Type(BASE_TYPE_UNION, nullptr, enum_def)))
+    return Error("datatype already exists: " + qualified_name);
   return NoError();
 }
 
 CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) {
   auto &struct_def = *LookupCreateStruct(name, true, true);
-  if (!struct_def.predecl) return Error("datatype already exists: " + name);
+  if (!struct_def.predecl)
+    return Error("datatype already exists: " +
+                 current_namespace_->GetFullyQualifiedName(name));
   struct_def.predecl = false;
   struct_def.name = name;
   struct_def.file = file_being_parsed_;
@@ -2446,11 +2645,20 @@ CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields,
   return NoError();
 }
 
+std::vector<IncludedFile> Parser::GetIncludedFiles() const {
+  const auto it = files_included_per_file_.find(file_being_parsed_);
+  if (it == files_included_per_file_.end()) { return {}; }
+
+  return { it->second.cbegin(), it->second.cend() };
+}
+
 bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions &opts) {
   static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
       IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster |
       IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava |
-      IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary;
+      IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary |
+      IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson |
+      IDLOptions::kNim;
   unsigned long langs = opts.lang_to_generate;
   return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs);
 }
@@ -2461,23 +2669,28 @@ bool Parser::SupportsOptionalScalars() const {
 
 bool Parser::SupportsDefaultVectorsAndStrings() const {
   static FLATBUFFERS_CONSTEXPR unsigned long supported_langs =
-      IDLOptions::kRust | IDLOptions::kSwift;
+      IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kNim;
   return !(opts.lang_to_generate & ~supported_langs);
 }
 
 bool Parser::SupportsAdvancedUnionFeatures() const {
-  return opts.lang_to_generate != 0 &&
-         (opts.lang_to_generate &
+  return (opts.lang_to_generate &
           ~(IDLOptions::kCpp | IDLOptions::kTs | IDLOptions::kPhp |
             IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kKotlin |
-            IDLOptions::kBinary | IDLOptions::kSwift)) == 0;
+            IDLOptions::kBinary | IDLOptions::kSwift | IDLOptions::kNim |
+            IDLOptions::kJson)) == 0;
 }
 
 bool Parser::SupportsAdvancedArrayFeatures() const {
   return (opts.lang_to_generate &
           ~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava |
             IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson |
-            IDLOptions::kBinary | IDLOptions::kRust)) == 0;
+            IDLOptions::kBinary | IDLOptions::kRust | IDLOptions::kTs)) == 0;
+}
+
+bool Parser::Supports64BitOffsets() const {
+  return (opts.lang_to_generate &
+          ~(IDLOptions::kCpp | IDLOptions::kJson | IDLOptions::kBinary)) == 0;
 }
 
 Namespace *Parser::UniqueNamespace(Namespace *ns) {
@@ -2506,13 +2719,7 @@ std::string Parser::UnqualifiedName(const std::string &full_qualified_name) {
   return full_qualified_name.substr(previous, current - previous);
 }
 
-static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) {
-  auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str());
-  auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str());
-  return a_id < b_id;
-}
-
-CheckedError Parser::ParseDecl() {
+CheckedError Parser::ParseDecl(const char *filename) {
   std::vector<std::string> dc = doc_comment_;
   bool fixed = IsIdent("struct");
   if (!fixed && !IsIdent("table")) return Error("declaration expected");
@@ -2523,6 +2730,10 @@ CheckedError Parser::ParseDecl() {
   ECHECK(StartStruct(name, &struct_def));
   struct_def->doc_comment = dc;
   struct_def->fixed = fixed;
+  if (filename && !opts.project_root.empty()) {
+    struct_def->declaration_file =
+        &GetPooledString(RelativeToRootPath(opts.project_root, filename));
+  }
   ECHECK(ParseMetaData(&struct_def->attributes));
   struct_def->sortbysize =
       struct_def->attributes.Lookup("original_order") == nullptr && !fixed;
@@ -2568,6 +2779,7 @@ CheckedError Parser::ParseDecl() {
       for (voffset_t i = 0; i < static_cast<voffset_t>(fields.size()); i++) {
         auto &field = *fields[i];
         const auto &id_str = field.attributes.Lookup("id")->constant;
+
         // Metadata values have a dynamic type, they can be `float`, 'int', or
         // 'string`.
         // The FieldIndexToOffset(i) expects the voffset_t so `id` is limited by
@@ -2594,12 +2806,15 @@ CheckedError Parser::ParseDecl() {
   ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING));
   ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING));
   EXPECT('}');
-  types_.Add(current_namespace_->GetFullyQualifiedName(struct_def->name),
-             new Type(BASE_TYPE_STRUCT, struct_def, nullptr));
+  const auto qualified_name =
+      current_namespace_->GetFullyQualifiedName(struct_def->name);
+  if (types_.Add(qualified_name,
+                 new Type(BASE_TYPE_STRUCT, struct_def, nullptr)))
+    return Error("datatype already exists: " + qualified_name);
   return NoError();
 }
 
-CheckedError Parser::ParseService() {
+CheckedError Parser::ParseService(const char *filename) {
   std::vector<std::string> service_comment = doc_comment_;
   NEXT();
   auto service_name = attribute_;
@@ -2609,6 +2824,10 @@ CheckedError Parser::ParseService() {
   service_def.file = file_being_parsed_;
   service_def.doc_comment = service_comment;
   service_def.defined_namespace = current_namespace_;
+  if (filename != nullptr && !opts.project_root.empty()) {
+    service_def.declaration_file =
+        &GetPooledString(RelativeToRootPath(opts.project_root, filename));
+  }
   if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name),
                     &service_def))
     return Error("service already exists: " + service_name);
@@ -2724,7 +2943,7 @@ CheckedError Parser::ParseProtoDecl() {
   } else if (IsIdent("enum")) {
     // These are almost the same, just with different terminator:
     EnumDef *enum_def;
-    ECHECK(ParseEnum(false, &enum_def));
+    ECHECK(ParseEnum(false, &enum_def, nullptr));
     if (Is(';')) NEXT();
     // Temp: remove any duplicates, as .fbs files can't handle them.
     enum_def->RemoveDuplicates();
@@ -2747,17 +2966,17 @@ CheckedError Parser::ParseProtoDecl() {
   return NoError();
 }
 
-CheckedError Parser::StartEnum(const std::string &enum_name, bool is_union,
+CheckedError Parser::StartEnum(const std::string &name, bool is_union,
                                EnumDef **dest) {
   auto &enum_def = *new EnumDef();
-  enum_def.name = enum_name;
+  enum_def.name = name;
   enum_def.file = file_being_parsed_;
   enum_def.doc_comment = doc_comment_;
   enum_def.is_union = is_union;
   enum_def.defined_namespace = current_namespace_;
-  if (enums_.Add(current_namespace_->GetFullyQualifiedName(enum_name),
-                 &enum_def))
-    return Error("enum already exists: " + enum_name);
+  const auto qualified_name = current_namespace_->GetFullyQualifiedName(name);
+  if (enums_.Add(qualified_name, &enum_def))
+    return Error("enum already exists: " + qualified_name);
   enum_def.underlying_type.base_type =
       is_union ? BASE_TYPE_UTYPE : BASE_TYPE_INT;
   enum_def.underlying_type.enum_def = &enum_def;
@@ -2784,9 +3003,43 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       ECHECK(ParseProtoOption());
       EXPECT(';');
     } else if (IsIdent("reserved")) {  // Skip these.
+      /**
+       * Reserved proto ids can be comma seperated (e.g. 1,2,4,5;)
+       * or range based (e.g. 9 to 11;)
+       * or combination of them (e.g. 1,2,9 to 11,4,5;)
+       * It will be ended by a semicolon.
+       */
       NEXT();
-      while (!Is(';')) { NEXT(); }  // A variety of formats, just skip.
+      bool range = false;
+      voffset_t from = 0;
+
+      while (!Is(';')) {
+        if (token_ == kTokenIntegerConstant) {
+          voffset_t attribute = 0;
+          bool done = StringToNumber(attribute_.c_str(), &attribute);
+          if (!done)
+            return Error("Protobuf has non positive number in reserved ids");
+
+          if (range) {
+            for (voffset_t id = from + 1; id <= attribute; id++)
+              struct_def->reserved_ids.push_back(id);
+
+            range = false;
+          } else {
+            struct_def->reserved_ids.push_back(attribute);
+          }
+
+          from = attribute;
+        }
+
+        if (attribute_ == "to") range = true;
+
+        NEXT();
+      }  // A variety of formats, just skip.
+
       NEXT();
+    } else if (IsIdent("map")) {
+      ECHECK(ParseProtoMapField(struct_def));
     } else {
       std::vector<std::string> field_comment = doc_comment_;
       // Parse the qualifier.
@@ -2816,7 +3069,7 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       if (IsIdent("group") || oneof) {
         if (!oneof) NEXT();
         if (oneof && opts.proto_oneof_union) {
-          auto name = MakeCamel(attribute_, true) + "Union";
+          auto name = ConvertCase(attribute_, Case::kUpperCamel) + "Union";
           ECHECK(StartEnum(name, true, &oneof_union));
           type = Type(BASE_TYPE_UNION, nullptr, oneof_union);
         } else {
@@ -2841,11 +3094,13 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       }
       std::string name = attribute_;
       EXPECT(kTokenIdentifier);
+      std::string proto_field_id;
       if (!oneof) {
         // Parse the field id. Since we're just translating schemas, not
         // any kind of binary compatibility, we can safely ignore these, and
         // assign our own.
         EXPECT('=');
+        proto_field_id = attribute_;
         EXPECT(kTokenIntegerConstant);
       }
       FieldDef *field = nullptr;
@@ -2856,6 +3111,11 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
       }
       if (!field) ECHECK(AddField(*struct_def, name, type, &field));
       field->doc_comment = field_comment;
+      if (!proto_field_id.empty() || oneof) {
+        auto val = new Value();
+        val->constant = proto_field_id;
+        field->attributes.Add("id", val);
+      }
       if (!IsScalar(type.base_type) && required) {
         field->presence = FieldDef::kRequired;
       }
@@ -2871,7 +3131,11 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
           if (key == "default") {
             // Temp: skip non-numeric and non-boolean defaults (enums).
             auto numeric = strpbrk(val.c_str(), "0123456789-+.");
-            if (IsScalar(type.base_type) && numeric == val.c_str()) {
+            if (IsFloat(type.base_type) &&
+                (val == "inf" || val == "+inf" || val == "-inf")) {
+              // Prefer to be explicit with +inf.
+              field->value.constant = val == "inf" ? "+inf" : val;
+            } else if (IsScalar(type.base_type) && numeric == val.c_str()) {
               field->value.constant = val;
             } else if (val == "true") {
               field->value.constant = val;
@@ -2917,6 +3181,47 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend,
   return NoError();
 }
 
+CheckedError Parser::ParseProtoMapField(StructDef *struct_def) {
+  NEXT();
+  EXPECT('<');
+  Type key_type;
+  ECHECK(ParseType(key_type));
+  EXPECT(',');
+  Type value_type;
+  ECHECK(ParseType(value_type));
+  EXPECT('>');
+  auto field_name = attribute_;
+  NEXT();
+  EXPECT('=');
+  std::string proto_field_id = attribute_;
+  EXPECT(kTokenIntegerConstant);
+  EXPECT(';');
+
+  auto entry_table_name = ConvertCase(field_name, Case::kUpperCamel) + "Entry";
+  StructDef *entry_table;
+  ECHECK(StartStruct(entry_table_name, &entry_table));
+  entry_table->has_key = true;
+  FieldDef *key_field;
+  ECHECK(AddField(*entry_table, "key", key_type, &key_field));
+  key_field->key = true;
+  FieldDef *value_field;
+  ECHECK(AddField(*entry_table, "value", value_type, &value_field));
+
+  Type field_type;
+  field_type.base_type = BASE_TYPE_VECTOR;
+  field_type.element = BASE_TYPE_STRUCT;
+  field_type.struct_def = entry_table;
+  FieldDef *field;
+  ECHECK(AddField(*struct_def, field_name, field_type, &field));
+  if (!proto_field_id.empty()) {
+    auto val = new Value();
+    val->constant = proto_field_id;
+    field->attributes.Add("id", val);
+  }
+
+  return NoError();
+}
+
 CheckedError Parser::ParseProtoKey() {
   if (token_ == '(') {
     NEXT();
@@ -3010,16 +3315,16 @@ CheckedError Parser::SkipAnyJsonValue() {
                                   });
     }
     case '[': {
-      uoffset_t count = 0;
-      return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
-        return SkipAnyJsonValue();
-      });
+      size_t count = 0;
+      return ParseVectorDelimiters(
+          count, [&](size_t &) -> CheckedError { return SkipAnyJsonValue(); });
     }
     case kTokenStringConstant:
     case kTokenIntegerConstant:
     case kTokenFloatConstant: NEXT(); break;
     default:
-      if (IsIdent("true") || IsIdent("false") || IsIdent("null")) {
+      if (IsIdent("true") || IsIdent("false") || IsIdent("null") ||
+          IsIdent("inf")) {
         NEXT();
       } else
         return TokenError();
@@ -3061,8 +3366,8 @@ CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) {
     }
     case '[': {
       auto start = builder->StartVector();
-      uoffset_t count = 0;
-      ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError {
+      size_t count = 0;
+      ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError {
         return ParseFlexBufferValue(builder);
       }));
       builder->EndVector(start, false, false);
@@ -3150,6 +3455,10 @@ bool Parser::ParseJson(const char *json, const char *json_filename) {
   return done;
 }
 
+std::ptrdiff_t Parser::BytesConsumed() const {
+  return std::distance(source_, prev_cursor_);
+}
+
 CheckedError Parser::StartParseFile(const char *source,
                                     const char *source_filename) {
   file_being_parsed_ = source_filename ? source_filename : "";
@@ -3232,30 +3541,76 @@ CheckedError Parser::ParseRoot(const char *source, const char **include_paths,
       for (auto val_it = enum_def.Vals().begin();
            val_it != enum_def.Vals().end(); ++val_it) {
         auto &val = **val_it;
-        if (!SupportsAdvancedUnionFeatures() &&
+
+        if (!(opts.lang_to_generate != 0 && SupportsAdvancedUnionFeatures()) &&
             (IsStruct(val.union_type) || IsString(val.union_type)))
+
           return Error(
               "only tables can be union elements in the generated language: " +
               val.name);
       }
     }
   }
+
+  auto err = CheckPrivateLeak();
+  if (err.Check()) return err;
+
   // Parse JSON object only if the scheme has been parsed.
   if (token_ == '{') { ECHECK(DoParseJson()); }
-  EXPECT(kTokenEof);
   return NoError();
 }
 
-// Generate a unique hash for a file based on its name and contents (if any).
-static uint64_t HashFile(const char *source_filename, const char *source) {
-  uint64_t hash = 0;
-
-  if (source_filename)
-    hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str());
-
-  if (source && *source) hash ^= HashFnv1a<uint64_t>(source);
+CheckedError Parser::CheckPrivateLeak() {
+  if (!opts.no_leak_private_annotations) return NoError();
+  // Iterate over all structs/tables to validate we arent leaking
+  // any private (structs/tables/enums)
+  for (auto it = structs_.vec.begin(); it != structs_.vec.end(); it++) {
+    auto &struct_def = **it;
+    for (auto fld_it = struct_def.fields.vec.begin();
+         fld_it != struct_def.fields.vec.end(); ++fld_it) {
+      auto &field = **fld_it;
+
+      if (field.value.type.enum_def) {
+        auto err =
+            CheckPrivatelyLeakedFields(struct_def, *field.value.type.enum_def);
+        if (err.Check()) { return err; }
+      } else if (field.value.type.struct_def) {
+        auto err = CheckPrivatelyLeakedFields(struct_def,
+                                              *field.value.type.struct_def);
+        if (err.Check()) { return err; }
+      }
+    }
+  }
+  // Iterate over all enums to validate we arent leaking
+  // any private (structs/tables)
+  for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
+    auto &enum_def = **it;
+    if (enum_def.is_union) {
+      for (auto val_it = enum_def.Vals().begin();
+           val_it != enum_def.Vals().end(); ++val_it) {
+        auto &val = **val_it;
+        if (val.union_type.struct_def) {
+          auto err =
+              CheckPrivatelyLeakedFields(enum_def, *val.union_type.struct_def);
+          if (err.Check()) { return err; }
+        }
+      }
+    }
+  }
+  return NoError();
+}
 
-  return hash;
+CheckedError Parser::CheckPrivatelyLeakedFields(const Definition &def,
+                                                const Definition &value_type) {
+  if (!opts.no_leak_private_annotations) return NoError();
+  const auto is_private = def.attributes.Lookup("private");
+  const auto is_field_private = value_type.attributes.Lookup("private");
+  if (!is_private && is_field_private) {
+    return Error(
+        "Leaking private implementation, verify all objects have similar "
+        "annotations");
+  }
+  return NoError();
 }
 
 CheckedError Parser::DoParse(const char *source, const char **include_paths,
@@ -3272,7 +3627,7 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
 
     if (included_files_.find(source_hash) == included_files_.end()) {
       included_files_[source_hash] = include_filename ? include_filename : "";
-      files_included_per_file_[source_filename] = std::set<std::string>();
+      files_included_per_file_[source_filename] = std::set<IncludedFile>();
     } else {
       return NoError();
     }
@@ -3296,7 +3651,7 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
       ECHECK(ParseProtoDecl());
     } else if (IsIdent("native_include")) {
       NEXT();
-      vector_emplace_back(&native_included_files_, attribute_);
+      native_included_files_.emplace_back(attribute_);
       EXPECT(kTokenStringConstant);
       EXPECT(';');
     } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) {
@@ -3320,8 +3675,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
       }
       if (filepath.empty())
         return Error("unable to locate include file: " + name);
-      if (source_filename)
-        files_included_per_file_[source_filename].insert(filepath);
+      if (source_filename) {
+        IncludedFile included_file;
+        included_file.filename = filepath;
+        included_file.schema_name = name;
+        files_included_per_file_[source_filename].insert(included_file);
+      }
 
       std::string contents;
       bool file_loaded = LoadFile(filepath.c_str(), true, &contents);
@@ -3364,9 +3723,9 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
     } else if (token_ == '{') {
       return NoError();
     } else if (IsIdent("enum")) {
-      ECHECK(ParseEnum(false, nullptr));
+      ECHECK(ParseEnum(false, nullptr, source_filename));
     } else if (IsIdent("union")) {
-      ECHECK(ParseEnum(true, nullptr));
+      ECHECK(ParseEnum(true, nullptr, source_filename));
     } else if (IsIdent("root_type")) {
       NEXT();
       auto root_type = attribute_;
@@ -3382,9 +3741,9 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
       NEXT();
       file_identifier_ = attribute_;
       EXPECT(kTokenStringConstant);
-      if (file_identifier_.length() != FlatBufferBuilder::kFileIdentifierLength)
+      if (file_identifier_.length() != flatbuffers::kFileIdentifierLength)
         return Error("file_identifier must be exactly " +
-                     NumToString(FlatBufferBuilder::kFileIdentifierLength) +
+                     NumToString(flatbuffers::kFileIdentifierLength) +
                      " characters");
       EXPECT(';');
     } else if (IsIdent("file_extension")) {
@@ -3405,11 +3764,15 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths,
       EXPECT(';');
       known_attributes_[name] = false;
     } else if (IsIdent("rpc_service")) {
-      ECHECK(ParseService());
+      ECHECK(ParseService(source_filename));
     } else {
-      ECHECK(ParseDecl());
+      ECHECK(ParseDecl(source_filename));
     }
   }
+  EXPECT(kTokenEof);
+  if (opts.warnings_as_errors && has_warning_) {
+    return Error("treating warnings as errors, failed due to above warnings");
+  }
   return NoError();
 }
 
@@ -3433,9 +3796,11 @@ CheckedError Parser::DoParseJson() {
                                                : nullptr);
     }
   }
-  // Check that JSON file doesn't contain more objects or IDL directives.
-  // Comments after JSON are allowed.
-  EXPECT(kTokenEof);
+  if (opts.require_json_eof) {
+    // Check that JSON file doesn't contain more objects or IDL directives.
+    // Comments after JSON are allowed.
+    EXPECT(kTokenEof);
+  }
   return NoError();
 }
 
@@ -3454,11 +3819,11 @@ std::set<std::string> Parser::GetIncludedFilesRecursive(
 
     // Workaround the lack of const accessor in C++98 maps.
     auto &new_files =
-        (*const_cast<std::map<std::string, std::set<std::string>> *>(
+        (*const_cast<std::map<std::string, std::set<IncludedFile>> *>(
             &files_included_per_file_))[current];
     for (auto it = new_files.begin(); it != new_files.end(); ++it) {
-      if (included_files.find(*it) == included_files.end())
-        to_process.push_back(*it);
+      if (included_files.find(it->filename) == included_files.end())
+        to_process.push_back(it->filename);
     }
   }
 
@@ -3467,16 +3832,42 @@ std::set<std::string> Parser::GetIncludedFilesRecursive(
 
 // Schema serialization functionality:
 
-template<typename T> bool compareName(const T *a, const T *b) {
-  return a->defined_namespace->GetFullyQualifiedName(a->name) <
-         b->defined_namespace->GetFullyQualifiedName(b->name);
+static flatbuffers::Offset<
+    flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
+SerializeAttributesCommon(const SymbolTable<Value> &attributes,
+                          FlatBufferBuilder *builder, const Parser &parser) {
+  std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
+  for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
+    auto it = parser.known_attributes_.find(kv->first);
+    FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
+    if (parser.opts.binary_schema_builtins || !it->second) {
+      auto key = builder->CreateString(kv->first);
+      auto val = builder->CreateString(kv->second->constant);
+      attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
+    }
+  }
+  if (attrs.size()) {
+    return builder->CreateVectorOfSortedTables(&attrs);
+  } else {
+    return 0;
+  }
 }
 
-template<typename T> void AssignIndices(const std::vector<T *> &defvec) {
-  // Pre-sort these vectors, such that we can set the correct indices for them.
-  auto vec = defvec;
-  std::sort(vec.begin(), vec.end(), compareName<T>);
-  for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
+static bool DeserializeAttributesCommon(
+    SymbolTable<Value> &attributes, Parser &parser,
+    const Vector<Offset<reflection::KeyValue>> *attrs) {
+  if (attrs == nullptr) return true;
+  for (uoffset_t i = 0; i < attrs->size(); ++i) {
+    auto kv = attrs->Get(i);
+    auto value = new Value();
+    if (kv->value()) { value->constant = kv->value()->str(); }
+    if (attributes.Add(kv->key()->str(), value)) {
+      delete value;
+      return false;
+    }
+    parser.known_attributes_[kv->key()->str()];
+  }
+  return true;
 }
 
 void Parser::Serialize() {
@@ -3484,32 +3875,63 @@ void Parser::Serialize() {
   AssignIndices(structs_.vec);
   AssignIndices(enums_.vec);
   std::vector<Offset<reflection::Object>> object_offsets;
+  std::set<std::string> files;
   for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
     auto offset = (*it)->Serialize(&builder_, *this);
     object_offsets.push_back(offset);
     (*it)->serialized_location = offset.o;
+    const std::string *file = (*it)->declaration_file;
+    if (file) files.insert(*file);
   }
   std::vector<Offset<reflection::Enum>> enum_offsets;
   for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
     auto offset = (*it)->Serialize(&builder_, *this);
     enum_offsets.push_back(offset);
-    (*it)->serialized_location = offset.o;
+    const std::string *file = (*it)->declaration_file;
+    if (file) files.insert(*file);
   }
   std::vector<Offset<reflection::Service>> service_offsets;
   for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
     auto offset = (*it)->Serialize(&builder_, *this);
     service_offsets.push_back(offset);
-    (*it)->serialized_location = offset.o;
+    const std::string *file = (*it)->declaration_file;
+    if (file) files.insert(*file);
+  }
+
+  // Create Schemafiles vector of tables.
+  flatbuffers::Offset<
+      flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>>
+      schema_files__;
+  if (!opts.project_root.empty()) {
+    std::vector<Offset<reflection::SchemaFile>> schema_files;
+    std::vector<Offset<flatbuffers::String>> included_files;
+    for (auto f = files_included_per_file_.begin();
+         f != files_included_per_file_.end(); f++) {
+      const auto filename__ = builder_.CreateSharedString(
+          RelativeToRootPath(opts.project_root, f->first));
+      for (auto i = f->second.begin(); i != f->second.end(); i++) {
+        included_files.push_back(builder_.CreateSharedString(
+            RelativeToRootPath(opts.project_root, i->filename)));
+      }
+      const auto included_files__ = builder_.CreateVector(included_files);
+      included_files.clear();
+
+      schema_files.push_back(
+          reflection::CreateSchemaFile(builder_, filename__, included_files__));
+    }
+    schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files);
   }
-  auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
-  auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets);
-  auto fiid__ = builder_.CreateString(file_identifier_);
-  auto fext__ = builder_.CreateString(file_extension_);
-  auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
-  auto schema_offset = reflection::CreateSchema(
+
+  const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets);
+  const auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets);
+  const auto fiid__ = builder_.CreateString(file_identifier_);
+  const auto fext__ = builder_.CreateString(file_extension_);
+  const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets);
+  const auto schema_offset = reflection::CreateSchema(
       builder_, objs__, enum__, fiid__, fext__,
       (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__,
-      static_cast<reflection::AdvancedFeatures>(advanced_features_));
+      static_cast<reflection::AdvancedFeatures>(advanced_features_),
+      schema_files__);
   if (opts.size_prefixed) {
     builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
   } else {
@@ -3517,32 +3939,6 @@ void Parser::Serialize() {
   }
 }
 
-static Namespace *GetNamespace(
-    const std::string &qualified_name, std::vector<Namespace *> &namespaces,
-    std::map<std::string, Namespace *> &namespaces_index) {
-  size_t dot = qualified_name.find_last_of('.');
-  std::string namespace_name = (dot != std::string::npos)
-                                   ? std::string(qualified_name.c_str(), dot)
-                                   : "";
-  Namespace *&ns = namespaces_index[namespace_name];
-
-  if (!ns) {
-    ns = new Namespace();
-    namespaces.push_back(ns);
-
-    size_t pos = 0;
-
-    for (;;) {
-      dot = qualified_name.find('.', pos);
-      if (dot == std::string::npos) { break; }
-      ns->components.push_back(qualified_name.substr(pos, dot - pos));
-      pos = dot + 1;
-    }
-  }
-
-  return ns;
-}
-
 Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
                                                 const Parser &parser) const {
   std::vector<Offset<reflection::Field>> field_offsets;
@@ -3550,16 +3946,18 @@ Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
     field_offsets.push_back((*it)->Serialize(
         builder, static_cast<uint16_t>(it - fields.vec.begin()), parser));
   }
-  auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
-  auto name__ = builder->CreateString(qualified_name);
-  auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets);
-  auto attr__ = SerializeAttributes(builder, parser);
-  auto docs__ = parser.opts.binary_schema_comments
-                    ? builder->CreateVectorOfStrings(doc_comment)
-                    : 0;
-  return reflection::CreateObject(*builder, name__, flds__, fixed,
-                                  static_cast<int>(minalign),
-                                  static_cast<int>(bytesize), attr__, docs__);
+  const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
+  const auto name__ = builder->CreateString(qualified_name);
+  const auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets);
+  const auto attr__ = SerializeAttributes(builder, parser);
+  const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
+                          ? builder->CreateVectorOfStrings(doc_comment)
+                          : 0;
+  std::string decl_file_in_project = declaration_file ? *declaration_file : "";
+  const auto file__ = builder->CreateSharedString(decl_file_in_project);
+  return reflection::CreateObject(
+      *builder, name__, flds__, fixed, static_cast<int>(minalign),
+      static_cast<int>(bytesize), attr__, docs__, file__);
 }
 
 bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) {
@@ -3580,6 +3978,14 @@ bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) {
       delete field_def;
       return false;
     }
+    if (field_def->key) {
+      if (has_key) {
+        // only one field may be set as key
+        delete field_def;
+        return false;
+      }
+      has_key = true;
+    }
     if (fixed) {
       // Recompute padding since that's currently not serialized.
       auto size = InlineSize(field_def->value.type);
@@ -3602,7 +4008,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
   auto name__ = builder->CreateString(name);
   auto type__ = value.type.Serialize(builder);
   auto attr__ = SerializeAttributes(builder, parser);
-  auto docs__ = parser.opts.binary_schema_comments
+  auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
                     ? builder->CreateVectorOfStrings(doc_comment)
                     : 0;
   double d;
@@ -3613,7 +4019,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
       IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0,
       // result may be platform-dependent if underlying is float (not double)
       IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key,
-      attr__, docs__, IsOptional());
+      attr__, docs__, IsOptional(), static_cast<uint16_t>(padding), offset64);
   // TODO: value.constant is almost always "0", we could save quite a bit of
   // space by sharing it. Same for common values of value.type.
 }
@@ -3626,10 +4032,12 @@ bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) {
   if (IsInteger(value.type.base_type)) {
     value.constant = NumToString(field->default_integer());
   } else if (IsFloat(value.type.base_type)) {
-    value.constant = FloatToString(field->default_real(), 16);
+    value.constant = FloatToString(field->default_real(), 17);
   }
   presence = FieldDef::MakeFieldPresence(field->optional(), field->required());
+  padding = field->padding();
   key = field->key();
+  offset64 = field->offset64();
   if (!DeserializeAttributes(parser, field->attributes())) return false;
   // TODO: this should probably be handled by a separate attribute
   if (attributes.Lookup("flexbuffer")) {
@@ -3654,7 +4062,7 @@ Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder,
                                                const Parser &parser) const {
   auto name__ = builder->CreateString(name);
   auto attr__ = SerializeAttributes(builder, parser);
-  auto docs__ = parser.opts.binary_schema_comments
+  auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
                     ? builder->CreateVectorOfStrings(doc_comment)
                     : 0;
   return reflection::CreateRPCCall(
@@ -3678,14 +4086,17 @@ Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder,
   for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) {
     servicecall_offsets.push_back((*it)->Serialize(builder, parser));
   }
-  auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
-  auto name__ = builder->CreateString(qualified_name);
-  auto call__ = builder->CreateVector(servicecall_offsets);
-  auto attr__ = SerializeAttributes(builder, parser);
-  auto docs__ = parser.opts.binary_schema_comments
-                    ? builder->CreateVectorOfStrings(doc_comment)
-                    : 0;
-  return reflection::CreateService(*builder, name__, call__, attr__, docs__);
+  const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
+  const auto name__ = builder->CreateString(qualified_name);
+  const auto call__ = builder->CreateVector(servicecall_offsets);
+  const auto attr__ = SerializeAttributes(builder, parser);
+  const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
+                          ? builder->CreateVectorOfStrings(doc_comment)
+                          : 0;
+  std::string decl_file_in_project = declaration_file ? *declaration_file : "";
+  const auto file__ = builder->CreateSharedString(decl_file_in_project);
+  return reflection::CreateService(*builder, name__, call__, attr__, docs__,
+                                   file__);
 }
 
 bool ServiceDef::Deserialize(Parser &parser,
@@ -3712,16 +4123,18 @@ Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
   for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
     enumval_offsets.push_back((*it)->Serialize(builder, parser));
   }
-  auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
-  auto name__ = builder->CreateString(qualified_name);
-  auto vals__ = builder->CreateVector(enumval_offsets);
-  auto type__ = underlying_type.Serialize(builder);
-  auto attr__ = SerializeAttributes(builder, parser);
-  auto docs__ = parser.opts.binary_schema_comments
-                    ? builder->CreateVectorOfStrings(doc_comment)
-                    : 0;
+  const auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
+  const auto name__ = builder->CreateString(qualified_name);
+  const auto vals__ = builder->CreateVector(enumval_offsets);
+  const auto type__ = underlying_type.Serialize(builder);
+  const auto attr__ = SerializeAttributes(builder, parser);
+  const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
+                          ? builder->CreateVectorOfStrings(doc_comment)
+                          : 0;
+  std::string decl_file_in_project = declaration_file ? *declaration_file : "";
+  const auto file__ = builder->CreateSharedString(decl_file_in_project);
   return reflection::CreateEnum(*builder, name__, vals__, is_union, type__,
-                                attr__, docs__);
+                                attr__, docs__, file__);
 }
 
 bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) {
@@ -3743,34 +4156,52 @@ bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) {
   return true;
 }
 
+flatbuffers::Offset<
+    flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
+EnumVal::SerializeAttributes(FlatBufferBuilder *builder,
+                             const Parser &parser) const {
+  return SerializeAttributesCommon(attributes, builder, parser);
+}
+
+bool EnumVal::DeserializeAttributes(
+    Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
+  return DeserializeAttributesCommon(attributes, parser, attrs);
+}
+
 Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
                                                const Parser &parser) const {
-  auto name__ = builder->CreateString(name);
-  auto type__ = union_type.Serialize(builder);
-  auto docs__ = parser.opts.binary_schema_comments
-                    ? builder->CreateVectorOfStrings(doc_comment)
-                    : 0;
-  return reflection::CreateEnumVal(
-      *builder, name__, value,
-      union_type.struct_def ? union_type.struct_def->serialized_location : 0,
-      type__, docs__);
+  const auto name__ = builder->CreateString(name);
+  const auto type__ = union_type.Serialize(builder);
+  const auto attr__ = SerializeAttributes(builder, parser);
+  const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty()
+                          ? builder->CreateVectorOfStrings(doc_comment)
+                          : 0;
+  return reflection::CreateEnumVal(*builder, name__, value, type__, docs__,
+                                   attr__);
 }
 
-bool EnumVal::Deserialize(const Parser &parser,
-                          const reflection::EnumVal *val) {
+bool EnumVal::Deserialize(Parser &parser, const reflection::EnumVal *val) {
   name = val->name()->str();
   value = val->value();
   if (!union_type.Deserialize(parser, val->union_type())) return false;
+  if (!DeserializeAttributes(parser, val->attributes())) return false;
   DeserializeDoc(doc_comment, val->documentation());
   return true;
 }
 
 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
+  size_t element_size = SizeOf(element);
+  if (base_type == BASE_TYPE_VECTOR && element == BASE_TYPE_STRUCT &&
+      struct_def->bytesize != 0) {
+    // struct_def->bytesize==0 means struct is table
+    element_size = struct_def->bytesize;
+  }
   return reflection::CreateType(
       *builder, static_cast<reflection::BaseType>(base_type),
       static_cast<reflection::BaseType>(element),
       struct_def ? struct_def->index : (enum_def ? enum_def->index : -1),
-      fixed_length);
+      fixed_length, static_cast<uint32_t>(SizeOf(base_type)),
+      static_cast<uint32_t>(element_size));
 }
 
 bool Type::Deserialize(const Parser &parser, const reflection::Type *type) {
@@ -3804,37 +4235,12 @@ flatbuffers::Offset<
     flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
 Definition::SerializeAttributes(FlatBufferBuilder *builder,
                                 const Parser &parser) const {
-  std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
-  for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
-    auto it = parser.known_attributes_.find(kv->first);
-    FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
-    if (parser.opts.binary_schema_builtins || !it->second) {
-      auto key = builder->CreateString(kv->first);
-      auto val = builder->CreateString(kv->second->constant);
-      attrs.push_back(reflection::CreateKeyValue(*builder, key, val));
-    }
-  }
-  if (attrs.size()) {
-    return builder->CreateVectorOfSortedTables(&attrs);
-  } else {
-    return 0;
-  }
+  return SerializeAttributesCommon(attributes, builder, parser);
 }
 
 bool Definition::DeserializeAttributes(
     Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) {
-  if (attrs == nullptr) return true;
-  for (uoffset_t i = 0; i < attrs->size(); ++i) {
-    auto kv = attrs->Get(i);
-    auto value = new Value();
-    if (kv->value()) { value->constant = kv->value()->str(); }
-    if (attributes.Add(kv->key()->str(), value)) {
-      delete value;
-      return false;
-    }
-    parser.known_attributes_[kv->key()->str()];
-  }
-  return true;
+  return DeserializeAttributesCommon(attributes, parser, attrs);
 }
 
 /************************************************************************/
@@ -3927,6 +4333,18 @@ bool Parser::Deserialize(const reflection::Schema *schema) {
     }
   }
   advanced_features_ = schema->advanced_features();
+
+  if (schema->fbs_files())
+    for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end();
+         ++s) {
+      for (auto f = s->included_filenames()->begin();
+           f != s->included_filenames()->end(); ++f) {
+        IncludedFile included_file;
+        included_file.filename = f->str();
+        files_included_per_file_[s->filename()->str()].insert(included_file);
+      }
+    }
+
   return true;
 }
 
@@ -3937,17 +4355,25 @@ std::string Parser::ConformTo(const Parser &base) {
         struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name);
     auto struct_def_base = base.LookupStruct(qualified_name);
     if (!struct_def_base) continue;
+    std::set<FieldDef *> renamed_fields;
     for (auto fit = struct_def.fields.vec.begin();
          fit != struct_def.fields.vec.end(); ++fit) {
       auto &field = **fit;
       auto field_base = struct_def_base->fields.Lookup(field.name);
+      const auto qualified_field_name = qualified_name + "." + field.name;
       if (field_base) {
-        if (field.value.offset != field_base->value.offset)
-          return "offsets differ for field: " + field.name;
-        if (field.value.constant != field_base->value.constant)
-          return "defaults differ for field: " + field.name;
-        if (!EqualByName(field.value.type, field_base->value.type))
-          return "types differ for field: " + field.name;
+        if (field.value.offset != field_base->value.offset) {
+          return "offsets differ for field: " + qualified_field_name;
+        }
+        if (field.value.constant != field_base->value.constant) {
+          return "defaults differ for field: " + qualified_field_name;
+        }
+        if (!EqualByName(field.value.type, field_base->value.type)) {
+          return "types differ for field: " + qualified_field_name;
+        }
+        if (field.offset64 != field_base->offset64) {
+          return "offset types differ for field: " + qualified_field_name;
+        }
       } else {
         // Doesn't have to exist, deleting fields is fine.
         // But we should check if there is a field that has the same offset
@@ -3956,14 +4382,33 @@ std::string Parser::ConformTo(const Parser &base) {
              fbit != struct_def_base->fields.vec.end(); ++fbit) {
           field_base = *fbit;
           if (field.value.offset == field_base->value.offset) {
-            if (!EqualByName(field.value.type, field_base->value.type))
-              return "field renamed to different type: " + field.name;
+            renamed_fields.insert(field_base);
+            if (!EqualByName(field.value.type, field_base->value.type)) {
+              const auto qualified_field_base =
+                  qualified_name + "." + field_base->name;
+              return "field renamed to different type: " +
+                     qualified_field_name + " (renamed from " +
+                     qualified_field_base + ")";
+            }
             break;
           }
         }
       }
     }
+    // deletion of trailing fields are not allowed
+    for (auto fit = struct_def_base->fields.vec.begin();
+         fit != struct_def_base->fields.vec.end(); ++fit) {
+      auto &field_base = **fit;
+      // not a renamed field
+      if (renamed_fields.find(&field_base) == renamed_fields.end()) {
+        auto field = struct_def.fields.Lookup(field_base.name);
+        if (!field) {
+          return "field deleted: " + qualified_name + "." + field_base.name;
+        }
+      }
+    }
   }
+
   for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) {
     auto &enum_def = **eit;
     auto qualified_name =